Mercurial > dive4elements > river

annotate backend/src/main/java/org/dive4elements/river/importer/common/AbstractParser.java @ 8989:2693bfaf503d

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Fixed several BigDecimal(double) creations by BigDecimal(String) parsing to avoid unnecessary decimal digits
author mschaefer
date Mon, 09 Apr 2018 09:07:00 +0200
parents 50416a0df385
children c43d8c1a4455
rev   line source
8971
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
1 /* Copyright (C) 2017 by Bundesanstalt für Gewässerkunde
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
2 * Software engineering by
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
3 * Björnsen Beratende Ingenieure GmbH
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
4 * Dr. Schumacher Ingenieurbüro für Wasser und Umwelt
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
5 *
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
6 * This file is Free Software under the GNU AGPL (>=v3)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
7 * and comes with ABSOLUTELY NO WARRANTY! Check out the
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
8 * documentation coming with Dive4Elements River for details.
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
9 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
10
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
11 package org.dive4elements.river.importer.common;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
12
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
13 import java.io.File;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
14 import java.io.FileInputStream;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
15 import java.io.FilenameFilter;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
16 import java.io.IOException;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
17 import java.io.InputStreamReader;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
18 import java.io.LineNumberReader;
8989
2693bfaf503d Fixed several BigDecimal(double) creations by BigDecimal(String) parsing to avoid unnecessary decimal digits
mschaefer
parents: 8971
diff changeset
19 import java.math.BigDecimal;
8971
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
20 import java.text.NumberFormat;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
21 import java.text.ParseException;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
22 import java.util.ArrayList;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
23 import java.util.List;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
24 import java.util.Locale;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
25 import java.util.TreeSet;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
26 import java.util.regex.Matcher;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
27 import java.util.regex.Pattern;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
28
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
29 import org.apache.log4j.Logger;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
30 import org.dive4elements.river.backend.utils.EpsilonComparator;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
31 import org.dive4elements.river.importer.ImportRiver;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
32
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
33 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
34 * Abstract base class for a parser of one FLYS csv data file.<br />
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
35 * The {@link parse} method creates a SERIES object for the meta data
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
36 * and a list of KMLINE objects for the km value lines read from the file.<br />
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
37 * The {@link store} method gets or creates the corresponding database objects
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
38 * by the hibernate binding classes DB_SERIES and DB_KMTUPLE,
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
39 * and updates or inserts them in the database.
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
40 * DB_SERIES has a one-to-many relationship with DB_KMTUPLE.<br />
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
41 * <br />
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
42 * The structure of the file is as follows:<br />
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
43 * <ul>
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
44 * <li>one or more comment lines (#) with the meta info of the data series</li>
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
45 * <li>the comment line with the column titles of values table, starting with the km column</li>
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
46 * <li>the rows of the values table, each one on its own line</li>
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
47 * </ul>
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
48 *
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
49 * @author Matthias Schäfer
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
50 *
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
51 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
52 public abstract class AbstractParser<DB_SERIES, DB_KMTUPLE, KMLINE extends AbstractKmLineImport<DB_SERIES, DB_KMTUPLE>, HEADER extends AbstractSeriesImport<DB_SERIES, DB_KMTUPLE, KMLINE>> implements ImportParser {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
53
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
54 /***** FIELDS *****/
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
55
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
56 public static final String ENCODING = "ISO-8859-1";
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
57
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
58 protected static final Locale DEFAULT_LOCALE = Locale.GERMAN;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
59
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
60 public static final String START_META_CHAR = "#";
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
61
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
62 protected static final String SEPARATOR_CHAR = ";";
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
63
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
64 protected static final Pattern META_RIVERNAME = Pattern.compile("^#\\s*((Gew.sser)|(Gewaesser)):\\s*(\\S[^;]*).*", Pattern.CASE_INSENSITIVE);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
65
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
66 protected static final Pattern META_KMRANGE_INFO = Pattern.compile("^#\\s*Strecke:\\s*(\\S[^;]*).*", Pattern.CASE_INSENSITIVE);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
67
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
68 protected static final Pattern META_COMMENTS = Pattern.compile("^#\\s*weitere Bemerkungen:\\s*(\\S[^;]*).*", Pattern.CASE_INSENSITIVE);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
69
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
70 private static final Pattern META_COLUMNTITLES = Pattern.compile("^#*\\s*Fluss.km\\s*;.+", Pattern.CASE_INSENSITIVE);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
71
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
72 private static final Pattern META_SUBGROUP = Pattern.compile("^##.*", Pattern.CASE_INSENSITIVE);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
73
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
74 private static NumberFormat numberFormat = NumberFormat.getInstance(Locale.ROOT);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
75
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
76 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
77 * Path of the file or directory to import from
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
78 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
79 protected final File importPath;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
80
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
81 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
82 * Part of {@link importPath} without the river root dir
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
83 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
84 protected final File rootRelativePath;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
85
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
86 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
87 * River for which the import runs
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
88 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
89 protected final ImportRiver river;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
90
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
91 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
92 * Reader during parse
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
93 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
94 protected LineNumberReader in;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
95
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
96 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
97 * Last line read from in
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
98 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
99 protected String currentLine;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
100
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
101 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
102 * State of the header lines parse loop
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
103 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
104 protected ParsingState headerParsingState;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
105
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
106 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
107 * Series header of the stations table, with the imported meta info.
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
108 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
109 protected HEADER seriesHeader;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
110
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
111 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
112 * List of meta info Pattern matched during {@link handleMetaLine}
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
113 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
114 protected final List<Pattern> metaPatternsMatched;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
115
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
116 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
117 * Column titles of the stations table, starting with the km column.
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
118 * All strings have been trimmed.
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
119 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
120 protected final List<String> columnTitles;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
121
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
122 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
123 * List of the km value tuples imported, no duplicate km
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
124 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
125 protected final List<KMLINE> values;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
126
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
127 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
128 * Ordered list with the imported km to check for duplicates.
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
129 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
130 protected final TreeSet<Double> kmExists;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
131
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
132
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
133 /***** CONSTRUCTORS *****/
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
134
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
135 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
136 * Constructs a parser for an import file
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
137 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
138 public AbstractParser(final File importPath, final File rootRelativePath, final ImportRiver river) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
139 this.importPath = importPath;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
140 this.rootRelativePath = rootRelativePath;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
141 this.river = river;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
142 this.metaPatternsMatched = new ArrayList<>();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
143 this.kmExists = new TreeSet<>(EpsilonComparator.CMP);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
144 this.columnTitles = new ArrayList<>();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
145 this.values = new ArrayList<>();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
146 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
147
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
148
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
149 /***** METHODS *****/
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
150
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
151 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
152 * Lists all files from a directory having a type extension (starting with dot)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
153 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
154 protected static List<File> listFiles(final File importDir, final String extension) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
155 final File[] files = importDir.listFiles(new FilenameFilter() {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
156 @Override
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
157 public boolean accept(final File dir, final String name) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
158 return name.toLowerCase().endsWith(extension);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
159 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
160 });
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
161 final List<File> fl = new ArrayList<>();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
162 if (files != null)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
163 for (final File file : files)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
164 fl.add(file);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
165 return fl;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
166 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
167
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
168 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
169 * Parses a file and adds series and values to the parser's collection
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
170 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
171 @Override
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
172 public void parse() throws IOException {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
173 logStartInfo();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
174 this.seriesHeader = createSeriesImport(this.importPath.getName().replaceAll("\\.csv", ""));
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
175 this.metaPatternsMatched.clear();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
176 this.kmExists.clear();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
177 this.headerParsingState = ParsingState.CONTINUE;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
178 try {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
179 try {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
180 this.in = new LineNumberReader(new InputStreamReader(new FileInputStream(this.importPath), ENCODING));
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
181 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
182 catch (final Exception e) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
183 logError("Could not open (" + e.getMessage() + ")");
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
184 this.headerParsingState = ParsingState.STOP;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
185 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
186 this.currentLine = null;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
187 while (this.headerParsingState != ParsingState.STOP) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
188 this.currentLine = this.in.readLine();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
189 if (this.currentLine == null)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
190 break;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
191 this.currentLine = this.currentLine.trim();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
192 if (this.currentLine.isEmpty())
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
193 continue;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
194 if (this.headerParsingState == ParsingState.CONTINUE)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
195 handleMetaLine();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
196 else
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
197 handleDataLine();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
198 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
199 if (this.headerParsingState != ParsingState.STOP)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
200 getLog().info("Number of values found: " + this.seriesHeader.getValueCount());
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
201 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
202 finally {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
203 if (this.in != null) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
204 this.in.close();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
205 this.in = null;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
206 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
207 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
208 if (this.headerParsingState == ParsingState.STOP)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
209 logError("Parsing of the file stopped due to a severe error");
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
210 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
211
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
212 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
213 * Writes the parse start info to the log
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
214 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
215 protected void logStartInfo() {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
216 getLog().info("Start parsing:;'" + this.rootRelativePath + "'");
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
217 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
218
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
219 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
220 * Stores the parsed series and values in the database
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
221 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
222 @Override
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
223 public void store() {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
224 if (this.headerParsingState != ParsingState.STOP) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
225 this.seriesHeader.store(this.river.getPeer());
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
226 final String counts = String.format("parse=%d, insert=%d, update/ignore=%d", this.seriesHeader.getValueCount(),
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
227 this.seriesHeader.getValueStoreCount(StoreMode.INSERT), this.seriesHeader.getValueStoreCount(StoreMode.UPDATE));
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
228 if (this.seriesHeader.getValueCount() > this.seriesHeader.getValueStoreCount(StoreMode.INSERT))
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
229 logWarning("Number of value inserts less than number parsed: " + counts);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
230 else
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
231 getLog().info("Number of values records: " + counts);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
232 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
233 else
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
234 logWarning("Severe parsing errors, not storing series '" + this.seriesHeader.getFilename() + "'");
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
235 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
236
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
237 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
238 * Strips separator chars from a meta info text, and trims leading and trailing whitespace
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
239 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
240 public static String parseMetaInfo(final String text) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
241 return text.replace(SEPARATOR_CHAR, "").trim();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
242 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
243
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
244 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
245 * Parses a number string with dot or comma as decimal char, and returning null in case of an error
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
246 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
247 public static Number parseDoubleWithNull(final String text) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
248 try {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
249 return parseDouble(text);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
250 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
251 catch (final Exception e) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
252 return null;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
253 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
254 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
255
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
256 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
257 * Parses a number string with dot or comma as decimal char
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
258 *
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
259 * @throws ParseException
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
260 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
261 public static Number parseDouble(final String text) throws ParseException {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
262 return numberFormat.parse(text.replace(',', '.'));
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
263 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
264
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
265 /**
8989
2693bfaf503d Fixed several BigDecimal(double) creations by BigDecimal(String) parsing to avoid unnecessary decimal digits
mschaefer
parents: 8971
diff changeset
266 * Parses a number string as a BigDecimal, replacing a comma with a dot first
2693bfaf503d Fixed several BigDecimal(double) creations by BigDecimal(String) parsing to avoid unnecessary decimal digits
mschaefer
parents: 8971
diff changeset
267 */
2693bfaf503d Fixed several BigDecimal(double) creations by BigDecimal(String) parsing to avoid unnecessary decimal digits
mschaefer
parents: 8971
diff changeset
268 public static BigDecimal parseDecimal(final String text) throws NumberFormatException {
2693bfaf503d Fixed several BigDecimal(double) creations by BigDecimal(String) parsing to avoid unnecessary decimal digits
mschaefer
parents: 8971
diff changeset
269 return new BigDecimal(text.replace(',', '.'));
2693bfaf503d Fixed several BigDecimal(double) creations by BigDecimal(String) parsing to avoid unnecessary decimal digits
mschaefer
parents: 8971
diff changeset
270 }
2693bfaf503d Fixed several BigDecimal(double) creations by BigDecimal(String) parsing to avoid unnecessary decimal digits
mschaefer
parents: 8971
diff changeset
271
2693bfaf503d Fixed several BigDecimal(double) creations by BigDecimal(String) parsing to avoid unnecessary decimal digits
mschaefer
parents: 8971
diff changeset
272 /**
8971
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
273 * Gets the class's logger
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
274 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
275 protected abstract Logger getLog();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
276
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
277 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
278 * Logs an error message, appending the relative file path
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
279 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
280 protected void logError(final String message) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
281 getLog().error(message + ";" + this.rootRelativePath);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
282 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
283
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
284 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
285 * Logs a warning message, appending the relative file path
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
286 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
287 protected void logWarning(final String message) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
288 getLog().warn(message + ";" + this.rootRelativePath);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
289 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
290
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
291 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
292 * Creates a new series import object
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
293 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
294 protected abstract HEADER createSeriesImport(final String filename);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
295
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
296 protected void handleMetaLine() {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
297 if (META_SUBGROUP.matcher(this.currentLine).matches())
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
298 return;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
299 else if (handleMetaRivername())
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
300 return;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
301 else if (handleMetaKmrange_info())
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
302 return;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
303 else if (handleMetaComment())
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
304 return;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
305 else if (handleMetaOther())
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
306 return;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
307 else if (handleMetaColumnTitles()) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
308 if (this.headerParsingState != ParsingState.STOP)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
309 this.headerParsingState = ParsingState.DONE;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
310 return;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
311 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
312 else {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
313 if (this.currentLine.startsWith(START_META_CHAR)) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
314 if (this.headerParsingState != ParsingState.IGNORE)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
315 logWarning("Not matching any known meta type in line " + this.in.getLineNumber() + ", ignored");
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
316 else
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
317 this.headerParsingState = ParsingState.CONTINUE;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
318 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
319 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
320 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
321
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
322 private boolean handleMetaRivername() {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
323 if (META_RIVERNAME.matcher(this.currentLine).matches()) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
324 this.metaPatternsMatched.add(META_RIVERNAME);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
325 return true;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
326 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
327 else
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
328 return false;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
329 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
330
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
331 private boolean handleMetaKmrange_info() {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
332 final Matcher m = META_KMRANGE_INFO.matcher(this.currentLine);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
333 if (m.matches()) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
334 this.metaPatternsMatched.add(META_KMRANGE_INFO);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
335 this.seriesHeader.setKmrange_info(parseMetaInfo(m.group(1)));
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
336 return true;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
337 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
338 return false;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
339 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
340
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
341 private boolean handleMetaComment() {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
342 final Matcher m = META_COMMENTS.matcher(this.currentLine);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
343 if (m.matches()) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
344 this.metaPatternsMatched.add(META_COMMENTS);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
345 this.seriesHeader.setComment(parseMetaInfo(m.group(1)));
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
346 return true;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
347 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
348 return false;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
349 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
350
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
351 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
352 * Parses currentLine for non-default meta info
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
353 *
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
354 * @return Whether the line has been handled
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
355 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
356 protected boolean handleMetaOther() {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
357 return false;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
358 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
359
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
360 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
361 * Parses a header line for the km table column header line
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
362 *
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
363 * @return Whether the line has been handled and we are ready for reading the km values lines
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
364 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
365 protected boolean handleMetaColumnTitles() {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
366 if (META_COLUMNTITLES.matcher(this.currentLine).matches()) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
367 this.metaPatternsMatched.add(META_COLUMNTITLES);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
368 this.columnTitles.clear();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
369 final String[] titles = this.currentLine.split(SEPARATOR_CHAR, 0);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
370 for (int i = 0; i <= titles.length - 1; i++)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
371 this.columnTitles.add(titles[i].trim());
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
372 return true;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
373 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
374 return false;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
375 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
376
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
377 private void handleDataLine() {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
378 final String[] values = this.currentLine.split(SEPARATOR_CHAR, 0);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
379 // Skip import line without data or only km
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
380 if (values.length < 2)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
381 return;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
382 Double km;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
383 try {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
384 km = Double.valueOf(parseDouble(values[0]).doubleValue());
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
385 if (kmMustBeUnique()) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
386 if (this.kmExists.contains(km)) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
387 logWarning("Ignoring duplicate station '" + values[0] + "' in line " + this.in.getLineNumber());
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
388 return;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
389 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
390 this.kmExists.add(km);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
391 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
392 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
393 catch (final Exception e) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
394 logError("Not parseable km in line " + this.in.getLineNumber() + ": " + e.getMessage());
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
395 return;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
396 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
397 final KMLINE value = createKmLineImport(km, values);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
398 if (value != null)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
399 this.seriesHeader.addValue(value);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
400 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
401
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
402 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
403 * Whether {@link handleDataLine} shall check for and reject km duplicates
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
404 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
405 protected boolean kmMustBeUnique() {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
406 return true;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
407 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
408
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
409 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
410 * Creates a value import item with the km and other fields of the current line;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
411 * the km has been validated
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
412 *
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
413 * @return value item, or null if parse error
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
414 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
415 protected abstract KMLINE createKmLineImport(final Double km, final String[] values);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
416 }
http://dive4elements.wald.intevation.org