Mercurial > dive4elements > river

annotate backend/src/main/java/org/dive4elements/river/importer/common/AbstractParser.java @ 8971:50416a0df385

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
author mschaefer
date Tue, 03 Apr 2018 10:18:30 +0200
parents
children 2693bfaf503d
rev   line source
8971
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
1 /* Copyright (C) 2017 by Bundesanstalt für Gewässerkunde
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
2 * Software engineering by
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
3 * Björnsen Beratende Ingenieure GmbH
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
4 * Dr. Schumacher Ingenieurbüro für Wasser und Umwelt
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
5 *
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
6 * This file is Free Software under the GNU AGPL (>=v3)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
7 * and comes with ABSOLUTELY NO WARRANTY! Check out the
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
8 * documentation coming with Dive4Elements River for details.
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
9 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
10
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
11 package org.dive4elements.river.importer.common;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
12
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
13 import java.io.File;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
14 import java.io.FileInputStream;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
15 import java.io.FilenameFilter;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
16 import java.io.IOException;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
17 import java.io.InputStreamReader;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
18 import java.io.LineNumberReader;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
19 import java.text.NumberFormat;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
20 import java.text.ParseException;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
21 import java.util.ArrayList;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
22 import java.util.List;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
23 import java.util.Locale;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
24 import java.util.TreeSet;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
25 import java.util.regex.Matcher;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
26 import java.util.regex.Pattern;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
27
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
28 import org.apache.log4j.Logger;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
29 import org.dive4elements.river.backend.utils.EpsilonComparator;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
30 import org.dive4elements.river.importer.ImportRiver;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
31
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
32 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
33 * Abstract base class for a parser of one FLYS csv data file.<br />
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
34 * The {@link parse} method creates a SERIES object for the meta data
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
35 * and a list of KMLINE objects for the km value lines read from the file.<br />
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
36 * The {@link store} method gets or creates the corresponding database objects
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
37 * by the hibernate binding classes DB_SERIES and DB_KMTUPLE,
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
38 * and updates or inserts them in the database.
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
39 * DB_SERIES has a one-to-many relationship with DB_KMTUPLE.<br />
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
40 * <br />
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
41 * The structure of the file is as follows:<br />
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
42 * <ul>
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
43 * <li>one or more comment lines (#) with the meta info of the data series</li>
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
44 * <li>the comment line with the column titles of values table, starting with the km column</li>
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
45 * <li>the rows of the values table, each one on its own line</li>
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
46 * </ul>
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
47 *
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
48 * @author Matthias Schäfer
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
49 *
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
50 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
51 public abstract class AbstractParser<DB_SERIES, DB_KMTUPLE, KMLINE extends AbstractKmLineImport<DB_SERIES, DB_KMTUPLE>, HEADER extends AbstractSeriesImport<DB_SERIES, DB_KMTUPLE, KMLINE>> implements ImportParser {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
52
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
53 /***** FIELDS *****/
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
54
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
55 public static final String ENCODING = "ISO-8859-1";
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
56
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
57 protected static final Locale DEFAULT_LOCALE = Locale.GERMAN;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
58
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
59 public static final String START_META_CHAR = "#";
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
60
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
61 protected static final String SEPARATOR_CHAR = ";";
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
62
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
63 protected static final Pattern META_RIVERNAME = Pattern.compile("^#\\s*((Gew.sser)|(Gewaesser)):\\s*(\\S[^;]*).*", Pattern.CASE_INSENSITIVE);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
64
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
65 protected static final Pattern META_KMRANGE_INFO = Pattern.compile("^#\\s*Strecke:\\s*(\\S[^;]*).*", Pattern.CASE_INSENSITIVE);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
66
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
67 protected static final Pattern META_COMMENTS = Pattern.compile("^#\\s*weitere Bemerkungen:\\s*(\\S[^;]*).*", Pattern.CASE_INSENSITIVE);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
68
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
69 private static final Pattern META_COLUMNTITLES = Pattern.compile("^#*\\s*Fluss.km\\s*;.+", Pattern.CASE_INSENSITIVE);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
70
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
71 private static final Pattern META_SUBGROUP = Pattern.compile("^##.*", Pattern.CASE_INSENSITIVE);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
72
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
73 private static NumberFormat numberFormat = NumberFormat.getInstance(Locale.ROOT);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
74
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
75 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
76 * Path of the file or directory to import from
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
77 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
78 protected final File importPath;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
79
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
80 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
81 * Part of {@link importPath} without the river root dir
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
82 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
83 protected final File rootRelativePath;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
84
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
85 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
86 * River for which the import runs
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
87 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
88 protected final ImportRiver river;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
89
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
90 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
91 * Reader during parse
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
92 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
93 protected LineNumberReader in;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
94
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
95 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
96 * Last line read from in
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
97 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
98 protected String currentLine;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
99
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
100 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
101 * State of the header lines parse loop
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
102 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
103 protected ParsingState headerParsingState;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
104
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
105 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
106 * Series header of the stations table, with the imported meta info.
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
107 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
108 protected HEADER seriesHeader;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
109
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
110 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
111 * List of meta info Pattern matched during {@link handleMetaLine}
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
112 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
113 protected final List<Pattern> metaPatternsMatched;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
114
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
115 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
116 * Column titles of the stations table, starting with the km column.
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
117 * All strings have been trimmed.
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
118 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
119 protected final List<String> columnTitles;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
120
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
121 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
122 * List of the km value tuples imported, no duplicate km
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
123 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
124 protected final List<KMLINE> values;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
125
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
126 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
127 * Ordered list with the imported km to check for duplicates.
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
128 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
129 protected final TreeSet<Double> kmExists;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
130
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
131
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
132 /***** CONSTRUCTORS *****/
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
133
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
134 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
135 * Constructs a parser for an import file
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
136 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
137 public AbstractParser(final File importPath, final File rootRelativePath, final ImportRiver river) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
138 this.importPath = importPath;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
139 this.rootRelativePath = rootRelativePath;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
140 this.river = river;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
141 this.metaPatternsMatched = new ArrayList<>();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
142 this.kmExists = new TreeSet<>(EpsilonComparator.CMP);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
143 this.columnTitles = new ArrayList<>();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
144 this.values = new ArrayList<>();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
145 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
146
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
147
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
148 /***** METHODS *****/
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
149
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
150 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
151 * Lists all files from a directory having a type extension (starting with dot)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
152 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
153 protected static List<File> listFiles(final File importDir, final String extension) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
154 final File[] files = importDir.listFiles(new FilenameFilter() {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
155 @Override
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
156 public boolean accept(final File dir, final String name) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
157 return name.toLowerCase().endsWith(extension);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
158 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
159 });
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
160 final List<File> fl = new ArrayList<>();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
161 if (files != null)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
162 for (final File file : files)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
163 fl.add(file);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
164 return fl;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
165 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
166
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
167 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
168 * Parses a file and adds series and values to the parser's collection
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
169 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
170 @Override
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
171 public void parse() throws IOException {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
172 logStartInfo();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
173 this.seriesHeader = createSeriesImport(this.importPath.getName().replaceAll("\\.csv", ""));
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
174 this.metaPatternsMatched.clear();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
175 this.kmExists.clear();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
176 this.headerParsingState = ParsingState.CONTINUE;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
177 try {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
178 try {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
179 this.in = new LineNumberReader(new InputStreamReader(new FileInputStream(this.importPath), ENCODING));
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
180 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
181 catch (final Exception e) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
182 logError("Could not open (" + e.getMessage() + ")");
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
183 this.headerParsingState = ParsingState.STOP;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
184 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
185 this.currentLine = null;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
186 while (this.headerParsingState != ParsingState.STOP) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
187 this.currentLine = this.in.readLine();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
188 if (this.currentLine == null)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
189 break;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
190 this.currentLine = this.currentLine.trim();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
191 if (this.currentLine.isEmpty())
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
192 continue;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
193 if (this.headerParsingState == ParsingState.CONTINUE)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
194 handleMetaLine();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
195 else
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
196 handleDataLine();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
197 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
198 if (this.headerParsingState != ParsingState.STOP)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
199 getLog().info("Number of values found: " + this.seriesHeader.getValueCount());
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
200 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
201 finally {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
202 if (this.in != null) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
203 this.in.close();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
204 this.in = null;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
205 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
206 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
207 if (this.headerParsingState == ParsingState.STOP)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
208 logError("Parsing of the file stopped due to a severe error");
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
209 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
210
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
211 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
212 * Writes the parse start info to the log
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
213 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
214 protected void logStartInfo() {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
215 getLog().info("Start parsing:;'" + this.rootRelativePath + "'");
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
216 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
217
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
218 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
219 * Stores the parsed series and values in the database
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
220 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
221 @Override
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
222 public void store() {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
223 if (this.headerParsingState != ParsingState.STOP) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
224 this.seriesHeader.store(this.river.getPeer());
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
225 final String counts = String.format("parse=%d, insert=%d, update/ignore=%d", this.seriesHeader.getValueCount(),
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
226 this.seriesHeader.getValueStoreCount(StoreMode.INSERT), this.seriesHeader.getValueStoreCount(StoreMode.UPDATE));
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
227 if (this.seriesHeader.getValueCount() > this.seriesHeader.getValueStoreCount(StoreMode.INSERT))
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
228 logWarning("Number of value inserts less than number parsed: " + counts);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
229 else
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
230 getLog().info("Number of values records: " + counts);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
231 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
232 else
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
233 logWarning("Severe parsing errors, not storing series '" + this.seriesHeader.getFilename() + "'");
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
234 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
235
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
236 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
237 * Strips separator chars from a meta info text, and trims leading and trailing whitespace
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
238 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
239 public static String parseMetaInfo(final String text) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
240 return text.replace(SEPARATOR_CHAR, "").trim();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
241 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
242
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
243 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
244 * Parses a number string with dot or comma as decimal char, and returning null in case of an error
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
245 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
246 public static Number parseDoubleWithNull(final String text) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
247 try {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
248 return parseDouble(text);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
249 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
250 catch (final Exception e) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
251 return null;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
252 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
253 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
254
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
255 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
256 * Parses a number string with dot or comma as decimal char
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
257 *
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
258 * @throws ParseException
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
259 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
260 public static Number parseDouble(final String text) throws ParseException {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
261 return numberFormat.parse(text.replace(',', '.'));
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
262 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
263
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
264 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
265 * Gets the class's logger
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
266 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
267 protected abstract Logger getLog();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
268
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
269 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
270 * Logs an error message, appending the relative file path
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
271 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
272 protected void logError(final String message) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
273 getLog().error(message + ";" + this.rootRelativePath);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
274 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
275
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
276 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
277 * Logs a warning message, appending the relative file path
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
278 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
279 protected void logWarning(final String message) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
280 getLog().warn(message + ";" + this.rootRelativePath);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
281 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
282
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
283 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
284 * Creates a new series import object
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
285 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
286 protected abstract HEADER createSeriesImport(final String filename);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
287
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
288 protected void handleMetaLine() {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
289 if (META_SUBGROUP.matcher(this.currentLine).matches())
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
290 return;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
291 else if (handleMetaRivername())
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
292 return;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
293 else if (handleMetaKmrange_info())
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
294 return;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
295 else if (handleMetaComment())
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
296 return;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
297 else if (handleMetaOther())
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
298 return;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
299 else if (handleMetaColumnTitles()) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
300 if (this.headerParsingState != ParsingState.STOP)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
301 this.headerParsingState = ParsingState.DONE;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
302 return;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
303 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
304 else {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
305 if (this.currentLine.startsWith(START_META_CHAR)) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
306 if (this.headerParsingState != ParsingState.IGNORE)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
307 logWarning("Not matching any known meta type in line " + this.in.getLineNumber() + ", ignored");
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
308 else
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
309 this.headerParsingState = ParsingState.CONTINUE;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
310 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
311 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
312 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
313
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
314 private boolean handleMetaRivername() {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
315 if (META_RIVERNAME.matcher(this.currentLine).matches()) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
316 this.metaPatternsMatched.add(META_RIVERNAME);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
317 return true;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
318 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
319 else
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
320 return false;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
321 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
322
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
323 private boolean handleMetaKmrange_info() {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
324 final Matcher m = META_KMRANGE_INFO.matcher(this.currentLine);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
325 if (m.matches()) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
326 this.metaPatternsMatched.add(META_KMRANGE_INFO);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
327 this.seriesHeader.setKmrange_info(parseMetaInfo(m.group(1)));
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
328 return true;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
329 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
330 return false;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
331 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
332
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
333 private boolean handleMetaComment() {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
334 final Matcher m = META_COMMENTS.matcher(this.currentLine);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
335 if (m.matches()) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
336 this.metaPatternsMatched.add(META_COMMENTS);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
337 this.seriesHeader.setComment(parseMetaInfo(m.group(1)));
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
338 return true;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
339 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
340 return false;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
341 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
342
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
343 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
344 * Parses currentLine for non-default meta info
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
345 *
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
346 * @return Whether the line has been handled
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
347 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
348 protected boolean handleMetaOther() {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
349 return false;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
350 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
351
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
352 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
353 * Parses a header line for the km table column header line
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
354 *
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
355 * @return Whether the line has been handled and we are ready for reading the km values lines
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
356 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
357 protected boolean handleMetaColumnTitles() {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
358 if (META_COLUMNTITLES.matcher(this.currentLine).matches()) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
359 this.metaPatternsMatched.add(META_COLUMNTITLES);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
360 this.columnTitles.clear();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
361 final String[] titles = this.currentLine.split(SEPARATOR_CHAR, 0);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
362 for (int i = 0; i <= titles.length - 1; i++)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
363 this.columnTitles.add(titles[i].trim());
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
364 return true;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
365 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
366 return false;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
367 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
368
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
369 private void handleDataLine() {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
370 final String[] values = this.currentLine.split(SEPARATOR_CHAR, 0);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
371 // Skip import line without data or only km
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
372 if (values.length < 2)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
373 return;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
374 Double km;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
375 try {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
376 km = Double.valueOf(parseDouble(values[0]).doubleValue());
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
377 if (kmMustBeUnique()) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
378 if (this.kmExists.contains(km)) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
379 logWarning("Ignoring duplicate station '" + values[0] + "' in line " + this.in.getLineNumber());
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
380 return;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
381 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
382 this.kmExists.add(km);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
383 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
384 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
385 catch (final Exception e) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
386 logError("Not parseable km in line " + this.in.getLineNumber() + ": " + e.getMessage());
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
387 return;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
388 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
389 final KMLINE value = createKmLineImport(km, values);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
390 if (value != null)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
391 this.seriesHeader.addValue(value);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
392 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
393
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
394 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
395 * Whether {@link handleDataLine} shall check for and reject km duplicates
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
396 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
397 protected boolean kmMustBeUnique() {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
398 return true;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
399 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
400
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
401 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
402 * Creates a value import item with the km and other fields of the current line;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
403 * the km has been validated
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
404 *
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
405 * @return value item, or null if parse error
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
406 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
407 protected abstract KMLINE createKmLineImport(final Double km, final String[] values);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
408 }
http://dive4elements.wald.intevation.org