Mercurial > dive4elements > river

annotate backend/src/main/java/org/dive4elements/river/importer/common/AbstractParser.java @ 9038:4c5eeaff554c

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Database column "comment" renamed to "notes" (restrictions in Oracle)
author mschaefer
date Wed, 02 May 2018 06:56:21 +0200
parents abe069eb180c
children ddebd4c2fe93
rev   line source
8971
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
1 /* Copyright (C) 2017 by Bundesanstalt für Gewässerkunde
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
2 * Software engineering by
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
3 * Björnsen Beratende Ingenieure GmbH
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
4 * Dr. Schumacher Ingenieurbüro für Wasser und Umwelt
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
5 *
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
6 * This file is Free Software under the GNU AGPL (>=v3)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
7 * and comes with ABSOLUTELY NO WARRANTY! Check out the
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
8 * documentation coming with Dive4Elements River for details.
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
9 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
10
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
11 package org.dive4elements.river.importer.common;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
12
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
13 import java.io.File;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
14 import java.io.FileInputStream;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
15 import java.io.FilenameFilter;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
16 import java.io.IOException;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
17 import java.io.InputStreamReader;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
18 import java.io.LineNumberReader;
8989
2693bfaf503d Fixed several BigDecimal(double) creations by BigDecimal(String) parsing to avoid unnecessary decimal digits
mschaefer
parents: 8971
diff changeset
19 import java.math.BigDecimal;
8991
c43d8c1a4455 Parse via decimalformat instead of bigdecimal constructor (to cope with strings with chars after the actual number)
mschaefer
parents: 8989
diff changeset
20 import java.text.DecimalFormat;
8971
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
21 import java.text.NumberFormat;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
22 import java.text.ParseException;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
23 import java.util.ArrayList;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
24 import java.util.List;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
25 import java.util.Locale;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
26 import java.util.TreeSet;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
27 import java.util.regex.Matcher;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
28 import java.util.regex.Pattern;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
29
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
30 import org.apache.log4j.Logger;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
31 import org.dive4elements.river.backend.utils.EpsilonComparator;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
32 import org.dive4elements.river.importer.ImportRiver;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
33
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
34 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
35 * Abstract base class for a parser of one FLYS csv data file.<br />
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
36 * The {@link parse} method creates a SERIES object for the meta data
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
37 * and a list of KMLINE objects for the km value lines read from the file.<br />
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
38 * The {@link store} method gets or creates the corresponding database objects
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
39 * by the hibernate binding classes DB_SERIES and DB_KMTUPLE,
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
40 * and updates or inserts them in the database.
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
41 * DB_SERIES has a one-to-many relationship with DB_KMTUPLE.<br />
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
42 * <br />
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
43 * The structure of the file is as follows:<br />
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
44 * <ul>
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
45 * <li>one or more comment lines (#) with the meta info of the data series</li>
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
46 * <li>the comment line with the column titles of values table, starting with the km column</li>
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
47 * <li>the rows of the values table, each one on its own line</li>
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
48 * </ul>
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
49 *
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
50 * @author Matthias Schäfer
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
51 *
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
52 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
53 public abstract class AbstractParser<DB_SERIES, DB_KMTUPLE, KMLINE extends AbstractKmLineImport<DB_SERIES, DB_KMTUPLE>, HEADER extends AbstractSeriesImport<DB_SERIES, DB_KMTUPLE, KMLINE>> implements ImportParser {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
54
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
55 /***** FIELDS *****/
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
56
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
57 public static final String ENCODING = "ISO-8859-1";
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
58
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
59 protected static final Locale DEFAULT_LOCALE = Locale.GERMAN;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
60
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
61 public static final String START_META_CHAR = "#";
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
62
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
63 protected static final String SEPARATOR_CHAR = ";";
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
64
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
65 protected static final Pattern META_RIVERNAME = Pattern.compile("^#\\s*((Gew.sser)|(Gewaesser)):\\s*(\\S[^;]*).*", Pattern.CASE_INSENSITIVE);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
66
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
67 protected static final Pattern META_KMRANGE_INFO = Pattern.compile("^#\\s*Strecke:\\s*(\\S[^;]*).*", Pattern.CASE_INSENSITIVE);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
68
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
69 protected static final Pattern META_COMMENTS = Pattern.compile("^#\\s*weitere Bemerkungen:\\s*(\\S[^;]*).*", Pattern.CASE_INSENSITIVE);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
70
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
71 private static final Pattern META_COLUMNTITLES = Pattern.compile("^#*\\s*Fluss.km\\s*;.+", Pattern.CASE_INSENSITIVE);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
72
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
73 private static final Pattern META_SUBGROUP = Pattern.compile("^##.*", Pattern.CASE_INSENSITIVE);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
74
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
75 private static NumberFormat numberFormat = NumberFormat.getInstance(Locale.ROOT);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
76
8991
c43d8c1a4455 Parse via decimalformat instead of bigdecimal constructor (to cope with strings with chars after the actual number)
mschaefer
parents: 8989
diff changeset
77 private static DecimalFormat bigDecimalFormat;
c43d8c1a4455 Parse via decimalformat instead of bigdecimal constructor (to cope with strings with chars after the actual number)
mschaefer
parents: 8989
diff changeset
78
c43d8c1a4455 Parse via decimalformat instead of bigdecimal constructor (to cope with strings with chars after the actual number)
mschaefer
parents: 8989
diff changeset
79 static {
c43d8c1a4455 Parse via decimalformat instead of bigdecimal constructor (to cope with strings with chars after the actual number)
mschaefer
parents: 8989
diff changeset
80 bigDecimalFormat = (DecimalFormat) NumberFormat.getInstance(Locale.ROOT);
c43d8c1a4455 Parse via decimalformat instead of bigdecimal constructor (to cope with strings with chars after the actual number)
mschaefer
parents: 8989
diff changeset
81 bigDecimalFormat.setParseBigDecimal(true);
c43d8c1a4455 Parse via decimalformat instead of bigdecimal constructor (to cope with strings with chars after the actual number)
mschaefer
parents: 8989
diff changeset
82 }
c43d8c1a4455 Parse via decimalformat instead of bigdecimal constructor (to cope with strings with chars after the actual number)
mschaefer
parents: 8989
diff changeset
83
8971
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
84 /**
9012
abe069eb180c Import of vegetation zones added
mschaefer
parents: 8991
diff changeset
85 * How the km column and its content are expected
abe069eb180c Import of vegetation zones added
mschaefer
parents: 8991
diff changeset
86 */
abe069eb180c Import of vegetation zones added
mschaefer
parents: 8991
diff changeset
87 protected enum KmMode {
abe069eb180c Import of vegetation zones added
mschaefer
parents: 8991
diff changeset
88 NONE, UNIQUE, DUPLICATES
abe069eb180c Import of vegetation zones added
mschaefer
parents: 8991
diff changeset
89 }
abe069eb180c Import of vegetation zones added
mschaefer
parents: 8991
diff changeset
90
abe069eb180c Import of vegetation zones added
mschaefer
parents: 8991
diff changeset
91 /**
8971
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
92 * Path of the file or directory to import from
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
93 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
94 protected final File importPath;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
95
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
96 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
97 * Part of {@link importPath} without the river root dir
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
98 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
99 protected final File rootRelativePath;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
100
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
101 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
102 * River for which the import runs
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
103 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
104 protected final ImportRiver river;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
105
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
106 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
107 * Reader during parse
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
108 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
109 protected LineNumberReader in;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
110
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
111 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
112 * Last line read from in
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
113 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
114 protected String currentLine;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
115
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
116 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
117 * State of the header lines parse loop
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
118 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
119 protected ParsingState headerParsingState;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
120
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
121 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
122 * Series header of the stations table, with the imported meta info.
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
123 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
124 protected HEADER seriesHeader;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
125
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
126 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
127 * List of meta info Pattern matched during {@link handleMetaLine}
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
128 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
129 protected final List<Pattern> metaPatternsMatched;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
130
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
131 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
132 * Column titles of the stations table, starting with the km column.
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
133 * All strings have been trimmed.
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
134 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
135 protected final List<String> columnTitles;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
136
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
137 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
138 * List of the km value tuples imported, no duplicate km
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
139 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
140 protected final List<KMLINE> values;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
141
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
142 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
143 * Ordered list with the imported km to check for duplicates.
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
144 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
145 protected final TreeSet<Double> kmExists;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
146
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
147
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
148 /***** CONSTRUCTORS *****/
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
149
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
150 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
151 * Constructs a parser for an import file
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
152 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
153 public AbstractParser(final File importPath, final File rootRelativePath, final ImportRiver river) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
154 this.importPath = importPath;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
155 this.rootRelativePath = rootRelativePath;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
156 this.river = river;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
157 this.metaPatternsMatched = new ArrayList<>();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
158 this.kmExists = new TreeSet<>(EpsilonComparator.CMP);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
159 this.columnTitles = new ArrayList<>();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
160 this.values = new ArrayList<>();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
161 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
162
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
163
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
164 /***** METHODS *****/
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
165
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
166 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
167 * Lists all files from a directory having a type extension (starting with dot)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
168 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
169 protected static List<File> listFiles(final File importDir, final String extension) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
170 final File[] files = importDir.listFiles(new FilenameFilter() {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
171 @Override
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
172 public boolean accept(final File dir, final String name) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
173 return name.toLowerCase().endsWith(extension);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
174 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
175 });
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
176 final List<File> fl = new ArrayList<>();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
177 if (files != null)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
178 for (final File file : files)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
179 fl.add(file);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
180 return fl;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
181 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
182
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
183 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
184 * Parses a file and adds series and values to the parser's collection
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
185 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
186 @Override
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
187 public void parse() throws IOException {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
188 logStartInfo();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
189 this.seriesHeader = createSeriesImport(this.importPath.getName().replaceAll("\\.csv", ""));
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
190 this.metaPatternsMatched.clear();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
191 this.kmExists.clear();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
192 this.headerParsingState = ParsingState.CONTINUE;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
193 try {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
194 try {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
195 this.in = new LineNumberReader(new InputStreamReader(new FileInputStream(this.importPath), ENCODING));
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
196 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
197 catch (final Exception e) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
198 logError("Could not open (" + e.getMessage() + ")");
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
199 this.headerParsingState = ParsingState.STOP;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
200 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
201 this.currentLine = null;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
202 while (this.headerParsingState != ParsingState.STOP) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
203 this.currentLine = this.in.readLine();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
204 if (this.currentLine == null)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
205 break;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
206 this.currentLine = this.currentLine.trim();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
207 if (this.currentLine.isEmpty())
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
208 continue;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
209 if (this.headerParsingState == ParsingState.CONTINUE)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
210 handleMetaLine();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
211 else
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
212 handleDataLine();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
213 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
214 if (this.headerParsingState != ParsingState.STOP)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
215 getLog().info("Number of values found: " + this.seriesHeader.getValueCount());
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
216 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
217 finally {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
218 if (this.in != null) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
219 this.in.close();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
220 this.in = null;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
221 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
222 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
223 if (this.headerParsingState == ParsingState.STOP)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
224 logError("Parsing of the file stopped due to a severe error");
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
225 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
226
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
227 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
228 * Writes the parse start info to the log
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
229 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
230 protected void logStartInfo() {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
231 getLog().info("Start parsing:;'" + this.rootRelativePath + "'");
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
232 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
233
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
234 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
235 * Stores the parsed series and values in the database
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
236 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
237 @Override
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
238 public void store() {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
239 if (this.headerParsingState != ParsingState.STOP) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
240 this.seriesHeader.store(this.river.getPeer());
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
241 final String counts = String.format("parse=%d, insert=%d, update/ignore=%d", this.seriesHeader.getValueCount(),
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
242 this.seriesHeader.getValueStoreCount(StoreMode.INSERT), this.seriesHeader.getValueStoreCount(StoreMode.UPDATE));
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
243 if (this.seriesHeader.getValueCount() > this.seriesHeader.getValueStoreCount(StoreMode.INSERT))
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
244 logWarning("Number of value inserts less than number parsed: " + counts);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
245 else
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
246 getLog().info("Number of values records: " + counts);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
247 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
248 else
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
249 logWarning("Severe parsing errors, not storing series '" + this.seriesHeader.getFilename() + "'");
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
250 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
251
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
252 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
253 * Strips separator chars from a meta info text, and trims leading and trailing whitespace
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
254 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
255 public static String parseMetaInfo(final String text) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
256 return text.replace(SEPARATOR_CHAR, "").trim();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
257 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
258
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
259 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
260 * Parses a number string with dot or comma as decimal char, and returning null in case of an error
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
261 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
262 public static Number parseDoubleWithNull(final String text) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
263 try {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
264 return parseDouble(text);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
265 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
266 catch (final Exception e) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
267 return null;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
268 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
269 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
270
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
271 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
272 * Parses a number string with dot or comma as decimal char
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
273 *
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
274 * @throws ParseException
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
275 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
276 public static Number parseDouble(final String text) throws ParseException {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
277 return numberFormat.parse(text.replace(',', '.'));
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
278 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
279
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
280 /**
8989
2693bfaf503d Fixed several BigDecimal(double) creations by BigDecimal(String) parsing to avoid unnecessary decimal digits
mschaefer
parents: 8971
diff changeset
281 * Parses a number string as a BigDecimal, replacing a comma with a dot first
2693bfaf503d Fixed several BigDecimal(double) creations by BigDecimal(String) parsing to avoid unnecessary decimal digits
mschaefer
parents: 8971
diff changeset
282 */
8991
c43d8c1a4455 Parse via decimalformat instead of bigdecimal constructor (to cope with strings with chars after the actual number)
mschaefer
parents: 8989
diff changeset
283 public static BigDecimal parseDecimal(final String text) throws ParseException {
c43d8c1a4455 Parse via decimalformat instead of bigdecimal constructor (to cope with strings with chars after the actual number)
mschaefer
parents: 8989
diff changeset
284 return (BigDecimal) bigDecimalFormat.parse(text.replace(',', '.'));
8989
2693bfaf503d Fixed several BigDecimal(double) creations by BigDecimal(String) parsing to avoid unnecessary decimal digits
mschaefer
parents: 8971
diff changeset
285 }
2693bfaf503d Fixed several BigDecimal(double) creations by BigDecimal(String) parsing to avoid unnecessary decimal digits
mschaefer
parents: 8971
diff changeset
286
2693bfaf503d Fixed several BigDecimal(double) creations by BigDecimal(String) parsing to avoid unnecessary decimal digits
mschaefer
parents: 8971
diff changeset
287 /**
8971
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
288 * Gets the class's logger
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
289 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
290 protected abstract Logger getLog();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
291
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
292 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
293 * Logs an error message, appending the relative file path
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
294 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
295 protected void logError(final String message) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
296 getLog().error(message + ";" + this.rootRelativePath);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
297 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
298
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
299 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
300 * Logs a warning message, appending the relative file path
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
301 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
302 protected void logWarning(final String message) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
303 getLog().warn(message + ";" + this.rootRelativePath);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
304 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
305
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
306 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
307 * Creates a new series import object
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
308 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
309 protected abstract HEADER createSeriesImport(final String filename);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
310
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
311 protected void handleMetaLine() {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
312 if (META_SUBGROUP.matcher(this.currentLine).matches())
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
313 return;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
314 else if (handleMetaRivername())
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
315 return;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
316 else if (handleMetaKmrange_info())
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
317 return;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
318 else if (handleMetaComment())
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
319 return;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
320 else if (handleMetaOther())
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
321 return;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
322 else if (handleMetaColumnTitles()) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
323 if (this.headerParsingState != ParsingState.STOP)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
324 this.headerParsingState = ParsingState.DONE;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
325 return;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
326 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
327 else {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
328 if (this.currentLine.startsWith(START_META_CHAR)) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
329 if (this.headerParsingState != ParsingState.IGNORE)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
330 logWarning("Not matching any known meta type in line " + this.in.getLineNumber() + ", ignored");
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
331 else
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
332 this.headerParsingState = ParsingState.CONTINUE;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
333 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
334 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
335 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
336
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
337 private boolean handleMetaRivername() {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
338 if (META_RIVERNAME.matcher(this.currentLine).matches()) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
339 this.metaPatternsMatched.add(META_RIVERNAME);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
340 return true;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
341 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
342 else
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
343 return false;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
344 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
345
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
346 private boolean handleMetaKmrange_info() {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
347 final Matcher m = META_KMRANGE_INFO.matcher(this.currentLine);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
348 if (m.matches()) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
349 this.metaPatternsMatched.add(META_KMRANGE_INFO);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
350 this.seriesHeader.setKmrange_info(parseMetaInfo(m.group(1)));
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
351 return true;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
352 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
353 return false;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
354 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
355
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
356 private boolean handleMetaComment() {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
357 final Matcher m = META_COMMENTS.matcher(this.currentLine);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
358 if (m.matches()) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
359 this.metaPatternsMatched.add(META_COMMENTS);
9038
4c5eeaff554c Database column "comment" renamed to "notes" (restrictions in Oracle)
mschaefer
parents: 9012
diff changeset
360 this.seriesHeader.setNotes(parseMetaInfo(m.group(1)));
8971
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
361 return true;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
362 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
363 return false;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
364 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
365
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
366 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
367 * Parses currentLine for non-default meta info
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
368 *
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
369 * @return Whether the line has been handled
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
370 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
371 protected boolean handleMetaOther() {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
372 return false;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
373 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
374
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
375 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
376 * Parses a header line for the km table column header line
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
377 *
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
378 * @return Whether the line has been handled and we are ready for reading the km values lines
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
379 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
380 protected boolean handleMetaColumnTitles() {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
381 if (META_COLUMNTITLES.matcher(this.currentLine).matches()) {
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
382 this.metaPatternsMatched.add(META_COLUMNTITLES);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
383 this.columnTitles.clear();
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
384 final String[] titles = this.currentLine.split(SEPARATOR_CHAR, 0);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
385 for (int i = 0; i <= titles.length - 1; i++)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
386 this.columnTitles.add(titles[i].trim());
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
387 return true;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
388 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
389 return false;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
390 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
391
9012
abe069eb180c Import of vegetation zones added
mschaefer
parents: 8991
diff changeset
392 /**
abe069eb180c Import of vegetation zones added
mschaefer
parents: 8991
diff changeset
393 * Parses a values line and adds the values record
abe069eb180c Import of vegetation zones added
mschaefer
parents: 8991
diff changeset
394 */
abe069eb180c Import of vegetation zones added
mschaefer
parents: 8991
diff changeset
395 protected void handleDataLine() {
8971
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
396 final String[] values = this.currentLine.split(SEPARATOR_CHAR, 0);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
397 // Skip import line without data or only km
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
398 if (values.length < 2)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
399 return;
9012
abe069eb180c Import of vegetation zones added
mschaefer
parents: 8991
diff changeset
400 Double km = Double.NaN;
abe069eb180c Import of vegetation zones added
mschaefer
parents: 8991
diff changeset
401 if (kmMode() != KmMode.NONE) {
abe069eb180c Import of vegetation zones added
mschaefer
parents: 8991
diff changeset
402 try {
abe069eb180c Import of vegetation zones added
mschaefer
parents: 8991
diff changeset
403 km = Double.valueOf(parseDouble(values[0]).doubleValue());
abe069eb180c Import of vegetation zones added
mschaefer
parents: 8991
diff changeset
404 if (kmMode() == KmMode.UNIQUE) {
abe069eb180c Import of vegetation zones added
mschaefer
parents: 8991
diff changeset
405 if (this.kmExists.contains(km)) {
abe069eb180c Import of vegetation zones added
mschaefer
parents: 8991
diff changeset
406 logWarning("Ignoring duplicate station '" + values[0] + "' in line " + this.in.getLineNumber());
abe069eb180c Import of vegetation zones added
mschaefer
parents: 8991
diff changeset
407 return;
abe069eb180c Import of vegetation zones added
mschaefer
parents: 8991
diff changeset
408 }
abe069eb180c Import of vegetation zones added
mschaefer
parents: 8991
diff changeset
409 this.kmExists.add(km);
8971
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
410 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
411 }
9012
abe069eb180c Import of vegetation zones added
mschaefer
parents: 8991
diff changeset
412 catch (final Exception e) {
abe069eb180c Import of vegetation zones added
mschaefer
parents: 8991
diff changeset
413 logError("Not parseable km in line " + this.in.getLineNumber() + ": " + e.getMessage());
abe069eb180c Import of vegetation zones added
mschaefer
parents: 8991
diff changeset
414 return;
abe069eb180c Import of vegetation zones added
mschaefer
parents: 8991
diff changeset
415 }
8971
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
416 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
417 final KMLINE value = createKmLineImport(km, values);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
418 if (value != null)
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
419 this.seriesHeader.addValue(value);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
420 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
421
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
422 /**
9012
abe069eb180c Import of vegetation zones added
mschaefer
parents: 8991
diff changeset
423 * How {@link handleDataLine} shall handle the km column (if any)
8971
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
424 */
9012
abe069eb180c Import of vegetation zones added
mschaefer
parents: 8991
diff changeset
425 protected KmMode kmMode() {
abe069eb180c Import of vegetation zones added
mschaefer
parents: 8991
diff changeset
426 return KmMode.UNIQUE;
8971
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
427 }
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
428
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
429 /**
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
430 * Creates a value import item with the km and other fields of the current line;
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
431 * the km has been validated
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
432 *
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
433 * @return value item, or null if parse error
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
434 */
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
435 protected abstract KMLINE createKmLineImport(final Double km, final String[] values);
50416a0df385 Importer for the Schifffahrt (S-INFO) and Oekologie (U-INFO) files
mschaefer
parents:
diff changeset
436 }
http://dive4elements.wald.intevation.org