Mercurial > dive4elements > river

annotate backend/src/main/java/org/dive4elements/river/importer/parsers/BedHeightSingleParser.java @ 6273:e7526a074c0d double-precision

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Bed height parser: check for duplicate stations
author Tom Gottfried <tom.gottfried@intevation.de>
date Tue, 11 Jun 2013 13:03:50 +0200
parents a3fb76cf79b6
children f095b58c95d9
rev   line source
5844
4dd33b86dc61 Added header to river backend.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5838
diff changeset
1 /* Copyright (C) 2011, 2012, 2013 by Bundesanstalt für Gewässerkunde
4dd33b86dc61 Added header to river backend.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5838
diff changeset
2 * Software engineering by Intevation GmbH
4dd33b86dc61 Added header to river backend.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5838
diff changeset
3 *
5992
4c3ccf2b0304 Removed trailing whitespace.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5844
diff changeset
4 * This file is Free Software under the GNU AGPL (>=v3)
5844
4dd33b86dc61 Added header to river backend.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5838
diff changeset
5 * and comes with ABSOLUTELY NO WARRANTY! Check out the
5992
4c3ccf2b0304 Removed trailing whitespace.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5844
diff changeset
6 * documentation coming with Dive4Elements River for details.
5844
4dd33b86dc61 Added header to river backend.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5838
diff changeset
7 */
4dd33b86dc61 Added header to river backend.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5838
diff changeset
8
5829
18619c1e7c2a Repaired internal references.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5828
diff changeset
9 package org.dive4elements.river.importer.parsers;
2806
33f40b23edd8 Initial checkin for parsing MINFO bed heights.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
10
2808
b57c95094b68 Finished work on parsing meta information and data specific to single bed heights files in MINFO.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2806
diff changeset
11 import java.math.BigDecimal;
b57c95094b68 Finished work on parsing meta information and data specific to single bed heights files in MINFO.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2806
diff changeset
12
b57c95094b68 Finished work on parsing meta information and data specific to single bed heights files in MINFO.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2806
diff changeset
13 import java.text.ParseException;
b57c95094b68 Finished work on parsing meta information and data specific to single bed heights files in MINFO.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2806
diff changeset
14
2806
33f40b23edd8 Initial checkin for parsing MINFO bed heights.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
15 import org.apache.log4j.Logger;
33f40b23edd8 Initial checkin for parsing MINFO bed heights.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
16
5829
18619c1e7c2a Repaired internal references.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5828
diff changeset
17 import org.dive4elements.river.importer.ImportBedHeight;
18619c1e7c2a Repaired internal references.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5828
diff changeset
18 import org.dive4elements.river.importer.ImportBedHeightSingle;
18619c1e7c2a Repaired internal references.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5828
diff changeset
19 import org.dive4elements.river.importer.ImportBedHeightSingleValue;
2806
33f40b23edd8 Initial checkin for parsing MINFO bed heights.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
20
33f40b23edd8 Initial checkin for parsing MINFO bed heights.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
21
2811
8926571e47fb Finished importing MINFO bed heights (single and epoch).
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2809
diff changeset
22 public class BedHeightSingleParser extends BedHeightParser {
2806
33f40b23edd8 Initial checkin for parsing MINFO bed heights.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
23
33f40b23edd8 Initial checkin for parsing MINFO bed heights.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
24 private static final Logger log =
33f40b23edd8 Initial checkin for parsing MINFO bed heights.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
25 Logger.getLogger(BedHeightSingleParser.class);
33f40b23edd8 Initial checkin for parsing MINFO bed heights.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
26
33f40b23edd8 Initial checkin for parsing MINFO bed heights.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
27
2808
b57c95094b68 Finished work on parsing meta information and data specific to single bed heights files in MINFO.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2806
diff changeset
28
2811
8926571e47fb Finished importing MINFO bed heights (single and epoch).
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2809
diff changeset
29 @Override
8926571e47fb Finished importing MINFO bed heights (single and epoch).
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2809
diff changeset
30 protected ImportBedHeight newImportBedHeight(String description) {
8926571e47fb Finished importing MINFO bed heights (single and epoch).
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2809
diff changeset
31 return new ImportBedHeightSingle(description);
2808
b57c95094b68 Finished work on parsing meta information and data specific to single bed heights files in MINFO.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2806
diff changeset
32 }
b57c95094b68 Finished work on parsing meta information and data specific to single bed heights files in MINFO.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2806
diff changeset
33
b57c95094b68 Finished work on parsing meta information and data specific to single bed heights files in MINFO.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2806
diff changeset
34
b57c95094b68 Finished work on parsing meta information and data specific to single bed heights files in MINFO.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2806
diff changeset
35
3955
26685b846a29 Let importer handle gappy single bed height values.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 3662
diff changeset
36 /**
26685b846a29 Let importer handle gappy single bed height values.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 3662
diff changeset
37 * Create ImportBedHeightSingleValue from a line of csv file, add
26685b846a29 Let importer handle gappy single bed height values.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 3662
diff changeset
38 * it to the ImportBedHeight.
26685b846a29 Let importer handle gappy single bed height values.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 3662
diff changeset
39 */
2811
8926571e47fb Finished importing MINFO bed heights (single and epoch).
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2809
diff changeset
40 @Override
8926571e47fb Finished importing MINFO bed heights (single and epoch).
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2809
diff changeset
41 protected void handleDataLine(ImportBedHeight obj, String line) {
2808
b57c95094b68 Finished work on parsing meta information and data specific to single bed heights files in MINFO.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2806
diff changeset
42 String[] values = line.split(SEPERATOR_CHAR);
b57c95094b68 Finished work on parsing meta information and data specific to single bed heights files in MINFO.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2806
diff changeset
43
3955
26685b846a29 Let importer handle gappy single bed height values.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 3662
diff changeset
44 if (values == null || (values.length != 1 && values.length < 6)) {
3662
0d27d02b1208 backend: Completed the error messages of the importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2811
diff changeset
45 //log.warn("BSP: Error while parsing data line: '" + line + "'");
2808
b57c95094b68 Finished work on parsing meta information and data specific to single bed heights files in MINFO.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2806
diff changeset
46 return;
b57c95094b68 Finished work on parsing meta information and data specific to single bed heights files in MINFO.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2806
diff changeset
47 }
b57c95094b68 Finished work on parsing meta information and data specific to single bed heights files in MINFO.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2806
diff changeset
48
6203
a3fb76cf79b6 backout backout as it is now safely in a branch
Tom Gottfried <tom.gottfried@intevation.de>
parents: 6201
diff changeset
49 Double km;
3955
26685b846a29 Let importer handle gappy single bed height values.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 3662
diff changeset
50
26685b846a29 Let importer handle gappy single bed height values.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 3662
diff changeset
51 try {
6203
a3fb76cf79b6 backout backout as it is now safely in a branch
Tom Gottfried <tom.gottfried@intevation.de>
parents: 6201
diff changeset
52 km = new Double(nf.parse(values[0]).doubleValue());
6273
e7526a074c0d Bed height parser: check for duplicate stations
Tom Gottfried <tom.gottfried@intevation.de>
parents: 6203
diff changeset
53
e7526a074c0d Bed height parser: check for duplicate stations
Tom Gottfried <tom.gottfried@intevation.de>
parents: 6203
diff changeset
54 Double key = Double.valueOf(km);
e7526a074c0d Bed height parser: check for duplicate stations
Tom Gottfried <tom.gottfried@intevation.de>
parents: 6203
diff changeset
55
e7526a074c0d Bed height parser: check for duplicate stations
Tom Gottfried <tom.gottfried@intevation.de>
parents: 6203
diff changeset
56 if (kmExists.contains(key)) {
e7526a074c0d Bed height parser: check for duplicate stations
Tom Gottfried <tom.gottfried@intevation.de>
parents: 6203
diff changeset
57 log.warn("duplicate station '" + km + "': -> ignored");
e7526a074c0d Bed height parser: check for duplicate stations
Tom Gottfried <tom.gottfried@intevation.de>
parents: 6203
diff changeset
58 return;
e7526a074c0d Bed height parser: check for duplicate stations
Tom Gottfried <tom.gottfried@intevation.de>
parents: 6203
diff changeset
59 }
e7526a074c0d Bed height parser: check for duplicate stations
Tom Gottfried <tom.gottfried@intevation.de>
parents: 6203
diff changeset
60
e7526a074c0d Bed height parser: check for duplicate stations
Tom Gottfried <tom.gottfried@intevation.de>
parents: 6203
diff changeset
61 kmExists.add(key);
e7526a074c0d Bed height parser: check for duplicate stations
Tom Gottfried <tom.gottfried@intevation.de>
parents: 6203
diff changeset
62 }
e7526a074c0d Bed height parser: check for duplicate stations
Tom Gottfried <tom.gottfried@intevation.de>
parents: 6203
diff changeset
63
3955
26685b846a29 Let importer handle gappy single bed height values.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 3662
diff changeset
64 catch (ParseException e) {
26685b846a29 Let importer handle gappy single bed height values.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 3662
diff changeset
65 // We expect a lot of ";;;;;;" lines.
26685b846a29 Let importer handle gappy single bed height values.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 3662
diff changeset
66 //log.warn("BSP: Error while parsing km of data row.", e);
26685b846a29 Let importer handle gappy single bed height values.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 3662
diff changeset
67 return;
26685b846a29 Let importer handle gappy single bed height values.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 3662
diff changeset
68 }
26685b846a29 Let importer handle gappy single bed height values.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 3662
diff changeset
69
26685b846a29 Let importer handle gappy single bed height values.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 3662
diff changeset
70 // Handle gaps like "10,0;;;;;".
26685b846a29 Let importer handle gappy single bed height values.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 3662
diff changeset
71 if (values.length == 1) {
6047
1bb4b51ff1e6 BHP: do not import lines without useful data
Tom Gottfried <tom.gottfried@intevation.de>
parents: 5992
diff changeset
72 // Do not import line without useful data
3955
26685b846a29 Let importer handle gappy single bed height values.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 3662
diff changeset
73 return;
26685b846a29 Let importer handle gappy single bed height values.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 3662
diff changeset
74 }
26685b846a29 Let importer handle gappy single bed height values.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 3662
diff changeset
75
26685b846a29 Let importer handle gappy single bed height values.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 3662
diff changeset
76 // Because we cannot enforce consistency of values with complete data
26685b846a29 Let importer handle gappy single bed height values.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 3662
diff changeset
77 // via null constraints in the database (as there are "gap" values),
26685b846a29 Let importer handle gappy single bed height values.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 3662
diff changeset
78 // do this checks manually.
26685b846a29 Let importer handle gappy single bed height values.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 3662
diff changeset
79 if (values[3].length() == 0 || values[4].length() == 0
26685b846a29 Let importer handle gappy single bed height values.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 3662
diff changeset
80 || values[5].length() == 0) {
26685b846a29 Let importer handle gappy single bed height values.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 3662
diff changeset
81 //log.warn("BSP: Error while parsing data row (manual null constraint violated).");
26685b846a29 Let importer handle gappy single bed height values.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 3662
diff changeset
82 return;
26685b846a29 Let importer handle gappy single bed height values.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 3662
diff changeset
83 }
26685b846a29 Let importer handle gappy single bed height values.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 3662
diff changeset
84
2808
b57c95094b68 Finished work on parsing meta information and data specific to single bed heights files in MINFO.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2806
diff changeset
85 try {
b57c95094b68 Finished work on parsing meta information and data specific to single bed heights files in MINFO.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2806
diff changeset
86 ImportBedHeightSingleValue value = new ImportBedHeightSingleValue(
2811
8926571e47fb Finished importing MINFO bed heights (single and epoch).
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2809
diff changeset
87 (ImportBedHeightSingle) obj,
3955
26685b846a29 Let importer handle gappy single bed height values.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 3662
diff changeset
88 km,
6203
a3fb76cf79b6 backout backout as it is now safely in a branch
Tom Gottfried <tom.gottfried@intevation.de>
parents: 6201
diff changeset
89 new Double(nf.parse(values[1]).doubleValue()),
a3fb76cf79b6 backout backout as it is now safely in a branch
Tom Gottfried <tom.gottfried@intevation.de>
parents: 6201
diff changeset
90 new Double(nf.parse(values[2]).doubleValue()),
a3fb76cf79b6 backout backout as it is now safely in a branch
Tom Gottfried <tom.gottfried@intevation.de>
parents: 6201
diff changeset
91 new Double(nf.parse(values[3]).doubleValue()),
5450
84beabb3897c Correctly parse bedheights.
Raimund Renkert <rrenkert@intevation.de>
parents: 3956
diff changeset
92 parseBigDecimal(values[4], line),
6203
a3fb76cf79b6 backout backout as it is now safely in a branch
Tom Gottfried <tom.gottfried@intevation.de>
parents: 6201
diff changeset
93 new Double(nf.parse(values[5]).doubleValue())
2808
b57c95094b68 Finished work on parsing meta information and data specific to single bed heights files in MINFO.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2806
diff changeset
94 );
b57c95094b68 Finished work on parsing meta information and data specific to single bed heights files in MINFO.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2806
diff changeset
95
b57c95094b68 Finished work on parsing meta information and data specific to single bed heights files in MINFO.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2806
diff changeset
96 obj.addValue(value);
b57c95094b68 Finished work on parsing meta information and data specific to single bed heights files in MINFO.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2806
diff changeset
97 }
b57c95094b68 Finished work on parsing meta information and data specific to single bed heights files in MINFO.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2806
diff changeset
98 catch (ParseException e) {
5490
b3dd14fc13a6 Parsers: do not call warnings error
Tom Gottfried <tom.gottfried@intevation.de>
parents: 5450
diff changeset
99 log.warn("BSP: unparseable value in data row.", e);
2808
b57c95094b68 Finished work on parsing meta information and data specific to single bed heights files in MINFO.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2806
diff changeset
100 }
b57c95094b68 Finished work on parsing meta information and data specific to single bed heights files in MINFO.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2806
diff changeset
101 }
5992
4c3ccf2b0304 Removed trailing whitespace.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5844
diff changeset
102
5450
84beabb3897c Correctly parse bedheights.
Raimund Renkert <rrenkert@intevation.de>
parents: 3956
diff changeset
103 private BigDecimal parseBigDecimal(String value, String line) {
84beabb3897c Correctly parse bedheights.
Raimund Renkert <rrenkert@intevation.de>
parents: 3956
diff changeset
104 BigDecimal result = null;
84beabb3897c Correctly parse bedheights.
Raimund Renkert <rrenkert@intevation.de>
parents: 3956
diff changeset
105 try {
84beabb3897c Correctly parse bedheights.
Raimund Renkert <rrenkert@intevation.de>
parents: 3956
diff changeset
106 Double dValue = Double.valueOf(value.replace(",", "."));
84beabb3897c Correctly parse bedheights.
Raimund Renkert <rrenkert@intevation.de>
parents: 3956
diff changeset
107 result = new BigDecimal(dValue.doubleValue());
84beabb3897c Correctly parse bedheights.
Raimund Renkert <rrenkert@intevation.de>
parents: 3956
diff changeset
108 }
84beabb3897c Correctly parse bedheights.
Raimund Renkert <rrenkert@intevation.de>
parents: 3956
diff changeset
109 catch (NumberFormatException nfe) {
84beabb3897c Correctly parse bedheights.
Raimund Renkert <rrenkert@intevation.de>
parents: 3956
diff changeset
110 log.warn(
5490
b3dd14fc13a6 Parsers: do not call warnings error
Tom Gottfried <tom.gottfried@intevation.de>
parents: 5450
diff changeset
111 "Could not parse " +
5450
84beabb3897c Correctly parse bedheights.
Raimund Renkert <rrenkert@intevation.de>
parents: 3956
diff changeset
112 value +
84beabb3897c Correctly parse bedheights.
Raimund Renkert <rrenkert@intevation.de>
parents: 3956
diff changeset
113 " in bed heigt single row: "
84beabb3897c Correctly parse bedheights.
Raimund Renkert <rrenkert@intevation.de>
parents: 3956
diff changeset
114 + line);
84beabb3897c Correctly parse bedheights.
Raimund Renkert <rrenkert@intevation.de>
parents: 3956
diff changeset
115 }
84beabb3897c Correctly parse bedheights.
Raimund Renkert <rrenkert@intevation.de>
parents: 3956
diff changeset
116 return result;
84beabb3897c Correctly parse bedheights.
Raimund Renkert <rrenkert@intevation.de>
parents: 3956
diff changeset
117 }
2806
33f40b23edd8 Initial checkin for parsing MINFO bed heights.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
118 }
33f40b23edd8 Initial checkin for parsing MINFO bed heights.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
119 // vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 :
http://dive4elements.wald.intevation.org