Mercurial > dive4elements > river

comparison backend/src/main/java/org/dive4elements/river/importer/parsers/WaterlevelDifferencesParser.java @ 8989:2693bfaf503d

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Fixed several BigDecimal(double) creations by BigDecimal(String) parsing to avoid unnecessary decimal digits
author mschaefer
date Mon, 09 Apr 2018 09:07:00 +0200
parents 5e38e2924c07
children c43d8c1a4455
comparison
equal deleted inserted replaced
8988:ae76f618d990 8989:2693bfaf503d
10 10
11 import java.io.File; 11 import java.io.File;
12 import java.io.IOException; 12 import java.io.IOException;
13 import java.math.BigDecimal; 13 import java.math.BigDecimal;
14 import java.text.NumberFormat; 14 import java.text.NumberFormat;
15 import java.text.ParseException;
16 import java.util.ArrayList; 15 import java.util.ArrayList;
17 import java.util.List; 16 import java.util.List;
18 import java.util.regex.Matcher; 17 import java.util.regex.Matcher;
19 import java.util.regex.Pattern; 18 import java.util.regex.Pattern;
20 19
21 import org.apache.log4j.Logger; 20 import org.apache.log4j.Logger;
22 21 import org.dive4elements.river.backend.utils.DateUtil;
23 import org.dive4elements.river.importer.ImportTimeInterval; 22 import org.dive4elements.river.importer.ImportTimeInterval;
24 import org.dive4elements.river.importer.ImportUnit; 23 import org.dive4elements.river.importer.ImportUnit;
25
26 import org.dive4elements.river.importer.ImportWst; 24 import org.dive4elements.river.importer.ImportWst;
27 import org.dive4elements.river.importer.ImportWstQRange;
28 import org.dive4elements.river.importer.ImportWstColumn; 25 import org.dive4elements.river.importer.ImportWstColumn;
29 import org.dive4elements.river.importer.ImportWstColumnValue; 26 import org.dive4elements.river.importer.ImportWstColumnValue;
30 import org.dive4elements.river.backend.utils.DateUtil; 27 import org.dive4elements.river.importer.ImportWstQRange;
28 import org.dive4elements.river.importer.common.AbstractParser;
31 29
32 30
33 /** 31 /**
34 * Parse WaterlevelDifferences CSV file. 32 * Parse WaterlevelDifferences CSV file.
35 */ 33 */
36 public class WaterlevelDifferencesParser extends LineParser { 34 public class WaterlevelDifferencesParser extends LineParser {
37 35
38 private static final Logger log = 36 private static final Logger log =
39 Logger.getLogger(WaterlevelDifferencesParser.class); 37 Logger.getLogger(WaterlevelDifferencesParser.class);
40 38
41 private static final NumberFormat nf = 39 private static final NumberFormat nf =
42 NumberFormat.getInstance(DEFAULT_LOCALE); 40 NumberFormat.getInstance(DEFAULT_LOCALE);
43 41
44 public static final Pattern META_UNIT = 42 public static final Pattern META_UNIT =
45 Pattern.compile("^Einheit: \\[(.*)\\].*"); 43 Pattern.compile("^Einheit: \\[(.*)\\].*");
46 44
47 public static final Pattern YEARS_IN_COLUMN = 45 public static final Pattern YEARS_IN_COLUMN =
48 Pattern.compile(".*(\\d{4})-(\\d{4})$"); 46 Pattern.compile(".*(\\d{4})-(\\d{4})$");
49 47
50 public static final double INTERVAL_GAP = 0.00001d; 48 public static final BigDecimal INTERVAL_GAP = new BigDecimal("0.00001");
51 49
52 /** List of parsed differences as ImportWst s. */ 50 /** List of parsed differences as ImportWst s. */
53 private List<ImportWst> differences; 51 private final List<ImportWst> differences;
54 52
55 private ImportWstColumn[] columns; 53 private ImportWstColumn[] columns;
56 54
57 /** The currently processed dataset. */ 55 /** The currently processed dataset. */
58 private ImportWst current; 56 private ImportWst current;
59 57
60 58
61 public WaterlevelDifferencesParser() { 59 public WaterlevelDifferencesParser() {
62 differences = new ArrayList<ImportWst>(); 60 this.differences = new ArrayList<>();
63 } 61 }
64 62
65 63
66 /** Get the differences as wst parsed so far. */ 64 /** Get the differences as wst parsed so far. */
67 public List<ImportWst> getDifferences() { 65 public List<ImportWst> getDifferences() {
68 return differences; 66 return this.differences;
69 } 67 }
70 68
71 69
72 /** 70 /**
73 * Parse a csv waterleveldifferenceparser and create a ImportWst object 71 * Parse a csv waterleveldifferenceparser and create a ImportWst object
74 * from it. 72 * from it.
75 */ 73 */
76 @Override 74 @Override
77 public void parse(File file) throws IOException { 75 public void parse(final File file) throws IOException {
78 current = new ImportWst(file.getName()); 76 this.current = new ImportWst(file.getName());
79 current.setKind(6); 77 this.current.setKind(6);
80 78
81 super.parse(file); 79 super.parse(file);
82 } 80 }
83 81
84 82
88 } 86 }
89 87
90 88
91 @Override 89 @Override
92 protected void finish() { 90 protected void finish() {
93 if (columns != null && current != null) { 91 if (this.columns != null && this.current != null) {
94 // TODO figure out if its needed, as the columns 92 // TODO figure out if its needed, as the columns
95 // are registered at their construction time. 93 // are registered at their construction time.
96 for (ImportWstColumn col: columns) { 94 for (final ImportWstColumn col: this.columns) {
97 // TODO place a current.addColumn(col); here? 95 // TODO place a current.addColumn(col); here?
98 } 96 }
99 97
100 differences.add(current); 98 this.differences.add(this.current);
101 } 99 }
102 100
103 // For all differences columns, add a single Q-Range with 101 // For all differences columns, add a single Q-Range with
104 // -1. 102 // -1.
105 // Expand range to minimal length in case it would be 0 103 // Expand range to minimal length in case it would be 0
106 // TODO: should otherwise be extended to 104 // TODO: should otherwise be extended to
107 // (first station of next range - INTERVAL_GAP), 105 // (first station of next range - INTERVAL_GAP),
108 // assuming always ascending stations 106 // assuming always ascending stations
109 for (ImportWstColumn column: columns) { 107 for (final ImportWstColumn column: this.columns) {
110 List<ImportWstColumnValue> cValues = column.getColumnValues(); 108 final List<ImportWstColumnValue> cValues = column.getColumnValues();
111 BigDecimal a = cValues.get(0).getPosition(); 109 final BigDecimal a = cValues.get(0).getPosition();
112 BigDecimal b = cValues.get(cValues.size() - 1).getPosition(); 110 BigDecimal b = cValues.get(cValues.size() - 1).getPosition();
113 if (a.compareTo(b) == 0) { 111 if (a.compareTo(b) == 0) {
114 b = new BigDecimal(b.doubleValue() + INTERVAL_GAP); 112 b = b.add(INTERVAL_GAP);
115 } 113 }
116 column.addColumnQRange( 114 column.addColumnQRange(
117 new ImportWstQRange( 115 new ImportWstQRange(
118 a, 116 a,
119 b, 117 b,
120 new BigDecimal(-1d)) 118 new BigDecimal(-1d))
121 ); 119 );
122 } 120 }
123 current = null; 121 this.current = null;
124 columns = null; 122 this.columns = null;
125 } 123 }
126 124
127 125
128 @Override 126 @Override
129 protected void handleLine(int lineNum, String line) { 127 protected void handleLine(final int lineNum, final String line) {
130 if (line.startsWith(START_META_CHAR)) { 128 if (line.startsWith(START_META_CHAR)) {
131 handleMetaLine(stripMetaLine(line)); 129 handleMetaLine(stripMetaLine(line));
132 } 130 }
133 else { 131 else {
134 handleDataLine(line); 132 handleDataLine(line);
135 } 133 }
136 } 134 }
137 135
138 136
139 private void handleMetaLine(String meta) { 137 private void handleMetaLine(final String meta) {
140 if (handleMetaUnit(meta)) { 138 if (handleMetaUnit(meta)) {
141 return; 139 return;
142 } 140 }
143 else { 141 else {
144 handleMetaColumnNames(meta); 142 handleMetaColumnNames(meta);
145 } 143 }
146 } 144 }
147 145
148 146
149 private boolean handleMetaUnit(String meta) { 147 private boolean handleMetaUnit(final String meta) {
150 Matcher m = META_UNIT.matcher(meta); 148 final Matcher m = META_UNIT.matcher(meta);
151 149
152 if (m.matches()) { 150 if (m.matches()) {
153 String unit = m.group(1); 151 final String unit = m.group(1);
154 log.debug("Found unit: '" + unit + "'"); 152 log.debug("Found unit: '" + unit + "'");
155 153
156 current.setUnit(new ImportUnit(unit)); 154 this.current.setUnit(new ImportUnit(unit));
157 155
158 return true; 156 return true;
159 } 157 }
160 158
161 return false; 159 return false;
162 } 160 }
163 161
164 162
165 private boolean handleMetaColumnNames(String meta) { 163 private boolean handleMetaColumnNames(final String meta) {
166 Pattern META_COLUMN_NAMES = Pattern.compile("Fluss-km;(.*)"); 164 final Pattern META_COLUMN_NAMES = Pattern.compile("Fluss-km;(.*)");
167 Matcher m = META_COLUMN_NAMES.matcher(meta); 165 final Matcher m = META_COLUMN_NAMES.matcher(meta);
168 166
169 if (m.matches()) { 167 if (m.matches()) {
170 String colStr = m.group(1); 168 final String colStr = m.group(1);
171 String[] cols = colStr.split(SEPERATOR_CHAR); 169 final String[] cols = colStr.split(SEPERATOR_CHAR);
172 170
173 log.debug("Found " + cols.length + " columns."); 171 log.debug("Found " + cols.length + " columns.");
174 172
175 initColumns(cols); 173 initColumns(cols);
176 174
180 return false; 178 return false;
181 } 179 }
182 180
183 181
184 /** Setup column structures with name, description and time interval. */ 182 /** Setup column structures with name, description and time interval. */
185 private void initColumns(String[] cols) { 183 private void initColumns(final String[] cols) {
186 current.setNumberColumns(cols.length); 184 this.current.setNumberColumns(cols.length);
187 columns = current.getColumns().toArray( 185 this.columns = this.current.getColumns().toArray(
188 new ImportWstColumn[cols.length]); 186 new ImportWstColumn[cols.length]);
189 187
190 for (int i = 0; i < cols.length; i++) { 188 for (int i = 0; i < cols.length; i++) {
191 String name = cols[i].replace("\"", ""); 189 final String name = cols[i].replace("\"", "");
192 190
193 log.debug("Create new column '" + name + "'"); 191 log.debug("Create new column '" + name + "'");
194 ImportWstColumn column = current.getColumn(i); 192 final ImportWstColumn column = this.current.getColumn(i);
195 column.setName(name); 193 column.setName(name);
196 column.setDescription(name); 194 column.setDescription(name);
197 195
198 Matcher m = YEARS_IN_COLUMN.matcher(name); 196 final Matcher m = YEARS_IN_COLUMN.matcher(name);
199 197
200 if (m.matches()) { 198 if (m.matches()) {
201 int startYear = Integer.parseInt(m.group(1)); 199 final int startYear = Integer.parseInt(m.group(1));
202 int endYear = Integer.parseInt(m.group(2)); 200 final int endYear = Integer.parseInt(m.group(2));
203 ImportTimeInterval time = new ImportTimeInterval( 201 final ImportTimeInterval time = new ImportTimeInterval(
204 DateUtil.getStartDateFromYear(startYear), 202 DateUtil.getStartDateFromYear(startYear),
205 DateUtil.getEndDateFromYear(endYear) 203 DateUtil.getEndDateFromYear(endYear)
206 ); 204 );
207 column.setTimeInterval(time); 205 column.setTimeInterval(time);
208 } else { 206 } else {
209 log.debug("No time interval in column header found: " + name); 207 log.debug("No time interval in column header found: " + name);
210 } 208 }
211 } 209 }
213 211
214 212
215 /** Handle one line of data, add one value for all columns. 213 /** Handle one line of data, add one value for all columns.
216 * @param line the line to parse 214 * @param line the line to parse
217 */ 215 */
218 private void handleDataLine(String line) { 216 private void handleDataLine(final String line) {
219 // Split by separator, do not exclude trailing empty string. 217 // Split by separator, do not exclude trailing empty string.
220 String[] cols = line.split(SEPERATOR_CHAR, -1); 218 final String[] cols = line.split(SEPERATOR_CHAR, -1);
221 219
222 if (cols == null || cols.length < 2) { 220 if (cols == null || cols.length < 2) {
223 log.warn("skip invalid waterlevel-diff line: '" + line + "'"); 221 log.warn("skip invalid waterlevel-diff line: '" + line + "'");
224 return; 222 return;
225 } 223 }
226 224
227 try { 225 try {
228 // The first value in a line like 12,9;4,3;4,5 is the station, 226 // The first value in a line like 12,9;4,3;4,5 is the station,
229 // later real values. 227 // later real values.
230 Double station = nf.parse(cols[0]).doubleValue(); 228 final BigDecimal station = AbstractParser.parseDecimal(cols[0]);
231 229
232 for (int i = 0; i < columns.length; i++) { 230 for (int i = 0; i < this.columns.length; i++) {
233 int idx = i+1; 231 final int idx = i+1;
234 232
235 if (idx >= cols.length) { 233 if (idx >= cols.length) {
236 log.warn("Insufficient column numbers: " + line); 234 log.warn("Insufficient column numbers: " + line);
237 continue; 235 continue;
238 } 236 }
239 237
240 String value = cols[idx]; 238 final String value = cols[idx];
241 239
242 if (value != null && !value.equals("")) { 240 if (value != null && !value.equals("")) {
243 try { 241 try {
244 columns[i].addColumnValue( 242 this.columns[i].addColumnValue(
245 new BigDecimal(station), 243 station,
246 new BigDecimal(nf.parse(value).doubleValue())); 244 AbstractParser.parseDecimal(value));
247 } 245 }
248 catch (ParseException pe) { 246 catch (final NumberFormatException pe) {
249 log.warn("Could not parse value: '" + value + "'"); 247 log.warn("Could not parse value: '" + value + "'");
250 } 248 }
251 } 249 }
252 } 250 }
253 } 251 }
254 catch (ParseException pe) { 252 catch (final NumberFormatException pe) {
255 log.warn("Could not parse station: '" + line + "'"); 253 log.warn("Could not parse station: '" + line + "'");
256 } 254 }
257 } 255 }
258 } 256 }
259 // vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 : 257 // vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 :
http://dive4elements.wald.intevation.org