Mercurial > dive4elements > river

comparison backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/FlowDepthParser.java @ 9656:31549fdfaf4f

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Importer (s/u-info) extensions: flow-depth: uniform formatting of from-to series names, warning instead of cancelling in case of missing column values, detecting, logging and skipping columns with wrong unit, better counting of inserted/updated values for each column
author mschaefer
date Mon, 23 Mar 2020 15:21:39 +0100
parents 4c5eeaff554c
children
comparison
equal deleted inserted replaced
9655:1f57381b3bb5 9656:31549fdfaf4f
10 10
11 package org.dive4elements.river.importer.sinfo.parsers; 11 package org.dive4elements.river.importer.sinfo.parsers;
12 12
13 import java.io.File; 13 import java.io.File;
14 import java.io.FileInputStream; 14 import java.io.FileInputStream;
15 import java.io.IOException;
16 import java.io.InputStreamReader; 15 import java.io.InputStreamReader;
17 import java.io.LineNumberReader; 16 import java.io.LineNumberReader;
18 import java.util.ArrayList; 17 import java.util.ArrayList;
19 import java.util.List; 18 import java.util.List;
20 import java.util.regex.Matcher; 19 import java.util.regex.Matcher;
21 import java.util.regex.Pattern; 20 import java.util.regex.Pattern;
22 21
23 import org.apache.log4j.Logger; 22 import org.apache.log4j.Logger;
24 import org.dive4elements.river.importer.Config; 23 import org.dive4elements.river.importer.Config;
25 import org.dive4elements.river.importer.ImportRiver; 24 import org.dive4elements.river.importer.ImportRiver;
25 import org.dive4elements.river.importer.ImporterSession;
26 import org.dive4elements.river.importer.common.AbstractParser; 26 import org.dive4elements.river.importer.common.AbstractParser;
27 import org.dive4elements.river.importer.common.ParsingState; 27 import org.dive4elements.river.importer.common.ParsingState;
28 import org.dive4elements.river.importer.sinfo.importitem.FlowDepthColumnSeriesImport; 28 import org.dive4elements.river.importer.sinfo.importitem.FlowDepthColumnSeriesImport;
29 import org.dive4elements.river.importer.sinfo.importitem.FlowDepthKmLineImport; 29 import org.dive4elements.river.importer.sinfo.importitem.FlowDepthKmLineImport;
30 import org.dive4elements.river.importer.sinfo.importitem.FlowDepthSeriesImport; 30 import org.dive4elements.river.importer.sinfo.importitem.FlowDepthSeriesImport;
31 import org.dive4elements.river.model.sinfo.FlowDepth;
31 import org.dive4elements.river.model.sinfo.FlowDepthColumn; 32 import org.dive4elements.river.model.sinfo.FlowDepthColumn;
32 import org.dive4elements.river.model.sinfo.FlowDepthValue; 33 import org.dive4elements.river.model.sinfo.FlowDepthValue;
34 import org.hibernate.Session;
33 35
34 /** 36 /**
35 * Reads and parses the header of a flow depth file and handles the parse and store of the columns 37 * Reads and parses the header of a flow depth file and handles the parse and store of the columns
36 * 38 *
37 * @author Matthias Schäfer 39 * @author Matthias Schäfer
51 53
52 private static final Pattern META_TYPE = Pattern.compile("^#\\sTyp:\\s*([^;]*).*", Pattern.CASE_INSENSITIVE); 54 private static final Pattern META_TYPE = Pattern.compile("^#\\sTyp:\\s*([^;]*).*", Pattern.CASE_INSENSITIVE);
53 55
54 private static final Pattern COLUMN_TITLE = Pattern.compile("Flie((.)|(ss))tiefe\\s*\\((.+?)\\)\\s*\\[m\\].*", Pattern.CASE_INSENSITIVE); 56 private static final Pattern COLUMN_TITLE = Pattern.compile("Flie((.)|(ss))tiefe\\s*\\((.+?)\\)\\s*\\[m\\].*", Pattern.CASE_INSENSITIVE);
55 57
56 private final FlowDepthSeriesImport tkhGroup; 58 private final FlowDepthSeriesImport flowdepthGroup;
57 59
58 private final List<FlowDepthColumnParser> colParsers; 60 private final List<FlowDepthColumnParser> colParsers;
59 61
60 62
61 /***** CONSTRUCTORS *****/ 63 /***** CONSTRUCTORS *****/
62 64
63 public FlowDepthParser(final File importPath, final File rootRelativePath, final ImportRiver river) { 65 public FlowDepthParser(final File importPath, final File rootRelativePath, final ImportRiver river) {
64 super(importPath, rootRelativePath, river); 66 super(importPath, rootRelativePath, river);
65 this.tkhGroup = new FlowDepthSeriesImport(importPath.getName().replaceAll("\\.csv", "")); 67 this.flowdepthGroup = new FlowDepthSeriesImport(importPath.getName().replaceAll("\\.csv", ""));
66 this.seriesHeader = new FlowDepthColumnSeriesImport(this.tkhGroup.getFilename(), this.tkhGroup, null, null); 68 this.seriesHeader = new FlowDepthColumnSeriesImport(this.flowdepthGroup.getFilename(), this.flowdepthGroup, null, null);
67 this.colParsers = new ArrayList<>(); 69 this.colParsers = new ArrayList<>();
68 } 70 }
69 71
70 72
71 /***** METHODS *****/ 73 /***** METHODS *****/
92 parsers.add(new FlowDepthParser(file, new File(relativeDir, file.getName()), river)); 94 parsers.add(new FlowDepthParser(file, new File(relativeDir, file.getName()), river));
93 return parsers; 95 return parsers;
94 } 96 }
95 97
96 @Override 98 @Override
97 public void parse() throws IOException { 99 public void parse() throws Exception {
98 getLog().info("Start parsing:;'" + this.rootRelativePath + "'"); 100 getLog().info("Start parsing:;'" + this.rootRelativePath + "'");
99 // this.seriesHeader = createSeriesImport(this.importPath.getName().replaceAll("\\.csv", "")); 101 // this.seriesHeader = createSeriesImport(this.importPath.getName().replaceAll("\\.csv", ""));
100 this.metaPatternsMatched.clear(); 102 this.metaPatternsMatched.clear();
101 this.kmExists.clear(); 103 this.kmExists.clear();
102 this.colParsers.clear(); 104 this.colParsers.clear();
104 try { 106 try {
105 try { 107 try {
106 this.in = new LineNumberReader(new InputStreamReader(new FileInputStream(this.importPath), ENCODING)); 108 this.in = new LineNumberReader(new InputStreamReader(new FileInputStream(this.importPath), ENCODING));
107 } 109 }
108 catch (final Exception e) { 110 catch (final Exception e) {
109 logError("Could not open (" + e.getMessage() + ")"); 111 logError("Could not open (%s)", e.getMessage());
110 this.headerParsingState = ParsingState.STOP; 112 this.headerParsingState = ParsingState.STOP;
111 } 113 }
112 this.currentLine = null; 114 this.currentLine = null;
113 while (this.headerParsingState == ParsingState.CONTINUE) { 115 while (this.headerParsingState == ParsingState.CONTINUE) {
114 this.currentLine = this.in.readLine(); 116 this.currentLine = this.in.readLine();
116 break; 118 break;
117 this.currentLine = this.currentLine.trim(); 119 this.currentLine = this.currentLine.trim();
118 if (this.currentLine.isEmpty()) 120 if (this.currentLine.isEmpty())
119 continue; 121 continue;
120 handleMetaLine(); 122 handleMetaLine();
123 if (this.headerParsingState == ParsingState.DONE)
124 checkMetaData();
121 } 125 }
122 } 126 }
123 finally { 127 finally {
124 if (this.in != null) { 128 if (this.in != null) {
125 this.in.close(); 129 this.in.close();
150 154
151 private boolean handleMetaYear() { 155 private boolean handleMetaYear() {
152 final Matcher m = META_YEAR.matcher(this.currentLine); 156 final Matcher m = META_YEAR.matcher(this.currentLine);
153 if (m.matches()) { 157 if (m.matches()) {
154 this.metaPatternsMatched.add(META_YEAR); 158 this.metaPatternsMatched.add(META_YEAR);
155 this.tkhGroup.setYear(Integer.parseInt(m.group(1))); 159 this.flowdepthGroup.setYear(Integer.parseInt(m.group(1)));
156 return true; 160 return true;
157 } 161 }
158 return false; 162 return false;
159 } 163 }
160 164
165 169
166 private boolean handleMetaSounding() { 170 private boolean handleMetaSounding() {
167 final Matcher m = META_SOUNDING.matcher(this.currentLine); 171 final Matcher m = META_SOUNDING.matcher(this.currentLine);
168 if (m.matches()) { 172 if (m.matches()) {
169 this.metaPatternsMatched.add(META_SOUNDING); 173 this.metaPatternsMatched.add(META_SOUNDING);
170 this.tkhGroup.setSounding_info(parseMetaInfo(m.group(1).trim())); 174 this.flowdepthGroup.setSounding_info(parseMetaInfo(m.group(1).trim()));
171 return true; 175 return true;
172 } 176 }
173 return false; 177 return false;
174 } 178 }
175 179
176 private boolean handleMetaEvaluator() { 180 private boolean handleMetaEvaluator() {
177 final Matcher m = META_EVALUATOR.matcher(this.currentLine); 181 final Matcher m = META_EVALUATOR.matcher(this.currentLine);
178 if (m.matches()) { 182 if (m.matches()) {
179 this.metaPatternsMatched.add(META_EVALUATOR); 183 this.metaPatternsMatched.add(META_EVALUATOR);
180 this.tkhGroup.setEvaluation_by(parseMetaInfo(m.group(1).trim())); 184 this.flowdepthGroup.setEvaluation_by(parseMetaInfo(m.group(1).trim()));
181 return true; 185 return true;
182 } 186 }
183 return false; 187 return false;
184 } 188 }
185 189
186 @Override 190 @Override
187 protected boolean handleMetaColumnTitles() { 191 protected boolean handleMetaColumnTitles() {
188 if (!super.handleMetaColumnTitles()) 192 if (!super.handleMetaColumnTitles())
189 return false; 193 return false;
190 this.tkhGroup.setKmrange_info(this.seriesHeader.getKmrange_info()); 194 this.flowdepthGroup.setKmrange_info(this.seriesHeader.getKmrange_info());
191 this.tkhGroup.setNotes(this.seriesHeader.getNotes()); 195 this.flowdepthGroup.setNotes(this.seriesHeader.getNotes());
192 for (int i = 1; i <= this.columnTitles.size() - 1; i++) { 196 for (int i = 1; i <= this.columnTitles.size() - 1; i++) {
193 final Matcher m = COLUMN_TITLE.matcher(this.columnTitles.get(i)); 197 final Matcher m = COLUMN_TITLE.matcher(this.columnTitles.get(i));
194 if (m.matches()) 198 if (m.matches())
195 this.colParsers.add(new FlowDepthColumnParser(this.importPath, this.rootRelativePath, this.river, this.tkhGroup, i, m.group(4).trim())); 199 this.colParsers.add(new FlowDepthColumnParser(this.importPath, this.rootRelativePath, this.river, this.flowdepthGroup, i, m.group(4).trim()));
196 else 200 else
197 logWarning("No title found in column " + i + ", skipped"); 201 logLineWarning("Invalid title/unit in column %d (%s)", i + 1, this.columnTitles.get(i));
198 } 202 }
199 return true; 203 return true;
200 } 204 }
205
206 /**
207 * Checks the existence of the active series in the database
208 */
209 @Override
210 protected boolean checkSeriesExistsAlready() {
211 if (!checkRiverExists())
212 return false;
213 final Session session = ImporterSession.getInstance().getDatabaseSession();
214 final List<FlowDepth> rows = this.flowdepthGroup.querySeriesItem(session, this.river.getPeer());
215 return !rows.isEmpty();
216 }
217
201 218
202 @Override 219 @Override
203 public void store() { 220 public void store() {
204 if (this.headerParsingState != ParsingState.STOP) { 221 if (this.headerParsingState != ParsingState.STOP) {
205 this.tkhGroup.getPeer(this.river.getPeer()); 222 this.flowdepthGroup.getPeer(this.river.getPeer());
206 for (final FlowDepthColumnParser colParser : this.colParsers) 223 for (final FlowDepthColumnParser colParser : this.colParsers)
207 colParser.store(); 224 colParser.store();
208 } 225 }
209 else 226 else
210 logWarning("Severe parsing errors, not storing series '" + this.tkhGroup.getFilename() + "'"); 227 logWarning("Severe parsing errors, not storing series '%s'", this.flowdepthGroup.getFilename());
211 } 228 }
212 229
213 @Override 230 @Override
214 protected FlowDepthColumnSeriesImport createSeriesImport(final String filename) { 231 protected FlowDepthColumnSeriesImport createSeriesImport(final String filename) {
215 throw new UnsupportedOperationException(); 232 throw new UnsupportedOperationException();
http://dive4elements.wald.intevation.org