Mercurial > dive4elements > river
changeset 9657:a79881a892c9
Importer (s/u-info) extensions: depth-evolution: corrected directory name Bezug_aktueller_GlW,
detecting and logging of wrong units, then cancelling,
various checks of the plausibility of the meta data year values and cancelling in case of errors,
detecting and logging missing change values, skipping those lines
| author | mschaefer |
|---|---|
| date | Mon, 23 Mar 2020 15:26:50 +0100 |
| parents | 31549fdfaf4f |
| children | d86c7cb68b41 |
| files | backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/DepthEvolutionSeriesImport.java backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/DepthEvolutionParser.java |
| diffstat | 2 files changed, 120 insertions(+), 52 deletions(-) [+] |
line wrap: on
line diff
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/DepthEvolutionSeriesImport.java Mon Mar 23 15:21:39 2020 +0100 +++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/DepthEvolutionSeriesImport.java Mon Mar 23 15:26:50 2020 +0100 @@ -18,7 +18,7 @@ import org.dive4elements.river.model.sinfo.DepthEvolution; import org.dive4elements.river.model.sinfo.DepthEvolution.Group; import org.dive4elements.river.model.sinfo.DepthEvolutionValue; -import org.hibernate.SQLQuery; +import org.hibernate.Query; import org.hibernate.Session; /** @@ -65,10 +65,18 @@ this.start_year = start_year; } + public Integer getStart_year() { + return this.start_year; + } + public void setReference_year(final Integer reference_year) { this.reference_year = reference_year; } + public Integer getReference_year() { + return this.reference_year; + } + public void setCurr_sounding(final String curr_sounding) { this.curr_sounding = curr_sounding; } @@ -91,15 +99,10 @@ } @Override - public List<DepthEvolution> querySeriesItem(final Session session, final River river) { - /* - * final Query query = session.createQuery("FROM DepthEvolution WHERE river=:river AND lower(filename)=:filename"); - * query.setParameter("river", river); - * query.setParameter("filename", this.filename.toLowerCase()); - */ - final SQLQuery query = session.createSQLQuery("SELECT * FROM depth_evolution WHERE (river_id=:river) AND (lower(filename)=:filename)"); - query.setParameter("river", river.getId()); - query.setParameter("filename", getFilename().toLowerCase()); + public List<DepthEvolution> querySeriesItem(final Session session, final River river, final boolean doQueryParent) { + final Query query = session.createQuery("FROM DepthEvolution WHERE river=:river AND lower(filename)=:filename"); + query.setParameter("river", river); + query.setParameter("filename", this.filename.toLowerCase()); return query.list(); }
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/DepthEvolutionParser.java Mon Mar 23 15:21:39 2020 +0100 +++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/DepthEvolutionParser.java Mon Mar 23 15:26:50 2020 +0100 @@ -34,21 +34,7 @@ */ public class DepthEvolutionParser extends AbstractParser<DepthEvolution, DepthEvolutionValue, DepthEvolutionKmLineImport, DepthEvolutionSeriesImport> { - /***** FIELDS *****/ - - private static final Logger log = Logger.getLogger(DepthEvolutionParser.class); - - protected static final Pattern META_REFERENCE_YEAR = Pattern.compile("^#\\sBezugsjahr:\\s*([12]\\d\\d\\d).*", Pattern.CASE_INSENSITIVE); - - protected static final Pattern META_START_YEAR = Pattern.compile("^#\\sAusgangsjahr:\\s*([12]\\d\\d\\d).*", Pattern.CASE_INSENSITIVE); - - private static final Pattern META_CURR_SOUNDING = Pattern.compile("^#\\sAktuelle Peilung\\s*/\\s*Epoche:\\s*([^;]*).*", Pattern.CASE_INSENSITIVE); - - private static final Pattern META_OLD_SOUNDING = Pattern.compile("^#\\sHistorische Peilung\\s*/\\s*Epoche:\\s*([^;]*).*", Pattern.CASE_INSENSITIVE); - - private static final Pattern META_CURR_WSP = Pattern.compile("^#\\sAktuelle Wasserspiegellage:\\s*([^;]*).*", Pattern.CASE_INSENSITIVE); - - private static final Pattern META_OLD_WSP = Pattern.compile("^#\\sHistorische Wasserspiegellage:\\s*([^;]*).*", Pattern.CASE_INSENSITIVE); + /***** TYPES *****/ public enum GroupDirectory { NONE(DepthEvolution.Group.NONE, ""), // @@ -81,6 +67,34 @@ } + /***** FIELDS *****/ + + private static final Logger log = Logger.getLogger(DepthEvolutionParser.class); + + protected static final Pattern META_REFERENCE_YEAR = Pattern.compile("^#\\sBezugsjahr:\\s*([12]\\d{3}).*", Pattern.CASE_INSENSITIVE); + + protected static final Pattern META_START_YEAR = Pattern.compile("^#\\sAusgangsjahr:\\s*([12]\\d{3}).*", Pattern.CASE_INSENSITIVE); + + private static final Pattern META_CURR_SOUNDING = Pattern.compile("^#\\sAktuelle Peilung\\s*\\/\\s*Epoche:\\s*([^;]*([12]\\d{3})[^;]*).*", + Pattern.CASE_INSENSITIVE); + + private static final Pattern META_OLD_SOUNDING = Pattern.compile("^#\\sHistorische Peilung\\s*\\/\\s*Epoche:\\s*([^;]*([12]\\d{3})[^;]*).*", + Pattern.CASE_INSENSITIVE); + + private static final Pattern META_CURR_WSP = Pattern.compile("^#\\sAktuelle Wasserspiegellage:\\s*([^;]*([12]\\d{3})[^;]*).*", Pattern.CASE_INSENSITIVE); + + private static final Pattern META_OLD_WSP = Pattern.compile("^#\\sHistorische Wasserspiegellage:\\s*([^;]*([12]\\d{3})[^;]*).*", Pattern.CASE_INSENSITIVE); + + private static final Pattern COLUMN_TC_TITLE = Pattern.compile("Flie((.)|(ss))tiefenentwicklung\\s*\\[cm\\].*", Pattern.CASE_INSENSITIVE); + + private static final Pattern COLUMN_CPY_TITLE = Pattern.compile("Flie((.)|(ss))tiefenentwicklung\\s*\\[cm\\/a\\].*", Pattern.CASE_INSENSITIVE); + + private int currSoundingYear; + private int oldSoundingYear; + private int currWspYear; + private int oldWspYear; + + /***** CONSTRUCTORS *****/ public DepthEvolutionParser(final File importPath, final File rootRelativePath, final ImportRiver river) { @@ -115,9 +129,9 @@ @Override protected boolean handleMetaOther() { - if (handleMetaStartYear()) + if (handleMetaReferenceYear()) return true; - else if (handleMetaReferenceYear()) + else if (handleMetaStartYear()) return true; else if (handleMetaCurrSounding()) return true; @@ -131,6 +145,16 @@ return false; } + private boolean handleMetaReferenceYear() { + final Matcher m = META_REFERENCE_YEAR.matcher(this.currentLine); + if (m.matches()) { + this.metaPatternsMatched.add(META_REFERENCE_YEAR); + this.seriesHeader.setReference_year(Integer.parseInt(m.group(1))); + return true; + } + return false; + } + private boolean handleMetaStartYear() { final Matcher m = META_START_YEAR.matcher(this.currentLine); if (m.matches()) { @@ -141,21 +165,12 @@ return false; } - private boolean handleMetaReferenceYear() { - final Matcher m = META_REFERENCE_YEAR.matcher(this.currentLine); - if (m.matches()) { - this.metaPatternsMatched.add(META_REFERENCE_YEAR); - this.seriesHeader.setReference_year(Integer.parseInt(m.group(1))); - return true; - } - return false; - } - private boolean handleMetaCurrSounding() { final Matcher m = META_CURR_SOUNDING.matcher(this.currentLine); if (m.matches()) { this.metaPatternsMatched.add(META_CURR_SOUNDING); this.seriesHeader.setCurr_sounding(parseMetaInfo(m.group(1).trim())); + this.currSoundingYear = Integer.valueOf(m.group(2)).intValue(); return true; } return false; @@ -166,6 +181,7 @@ if (m.matches()) { this.metaPatternsMatched.add(META_OLD_SOUNDING); this.seriesHeader.setOld_sounding(parseMetaInfo(m.group(1).trim())); + this.oldSoundingYear = Integer.valueOf(m.group(2)).intValue(); return true; } return false; @@ -176,6 +192,7 @@ if (m.matches()) { this.metaPatternsMatched.add(META_CURR_WSP); this.seriesHeader.setCurr_glw(parseMetaInfo(m.group(1).trim())); + this.currWspYear = Integer.valueOf(m.group(2)).intValue(); return true; } return false; @@ -186,24 +203,71 @@ if (m.matches()) { this.metaPatternsMatched.add(META_OLD_WSP); this.seriesHeader.setOld_glw(parseMetaInfo(m.group(1).trim())); + this.oldWspYear = Integer.valueOf(m.group(2)).intValue(); return true; } return false; } @Override + protected boolean checkMetaData() { + if (super.checkMetaData() == false) + return false; + if (this.seriesHeader.getReference_year() == null) { + logError("Reference year is missing"); + this.headerParsingState = ParsingState.STOP; + return false; + } + if (this.seriesHeader.getStart_year() == null) { + logError("Start year is missing"); + this.headerParsingState = ParsingState.STOP; + return false; + } + if (this.seriesHeader.getReference_year() <= this.seriesHeader.getStart_year()) { + logError("Reference year not greater than start year"); + this.headerParsingState = ParsingState.STOP; + return false; + } + if (this.currSoundingYear <= this.oldSoundingYear) { + logError("Current sounding year not greater than historical sounding year"); + this.headerParsingState = ParsingState.STOP; + return false; + } + if (this.currWspYear != this.seriesHeader.getReference_year()) { + logError("Current waterlevel year differs from reference year"); + this.headerParsingState = ParsingState.STOP; + return false; + } + if (this.oldWspYear != this.seriesHeader.getStart_year()) { + logError("Historical waterlevel year differs from start year"); + this.headerParsingState = ParsingState.STOP; + return false; + } + if (!this.metaPatternsMatched.contains(META_START_YEAR) || !this.metaPatternsMatched.contains(META_REFERENCE_YEAR) + || !this.metaPatternsMatched.contains(META_CURR_SOUNDING) || !this.metaPatternsMatched.contains(META_OLD_SOUNDING) + || !this.metaPatternsMatched.contains(META_CURR_WSP) || !this.metaPatternsMatched.contains(META_OLD_WSP)) { + logError("One or more of the required meta infos are missing"); + this.headerParsingState = ParsingState.STOP; + } + return true; + } + + @Override protected boolean handleMetaColumnTitles() { - if (super.handleMetaColumnTitles()) { - if (!this.metaPatternsMatched.contains(META_START_YEAR) || !this.metaPatternsMatched.contains(META_REFERENCE_YEAR) - || !this.metaPatternsMatched.contains(META_CURR_SOUNDING) || !this.metaPatternsMatched.contains(META_OLD_SOUNDING) - || !this.metaPatternsMatched.contains(META_CURR_WSP) || !this.metaPatternsMatched.contains(META_OLD_WSP)) { - logError("One or more of the required meta infos are missing"); - this.headerParsingState = ParsingState.STOP; - } + if (!super.handleMetaColumnTitles()) + return false; + final Matcher tcm = COLUMN_TC_TITLE.matcher(this.columnTitles.get(1)); + if (!tcm.matches()) { + logLineError("Column 2: Invalid column title and/or unit"); + this.headerParsingState = ParsingState.STOP; return true; } - else - return false; + final Matcher cpym = COLUMN_CPY_TITLE.matcher(this.columnTitles.get(2)); + if (!cpym.matches()) { + logLineError("Column 3: Invalid column title and/or unit"); + this.headerParsingState = ParsingState.STOP; + } + return true; } @Override @@ -215,16 +279,17 @@ @Override protected DepthEvolutionKmLineImport createKmLineImport(final Double km, final String[] values) { - if (parseDoubleWithNull(values[1]) == null) { - logError("Invalid total change in line " + this.in.getLineNumber()); + final Number tc = parseDoubleCheckNull(values, 1); + if ((tc == null) || Double.isNaN(tc.doubleValue())) { + logLineWarning(INVALID_VALUE_ERROR_FORMAT, "total change"); return null; } - if (parseDoubleWithNull(values[2]) == null) { - logError("Invalid change per year in line " + this.in.getLineNumber()); + final Number cpy = parseDoubleCheckNull(values, 2); + if ((cpy == null) || Double.isNaN(cpy.doubleValue())) { + logLineWarning(INVALID_VALUE_ERROR_FORMAT, "change per year"); return null; } // cm to m - return new DepthEvolutionKmLineImport(km, parseDoubleWithNull(values[1]).doubleValue() / 100.0, - parseDoubleWithNull(values[2]).doubleValue() / 100.0); + return new DepthEvolutionKmLineImport(km, tc.doubleValue() / 100.0, cpy.doubleValue() / 100.0); } }