Mercurial > dive4elements > river

comparison backend/src/main/java/org/dive4elements/river/importer/uinfo/parsers/VegetationParser.java @ 9661:9b8ba3b83a15

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Importer (s/u-info) vegetation zones: new database column in vegetation_type table for german type name, localized vegetation type names by querying the database instead of translating by resource property, detecting and cancelling the import of a second vegetation zone file for a river, detecting, logging, cancelling in case of wrong column titles, detecting, logging and ignoring lines with missing (color) values, comparing vegetation zone name and class with the database and logging+ignoring in case of inconsistencies, starting the most elevated zone with 0 instead of -1 overflow days
author mschaefer
date Mon, 23 Mar 2020 16:38:12 +0100
parents 6146358c4842
children
comparison
equal deleted inserted replaced
9660:f0cad5212f49 9661:9b8ba3b83a15
12 12
13 import java.io.File; 13 import java.io.File;
14 import java.util.ArrayList; 14 import java.util.ArrayList;
15 import java.util.EnumMap; 15 import java.util.EnumMap;
16 import java.util.List; 16 import java.util.List;
17 import java.util.Locale;
17 import java.util.regex.Matcher; 18 import java.util.regex.Matcher;
18 import java.util.regex.Pattern; 19 import java.util.regex.Pattern;
19 20
20 import org.apache.log4j.Logger; 21 import org.apache.log4j.Logger;
21 import org.dive4elements.river.importer.Config; 22 import org.dive4elements.river.importer.Config;
22 import org.dive4elements.river.importer.ImportRiver; 23 import org.dive4elements.river.importer.ImportRiver;
24 import org.dive4elements.river.importer.ImporterSession;
23 import org.dive4elements.river.importer.common.AbstractParser; 25 import org.dive4elements.river.importer.common.AbstractParser;
24 import org.dive4elements.river.importer.common.ParsingState; 26 import org.dive4elements.river.importer.common.ParsingState;
25 import org.dive4elements.river.importer.uinfo.importitem.VegetationSeriesImport; 27 import org.dive4elements.river.importer.uinfo.importitem.VegetationSeriesImport;
26 import org.dive4elements.river.importer.uinfo.importitem.VegetationZoneImport; 28 import org.dive4elements.river.importer.uinfo.importitem.VegetationZoneImport;
27 import org.dive4elements.river.model.uinfo.Vegetation; 29 import org.dive4elements.river.model.uinfo.Vegetation;
30 import org.dive4elements.river.model.uinfo.VegetationType;
28 import org.dive4elements.river.model.uinfo.VegetationZone; 31 import org.dive4elements.river.model.uinfo.VegetationZone;
32 import org.hibernate.Session;
29 33
30 /** 34 /**
31 * Reads and parses a vegetation zones file 35 * Reads and parses a vegetation zones file
32 * 36 *
33 * @author Matthias Schäfer 37 * @author Matthias Schäfer
37 41
38 /***** FIELDS *****/ 42 /***** FIELDS *****/
39 43
40 private static final Logger log = Logger.getLogger(VegetationParser.class); 44 private static final Logger log = Logger.getLogger(VegetationParser.class);
41 45
46 private static final String IMPORT_FILENAME = "Standardvegetationszonen.csv";
47
48 private static final Pattern META_FIRST = Pattern.compile("^#\\sVegetationszonen.*", Pattern.CASE_INSENSITIVE);
49
42 private static final Pattern META_NAME = Pattern.compile("^#\\sEinteilung:\\s*([^;]*).*", Pattern.CASE_INSENSITIVE); 50 private static final Pattern META_NAME = Pattern.compile("^#\\sEinteilung:\\s*([^;]*).*", Pattern.CASE_INSENSITIVE);
43 51
44 private static final Pattern META_COLUMNTITLES = Pattern.compile("^#*\\s*Vegetationstyp\\s*;.+", Pattern.CASE_INSENSITIVE); 52 private static final Pattern META_COLUMNTITLES = Pattern.compile("^#*\\s*Vegetationstyp\\s*;.+", Pattern.CASE_INSENSITIVE);
45 53
46 private enum ColTitlePattern { 54 private enum ColTitlePattern {
94 /** 102 /**
95 * Creates a list of parsers for all vegetation import files in a directory 103 * Creates a list of parsers for all vegetation import files in a directory
96 */ 104 */
97 public static List<VegetationParser> createParsers(final File importDir, final File relativeDir, final ImportRiver river) { 105 public static List<VegetationParser> createParsers(final File importDir, final File relativeDir, final ImportRiver river) {
98 final List<VegetationParser> parsers = new ArrayList<>(); 106 final List<VegetationParser> parsers = new ArrayList<>();
99 if (importDir.exists()) { 107 final File importFile = new File(importDir, IMPORT_FILENAME);
100 for (final File file : listFiles(importDir, ".csv")) 108 if (importFile.exists())
101 parsers.add(new VegetationParser(file, new File(relativeDir, file.getName()), river)); 109 parsers.add(new VegetationParser(importFile, new File(relativeDir, IMPORT_FILENAME), river));
102 }
103 return parsers; 110 return parsers;
104 } 111 }
105 112
106 @Override 113 @Override
107 protected KmMode kmMode() { 114 protected KmMode kmMode() {
113 return new VegetationSeriesImport(filename); 120 return new VegetationSeriesImport(filename);
114 } 121 }
115 122
116 @Override 123 @Override
117 protected boolean handleMetaOther() { 124 protected boolean handleMetaOther() {
118 if (handleMetaName()) 125 if (handleMetaFirst())
126 return true;
127 else if (handleMetaName())
119 return true; 128 return true;
120 else 129 else
121 return false; 130 return false;
131 }
132
133 private boolean handleMetaFirst() {
134 final Matcher m = META_FIRST.matcher(this.currentLine);
135 if (m.matches()) {
136 this.metaPatternsMatched.add(META_FIRST);
137 return true;
138 }
139 return false;
122 } 140 }
123 141
124 private boolean handleMetaName() { 142 private boolean handleMetaName() {
125 final Matcher m = META_NAME.matcher(this.currentLine); 143 final Matcher m = META_NAME.matcher(this.currentLine);
126 if (m.matches()) { 144 if (m.matches()) {
148 this.cols.put(col, i); 166 this.cols.put(col, i);
149 break; 167 break;
150 } 168 }
151 } 169 }
152 } 170 }
153 if ((this.cols.get(ColTitlePattern.OVERFLOW_LIMIT) < 0) || (this.cols.get(ColTitlePattern.CLASSNO) < 0)) {
154 logError("Column of the overflow duration limit and/or vegetation zone class could not be identified");
155 this.headerParsingState = ParsingState.STOP;
156 return true;
157 }
158 this.previousClassNo = 0; 171 this.previousClassNo = 0;
159 this.previousDaysLimit = -1; 172 this.previousDaysLimit = 0;
173 return true;
174 }
175
176 @Override
177 protected boolean checkMetaData() {
178 if (super.checkMetaData() == false)
179 return false;
180 for (final ColTitlePattern value : ColTitlePattern.values()) {
181 if (this.cols.get(value) < 0) {
182 logError("No valid header line with the column titles found");
183 this.headerParsingState = ParsingState.STOP;
184 return false;
185 }
186 }
160 return true; 187 return true;
161 } 188 }
162 189
163 @Override 190 @Override
164 protected VegetationZoneImport createKmLineImport(final Double km, final String[] values) { 191 protected VegetationZoneImport createKmLineImport(final Double km, final String[] values) {
168 if (!values[this.cols.get(ColTitlePattern.OVERFLOW_LIMIT)].trim().isEmpty()) 195 if (!values[this.cols.get(ColTitlePattern.OVERFLOW_LIMIT)].trim().isEmpty())
169 daysLimit = Integer.parseInt(values[this.cols.get(ColTitlePattern.OVERFLOW_LIMIT)]); 196 daysLimit = Integer.parseInt(values[this.cols.get(ColTitlePattern.OVERFLOW_LIMIT)]);
170 classNo = Integer.parseInt(values[this.cols.get(ColTitlePattern.CLASSNO)]); 197 classNo = Integer.parseInt(values[this.cols.get(ColTitlePattern.CLASSNO)]);
171 } 198 }
172 catch (final Exception e) { 199 catch (final Exception e) {
173 logError("Overflow days limit and/or vegetation zone class could not be parsed: line " + this.in.getLineNumber()); 200 logLineWarning("Invalid overflow days limit and/or vegetation zone class (%s)", e.getMessage());
174 return null; 201 return null;
175 } 202 }
176 // Check completeness of vegetation zone type set, if needed 203 // Check completeness of vegetation zone type set, if needed
177 // if (classNo != this.previousClassNo + 1) { 204 // if (classNo != this.previousClassNo + 1) {
178 // logError("Wrong vegetation zone class number or wrong class order: line " + this.in.getLineNumber()); 205 // logError("Wrong vegetation zone class number or wrong class order: line " + this.in.getLineNumber());
180 // } 207 // }
181 // if (!this.types.containsKey(Integer.valueOf(classNo))) { 208 // if (!this.types.containsKey(Integer.valueOf(classNo))) {
182 // logError("Unknown vegetation zone class: line " + this.in.getLineNumber()); 209 // logError("Unknown vegetation zone class: line " + this.in.getLineNumber());
183 // return null; 210 // return null;
184 // } 211 // }
212 boolean classokay = false;
213 final Session session = ImporterSession.getInstance().getDatabaseSession();
214 for (final VegetationType vt : VegetationType.getTypes(session)) {
215 if (classNo == vt.getId().intValue()) {
216 final String dbname = vt.getLocalizedName(Locale.GERMAN);
217 if (!values[0].equalsIgnoreCase(dbname)) {
218 logLineWarning("Wrong Vegetationstyp or Vegetationsklasse (%d is '%s' in the database)", classNo, dbname);
219 return null;
220 }
221 classokay = true;
222 break;
223 }
224 }
225 if (!classokay) {
226 logLineWarning("Unknown Vegetationsklasse");
227 return null;
228 }
229
185 this.previousClassNo = classNo; 230 this.previousClassNo = classNo;
186 final int minDays = this.previousDaysLimit; 231 final int minDays = this.previousDaysLimit;
187 this.previousDaysLimit = daysLimit; 232 this.previousDaysLimit = daysLimit;
188 final int red = (this.cols.get(ColTitlePattern.COLOR_R) >= 0) ? Integer.parseInt(values[this.cols.get(ColTitlePattern.COLOR_R)]) : 0; 233 final Integer red = parseIntegerCheckNull(values, this.cols.get(ColTitlePattern.COLOR_R));
189 final int green = (this.cols.get(ColTitlePattern.COLOR_G) >= 0) ? Integer.parseInt(values[this.cols.get(ColTitlePattern.COLOR_G)]) : 0; 234 final Integer green = parseIntegerCheckNull(values, this.cols.get(ColTitlePattern.COLOR_G));
190 final int blue = (this.cols.get(ColTitlePattern.COLOR_B) >= 0) ? Integer.parseInt(values[this.cols.get(ColTitlePattern.COLOR_B)]) : 0; 235 final Integer blue = parseIntegerCheckNull(values, this.cols.get(ColTitlePattern.COLOR_B));
191 return new VegetationZoneImport(classNo, minDays, daysLimit, red, green, blue); 236 if ((red == null) || (green == null) || (blue == null)) {
237 logLineWarning("One or more color components invalid or missing");
238 return null;
239 }
240 if ((red.intValue() < 0) || (red.intValue() > 255)) {
241 logLineWarning("Invalid red value");
242 return null;
243 }
244 if ((green.intValue() < 0) || (green.intValue() > 255)) {
245 logLineError("Invalid green value");
246 return null;
247 }
248 if ((blue.intValue() < 0) || (blue.intValue() > 255)) {
249 logLineError("Invalid blue value");
250 return null;
251 }
252 return new VegetationZoneImport(classNo, minDays, daysLimit, red.intValue(), green.intValue(), blue.intValue());
192 } 253 }
193 } 254 }
http://dive4elements.wald.intevation.org