Mercurial > dive4elements > river
changeset 9661:9b8ba3b83a15
Importer (s/u-info) vegetation zones: new database column in vegetation_type table for german type name,
localized vegetation type names by querying the database instead of translating by resource property,
detecting and cancelling the import of a second vegetation zone file for a river,
detecting, logging, cancelling in case of wrong column titles,
detecting, logging and ignoring lines with missing (color) values,
comparing vegetation zone name and class with the database and logging+ignoring in case of inconsistencies,
starting the most elevated zone with 0 instead of -1 overflow days
line wrap: on
line diff
--- a/artifacts/src/main/java/org/dive4elements/river/artifacts/uinfo/vegetationzones/VegetationZoneServerClientXChange.java Mon Mar 23 15:40:12 2020 +0100 +++ b/artifacts/src/main/java/org/dive4elements/river/artifacts/uinfo/vegetationzones/VegetationZoneServerClientXChange.java Mon Mar 23 16:38:12 2020 +0100 @@ -102,7 +102,7 @@ } public static String getLocalizedVegetationTypeName(final CallContext context, final VegetationType zonetype) { - return Resources.getMsg(context.getMeta(), "uinfo_vegetation_type_" + zonetype.getId().toString()); + return zonetype.getLocalizedName(Resources.getLocale(context.getMeta())); } public static final List<VegetationZoneServerClientXChange> getStandardList(final River river, final CallContext context) {
--- a/artifacts/src/main/java/org/dive4elements/river/artifacts/uinfo/vegetationzones/VegetationZonesCrossSectionProcessor.java Mon Mar 23 15:40:12 2020 +0100 +++ b/artifacts/src/main/java/org/dive4elements/river/artifacts/uinfo/vegetationzones/VegetationZonesCrossSectionProcessor.java Mon Mar 23 16:38:12 2020 +0100 @@ -87,9 +87,7 @@ for (int i = 0; i < zones.size(); i++) { final VegetationZoneServerClientXChange zone = zones.get(i); - Integer lowerFromTo = zone.getLowerFromTo(); - if (i == 0) // Hack; turning "-1" invisible - lowerFromTo = 0; + final Integer lowerFromTo = zone.getLowerFromTo(); final double lower = uefdToHeight(river, currentStation, lowerFromTo); final double upper = uefdToHeight(river, currentStation, zone.getUpperFromTo());
--- a/artifacts/src/main/resources/messages.properties Mon Mar 23 15:40:12 2020 +0100 +++ b/artifacts/src/main/resources/messages.properties Mon Mar 23 16:38:12 2020 +0100 @@ -1165,15 +1165,6 @@ uinfo_salix_line = Iota (River/Floodplain Connectivity) uinfo_inundation_duration = Flood Duration over Floodplain uinfo_vegetation_zones = Vegetation Zones -uinfo_vegetation_type_1 = zonal forest -uinfo_vegetation_type_2 = dry hartwood forest floodplain -uinfo_vegetation_type_3 = wet hartwood forest floodplain -uinfo_vegetation_type_4 = salix alba forest -uinfo_vegetation_type_5 = salix shrubs -uinfo_vegetation_type_6 = reed bed -uinfo_vegetation_type_7 = bank pioneers -uinfo_vegetation_type_8 = no vegetation -uinfo_vegetation_type_9 = water state.uinfo.river = River state.uinfo.calculation_mode=Calculation Mode
--- a/artifacts/src/main/resources/messages_de.properties Mon Mar 23 15:40:12 2020 +0100 +++ b/artifacts/src/main/resources/messages_de.properties Mon Mar 23 16:38:12 2020 +0100 @@ -1165,15 +1165,6 @@ uinfo_salix_line = Iota (Fluss/Aue-Konnektivit\u00e4t) uinfo_inundation_duration = \u00dcberflutungsdauern Aue uinfo_vegetation_zones = Vegetationszonen -uinfo_vegetation_type_1 = Zonaler Wald -uinfo_vegetation_type_2 = Hartholzaue, trocken -uinfo_vegetation_type_3 = Hartholzaue, feucht -uinfo_vegetation_type_4 = Silberweidenwald -uinfo_vegetation_type_5 = Weidengeb\u00fcsch -uinfo_vegetation_type_6 = Uferr\u00f6hricht -uinfo_vegetation_type_7 = Uferpioniere -uinfo_vegetation_type_8 = Vegetationslos -uinfo_vegetation_type_9 = Wasserfl\u00e4che state.uinfo.river = Gew\u00e4sser state.uinfo.calculation_mode=Berechnungsart
--- a/backend/doc/schema/oracle-sinfo-uinfo.sql Mon Mar 23 15:40:12 2020 +0100 +++ b/backend/doc/schema/oracle-sinfo-uinfo.sql Mon Mar 23 16:38:12 2020 +0100 @@ -429,17 +429,19 @@ CREATE TABLE vegetation_type ( id NUMBER(9,0) PRIMARY KEY, - name VARCHAR2(256) NOT NULL + name VARCHAR2(256) NOT NULL, + de_name VARCHAR2(256) NOT NULL ); -INSERT INTO vegetation_type (id, name) VALUES (1, 'zonal forest'); -INSERT INTO vegetation_type (id, name) VALUES (2, 'dry hartwood forest floodplain'); -INSERT INTO vegetation_type (id, name) VALUES (3, 'wet hartwood forest floodplain'); -INSERT INTO vegetation_type (id, name) VALUES (4, 'salix alba forest'); -INSERT INTO vegetation_type (id, name) VALUES (5, 'salix shrubs'); -INSERT INTO vegetation_type (id, name) VALUES (6, 'reed bed'); -INSERT INTO vegetation_type (id, name) VALUES (7, 'bank pioneers'); -INSERT INTO vegetation_type (id, name) VALUES (8, 'no vegetation'); -INSERT INTO vegetation_type (id, name) VALUES (9, 'water'); +INSERT INTO vegetation_type (id, name, de_name) VALUES (1, 'zonal forest', 'Zonaler Wald'); +INSERT INTO vegetation_type (id, name, de_name) VALUES (1, 'zonal forest', 'Zonaler Wald'); +INSERT INTO vegetation_type (id, name, de_name) VALUES (2, 'dry hartwood forest floodplain', 'Hartholzaue, trocken'); +INSERT INTO vegetation_type (id, name, de_name) VALUES (3, 'wet hartwood forest floodplain', 'Hartholzaue, feucht'); +INSERT INTO vegetation_type (id, name, de_name) VALUES (4, 'salix alba forest', 'Silberweidenwald'); +INSERT INTO vegetation_type (id, name, de_name) VALUES (5, 'salix shrubs', 'Weidengebüsch'); +INSERT INTO vegetation_type (id, name, de_name) VALUES (6, 'reed bed', 'Uferröhricht'); +INSERT INTO vegetation_type (id, name, de_name) VALUES (7, 'bank pioneers', 'Uferpioniere'); +INSERT INTO vegetation_type (id, name, de_name) VALUES (8, 'no vegetation', 'vegetationslos'); +INSERT INTO vegetation_type (id, name, de_name) VALUES (9, 'water', 'Wasserfläche'); CREATE TABLE vegetation (
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/backend/doc/schema/oracle_migrations/update_bedarf_2-3-6-3-B.sql Mon Mar 23 16:38:12 2020 +0100 @@ -0,0 +1,19 @@ +--Neues Feld vegetation_type.de_name + +SET AUTOCOMMIT ON; + +ALTER TABLE vegetation_type ADD (de_name VARCHAR2(256)); + +UPDATE vegetation_type SET de_name='Zonaler Wald' WHERE id=1; +UPDATE vegetation_type SET de_name='Hartholzaue, trocken' WHERE id=2; +UPDATE vegetation_type SET de_name='Hartholzaue, feucht' WHERE id=3; +UPDATE vegetation_type SET de_name='Silberweidenwald' WHERE id=4; +UPDATE vegetation_type SET de_name='Weidengebüsch' WHERE id=5; +UPDATE vegetation_type SET de_name='Uferröhricht' WHERE id=6; +UPDATE vegetation_type SET de_name='Uferpioniere' WHERE id=7; +UPDATE vegetation_type SET de_name='vegetationslos' WHERE id=8; +UPDATE vegetation_type SET de_name='Wasserfläche' WHERE id=9; + +ALTER TABLE vegetation_type MODIFY (de_name NOT NULL); + +COMMIT;
--- a/backend/doc/schema/postgresql-sinfo-uinfo.sql Mon Mar 23 15:40:12 2020 +0100 +++ b/backend/doc/schema/postgresql-sinfo-uinfo.sql Mon Mar 23 16:38:12 2020 +0100 @@ -425,17 +425,18 @@ CREATE TABLE vegetation_type ( id NUMERIC(9,0) PRIMARY KEY, - name VARCHAR(256) NOT NULL + name VARCHAR(256) NOT NULL, + de_name VARCHAR(256) NOT NULL ); -INSERT INTO vegetation_type (id, name) VALUES (1, 'zonal forest'); -INSERT INTO vegetation_type (id, name) VALUES (2, 'dry hartwood forest floodplain'); -INSERT INTO vegetation_type (id, name) VALUES (3, 'wet hartwood forest floodplain'); -INSERT INTO vegetation_type (id, name) VALUES (4, 'salix alba forest'); -INSERT INTO vegetation_type (id, name) VALUES (5, 'salix shrubs'); -INSERT INTO vegetation_type (id, name) VALUES (6, 'reed bed'); -INSERT INTO vegetation_type (id, name) VALUES (7, 'bank pioneers'); -INSERT INTO vegetation_type (id, name) VALUES (8, 'no vegetation'); -INSERT INTO vegetation_type (id, name) VALUES (9, 'water'); +INSERT INTO vegetation_type (id, name, de_name) VALUES (1, 'zonal forest', 'Zonaler Wald'); +INSERT INTO vegetation_type (id, name, de_name) VALUES (2, 'dry hartwood forest floodplain', 'Hartholzaue, trocken'); +INSERT INTO vegetation_type (id, name, de_name) VALUES (3, 'wet hartwood forest floodplain', 'Hartholzaue, feucht'); +INSERT INTO vegetation_type (id, name, de_name) VALUES (4, 'salix alba forest', 'Silberweidenwald'); +INSERT INTO vegetation_type (id, name, de_name) VALUES (5, 'salix shrubs', 'Weidengebüsch'); +INSERT INTO vegetation_type (id, name, de_name) VALUES (6, 'reed bed', 'Uferröhricht'); +INSERT INTO vegetation_type (id, name, de_name) VALUES (7, 'bank pioneers', 'Uferpioniere'); +INSERT INTO vegetation_type (id, name, de_name) VALUES (8, 'no vegetation', 'vegetationslos'); +INSERT INTO vegetation_type (id, name, de_name) VALUES (9, 'water', 'Wasserfläche'); CREATE TABLE vegetation (
--- a/backend/src/main/java/org/dive4elements/river/importer/uinfo/importitem/VegetationSeriesImport.java Mon Mar 23 15:40:12 2020 +0100 +++ b/backend/src/main/java/org/dive4elements/river/importer/uinfo/importitem/VegetationSeriesImport.java Mon Mar 23 16:38:12 2020 +0100 @@ -54,10 +54,9 @@ } @Override - public List<Vegetation> querySeriesItem(final Session session, final River river) { - final Query query = session.createQuery("FROM Vegetation WHERE river=:river AND lower(filename)=:filename"); + public List<Vegetation> querySeriesItem(final Session session, final River river, final boolean doQueryParent) { + final Query query = session.createQuery("FROM Vegetation WHERE river=:river"); query.setParameter("river", river); - query.setParameter("filename", this.filename.toLowerCase()); return query.list(); }
--- a/backend/src/main/java/org/dive4elements/river/importer/uinfo/parsers/VegetationParser.java Mon Mar 23 15:40:12 2020 +0100 +++ b/backend/src/main/java/org/dive4elements/river/importer/uinfo/parsers/VegetationParser.java Mon Mar 23 16:38:12 2020 +0100 @@ -14,18 +14,22 @@ import java.util.ArrayList; import java.util.EnumMap; import java.util.List; +import java.util.Locale; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.log4j.Logger; import org.dive4elements.river.importer.Config; import org.dive4elements.river.importer.ImportRiver; +import org.dive4elements.river.importer.ImporterSession; import org.dive4elements.river.importer.common.AbstractParser; import org.dive4elements.river.importer.common.ParsingState; import org.dive4elements.river.importer.uinfo.importitem.VegetationSeriesImport; import org.dive4elements.river.importer.uinfo.importitem.VegetationZoneImport; import org.dive4elements.river.model.uinfo.Vegetation; +import org.dive4elements.river.model.uinfo.VegetationType; import org.dive4elements.river.model.uinfo.VegetationZone; +import org.hibernate.Session; /** * Reads and parses a vegetation zones file @@ -39,6 +43,10 @@ private static final Logger log = Logger.getLogger(VegetationParser.class); + private static final String IMPORT_FILENAME = "Standardvegetationszonen.csv"; + + private static final Pattern META_FIRST = Pattern.compile("^#\\sVegetationszonen.*", Pattern.CASE_INSENSITIVE); + private static final Pattern META_NAME = Pattern.compile("^#\\sEinteilung:\\s*([^;]*).*", Pattern.CASE_INSENSITIVE); private static final Pattern META_COLUMNTITLES = Pattern.compile("^#*\\s*Vegetationstyp\\s*;.+", Pattern.CASE_INSENSITIVE); @@ -96,10 +104,9 @@ */ public static List<VegetationParser> createParsers(final File importDir, final File relativeDir, final ImportRiver river) { final List<VegetationParser> parsers = new ArrayList<>(); - if (importDir.exists()) { - for (final File file : listFiles(importDir, ".csv")) - parsers.add(new VegetationParser(file, new File(relativeDir, file.getName()), river)); - } + final File importFile = new File(importDir, IMPORT_FILENAME); + if (importFile.exists()) + parsers.add(new VegetationParser(importFile, new File(relativeDir, IMPORT_FILENAME), river)); return parsers; } @@ -115,12 +122,23 @@ @Override protected boolean handleMetaOther() { - if (handleMetaName()) + if (handleMetaFirst()) + return true; + else if (handleMetaName()) return true; else return false; } + private boolean handleMetaFirst() { + final Matcher m = META_FIRST.matcher(this.currentLine); + if (m.matches()) { + this.metaPatternsMatched.add(META_FIRST); + return true; + } + return false; + } + private boolean handleMetaName() { final Matcher m = META_NAME.matcher(this.currentLine); if (m.matches()) { @@ -150,13 +168,22 @@ } } } - if ((this.cols.get(ColTitlePattern.OVERFLOW_LIMIT) < 0) || (this.cols.get(ColTitlePattern.CLASSNO) < 0)) { - logError("Column of the overflow duration limit and/or vegetation zone class could not be identified"); - this.headerParsingState = ParsingState.STOP; - return true; + this.previousClassNo = 0; + this.previousDaysLimit = 0; + return true; + } + + @Override + protected boolean checkMetaData() { + if (super.checkMetaData() == false) + return false; + for (final ColTitlePattern value : ColTitlePattern.values()) { + if (this.cols.get(value) < 0) { + logError("No valid header line with the column titles found"); + this.headerParsingState = ParsingState.STOP; + return false; + } } - this.previousClassNo = 0; - this.previousDaysLimit = -1; return true; } @@ -170,7 +197,7 @@ classNo = Integer.parseInt(values[this.cols.get(ColTitlePattern.CLASSNO)]); } catch (final Exception e) { - logError("Overflow days limit and/or vegetation zone class could not be parsed: line " + this.in.getLineNumber()); + logLineWarning("Invalid overflow days limit and/or vegetation zone class (%s)", e.getMessage()); return null; } // Check completeness of vegetation zone type set, if needed @@ -182,12 +209,46 @@ // logError("Unknown vegetation zone class: line " + this.in.getLineNumber()); // return null; // } + boolean classokay = false; + final Session session = ImporterSession.getInstance().getDatabaseSession(); + for (final VegetationType vt : VegetationType.getTypes(session)) { + if (classNo == vt.getId().intValue()) { + final String dbname = vt.getLocalizedName(Locale.GERMAN); + if (!values[0].equalsIgnoreCase(dbname)) { + logLineWarning("Wrong Vegetationstyp or Vegetationsklasse (%d is '%s' in the database)", classNo, dbname); + return null; + } + classokay = true; + break; + } + } + if (!classokay) { + logLineWarning("Unknown Vegetationsklasse"); + return null; + } + this.previousClassNo = classNo; final int minDays = this.previousDaysLimit; this.previousDaysLimit = daysLimit; - final int red = (this.cols.get(ColTitlePattern.COLOR_R) >= 0) ? Integer.parseInt(values[this.cols.get(ColTitlePattern.COLOR_R)]) : 0; - final int green = (this.cols.get(ColTitlePattern.COLOR_G) >= 0) ? Integer.parseInt(values[this.cols.get(ColTitlePattern.COLOR_G)]) : 0; - final int blue = (this.cols.get(ColTitlePattern.COLOR_B) >= 0) ? Integer.parseInt(values[this.cols.get(ColTitlePattern.COLOR_B)]) : 0; - return new VegetationZoneImport(classNo, minDays, daysLimit, red, green, blue); + final Integer red = parseIntegerCheckNull(values, this.cols.get(ColTitlePattern.COLOR_R)); + final Integer green = parseIntegerCheckNull(values, this.cols.get(ColTitlePattern.COLOR_G)); + final Integer blue = parseIntegerCheckNull(values, this.cols.get(ColTitlePattern.COLOR_B)); + if ((red == null) || (green == null) || (blue == null)) { + logLineWarning("One or more color components invalid or missing"); + return null; + } + if ((red.intValue() < 0) || (red.intValue() > 255)) { + logLineWarning("Invalid red value"); + return null; + } + if ((green.intValue() < 0) || (green.intValue() > 255)) { + logLineError("Invalid green value"); + return null; + } + if ((blue.intValue() < 0) || (blue.intValue() > 255)) { + logLineError("Invalid blue value"); + return null; + } + return new VegetationZoneImport(classNo, minDays, daysLimit, red.intValue(), green.intValue(), blue.intValue()); } }
--- a/backend/src/main/java/org/dive4elements/river/model/uinfo/VegetationType.java Mon Mar 23 15:40:12 2020 +0100 +++ b/backend/src/main/java/org/dive4elements/river/model/uinfo/VegetationType.java Mon Mar 23 16:38:12 2020 +0100 @@ -13,6 +13,7 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.List; +import java.util.Locale; import javax.persistence.Column; import javax.persistence.Entity; @@ -41,6 +42,7 @@ private String name; + private String de_name; /***** CONSTRUCTORS *****/ @@ -72,12 +74,37 @@ this.name = name; } + @Column(name = "de_name") + public String getDe_name() { + return this.de_name; + } + + public void setDe_name(final String de_name) { + this.de_name = de_name; + } + /** * Queries all vegetation types from the database, ordered by id */ public static List<VegetationType> getTypes() { - final Session session = SessionHolder.HOLDER.get(); + return getTypes(SessionHolder.HOLDER.get()); + } + + /** + * Queries all vegetation types from the database, ordered by id + */ + public static List<VegetationType> getTypes(final Session session) { final Query query = session.createQuery("FROM VegetationType ORDER BY id"); return new ArrayList<>(query.list()); } + + /** + * Localized name of the vegetation type + */ + public String getLocalizedName(final Locale locale) { + if ((locale == Locale.GERMAN) || (locale == Locale.GERMANY)) + return getDe_name(); + else + return getName(); + } }
--- a/gwt-client/src/main/java/org/dive4elements/river/client/client/FLYSConstants.properties Mon Mar 23 15:40:12 2020 +0100 +++ b/gwt-client/src/main/java/org/dive4elements/river/client/client/FLYSConstants.properties Mon Mar 23 16:38:12 2020 +0100 @@ -836,14 +836,14 @@ vegetation_zones = Vegetation Zones uinfo_vegetation_zone_overlap = Ranges are overlapping. -uinfo_vegetation_zone_has_gaps = The input (-1-366) has gaps. +uinfo_vegetation_zone_has_gaps = The input (0-366) has gaps. uinfo_vegetation_zone_color = Color uinfo_vegetation_zone_label = Vegetation Zone uinfo_vegetation_zones_label = Vegetation Zones uinfo_vegetation_zones_from = FD from [d/a] uinfo_vegetation_zones_to = FD to [d/a] uinfo_vegetation_zones_validation_empty = Input box empty. -uinfo_vegetation_zones_validation_range = Values have to be within the range of -1 and 366. +uinfo_vegetation_zones_validation_range = Values have to be within the range of 0 and 366. uinfo_vegetation_zones_validation_from_greater_to = The value "FD from" must be smaller than "FD to". uinfo_salix_km_limit_exceed = The number is out of the valid km-stretch. uinfo_salix_km_overlap = Km-stretches are overlapping.
--- a/gwt-client/src/main/java/org/dive4elements/river/client/client/FLYSConstants_de.properties Mon Mar 23 15:40:12 2020 +0100 +++ b/gwt-client/src/main/java/org/dive4elements/river/client/client/FLYSConstants_de.properties Mon Mar 23 16:38:12 2020 +0100 @@ -836,14 +836,14 @@ vegetation_zones = Vegetationszonen uinfo_vegetation_zone_overlap = Bereiche \u00fcberlappen. -uinfo_vegetation_zone_has_gaps = Die Wertebelegung (-1-366) weist L\u00fccken auf. +uinfo_vegetation_zone_has_gaps = Die Wertebelegung (0-366) weist L\u00fccken auf. uinfo_vegetation_zone_color = Farbe uinfo_vegetation_zone_label = Vegetationszone uinfo_vegetation_zones_label = Vegetationszonen uinfo_vegetation_zones_from = \u00dcfd von [d/a] uinfo_vegetation_zones_to = \u00dcfd bis [d/a] uinfo_vegetation_zones_validation_empty = Eingabefeld leer. -uinfo_vegetation_zones_validation_range = Werte m\u00fcssen zwischen -1 und 366 liegen. +uinfo_vegetation_zones_validation_range = Werte m\u00fcssen zwischen 0 und 366 liegen. uinfo_vegetation_zones_validation_from_greater_to = Der Wert f\u00fcr "\u00dcfd von" muss kleiner als "\u00dcfd bis". uinfo_salix_km_limit_exceed = Die Zahl ist au\u00dferhalb des g\u00fcltigen km-Bereichs. uinfo_salix_km_overlap = Km-Bereiche \u00fcberlappen.
--- a/gwt-client/src/main/java/org/dive4elements/river/client/client/ui/uinfo/AbstractVegZonesTablePanel.java Mon Mar 23 15:40:12 2020 +0100 +++ b/gwt-client/src/main/java/org/dive4elements/river/client/client/ui/uinfo/AbstractVegZonesTablePanel.java Mon Mar 23 16:38:12 2020 +0100 @@ -50,7 +50,7 @@ public abstract class AbstractVegZonesTablePanel extends AbstractUIProvider { private static final long serialVersionUID = 1L; - private static final int LOWER = -1; + private static final int LOWER = 0; private static final int UPPER = 366; private static final String datakey = "vegzones";