Mercurial > dive4elements > river
changeset 6507:f4fb7e38a288 double-precision
merged changes from default into double-precision branch
| author | Tom Gottfried <tom.gottfried@intevation.de> |
|---|---|
| date | Fri, 28 Jun 2013 17:48:04 +0200 |
| parents | 128ab9169db1 (diff) 1c68d4b2a1c0 (current diff) |
| children | 49dd3e478ccd |
| files | backend/src/main/java/org/dive4elements/river/importer/parsers/WstParser.java |
| diffstat | 15 files changed, 207 insertions(+), 114 deletions(-) [+] |
line wrap: on
line diff
--- a/backend/contrib/import_river.sh Fri Jun 28 17:47:27 2013 +0200 +++ b/backend/contrib/import_river.sh Fri Jun 28 17:48:04 2013 +0200 @@ -32,7 +32,7 @@ DEFAULT_BACKEND_NAME="XE" JAR="hydr_morph/importer.jar" IMPORTER_DRY_RUN=false -IMPORTER_MAINVALUE_TYPES=QWTD +IMPORTER_MAINVALUE_TYPES=QWTD- IMPORTER_ANNOTATION_TYPES="conf/annotation-types.xml"
--- a/backend/contrib/run_hydr_morph.sh Fri Jun 28 17:47:27 2013 +0200 +++ b/backend/contrib/run_hydr_morph.sh Fri Jun 28 17:48:04 2013 +0200 @@ -22,7 +22,7 @@ ######################## Custom Importer Settings ################### IMPORTER_DRY_RUN=false -IMPORTER_MAINVALUE_TYPES=QWTD +IMPORTER_MAINVALUE_TYPES=QWTD- IMPORTER_ANNOTATION_TYPES="conf/annotation-types.xml" IMPORTER_SKIP_ANNOTATIONS=false
--- a/backend/doc/schema/oracle-minfo.sql Fri Jun 28 17:47:27 2013 +0200 +++ b/backend/doc/schema/oracle-minfo.sql Fri Jun 28 17:48:04 2013 +0200 @@ -85,12 +85,12 @@ CREATE TABLE bed_height_single_values ( id NUMBER(38,0) NOT NULL, bed_height_single_id NUMBER(38,0) NOT NULL, - station NUMBER(38,2) NOT NULL, - height NUMBER(38,2), - uncertainty NUMBER(38,2), - data_gap NUMBER(38,2), - sounding_width NUMBER(38,2), - width NUMBER(38,2), + station DOUBLE PRECISION NOT NULL, + height DOUBLE PRECISION, + uncertainty DOUBLE PRECISION, + data_gap DOUBLE PRECISION, + sounding_width DOUBLE PRECISION, + width DOUBLE PRECISION, PRIMARY KEY(id), UNIQUE (station, bed_height_single_id), CONSTRAINT fk_bed_single_values_parent FOREIGN KEY (bed_height_single_id) REFERENCES bed_height_single(id) ON DELETE CASCADE @@ -179,8 +179,8 @@ river_id NUMBER(38,0) NOT NULL, gauge_name VARCHAR(64) NOT NULL, -- this is not very proper, but there are gauges with no db instance value NUMBER(38,3) NOT NULL, - lower_discharge VARCHAR(16) NOT NULL, - upper_discharge VARCHAR(16), + lower_discharge VARCHAR(64) NOT NULL, + upper_discharge VARCHAR(64), PRIMARY KEY(id), CONSTRAINT fk_dz_river_id FOREIGN KEY (river_id) REFERENCES rivers(id) ON DELETE CASCADE );
--- a/backend/doc/schema/oracle.sql Fri Jun 28 17:47:27 2013 +0200 +++ b/backend/doc/schema/oracle.sql Fri Jun 28 17:48:04 2013 +0200 @@ -325,6 +325,7 @@ id NUMBER(38,0) NOT NULL, description VARCHAR2(255), name VARCHAR2(255) NOT NULL, + source VARCHAR(256), position NUMBER(38,0) NOT NULL DEFAULT 0, time_interval_id NUMBER(38,0), wst_id NUMBER(38,0) NOT NULL,
--- a/backend/doc/schema/postgresql-minfo.sql Fri Jun 28 17:47:27 2013 +0200 +++ b/backend/doc/schema/postgresql-minfo.sql Fri Jun 28 17:48:04 2013 +0200 @@ -179,8 +179,8 @@ river_id int NOT NULL, gauge_name VARCHAR(64) NOT NULL, -- this is not very proper, but there are gauges with no db instance value NUMERIC NOT NULL, - lower_discharge VARCHAR(16) NOT NULL, - upper_discharge VARCHAR(16), + lower_discharge VARCHAR(64) NOT NULL, + upper_discharge VARCHAR(64), PRIMARY KEY(id), CONSTRAINT fk_dz_river_id FOREIGN KEY (river_id) REFERENCES rivers(id) ON DELETE CASCADE );
--- a/backend/doc/schema/postgresql.sql Fri Jun 28 17:47:27 2013 +0200 +++ b/backend/doc/schema/postgresql.sql Fri Jun 28 17:48:04 2013 +0200 @@ -201,6 +201,7 @@ wst_id int NOT NULL REFERENCES wsts(id) ON DELETE CASCADE, name VARCHAR(256) NOT NULL, description VARCHAR(256), + source VARCHAR(256), position int NOT NULL DEFAULT 0, time_interval_id int REFERENCES time_intervals(id),
--- a/backend/src/main/java/org/dive4elements/river/importer/ImportBedHeightSingleValue.java Fri Jun 28 17:47:27 2013 +0200 +++ b/backend/src/main/java/org/dive4elements/river/importer/ImportBedHeightSingleValue.java Fri Jun 28 17:48:04 2013 +0200 @@ -10,7 +10,7 @@ import java.util.List; -import java.math.BigDecimal; +//import java.math.BigDecimal; import org.apache.log4j.Logger; @@ -29,24 +29,26 @@ protected ImportBedHeightSingle bedHeight; - protected BigDecimal station; - protected BigDecimal height; - protected BigDecimal uncertainty; - protected BigDecimal dataGap; - protected BigDecimal soundingWidth; - protected BigDecimal width; + protected Double station; + protected Double height; + protected Double uncertainty; + protected Double dataGap; + // protected BigDecimal soundingWidth; + protected Double soundingWidth; + protected Double width; protected BedHeightSingleValue peer; public ImportBedHeightSingleValue( ImportBedHeightSingle bedHeight, - BigDecimal station, - BigDecimal height, - BigDecimal uncertainty, - BigDecimal dataGap, - BigDecimal soundingWidth, - BigDecimal width + Double station, + Double height, + Double uncertainty, + Double dataGap, + // BigDecimal soundingWidth, + Double soundingWidth, + Double width ) { this.bedHeight = bedHeight; this.station = station;
--- a/backend/src/main/java/org/dive4elements/river/importer/ImportOfficialLine.java Fri Jun 28 17:47:27 2013 +0200 +++ b/backend/src/main/java/org/dive4elements/river/importer/ImportOfficialLine.java Fri Jun 28 17:48:04 2013 +0200 @@ -45,7 +45,7 @@ // XXX: This is a bit odd. We do not have not enough infos here // to create a new NamedMainValue. So we just look for existing ones. Session session = ImporterSession.getInstance().getDatabaseSession(); - NamedMainValue nmv = NamedMainValue.fetchByName(name, session); + NamedMainValue nmv = NamedMainValue.fetchByNameAndType(name, "Q", session); if (nmv == null) { // failed -> failed to create OfficialLine return null;
--- a/backend/src/main/java/org/dive4elements/river/importer/parsers/BedHeightParser.java Fri Jun 28 17:47:27 2013 +0200 +++ b/backend/src/main/java/org/dive4elements/river/importer/parsers/BedHeightParser.java Fri Jun 28 17:48:04 2013 +0200 @@ -18,6 +18,7 @@ import java.util.ArrayList; import java.util.Date; import java.util.List; +import java.util.TreeSet; import java.util.Locale; import java.util.regex.Matcher; @@ -39,6 +40,7 @@ import org.dive4elements.river.importer.ImportUnit; import org.dive4elements.river.model.BedHeightType; import org.dive4elements.river.importer.ImporterSession; +import org.dive4elements.river.utils.EpsilonComparator; import org.dive4elements.river.importer.parsers.LineParser; public abstract class BedHeightParser { @@ -98,10 +100,12 @@ String line ); + protected TreeSet<Double> kmExists; public BedHeightParser() { - this.bedHeights = new ArrayList<ImportBedHeight>(); + bedHeights = new ArrayList<ImportBedHeight>(); + kmExists = new TreeSet<Double>(EpsilonComparator.CMP); } @@ -115,12 +119,15 @@ ImportBedHeight obj = newImportBedHeight(file.getName().replaceAll("\\.csv", "")); - LineNumberReader in = - new LineNumberReader( - new InputStreamReader( - new FileInputStream(file), ENCODING)); + kmExists.clear(); + LineNumberReader in = null; try { + in = + new LineNumberReader( + new InputStreamReader( + new FileInputStream(file), ENCODING)); + String line = null; while ((line = in.readLine()) != null) { if ((line = line.trim()).length() == 0) { @@ -139,7 +146,9 @@ bedHeights.add(obj); } finally { - in.close(); + if (in != null) { + in.close(); + } } } @@ -329,7 +338,7 @@ String tmp = m.group(1).replace(";", "").trim(); BedHeightType bht = BedHeightType.fetchBedHeightTypeForType( - tmp, ImporterSession.getInstance().getDatabaseSession()); + tmp, ImporterSession.getInstance().getDatabaseSession()); if (bht != null) { obj.setType(new ImportBedHeightType(bht));
--- a/backend/src/main/java/org/dive4elements/river/importer/parsers/BedHeightSingleParser.java Fri Jun 28 17:47:27 2013 +0200 +++ b/backend/src/main/java/org/dive4elements/river/importer/parsers/BedHeightSingleParser.java Fri Jun 28 17:48:04 2013 +0200 @@ -39,6 +39,7 @@ */ @Override protected void handleDataLine(ImportBedHeight obj, String line) { + log.debug(line); String[] values = line.split(SEPERATOR_CHAR); if (values == null || (values.length != 1 && values.length < 6)) { @@ -46,11 +47,21 @@ return; } - BigDecimal km; + Double km; try { - km = new BigDecimal(nf.parse(values[0]).doubleValue()); - } + km = new Double(nf.parse(values[0]).doubleValue()); + + Double key = Double.valueOf(km); + + if (kmExists.contains(key)) { + log.warn("duplicate station '" + km + "': -> ignored"); + return; + } + + kmExists.add(key); + } + catch (ParseException e) { // We expect a lot of ";;;;;;" lines. //log.warn("BSP: Error while parsing km of data row.", e); @@ -66,44 +77,95 @@ // Because we cannot enforce consistency of values with complete data // via null constraints in the database (as there are "gap" values), // do this checks manually. - if (values[3].length() == 0 || values[4].length() == 0 - || values[5].length() == 0) { + // if (values[3].length() == 0 || values[4].length() == 0 + // || values[5].length() == 0) { //log.warn("BSP: Error while parsing data row (manual null constraint violated)."); - return; - } + // return; + //} - try { - ImportBedHeightSingleValue value = new ImportBedHeightSingleValue( - (ImportBedHeightSingle) obj, - km, - new BigDecimal(nf.parse(values[1]).doubleValue()), - new BigDecimal(nf.parse(values[2]).doubleValue()), - new BigDecimal(nf.parse(values[3]).doubleValue()), - parseBigDecimal(values[4], line), - new BigDecimal(nf.parse(values[5]).doubleValue()) - ); + Double height = null; + if (values[1].length() > 0) { + try { + height = new Double(nf.parse(values[1]).doubleValue()); + } + catch (ParseException e) { + log.warn("BSP: unparseable height " + values[1]); + } + } - obj.addValue(value); - } - catch (ParseException e) { - log.warn("BSP: unparseable value in data row.", e); - } + Double uncertainty = null; + if (values[2].length() > 0) { + try { + uncertainty = new Double(nf.parse(values[2]).doubleValue()); + } + catch (ParseException e) { + log.warn("BSP: unparseable uncertainty value " + values[2]); + } + } + + Double dataGap = null; + if (values[3].length() > 0) { + try { + dataGap = new Double(nf.parse(values[3]).doubleValue()); + } + catch (ParseException e) { + log.warn("BSP: unparseable data gap " + values[3]); + } + } + + Double soundingWidth = null; + if (values[4].length() > 0) { + try { + soundingWidth = new Double(nf.parse(values[4]).doubleValue()); + } + catch (ParseException e) { + log.warn("BSP: unparseable sounding width " + values[4]); + } + } + + Double width = null; + if (values[5].length() > 0) { + try { + width = new Double(nf.parse(values[5]).doubleValue()); + } + catch (ParseException e) { + log.warn("BSP: unparseable width " + values[5]); + } + } + + // try { + ImportBedHeightSingleValue value = new ImportBedHeightSingleValue( + (ImportBedHeightSingle) obj, + km, + height,//new Double(nf.parse(values[1]).doubleValue()), + uncertainty, + dataGap, + soundingWidth, + // parseBigDecimal(values[4], line), + width + ); + + obj.addValue(value); + // } +// catch (ParseException e) { +// log.warn("BSP: unparseable value in data row.", e); + //} } - private BigDecimal parseBigDecimal(String value, String line) { - BigDecimal result = null; - try { - Double dValue = Double.valueOf(value.replace(",", ".")); - result = new BigDecimal(dValue.doubleValue()); - } - catch (NumberFormatException nfe) { - log.warn( - "Could not parse " + - value + - " in bed heigt single row: " - + line); - } - return result; - } +// private BigDecimal parseBigDecimal(String value, String line) { +// BigDecimal result = null; +// try { +// Double dValue = Double.valueOf(value.replace(",", ".")); +// result = new BigDecimal(dValue.doubleValue()); +// } +// catch (NumberFormatException nfe) { +// log.warn( +// "Could not parse " + +// value + +// " in bed heigt single row: " +// + line); +// } +// return result; +// } } // vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 :
--- a/backend/src/main/java/org/dive4elements/river/importer/parsers/FlowVelocityModelParser.java Fri Jun 28 17:47:27 2013 +0200 +++ b/backend/src/main/java/org/dive4elements/river/importer/parsers/FlowVelocityModelParser.java Fri Jun 28 17:48:04 2013 +0200 @@ -34,25 +34,25 @@ Logger.getLogger(FlowVelocityModelParser.class); private static final Pattern META_REGEX = - Pattern.compile(".*Rechnung (.*) \\(Pegel (.*)\\).*"); + Pattern.compile(".*Rechnung [unter ]*(.*) \\(Pegel (.*)\\).*"); private static final Pattern META_GAUGE = Pattern.compile("(.*) Q=(\\w*)m3/s"); private static final Pattern META_MAINVALUE_A = - Pattern.compile("([a-zA-Z]+)+(\\d+)*"); + Pattern.compile("([a-zA-Z]+)+(\\d+)*[\\w()]*"); private static final Pattern META_MAINVALUE_B = - Pattern.compile("(([a-zA-Z]+)+(\\d+)*)\\s*-\\s*(([a-zA-Z]+)+(\\d+)*)"); + Pattern.compile("(([a-zA-Z]+)+(\\d+)*)\\s*-\\s*(([a-zA-Z]+)+(\\d+)*\\S*)"); private static final Pattern META_MAINVALUE_C = - Pattern.compile("([0-9]++)\\s?(\\w*)|([0-9]++,[0-9]++)\\s?(\\w*)"); + Pattern.compile("([0-9]++)\\s?(\\S*)|([0-9]++,[0-9]++)\\s?(\\S*)"); private static final Pattern META_MAINVALUE_D = - Pattern.compile("(([0-9]*)\\s?(\\w*)|([0-9]++,[0-9]++)\\s?(\\w*)) bis (([0-9]*)\\s?(\\w*)|([0-9]++,[0-9]++)\\s?(\\w*))"); + Pattern.compile("(([0-9]*)\\s?(\\w*)|([0-9]++,[0-9]++)\\s?(\\w*))\\s*bis (([0-9]*)\\s?(\\S*)|([0-9]++,[0-9]++)\\s?(\\S*))"); private static final Pattern META_MAINVALUE_E = - Pattern.compile("(([a-zA-Z]+)+(\\d+)*) bis (([a-zA-Z]+)+(\\d+)*)"); + Pattern.compile("(([a-zA-Z]+)+(\\d+)*)\\s*bis (([a-zA-Z]+)+(\\d+)*\\S*)"); private static final NumberFormat nf = NumberFormat.getInstance(DEFAULT_LOCALE); @@ -115,18 +115,19 @@ if (m.matches()) { String mainValueStr = m.group(1); + log.debug("mainValueStr = '" + mainValueStr + "'"); String gaugeStr = m.group(2); Object[] valueData = handleMainValueString(mainValueStr); Object[] gaugeData = handleGaugeString(gaugeStr); if (valueData == null || valueData.length < 2) { - log.warn("skip invalid MainValue part: '" + line + "'"); + log.warn("skip invalid MainValue part in '" + line + "'"); return; } if (gaugeData == null || gaugeData.length < 2) { - log.warn("skip invalid gauge part: '" + line + "'"); + log.warn("skip invalid gauge part in '" + line + "'"); return; } @@ -149,23 +150,26 @@ protected Object[] handleMainValueString(String mainValueStr) { - Matcher mA = META_MAINVALUE_A.matcher(mainValueStr); + Matcher mA = META_MAINVALUE_A.matcher(mainValueStr.trim()); if (mA.matches()) { + log.debug("mainValueStr matches META_MAINVALUE_A"); String name = mA.group(0); return new Object[] { name, name }; } - Matcher mB = META_MAINVALUE_B.matcher(mainValueStr); + Matcher mB = META_MAINVALUE_B.matcher(mainValueStr.trim()); if (mB.matches()) { + log.debug("mainValueStr matches META_MAINVALUE_B"); String lower = mB.group(1); String upper = mB.group(4); return new Object[] { lower, upper }; } - Matcher mC = META_MAINVALUE_C.matcher(mainValueStr); + Matcher mC = META_MAINVALUE_C.matcher(mainValueStr.trim()); if (mC.matches()) { + log.debug("mainValueStr matches META_MAINVALUE_C"); String facA = mC.group(1); String nameA = mC.group(2); String facB = mC.group(3); @@ -179,8 +183,9 @@ return new Object[] { mainValue, mainValue }; } - Matcher mD = META_MAINVALUE_D.matcher(mainValueStr); + Matcher mD = META_MAINVALUE_D.matcher(mainValueStr.trim()); if (mD.matches()) { + log.debug("mainValueStr matches META_MAINVALUE_D"); String loFacA = mD.group(2); String loNameA = mD.group(3); String loFacB = mD.group(4); @@ -203,14 +208,16 @@ return new Object[] { loMainValue, upMainValue }; } - Matcher mE = META_MAINVALUE_E.matcher(mainValueStr); + Matcher mE = META_MAINVALUE_E.matcher(mainValueStr.trim()); if (mE.matches()) { + log.debug("mainValueStr matches META_MAINVALUE_E"); String lower = mE.group(1); String upper = mE.group(4); return new Object[] { lower, upper }; } + log.debug("mainValueStr not matched"); return null; }
--- a/backend/src/main/java/org/dive4elements/river/importer/parsers/WstParser.java Fri Jun 28 17:47:27 2013 +0200 +++ b/backend/src/main/java/org/dive4elements/river/importer/parsers/WstParser.java Fri Jun 28 17:48:04 2013 +0200 @@ -98,7 +98,12 @@ log.info("Parsing WST file '" + file + "'"); - wst = new ImportWst(file.getName()); + if (wst == null) { + wst = new ImportWst(file.getName()); + } + else { + wst.setDescription(file.getName()); + } LineNumberReader in = new LineNumberReader(
--- a/backend/src/main/java/org/dive4elements/river/model/BedHeightSingleValue.java Fri Jun 28 17:47:27 2013 +0200 +++ b/backend/src/main/java/org/dive4elements/river/model/BedHeightSingleValue.java Fri Jun 28 17:48:04 2013 +0200 @@ -11,7 +11,7 @@ import java.util.List; import java.io.Serializable; -import java.math.BigDecimal; +//import java.math.BigDecimal; import javax.persistence.Entity; import javax.persistence.Id; @@ -43,12 +43,13 @@ private BedHeightSingle bedHeight; - private BigDecimal station; - private BigDecimal height; - private BigDecimal uncertainty; - private BigDecimal dataGap; - private BigDecimal soundingWidth; - private BigDecimal width; + private Double station; + private Double height; + private Double uncertainty; + private Double dataGap; + // private BigDecimal soundingWidth; + private Double soundingWidth; + private Double width; public BedHeightSingleValue() { @@ -56,12 +57,13 @@ public BedHeightSingleValue( BedHeightSingle bedHeight, - BigDecimal station, - BigDecimal height, - BigDecimal uncertainty, - BigDecimal dataGap, - BigDecimal soundingWidth, - BigDecimal width + Double station, + Double height, + Double uncertainty, + Double dataGap, + // BigDecimal soundingWidth, + Double soundingWidth, + Double width ) { this.bedHeight = bedHeight; this.station = station; @@ -100,56 +102,58 @@ } @Column(name = "station") - public BigDecimal getStation() { + public Double getStation() { return station; } - public void setStation(BigDecimal station) { + public void setStation(Double station) { this.station = station; } @Column(name = "height") - public BigDecimal getHeight() { + public Double getHeight() { return height; } - public void setHeight(BigDecimal height) { + public void setHeight(Double height) { this.height = height; } @Column(name="uncertainty") - public BigDecimal getUncertainty() { + public Double getUncertainty() { return uncertainty; } - public void setUncertainty(BigDecimal uncertainty) { + public void setUncertainty(Double uncertainty) { this.uncertainty = uncertainty; } @Column(name="data_gap") - public BigDecimal getDataGap() { + public Double getDataGap() { return dataGap; } - public void setDataGap(BigDecimal dataGap) { + public void setDataGap(Double dataGap) { this.dataGap = dataGap; } @Column(name="sounding_width") - public BigDecimal getSoundingWidth() { + // public BigDecimal getSoundingWidth() { + public Double getSoundingWidth() { return soundingWidth; } - public void setSoundingWidth(BigDecimal soundingWidth) { + //public void setSoundingWidth(BigDecimal soundingWidth) { + public void setSoundingWidth(Double soundingWidth) { this.soundingWidth = soundingWidth; } @Column(name="width") - public BigDecimal getWidth() { + public Double getWidth() { return width; } - public void setWidth(BigDecimal width) { + public void setWidth(Double width) { this.width = width; } @@ -166,8 +170,8 @@ " and station >= :kmLo and station <= :kmHi"); query.setParameter("single", single); - query.setParameter("kmLo", new BigDecimal(kmLo)); - query.setParameter("kmHi", new BigDecimal(kmHi)); + query.setParameter("kmLo", new Double(kmLo)); + query.setParameter("kmHi", new Double(kmHi)); return query.list(); }
--- a/backend/src/main/java/org/dive4elements/river/model/NamedMainValue.java Fri Jun 28 17:47:27 2013 +0200 +++ b/backend/src/main/java/org/dive4elements/river/model/NamedMainValue.java Fri Jun 28 17:48:04 2013 +0200 @@ -90,10 +90,12 @@ this.officialLines = officialLines; } - public static NamedMainValue fetchByName(String name, Session session) { + public static NamedMainValue fetchByNameAndType( + String name, String type, Session session) { Query query = session.createQuery( - "from NamedMainValue where name=:name"); + "from NamedMainValue where name=:name and type.name = :type"); query.setString("name", name); + query.setString("type", type); List<NamedMainValue> named = query.list(); return named.isEmpty() ? null : named.get(0); }
--- a/backend/src/main/java/org/dive4elements/river/model/OfficialLine.java Fri Jun 28 17:47:27 2013 +0200 +++ b/backend/src/main/java/org/dive4elements/river/model/OfficialLine.java Fri Jun 28 17:48:04 2013 +0200 @@ -45,7 +45,7 @@ @Id @SequenceGenerator( name = "SEQUENCE_OFFICIAL_LINES_ID_SEQ", - sequenceName = "OFFICIAL_LINES_ID_SEQ_ID_SEQ", + sequenceName = "OFFICIAL_LINES_ID_SEQ", allocationSize = 1) @GeneratedValue( strategy = GenerationType.SEQUENCE,