Mercurial > dive4elements > river
changeset 2833:5b54a648f702
Finished flow velocity data import: finished parsing meta data of model files and repaired broken HQL statements.
flys-backend/trunk@4259 c6561f87-3c4e-4783-a992-168aeb5c3f6f
| author | Ingo Weinzierl <ingo.weinzierl@intevation.de> |
|---|---|
| date | Wed, 18 Apr 2012 08:54:55 +0000 |
| parents | ac5bd90697c1 |
| children | daba3f895eb3 |
| files | flys-backend/ChangeLog flys-backend/doc/schema/oracle-drop-minfo.sql flys-backend/doc/schema/oracle-minfo.sql flys-backend/src/main/java/de/intevation/flys/importer/ImportDischargeZone.java flys-backend/src/main/java/de/intevation/flys/importer/ImportFlowVelocityModel.java flys-backend/src/main/java/de/intevation/flys/importer/ImportFlowVelocityModelValue.java flys-backend/src/main/java/de/intevation/flys/importer/parsers/FlowVelocityModelParser.java flys-backend/src/main/java/de/intevation/flys/model/DischargeZone.java |
| diffstat | 8 files changed, 215 insertions(+), 70 deletions(-) [+] |
line wrap: on
line diff
--- a/flys-backend/ChangeLog Tue Apr 17 13:00:04 2012 +0000 +++ b/flys-backend/ChangeLog Wed Apr 18 08:54:55 2012 +0000 @@ -1,3 +1,24 @@ +2012-04-18 Ingo Weinzierl <ingo@intevation.de> + + * doc/schema/oracle-minfo.sql, + doc/schema/oracle-drop-minfo.sql: The discharge_zone relation has no + longer a reference to a named main value but stores a lower and upper + discharge as string. + + * src/main/java/de/intevation/flys/model/DischargeZone.java, + src/main/java/de/intevation/flys/importer/ImportDischargeZone.java: + Adapted the code to the changes in the db schema. + + * src/main/java/de/intevation/flys/importer/parsers/FlowVelocityModelParser.java: + This parser now reads the meta information properly. + + * src/main/java/de/intevation/flys/importer/ImportFlowVelocityModelValue.java: + Repaired broken HQL statement. + + * src/main/java/de/intevation/flys/importer/ImportFlowVelocityModel.java: + Log the number of flow velocity model values that have been written into + database. + 2012-04-17 Ingo Weinzierl <ingo@intevation.de> * src/main/java/de/intevation/flys/importer/parsers/FlowVelocityMeasurementParser.java:
--- a/flys-backend/doc/schema/oracle-drop-minfo.sql Tue Apr 17 13:00:04 2012 +0000 +++ b/flys-backend/doc/schema/oracle-drop-minfo.sql Wed Apr 18 08:54:55 2012 +0000 @@ -23,7 +23,6 @@ ALTER TABLE flow_velocity_model DROP CONSTRAINT fk_fvm_river_id; ALTER TABLE flow_velocity_model DROP CONSTRAINT fk_fvm_discharge_zone_id; ALTER TABLE discharge_zone DROP CONSTRAINT fk_dz_river_id; -ALTER TABLE discharge_zone DROP CONSTRAINT fk_dz_nmv_id; ALTER TABLE flow_velocity_measurements DROP CONSTRAINT fk_fvm_rivers_id; ALTER TABLE flow_velocity_measure_values DROP CONSTRAINT fk_fvmv_measurements_id;
--- a/flys-backend/doc/schema/oracle-minfo.sql Tue Apr 17 13:00:04 2012 +0000 +++ b/flys-backend/doc/schema/oracle-minfo.sql Wed Apr 18 08:54:55 2012 +0000 @@ -174,13 +174,12 @@ CREATE TABLE discharge_zone ( id NUMBER(38,0) NOT NULL, river_id NUMBER(38,0) NOT NULL, - named_main_value_id NUMBER(38,0) NOT NULL, gauge_name VARCHAR(64) NOT NULL, -- this is not very proper, but there are gauges with no db instance - lower_factor NUMBER(38,2) NOT NULL, - upper_factor NUMBER(38,2) NOT NULL, + value NUMBER(38,3) NOT NULL, + lower_discharge VARCHAR(16) NOT NULL, + upper_discharge VARCHAR(16), PRIMARY KEY(id), - CONSTRAINT fk_dz_river_id FOREIGN KEY (river_id) REFERENCES rivers(id), - CONSTRAINT fk_dz_nmv_id FOREIGN KEY (named_main_value_id) REFERENCES named_main_values(id) + CONSTRAINT fk_dz_river_id FOREIGN KEY (river_id) REFERENCES rivers(id) );
--- a/flys-backend/src/main/java/de/intevation/flys/importer/ImportDischargeZone.java Tue Apr 17 13:00:04 2012 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImportDischargeZone.java Wed Apr 18 08:54:55 2012 +0000 @@ -23,25 +23,24 @@ private String gaugeName; - private ImportNamedMainValue mainValue; + private BigDecimal value; - private BigDecimal lowerFactor; - - private BigDecimal upperFactor; + private String lowerDischarge; + private String upperDischarge; private DischargeZone peer; public ImportDischargeZone( - ImportNamedMainValue mainValue, - String gaugeName, - BigDecimal lowerFactor, - BigDecimal upperFactor + String gaugeName, + BigDecimal value, + String lowerDischarge, + String upperDischarge ) { - this.mainValue = mainValue; - this.gaugeName = gaugeName; - this.lowerFactor = lowerFactor; - this.upperFactor = upperFactor; + this.gaugeName = gaugeName; + this.value = value; + this.lowerDischarge = lowerDischarge; + this.upperDischarge = upperDischarge; } @@ -50,8 +49,6 @@ { log.debug("store dependencies"); - mainValue.getPeer(); - getPeer(river); } @@ -60,28 +57,26 @@ if (peer == null) { Session session = ImporterSession.getInstance().getDatabaseSession(); - NamedMainValue namedMainValue = mainValue.getPeer(); - Query query = session.createQuery( "from DischargeZone where " + - " river:=river and " + + " river=:river and " + " gaugeName=:gaugeName and " + - " mainValue=:mainValue" + " value=:value" ); query.setParameter("river", river); query.setParameter("gaugeName", gaugeName); - query.setParameter("mainValue", namedMainValue); + query.setParameter("value", value); List<DischargeZone> zone = query.list(); if (zone.isEmpty()) { peer = new DischargeZone( - gaugeName, river, - namedMainValue, - lowerFactor, - upperFactor); + gaugeName, + value, + lowerDischarge, + upperDischarge); session.save(peer); }
--- a/flys-backend/src/main/java/de/intevation/flys/importer/ImportFlowVelocityModel.java Tue Apr 17 13:00:04 2012 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImportFlowVelocityModel.java Wed Apr 18 08:54:55 2012 +0000 @@ -75,9 +75,14 @@ FlowVelocityModel peer = getPeer(river); + int i = 0; + for (ImportFlowVelocityModelValue value: values) { value.storeDependencies(peer); + i++; } + + log.info("stored " + i + " flow velocity model values."); }
--- a/flys-backend/src/main/java/de/intevation/flys/importer/ImportFlowVelocityModelValue.java Tue Apr 17 13:00:04 2012 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImportFlowVelocityModelValue.java Wed Apr 18 08:54:55 2012 +0000 @@ -51,7 +51,7 @@ Query query = session.createQuery( "from FlowVelocityModelValue where " + - " model=:model and " + + " flowVelocity=:model and " + " station=:station" );
--- a/flys-backend/src/main/java/de/intevation/flys/importer/parsers/FlowVelocityModelParser.java Tue Apr 17 13:00:04 2012 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/importer/parsers/FlowVelocityModelParser.java Wed Apr 18 08:54:55 2012 +0000 @@ -8,10 +8,9 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import java.text.ParseException; - import org.apache.log4j.Logger; +import de.intevation.flys.importer.ImportDischargeZone; import de.intevation.flys.importer.ImportFlowVelocityModel; import de.intevation.flys.importer.ImportFlowVelocityModelValue; @@ -24,6 +23,24 @@ private static final Pattern META_REGEX = Pattern.compile(".*Rechnung (.*) \\(Pegel (.*)\\).*"); + private static final Pattern META_GAUGE = + Pattern.compile("(.*) Q=(\\w*)m3/s"); + + private static final Pattern META_MAINVALUE_A = + Pattern.compile("([a-zA-Z]+)+(\\d+)*"); + + private static final Pattern META_MAINVALUE_B = + Pattern.compile("(([a-zA-Z]+)+(\\d+)*)\\s*-\\s*(([a-zA-Z]+)+(\\d+)*)"); + + private static final Pattern META_MAINVALUE_C = + Pattern.compile("([0-9]++)\\s?(\\w*)|([0-9]++,[0-9]++)\\s?(\\w*)"); + + private static final Pattern META_MAINVALUE_D = + Pattern.compile("(([0-9]*)\\s?(\\w*)|([0-9]++,[0-9]++)\\s?(\\w*)) bis (([0-9]*)\\s?(\\w*)|([0-9]++,[0-9]++)\\s?(\\w*))"); + + private static final Pattern META_MAINVALUE_E = + Pattern.compile("(([a-zA-Z]+)+(\\d+)*) bis (([a-zA-Z]+)+(\\d+)*)"); + private static final NumberFormat nf = NumberFormat.getInstance(DEFAULT_LOCALE); @@ -65,22 +82,133 @@ } - public void handleMetaLine(String line) { + protected void handleMetaLine(String line) { Matcher m = META_REGEX.matcher(line); if (m.matches()) { - String zoneStr = m.group(1); - String gaugeStr = m.group(2); + String mainValueStr = m.group(1); + String gaugeStr = m.group(2); - log.debug("Found zone string: '" + zoneStr + "'"); - log.debug("Found gauge string: '" + gaugeStr + "'"); + Object[] valueData = handleMainValueString(mainValueStr); + Object[] gaugeData = handleGaugeString(gaugeStr); - // TODO Do something with these information + if (valueData == null || valueData.length < 2) { + log.warn("skip invalid MainValue part: '" + line + "'"); + return; + } + + if (gaugeData == null || gaugeData.length < 2) { + log.warn("skip invalid gauge part: '" + line + "'"); + return; + } + + if (log.isDebugEnabled()) { + log.debug("Found meta information:"); + log.debug(" Gauge: " + gaugeData[0]); + log.debug(" Value: " + gaugeData[1]); + log.debug(" Lower: " + valueData[0]); + log.debug(" upper: " + valueData[1]); + } + + current.setDischargeZone(new ImportDischargeZone( + (String) gaugeData[0], + (BigDecimal) gaugeData[1], + (String) valueData[0], + (String) valueData[1] + )); } } - public void handleDataLine(String line) { + protected Object[] handleMainValueString(String mainValueStr) { + Matcher mA = META_MAINVALUE_A.matcher(mainValueStr); + if (mA.matches()) { + String name = mA.group(0); + + return new Object[] { name, name }; + } + + Matcher mB = META_MAINVALUE_B.matcher(mainValueStr); + if (mB.matches()) { + String lower = mB.group(1); + String upper = mB.group(4); + + return new Object[] { lower, upper }; + } + + Matcher mC = META_MAINVALUE_C.matcher(mainValueStr); + if (mC.matches()) { + String facA = mC.group(1); + String nameA = mC.group(2); + String facB = mC.group(3); + String nameB = mC.group(4); + + String fac = facA != null ? facA : facB; + String name = nameA != null ? nameA : nameB; + + String mainValue = fac + " " + name; + + return new Object[] { mainValue, mainValue }; + } + + Matcher mD = META_MAINVALUE_D.matcher(mainValueStr); + if (mD.matches()) { + String loFacA = mD.group(2); + String loNameA = mD.group(3); + String loFacB = mD.group(4); + String loNameB = mD.group(5); + + String upFacA = mD.group(7); + String upNameA = mD.group(8); + String upFacB = mD.group(9); + String upNameB = mD.group(10); + + String loFac = loFacA != null ? loFacA : loFacB; + String loName = loNameA != null ? loNameA : loNameB; + + String upFac = upFacA != null ? upFacA : upFacB; + String upName = upNameA != null ? upNameA : upNameB; + + String loMainValue = loFac + " " + loName; + String upMainValue = upFac + " " + upName; + + return new Object[] { loMainValue, upMainValue }; + } + + Matcher mE = META_MAINVALUE_E.matcher(mainValueStr); + if (mE.matches()) { + String lower = mE.group(1); + String upper = mE.group(4); + + return new Object[] { lower, upper }; + } + + return null; + } + + + protected Object[] handleGaugeString(String gaugeStr) { + Matcher m = META_GAUGE.matcher(gaugeStr); + + if (m.matches()) { + String name = m.group(1); + String qStr = m.group(2); + + try { + return new Object[] { + name, + new BigDecimal(nf.parse(qStr).doubleValue()) }; + } + catch (ParseException pe) { + log.warn("Error while parsing Q value: '" + qStr + "'"); + } + } + + return null; + } + + + protected void handleDataLine(String line) { String[] cols = line.split(SEPERATOR_CHAR); if (cols.length < 5) {
--- a/flys-backend/src/main/java/de/intevation/flys/model/DischargeZone.java Tue Apr 17 13:00:04 2012 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/model/DischargeZone.java Wed Apr 18 08:54:55 2012 +0000 @@ -26,15 +26,14 @@ private Integer id; - private String gaugeName; - private River river; - private NamedMainValue mainValue; + private String gaugeName; - private BigDecimal lowerFactor; + private BigDecimal value; - private BigDecimal upperFactor; + private String lowerDischarge; + private String upperDischarge; public DischargeZone() { @@ -42,17 +41,17 @@ public DischargeZone( - String gaugeName, - River river, - NamedMainValue mainValue, - BigDecimal lowerFactor, - BigDecimal upperFactor + River river, + String gaugeName, + BigDecimal value, + String lowerDischarge, + String upperDischarge ) { - this.gaugeName = gaugeName; - this.river = river; - this.mainValue = mainValue; - this.lowerFactor = lowerFactor; - this.upperFactor = upperFactor; + this.river = river; + this.gaugeName = gaugeName; + this.value = value; + this.lowerDischarge = lowerDischarge; + this.upperDischarge = upperDischarge; } @Id @@ -82,14 +81,13 @@ this.river = river; } - @OneToOne - @JoinColumn(name = "named_main_value_id") - public NamedMainValue getMainValue() { - return mainValue; + @Column(name = "value") + public BigDecimal getValue() { + return value; } - public void setMainValue(NamedMainValue mainValue) { - this.mainValue = mainValue; + public void setValue(BigDecimal value) { + this.value = value; } @Column(name = "gauge_name") @@ -101,22 +99,22 @@ this.gaugeName = gaugeName; } - @Column(name = "lower_factor") - public BigDecimal getLowerFactor() { - return lowerFactor; + @Column(name = "lower_discharge") + public String getLowerDischarge() { + return lowerDischarge; } - public void setLowerFactor(BigDecimal lowerFactor) { - this.lowerFactor = lowerFactor; + public void setLowerDischarge(String lowerDischarge) { + this.lowerDischarge = lowerDischarge; } - @Column(name = "upper_factor") - public BigDecimal getUpperFactor() { - return upperFactor; + @Column(name = "upper_discharge") + public String getUpperDischarge() { + return upperDischarge; } - public void setUpperFactor(BigDecimal upperFactor) { - this.upperFactor = upperFactor; + public void setUpperDischarge(String upperDischarge) { + this.upperDischarge = upperDischarge; } } // vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 :