changeset 2833:5b54a648f702

Finished flow velocity data import: finished parsing meta data of model files and repaired broken HQL statements. flys-backend/trunk@4259 c6561f87-3c4e-4783-a992-168aeb5c3f6f
author Ingo Weinzierl <ingo.weinzierl@intevation.de>
date Wed, 18 Apr 2012 08:54:55 +0000
parents ac5bd90697c1
children daba3f895eb3
files flys-backend/ChangeLog flys-backend/doc/schema/oracle-drop-minfo.sql flys-backend/doc/schema/oracle-minfo.sql flys-backend/src/main/java/de/intevation/flys/importer/ImportDischargeZone.java flys-backend/src/main/java/de/intevation/flys/importer/ImportFlowVelocityModel.java flys-backend/src/main/java/de/intevation/flys/importer/ImportFlowVelocityModelValue.java flys-backend/src/main/java/de/intevation/flys/importer/parsers/FlowVelocityModelParser.java flys-backend/src/main/java/de/intevation/flys/model/DischargeZone.java
diffstat 8 files changed, 215 insertions(+), 70 deletions(-) [+]
line wrap: on
line diff
--- a/flys-backend/ChangeLog	Tue Apr 17 13:00:04 2012 +0000
+++ b/flys-backend/ChangeLog	Wed Apr 18 08:54:55 2012 +0000
@@ -1,3 +1,24 @@
+2012-04-18  Ingo Weinzierl <ingo@intevation.de>
+
+	* doc/schema/oracle-minfo.sql,
+	  doc/schema/oracle-drop-minfo.sql: The discharge_zone relation has no
+	  longer a reference to a named main value but stores a lower and upper
+	  discharge as string.
+
+	* src/main/java/de/intevation/flys/model/DischargeZone.java,
+	  src/main/java/de/intevation/flys/importer/ImportDischargeZone.java:
+	  Adapted the code to the changes in the db schema.
+
+	* src/main/java/de/intevation/flys/importer/parsers/FlowVelocityModelParser.java:
+	  This parser now reads the meta information properly.
+
+	* src/main/java/de/intevation/flys/importer/ImportFlowVelocityModelValue.java:
+	  Repaired broken HQL statement.
+
+	* src/main/java/de/intevation/flys/importer/ImportFlowVelocityModel.java:
+	  Log the number of flow velocity model values that have been written into
+	  database.
+
 2012-04-17  Ingo Weinzierl <ingo@intevation.de>
 
 	* src/main/java/de/intevation/flys/importer/parsers/FlowVelocityMeasurementParser.java:
--- a/flys-backend/doc/schema/oracle-drop-minfo.sql	Tue Apr 17 13:00:04 2012 +0000
+++ b/flys-backend/doc/schema/oracle-drop-minfo.sql	Wed Apr 18 08:54:55 2012 +0000
@@ -23,7 +23,6 @@
 ALTER TABLE flow_velocity_model DROP CONSTRAINT fk_fvm_river_id;
 ALTER TABLE flow_velocity_model DROP CONSTRAINT fk_fvm_discharge_zone_id;
 ALTER TABLE discharge_zone DROP CONSTRAINT fk_dz_river_id;
-ALTER TABLE discharge_zone DROP CONSTRAINT fk_dz_nmv_id;
 ALTER TABLE flow_velocity_measurements DROP CONSTRAINT fk_fvm_rivers_id;
 ALTER TABLE flow_velocity_measure_values DROP CONSTRAINT fk_fvmv_measurements_id;
 
--- a/flys-backend/doc/schema/oracle-minfo.sql	Tue Apr 17 13:00:04 2012 +0000
+++ b/flys-backend/doc/schema/oracle-minfo.sql	Wed Apr 18 08:54:55 2012 +0000
@@ -174,13 +174,12 @@
 CREATE TABLE discharge_zone (
     id                      NUMBER(38,0) NOT NULL,
     river_id                NUMBER(38,0) NOT NULL,
-    named_main_value_id     NUMBER(38,0) NOT NULL,
     gauge_name              VARCHAR(64)  NOT NULL, -- this is not very proper, but there are gauges with no db instance
-    lower_factor            NUMBER(38,2) NOT NULL,
-    upper_factor            NUMBER(38,2) NOT NULL,
+    value                   NUMBER(38,3) NOT NULL,
+    lower_discharge         VARCHAR(16)  NOT NULL,
+    upper_discharge         VARCHAR(16),
     PRIMARY KEY(id),
-    CONSTRAINT fk_dz_river_id FOREIGN KEY (river_id) REFERENCES rivers(id),
-    CONSTRAINT fk_dz_nmv_id FOREIGN KEY (named_main_value_id) REFERENCES named_main_values(id)
+    CONSTRAINT fk_dz_river_id FOREIGN KEY (river_id) REFERENCES rivers(id)
 );
 
 
--- a/flys-backend/src/main/java/de/intevation/flys/importer/ImportDischargeZone.java	Tue Apr 17 13:00:04 2012 +0000
+++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImportDischargeZone.java	Wed Apr 18 08:54:55 2012 +0000
@@ -23,25 +23,24 @@
 
     private String gaugeName;
 
-    private ImportNamedMainValue mainValue;
+    private BigDecimal value;
 
-    private BigDecimal lowerFactor;
-
-    private BigDecimal upperFactor;
+    private String lowerDischarge;
+    private String upperDischarge;
 
     private DischargeZone peer;
 
 
     public ImportDischargeZone(
-        ImportNamedMainValue mainValue,
-        String               gaugeName,
-        BigDecimal           lowerFactor,
-        BigDecimal           upperFactor
+        String     gaugeName,
+        BigDecimal value,
+        String     lowerDischarge,
+        String     upperDischarge
     ) {
-        this.mainValue   = mainValue;
-        this.gaugeName   = gaugeName;
-        this.lowerFactor = lowerFactor;
-        this.upperFactor = upperFactor;
+        this.gaugeName      = gaugeName;
+        this.value          = value;
+        this.lowerDischarge = lowerDischarge;
+        this.upperDischarge = upperDischarge;
     }
 
 
@@ -50,8 +49,6 @@
     {
         log.debug("store dependencies");
 
-        mainValue.getPeer();
-
         getPeer(river);
     }
 
@@ -60,28 +57,26 @@
         if (peer == null) {
             Session session = ImporterSession.getInstance().getDatabaseSession();
 
-            NamedMainValue namedMainValue = mainValue.getPeer();
-
             Query query = session.createQuery(
                 "from DischargeZone where " +
-                "   river:=river and " +
+                "   river=:river and " +
                 "   gaugeName=:gaugeName and " +
-                "   mainValue=:mainValue"
+                "   value=:value"
             );
 
             query.setParameter("river", river);
             query.setParameter("gaugeName", gaugeName);
-            query.setParameter("mainValue", namedMainValue);
+            query.setParameter("value", value);
 
             List<DischargeZone> zone = query.list();
 
             if (zone.isEmpty()) {
                 peer = new DischargeZone(
-                    gaugeName,
                     river,
-                    namedMainValue,
-                    lowerFactor,
-                    upperFactor);
+                    gaugeName,
+                    value,
+                    lowerDischarge,
+                    upperDischarge);
 
                 session.save(peer);
             }
--- a/flys-backend/src/main/java/de/intevation/flys/importer/ImportFlowVelocityModel.java	Tue Apr 17 13:00:04 2012 +0000
+++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImportFlowVelocityModel.java	Wed Apr 18 08:54:55 2012 +0000
@@ -75,9 +75,14 @@
 
         FlowVelocityModel peer = getPeer(river);
 
+        int i = 0;
+
         for (ImportFlowVelocityModelValue value: values) {
             value.storeDependencies(peer);
+            i++;
         }
+
+        log.info("stored " + i + " flow velocity model values.");
     }
 
 
--- a/flys-backend/src/main/java/de/intevation/flys/importer/ImportFlowVelocityModelValue.java	Tue Apr 17 13:00:04 2012 +0000
+++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImportFlowVelocityModelValue.java	Wed Apr 18 08:54:55 2012 +0000
@@ -51,7 +51,7 @@
 
             Query query = session.createQuery(
                 "from FlowVelocityModelValue where " +
-                "   model=:model and " +
+                "   flowVelocity=:model and " +
                 "   station=:station"
             );
 
--- a/flys-backend/src/main/java/de/intevation/flys/importer/parsers/FlowVelocityModelParser.java	Tue Apr 17 13:00:04 2012 +0000
+++ b/flys-backend/src/main/java/de/intevation/flys/importer/parsers/FlowVelocityModelParser.java	Wed Apr 18 08:54:55 2012 +0000
@@ -8,10 +8,9 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import java.text.ParseException;
-
 import org.apache.log4j.Logger;
 
+import de.intevation.flys.importer.ImportDischargeZone;
 import de.intevation.flys.importer.ImportFlowVelocityModel;
 import de.intevation.flys.importer.ImportFlowVelocityModelValue;
 
@@ -24,6 +23,24 @@
     private static final Pattern META_REGEX =
         Pattern.compile(".*Rechnung (.*) \\(Pegel (.*)\\).*");
 
+    private static final Pattern META_GAUGE =
+        Pattern.compile("(.*) Q=(\\w*)m3/s");
+
+    private static final Pattern META_MAINVALUE_A =
+        Pattern.compile("([a-zA-Z]+)+(\\d+)*");
+
+    private static final Pattern META_MAINVALUE_B =
+        Pattern.compile("(([a-zA-Z]+)+(\\d+)*)\\s*-\\s*(([a-zA-Z]+)+(\\d+)*)");
+
+    private static final Pattern META_MAINVALUE_C =
+        Pattern.compile("([0-9]++)\\s?(\\w*)|([0-9]++,[0-9]++)\\s?(\\w*)");
+
+    private static final Pattern META_MAINVALUE_D =
+        Pattern.compile("(([0-9]*)\\s?(\\w*)|([0-9]++,[0-9]++)\\s?(\\w*)) bis (([0-9]*)\\s?(\\w*)|([0-9]++,[0-9]++)\\s?(\\w*))");
+
+    private static final Pattern META_MAINVALUE_E =
+        Pattern.compile("(([a-zA-Z]+)+(\\d+)*) bis (([a-zA-Z]+)+(\\d+)*)");
+
     private static final NumberFormat nf =
         NumberFormat.getInstance(DEFAULT_LOCALE);
 
@@ -65,22 +82,133 @@
     }
 
 
-    public void handleMetaLine(String line) {
+    protected void handleMetaLine(String line) {
         Matcher m = META_REGEX.matcher(line);
 
         if (m.matches()) {
-            String zoneStr  = m.group(1);
-            String gaugeStr = m.group(2);
+            String mainValueStr = m.group(1);
+            String gaugeStr     = m.group(2);
 
-            log.debug("Found zone string: '" + zoneStr + "'");
-            log.debug("Found gauge string: '" + gaugeStr + "'");
+            Object[] valueData = handleMainValueString(mainValueStr);
+            Object[] gaugeData = handleGaugeString(gaugeStr);
 
-            // TODO Do something with these information
+            if (valueData == null || valueData.length < 2) {
+                log.warn("skip invalid MainValue part: '" + line + "'");
+                return;
+            }
+
+            if (gaugeData == null || gaugeData.length < 2) {
+                log.warn("skip invalid gauge part: '" + line + "'");
+                return;
+            }
+
+            if (log.isDebugEnabled()) {
+                log.debug("Found meta information:");
+                log.debug("   Gauge: " + gaugeData[0]);
+                log.debug("   Value: " + gaugeData[1]);
+                log.debug("   Lower: " + valueData[0]);
+                log.debug("   upper: " + valueData[1]);
+            }
+
+            current.setDischargeZone(new ImportDischargeZone(
+                (String) gaugeData[0],
+                (BigDecimal) gaugeData[1],
+                (String) valueData[0],
+                (String) valueData[1]
+            ));
         }
     }
 
 
-    public void handleDataLine(String line) {
+    protected Object[] handleMainValueString(String mainValueStr) {
+        Matcher mA = META_MAINVALUE_A.matcher(mainValueStr);
+        if (mA.matches()) {
+            String name = mA.group(0);
+
+            return new Object[] { name, name };
+        }
+
+        Matcher mB = META_MAINVALUE_B.matcher(mainValueStr);
+        if (mB.matches()) {
+            String lower = mB.group(1);
+            String upper = mB.group(4);
+
+            return new Object[] { lower, upper };
+        }
+
+        Matcher mC = META_MAINVALUE_C.matcher(mainValueStr);
+        if (mC.matches()) {
+            String facA  = mC.group(1);
+            String nameA = mC.group(2);
+            String facB  = mC.group(3);
+            String nameB = mC.group(4);
+
+            String fac  = facA  != null ? facA  : facB;
+            String name = nameA != null ? nameA : nameB;
+
+            String mainValue = fac + " " + name;
+
+            return new Object[] { mainValue, mainValue };
+        }
+
+        Matcher mD = META_MAINVALUE_D.matcher(mainValueStr);
+        if (mD.matches()) {
+            String loFacA  = mD.group(2);
+            String loNameA = mD.group(3);
+            String loFacB  = mD.group(4);
+            String loNameB = mD.group(5);
+
+            String upFacA  = mD.group(7);
+            String upNameA = mD.group(8);
+            String upFacB  = mD.group(9);
+            String upNameB = mD.group(10);
+
+            String loFac  = loFacA  != null ? loFacA  : loFacB;
+            String loName = loNameA != null ? loNameA : loNameB;
+
+            String upFac  = upFacA  != null ? upFacA  : upFacB;
+            String upName = upNameA != null ? upNameA : upNameB;
+
+            String loMainValue = loFac + " " + loName;
+            String upMainValue = upFac + " " + upName;
+
+            return new Object[] { loMainValue, upMainValue };
+        }
+
+        Matcher mE = META_MAINVALUE_E.matcher(mainValueStr);
+        if (mE.matches()) {
+            String lower = mE.group(1);
+            String upper = mE.group(4);
+
+            return new Object[] { lower, upper };
+        }
+
+        return null;
+    }
+
+
+    protected Object[] handleGaugeString(String gaugeStr) {
+        Matcher m = META_GAUGE.matcher(gaugeStr);
+
+        if (m.matches()) {
+            String name = m.group(1);
+            String qStr = m.group(2);
+
+            try {
+                return new Object[] {
+                    name,
+                    new BigDecimal(nf.parse(qStr).doubleValue()) };
+            }
+            catch (ParseException pe) {
+                log.warn("Error while parsing Q value: '" + qStr + "'");
+            }
+        }
+
+        return null;
+    }
+
+
+    protected void handleDataLine(String line) {
         String[] cols = line.split(SEPERATOR_CHAR);
 
         if (cols.length < 5) {
--- a/flys-backend/src/main/java/de/intevation/flys/model/DischargeZone.java	Tue Apr 17 13:00:04 2012 +0000
+++ b/flys-backend/src/main/java/de/intevation/flys/model/DischargeZone.java	Wed Apr 18 08:54:55 2012 +0000
@@ -26,15 +26,14 @@
 
     private Integer id;
 
-    private String gaugeName;
-
     private River river;
 
-    private NamedMainValue mainValue;
+    private String gaugeName;
 
-    private BigDecimal lowerFactor;
+    private BigDecimal value;
 
-    private BigDecimal upperFactor;
+    private String lowerDischarge;
+    private String upperDischarge;
 
 
     public DischargeZone() {
@@ -42,17 +41,17 @@
 
 
     public DischargeZone(
-        String         gaugeName,
-        River          river,
-        NamedMainValue mainValue,
-        BigDecimal     lowerFactor,
-        BigDecimal     upperFactor
+        River       river,
+        String      gaugeName,
+        BigDecimal  value,
+        String      lowerDischarge,
+        String      upperDischarge
     ) {
-        this.gaugeName   = gaugeName;
-        this.river       = river;
-        this.mainValue   = mainValue;
-        this.lowerFactor = lowerFactor;
-        this.upperFactor = upperFactor;
+        this.river          = river;
+        this.gaugeName      = gaugeName;
+        this.value          = value;
+        this.lowerDischarge = lowerDischarge;
+        this.upperDischarge = upperDischarge;
     }
 
     @Id
@@ -82,14 +81,13 @@
         this.river = river;
     }
 
-    @OneToOne
-    @JoinColumn(name = "named_main_value_id")
-    public NamedMainValue getMainValue() {
-        return mainValue;
+    @Column(name = "value")
+    public BigDecimal getValue() {
+        return value;
     }
 
-    public void setMainValue(NamedMainValue mainValue) {
-        this.mainValue = mainValue;
+    public void setValue(BigDecimal value) {
+        this.value = value;
     }
 
     @Column(name = "gauge_name")
@@ -101,22 +99,22 @@
         this.gaugeName = gaugeName;
     }
 
-    @Column(name = "lower_factor")
-    public BigDecimal getLowerFactor() {
-        return lowerFactor;
+    @Column(name = "lower_discharge")
+    public String getLowerDischarge() {
+        return lowerDischarge;
     }
 
-    public void setLowerFactor(BigDecimal lowerFactor) {
-        this.lowerFactor = lowerFactor;
+    public void setLowerDischarge(String lowerDischarge) {
+        this.lowerDischarge = lowerDischarge;
     }
 
-    @Column(name = "upper_factor")
-    public BigDecimal getUpperFactor() {
-        return upperFactor;
+    @Column(name = "upper_discharge")
+    public String getUpperDischarge() {
+        return upperDischarge;
     }
 
-    public void setUpperFactor(BigDecimal upperFactor) {
-        this.upperFactor = upperFactor;
+    public void setUpperDischarge(String upperDischarge) {
+        this.upperDischarge = upperDischarge;
     }
 }
 // vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 :

http://dive4elements.wald.intevation.org