Mercurial > dive4elements > river

changeset 6443:28c390debd9b double-precision

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
merged changes from default into double-precision branch
author Tom Gottfried <tom.gottfried@intevation.de>
date Wed, 26 Jun 2013 16:40:25 +0200
parents 9787773e7bee (diff) f3078357ec65 (current diff)
children 25912df0facf
files
diffstat 9 files changed, 198 insertions(+), 109 deletions(-) [+]
line wrap: on
line diff
--- a/backend/doc/schema/oracle-minfo.sql	Wed Jun 26 16:36:30 2013 +0200
+++ b/backend/doc/schema/oracle-minfo.sql	Wed Jun 26 16:40:25 2013 +0200
@@ -85,12 +85,12 @@
 CREATE TABLE bed_height_single_values (
     id                      NUMBER(38,0) NOT NULL,
     bed_height_single_id    NUMBER(38,0) NOT NULL,
-    station                 NUMBER(38,2) NOT NULL,
-    height                  NUMBER(38,2),
-    uncertainty             NUMBER(38,2),
-    data_gap                NUMBER(38,2),
-    sounding_width          NUMBER(38,2),
-    width                   NUMBER(38,2),
+    station                 DOUBLE PRECISION NOT NULL,
+    height                  DOUBLE PRECISION,
+    uncertainty             DOUBLE PRECISION,
+    data_gap                DOUBLE PRECISION,
+    sounding_width          DOUBLE PRECISION,
+    width                   DOUBLE PRECISION,
     PRIMARY KEY(id),
     UNIQUE (station, bed_height_single_id),
     CONSTRAINT fk_bed_single_values_parent FOREIGN KEY (bed_height_single_id) REFERENCES bed_height_single(id) ON DELETE CASCADE
@@ -179,8 +179,8 @@
     river_id                NUMBER(38,0) NOT NULL,
     gauge_name              VARCHAR(64)  NOT NULL, -- this is not very proper, but there are gauges with no db instance
     value                   NUMBER(38,3) NOT NULL,
-    lower_discharge         VARCHAR(16)  NOT NULL,
-    upper_discharge         VARCHAR(16),
+    lower_discharge         VARCHAR(64)  NOT NULL,
+    upper_discharge         VARCHAR(64),
     PRIMARY KEY(id),
     CONSTRAINT fk_dz_river_id FOREIGN KEY (river_id) REFERENCES rivers(id) ON DELETE CASCADE
 );
--- a/backend/doc/schema/postgresql-minfo.sql	Wed Jun 26 16:36:30 2013 +0200
+++ b/backend/doc/schema/postgresql-minfo.sql	Wed Jun 26 16:40:25 2013 +0200
@@ -179,8 +179,8 @@
     river_id                int NOT NULL,
     gauge_name              VARCHAR(64)  NOT NULL, -- this is not very proper, but there are gauges with no db instance
     value                   NUMERIC NOT NULL,
-    lower_discharge         VARCHAR(16)  NOT NULL,
-    upper_discharge         VARCHAR(16),
+    lower_discharge         VARCHAR(64)  NOT NULL,
+    upper_discharge         VARCHAR(64),
     PRIMARY KEY(id),
     CONSTRAINT fk_dz_river_id FOREIGN KEY (river_id) REFERENCES rivers(id) ON DELETE CASCADE
 );
--- a/backend/src/main/java/org/dive4elements/river/importer/ImportBedHeightSingleValue.java	Wed Jun 26 16:36:30 2013 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/ImportBedHeightSingleValue.java	Wed Jun 26 16:40:25 2013 +0200
@@ -10,7 +10,7 @@
 
 import java.util.List;
 
-import java.math.BigDecimal;
+//import java.math.BigDecimal;
 
 import org.apache.log4j.Logger;
 
@@ -29,24 +29,26 @@
 
     protected ImportBedHeightSingle bedHeight;
 
-    protected BigDecimal station;
-    protected BigDecimal height;
-    protected BigDecimal uncertainty;
-    protected BigDecimal dataGap;
-    protected BigDecimal soundingWidth;
-    protected BigDecimal width;
+    protected Double station;
+    protected Double height;
+    protected Double uncertainty;
+    protected Double dataGap;
+    //    protected BigDecimal soundingWidth;
+    protected Double soundingWidth;
+    protected Double width;
 
     protected BedHeightSingleValue peer;
 
 
     public ImportBedHeightSingleValue(
         ImportBedHeightSingle bedHeight,
-        BigDecimal station,
-        BigDecimal height,
-        BigDecimal uncertainty,
-        BigDecimal dataGap,
-        BigDecimal soundingWidth,
-        BigDecimal width
+        Double station,
+        Double height,
+        Double uncertainty,
+        Double dataGap,
+	//        BigDecimal soundingWidth,
+	Double soundingWidth,
+        Double width
     ) {
         this.bedHeight     = bedHeight;
         this.station       = station;
--- a/backend/src/main/java/org/dive4elements/river/importer/parsers/BedHeightParser.java	Wed Jun 26 16:36:30 2013 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/parsers/BedHeightParser.java	Wed Jun 26 16:40:25 2013 +0200
@@ -18,6 +18,7 @@
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
+import java.util.TreeSet;
 import java.util.Locale;
 
 import java.util.regex.Matcher;
@@ -39,6 +40,7 @@
 import org.dive4elements.river.importer.ImportUnit;
 import org.dive4elements.river.model.BedHeightType;
 import org.dive4elements.river.importer.ImporterSession;
+import org.dive4elements.river.utils.EpsilonComparator;
 import org.dive4elements.river.importer.parsers.LineParser;
 
 public abstract class BedHeightParser {
@@ -98,10 +100,12 @@
         String          line
     );
 
+    protected TreeSet<Double> kmExists;
 
 
     public BedHeightParser() {
-        this.bedHeights = new ArrayList<ImportBedHeight>();
+        bedHeights = new ArrayList<ImportBedHeight>();
+        kmExists = new TreeSet<Double>(EpsilonComparator.CMP);
     }
 
 
@@ -115,12 +119,15 @@
 
         ImportBedHeight obj = newImportBedHeight(file.getName().replaceAll("\\.csv", ""));
 
-        LineNumberReader in =
-            new LineNumberReader(
-            new InputStreamReader(
-            new FileInputStream(file), ENCODING));
+        kmExists.clear();
 
+        LineNumberReader in = null;
         try {
+            in =
+                new LineNumberReader(
+                new InputStreamReader(
+                new FileInputStream(file), ENCODING));
+
             String line = null;
             while ((line = in.readLine()) != null) {
                 if ((line = line.trim()).length() == 0) {
@@ -139,7 +146,9 @@
             bedHeights.add(obj);
         }
         finally {
-            in.close();
+            if (in != null) {
+                in.close();
+            }
         }
     }
 
@@ -329,7 +338,7 @@
             String tmp = m.group(1).replace(";", "").trim();
 
             BedHeightType bht = BedHeightType.fetchBedHeightTypeForType(
-                tmp, ImporterSession.getInstance().getDatabaseSession());
+		tmp, ImporterSession.getInstance().getDatabaseSession());
 
             if (bht != null) {
                 obj.setType(new ImportBedHeightType(bht));
--- a/backend/src/main/java/org/dive4elements/river/importer/parsers/BedHeightSingleParser.java	Wed Jun 26 16:36:30 2013 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/parsers/BedHeightSingleParser.java	Wed Jun 26 16:40:25 2013 +0200
@@ -39,6 +39,7 @@
      */
     @Override
     protected void handleDataLine(ImportBedHeight obj, String line) {
+	log.debug(line);
         String[] values = line.split(SEPERATOR_CHAR);
 
         if (values == null || (values.length != 1 && values.length < 6)) {
@@ -46,11 +47,21 @@
             return;
         }
 
-        BigDecimal km;
+        Double km;
 
         try {
-            km = new BigDecimal(nf.parse(values[0]).doubleValue());
-        }
+            km = new Double(nf.parse(values[0]).doubleValue());
+	    
+            Double key = Double.valueOf(km);
+
+            if (kmExists.contains(key)) {
+                log.warn("duplicate station '" + km + "': -> ignored");
+                return;
+            }
+
+            kmExists.add(key);
+	}
+
         catch (ParseException e) {
             // We expect a lot of ";;;;;;" lines.
             //log.warn("BSP: Error while parsing km of data row.", e);
@@ -66,44 +77,95 @@
         // Because we cannot enforce consistency of values with complete data
         // via null constraints in the database (as there are "gap" values),
         // do this checks manually.
-        if (values[3].length() == 0 || values[4].length() == 0
-            || values[5].length() == 0) {
+	//        if (values[3].length() == 0 || values[4].length() == 0
+	//  || values[5].length() == 0) {
             //log.warn("BSP: Error while parsing data row (manual null constraint violated).");
-            return;
-        }
+	//  return;
+        //}
 
-        try {
-            ImportBedHeightSingleValue value = new ImportBedHeightSingleValue(
-                (ImportBedHeightSingle) obj,
-                km,
-                new BigDecimal(nf.parse(values[1]).doubleValue()),
-                new BigDecimal(nf.parse(values[2]).doubleValue()),
-                new BigDecimal(nf.parse(values[3]).doubleValue()),
-                parseBigDecimal(values[4], line),
-                new BigDecimal(nf.parse(values[5]).doubleValue())
-            );
+	Double height = null;
+	if (values[1].length() > 0) {
+	    try {
+		height = new Double(nf.parse(values[1]).doubleValue());
+	    }
+	    catch (ParseException e) {
+		log.warn("BSP: unparseable height " + values[1]);
+	    }
+	}
 
-            obj.addValue(value);
-        }
-        catch (ParseException e) {
-            log.warn("BSP: unparseable value in data row.", e);
-        }
+	Double uncertainty = null;
+	if (values[2].length() > 0) {
+	    try {
+		uncertainty = new Double(nf.parse(values[2]).doubleValue());
+	    }
+	    catch (ParseException e) {
+		log.warn("BSP: unparseable uncertainty value " + values[2]);
+	    }
+	}
+
+	Double dataGap = null;
+	if (values[3].length() > 0) {
+	    try {
+		dataGap = new Double(nf.parse(values[3]).doubleValue());
+	    }
+	    catch (ParseException e) {
+		log.warn("BSP: unparseable data gap " + values[3]);
+	    }
+	}
+
+	Double soundingWidth = null;
+	if (values[4].length() > 0) {
+	    try {
+		soundingWidth = new Double(nf.parse(values[4]).doubleValue());
+	    }
+	    catch (ParseException e) {
+		log.warn("BSP: unparseable sounding width " + values[4]);
+	    }
+	}
+
+	Double width = null;
+	if (values[5].length() > 0) {
+	    try {
+		width = new Double(nf.parse(values[5]).doubleValue());
+	    }
+	    catch (ParseException e) {
+		log.warn("BSP: unparseable width " + values[5]);
+	    }
+	}
+
+	//        try {
+	ImportBedHeightSingleValue value = new ImportBedHeightSingleValue(
+	    (ImportBedHeightSingle) obj,
+	    km,
+	    height,//new Double(nf.parse(values[1]).doubleValue()),
+	    uncertainty,
+	    dataGap,
+	    soundingWidth,
+	    //                parseBigDecimal(values[4], line),
+	    width
+	);
+
+	obj.addValue(value);
+        // }
+//         catch (ParseException e) {
+//             log.warn("BSP: unparseable value in data row.", e);
+        //}
     }
 
-    private BigDecimal parseBigDecimal(String value, String line) {
-        BigDecimal result = null;
-        try {
-            Double dValue = Double.valueOf(value.replace(",", "."));
-            result = new BigDecimal(dValue.doubleValue());
-        }
-        catch (NumberFormatException nfe) {
-            log.warn(
-                "Could not parse " +
-                value +
-                " in bed heigt single row: "
-                + line);
-        }
-        return result;
-    }
+//     private BigDecimal parseBigDecimal(String value, String line) {
+//         BigDecimal result = null;
+//         try {
+//             Double dValue = Double.valueOf(value.replace(",", "."));
+//             result = new BigDecimal(dValue.doubleValue());
+//         }
+//         catch (NumberFormatException nfe) {
+//             log.warn(
+//                 "Could not parse " +
+//                 value +
+//                 " in bed heigt single row: "
+//                 + line);
+//         }
+//        return result;
+//    }
 }
 // vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 :
--- a/backend/src/main/java/org/dive4elements/river/importer/parsers/FlowVelocityModelParser.java	Wed Jun 26 16:36:30 2013 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/parsers/FlowVelocityModelParser.java	Wed Jun 26 16:40:25 2013 +0200
@@ -34,25 +34,25 @@
         Logger.getLogger(FlowVelocityModelParser.class);
 
     private static final Pattern META_REGEX =
-        Pattern.compile(".*Rechnung (.*) \\(Pegel (.*)\\).*");
+        Pattern.compile(".*Rechnung [unter ]*(.*) \\(Pegel (.*)\\).*");
 
     private static final Pattern META_GAUGE =
         Pattern.compile("(.*) Q=(\\w*)m3/s");
 
     private static final Pattern META_MAINVALUE_A =
-        Pattern.compile("([a-zA-Z]+)+(\\d+)*");
+        Pattern.compile("([a-zA-Z]+)+(\\d+)*[\\w()]*");
 
     private static final Pattern META_MAINVALUE_B =
-        Pattern.compile("(([a-zA-Z]+)+(\\d+)*)\\s*-\\s*(([a-zA-Z]+)+(\\d+)*)");
+        Pattern.compile("(([a-zA-Z]+)+(\\d+)*)\\s*-\\s*(([a-zA-Z]+)+(\\d+)*\\S*)");
 
     private static final Pattern META_MAINVALUE_C =
-        Pattern.compile("([0-9]++)\\s?(\\w*)|([0-9]++,[0-9]++)\\s?(\\w*)");
+        Pattern.compile("([0-9]++)\\s?(\\S*)|([0-9]++,[0-9]++)\\s?(\\S*)");
 
     private static final Pattern META_MAINVALUE_D =
-        Pattern.compile("(([0-9]*)\\s?(\\w*)|([0-9]++,[0-9]++)\\s?(\\w*)) bis (([0-9]*)\\s?(\\w*)|([0-9]++,[0-9]++)\\s?(\\w*))");
+        Pattern.compile("(([0-9]*)\\s?(\\w*)|([0-9]++,[0-9]++)\\s?(\\w*))\\s*bis (([0-9]*)\\s?(\\S*)|([0-9]++,[0-9]++)\\s?(\\S*))");
 
     private static final Pattern META_MAINVALUE_E =
-        Pattern.compile("(([a-zA-Z]+)+(\\d+)*) bis (([a-zA-Z]+)+(\\d+)*)");
+        Pattern.compile("(([a-zA-Z]+)+(\\d+)*)\\s*bis (([a-zA-Z]+)+(\\d+)*\\S*)");
 
     private static final NumberFormat nf =
         NumberFormat.getInstance(DEFAULT_LOCALE);
@@ -115,18 +115,19 @@
 
         if (m.matches()) {
             String mainValueStr = m.group(1);
+	    log.debug("mainValueStr = '" + mainValueStr + "'");
             String gaugeStr     = m.group(2);
 
             Object[] valueData = handleMainValueString(mainValueStr);
             Object[] gaugeData = handleGaugeString(gaugeStr);
 
             if (valueData == null || valueData.length < 2) {
-                log.warn("skip invalid MainValue part: '" + line + "'");
+                log.warn("skip invalid MainValue part in '" + line + "'");
                 return;
             }
 
             if (gaugeData == null || gaugeData.length < 2) {
-                log.warn("skip invalid gauge part: '" + line + "'");
+                log.warn("skip invalid gauge part in '" + line + "'");
                 return;
             }
 
@@ -149,23 +150,26 @@
 
 
     protected Object[] handleMainValueString(String mainValueStr) {
-        Matcher mA = META_MAINVALUE_A.matcher(mainValueStr);
+        Matcher mA = META_MAINVALUE_A.matcher(mainValueStr.trim());
         if (mA.matches()) {
+	    log.debug("mainValueStr matches META_MAINVALUE_A");
             String name = mA.group(0);
 
             return new Object[] { name, name };
         }
 
-        Matcher mB = META_MAINVALUE_B.matcher(mainValueStr);
+        Matcher mB = META_MAINVALUE_B.matcher(mainValueStr.trim());
         if (mB.matches()) {
+	    log.debug("mainValueStr matches META_MAINVALUE_B");
             String lower = mB.group(1);
             String upper = mB.group(4);
 
             return new Object[] { lower, upper };
         }
 
-        Matcher mC = META_MAINVALUE_C.matcher(mainValueStr);
+        Matcher mC = META_MAINVALUE_C.matcher(mainValueStr.trim());
         if (mC.matches()) {
+	    log.debug("mainValueStr matches META_MAINVALUE_C");
             String facA  = mC.group(1);
             String nameA = mC.group(2);
             String facB  = mC.group(3);
@@ -179,8 +183,9 @@
             return new Object[] { mainValue, mainValue };
         }
 
-        Matcher mD = META_MAINVALUE_D.matcher(mainValueStr);
+        Matcher mD = META_MAINVALUE_D.matcher(mainValueStr.trim());
         if (mD.matches()) {
+	    log.debug("mainValueStr matches META_MAINVALUE_D");
             String loFacA  = mD.group(2);
             String loNameA = mD.group(3);
             String loFacB  = mD.group(4);
@@ -203,14 +208,16 @@
             return new Object[] { loMainValue, upMainValue };
         }
 
-        Matcher mE = META_MAINVALUE_E.matcher(mainValueStr);
+        Matcher mE = META_MAINVALUE_E.matcher(mainValueStr.trim());
         if (mE.matches()) {
+	    log.debug("mainValueStr matches META_MAINVALUE_E");
             String lower = mE.group(1);
             String upper = mE.group(4);
 
             return new Object[] { lower, upper };
         }
 
+	log.debug("mainValueStr not matched");
         return null;
     }
 
--- a/backend/src/main/java/org/dive4elements/river/importer/parsers/WstParser.java	Wed Jun 26 16:36:30 2013 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/parsers/WstParser.java	Wed Jun 26 16:40:25 2013 +0200
@@ -97,7 +97,12 @@
 
         log.info("Parsing WST file '" + file + "'");
 
-        wst = new ImportWst(file.getName());
+	if (wst == null) {
+	    wst = new ImportWst(file.getName());
+	}
+	else {
+	    wst.setDescription(file.getName());
+	}
 
         LineNumberReader in =
             new LineNumberReader(
--- a/backend/src/main/java/org/dive4elements/river/model/BedHeightSingleValue.java	Wed Jun 26 16:36:30 2013 +0200
+++ b/backend/src/main/java/org/dive4elements/river/model/BedHeightSingleValue.java	Wed Jun 26 16:40:25 2013 +0200
@@ -11,7 +11,7 @@
 import java.util.List;
 
 import java.io.Serializable;
-import java.math.BigDecimal;
+//import java.math.BigDecimal;
 
 import javax.persistence.Entity;
 import javax.persistence.Id;
@@ -43,12 +43,13 @@
 
     private BedHeightSingle bedHeight;
 
-    private BigDecimal station;
-    private BigDecimal height;
-    private BigDecimal uncertainty;
-    private BigDecimal dataGap;
-    private BigDecimal soundingWidth;
-    private BigDecimal width;
+    private Double station;
+    private Double height;
+    private Double uncertainty;
+    private Double dataGap;
+    //    private BigDecimal soundingWidth;
+    private Double soundingWidth;
+    private Double width;
 
 
     public BedHeightSingleValue() {
@@ -56,12 +57,13 @@
 
     public BedHeightSingleValue(
         BedHeightSingle bedHeight,
-        BigDecimal station,
-        BigDecimal height,
-        BigDecimal uncertainty,
-        BigDecimal dataGap,
-        BigDecimal soundingWidth,
-        BigDecimal width
+        Double station,
+        Double height,
+        Double uncertainty,
+        Double dataGap,
+	//        BigDecimal soundingWidth,
+	Double soundingWidth,
+        Double width
     ) {
         this.bedHeight     = bedHeight;
         this.station       = station;
@@ -100,56 +102,58 @@
     }
 
     @Column(name = "station")
-    public BigDecimal getStation() {
+    public Double getStation() {
         return station;
     }
 
-    public void setStation(BigDecimal station) {
+    public void setStation(Double station) {
         this.station = station;
     }
 
     @Column(name = "height")
-    public BigDecimal getHeight() {
+    public Double getHeight() {
         return height;
     }
 
-    public void setHeight(BigDecimal height) {
+    public void setHeight(Double height) {
         this.height = height;
     }
 
     @Column(name="uncertainty")
-    public BigDecimal getUncertainty() {
+    public Double getUncertainty() {
         return uncertainty;
     }
 
-    public void setUncertainty(BigDecimal uncertainty) {
+    public void setUncertainty(Double uncertainty) {
         this.uncertainty = uncertainty;
     }
 
     @Column(name="data_gap")
-    public BigDecimal getDataGap() {
+    public Double getDataGap() {
         return dataGap;
     }
 
-    public void setDataGap(BigDecimal dataGap) {
+    public void setDataGap(Double dataGap) {
         this.dataGap = dataGap;
     }
 
     @Column(name="sounding_width")
-    public BigDecimal getSoundingWidth() {
+    //    public BigDecimal getSoundingWidth() {
+    public Double getSoundingWidth() {
         return soundingWidth;
     }
 
-    public void setSoundingWidth(BigDecimal soundingWidth) {
+    //public void setSoundingWidth(BigDecimal soundingWidth) {
+    public void setSoundingWidth(Double soundingWidth) {
         this.soundingWidth = soundingWidth;
     }
 
     @Column(name="width")
-    public BigDecimal getWidth() {
+    public Double getWidth() {
         return width;
     }
 
-    public void setWidth(BigDecimal width) {
+    public void setWidth(Double width) {
         this.width = width;
     }
 
@@ -166,8 +170,8 @@
             "   and station >= :kmLo and station <= :kmHi");
 
         query.setParameter("single", single);
-        query.setParameter("kmLo", new BigDecimal(kmLo));
-        query.setParameter("kmHi", new BigDecimal(kmHi));
+        query.setParameter("kmLo", new Double(kmLo));
+        query.setParameter("kmHi", new Double(kmHi));
 
         return query.list();
     }
--- a/backend/src/main/java/org/dive4elements/river/model/OfficialLine.java	Wed Jun 26 16:36:30 2013 +0200
+++ b/backend/src/main/java/org/dive4elements/river/model/OfficialLine.java	Wed Jun 26 16:40:25 2013 +0200
@@ -45,7 +45,7 @@
     @Id
     @SequenceGenerator(
         name           = "SEQUENCE_OFFICIAL_LINES_ID_SEQ",
-        sequenceName   = "OFFICIAL_LINES_ID_SEQ_ID_SEQ",
+        sequenceName   = "OFFICIAL_LINES_ID_SEQ",
         allocationSize = 1)
     @GeneratedValue(
         strategy  = GenerationType.SEQUENCE,
http://dive4elements.wald.intevation.org