changeset 503:bcc18293a547

Importer: Added unsharp lookup to avoid numerical problems. flys-backend/trunk@1859 c6561f87-3c4e-4783-a992-168aeb5c3f6f
author Sascha L. Teichmann <sascha.teichmann@intevation.de>
date Mon, 09 May 2011 10:04:23 +0000 (2011-05-09)
parents c6889097f81f
children ee693b8fbb55
files flys-backend/ChangeLog flys-backend/src/main/java/de/intevation/flys/importer/IdValueKey.java flys-backend/src/main/java/de/intevation/flys/importer/ImportRiver.java flys-backend/src/main/java/de/intevation/flys/importer/ImportWst.java flys-backend/src/main/java/de/intevation/flys/importer/ImporterSession.java flys-backend/src/main/java/de/intevation/flys/importer/ValueKey.java
diffstat 6 files changed, 86 insertions(+), 110 deletions(-) [+]
line wrap: on
line diff
--- a/flys-backend/ChangeLog	Mon May 09 00:38:13 2011 +0000
+++ b/flys-backend/ChangeLog	Mon May 09 10:04:23 2011 +0000
@@ -1,3 +1,20 @@
+2011-05-09	Sascha L. Teichmann	<sascha.teichmann@intevation.de>
+
+	* src/main/java/de/intevation/flys/importer/ValueKey.java:
+	  Add some unsharp comparison (eps = 1e-6) to avoid 
+	  numerical problems.
+
+	* src/main/java/de/intevation/flys/importer/IdValueKey.java:
+	  Removed. Not need any longer.
+
+	* src/main/java/de/intevation/flys/importer/ImporterSession.java:
+	  Use unsharp lookup.
+
+	* src/main/java/de/intevation/flys/importer/ImportWst.java,
+	  src/main/java/de/intevation/flys/importer/ImportRiver.java:
+	  Flush more often. Hopefully this reduces hibernate sync
+	  problems?!
+
 2011-05-09	Sascha L. Teichmann	<sascha.teichmann@intevation.de>
 
 	* src/main/java/de/intevation/flys/importer/ImporterSession.java:
--- a/flys-backend/src/main/java/de/intevation/flys/importer/IdValueKey.java	Mon May 09 00:38:13 2011 +0000
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,67 +0,0 @@
-package de.intevation.flys.importer;
-
-import de.intevation.flys.model.WstColumnValue;
-import de.intevation.flys.model.DischargeTableValue;
-import de.intevation.flys.model.Range;
-
-import java.math.BigDecimal;
-
-public class IdValueKey {
-
-    protected int        id;
-    protected BigDecimal a;
-    protected BigDecimal b;
-
-
-    public IdValueKey(WstColumnValue value) {
-        this.id = value.getWstColumn().getId();
-        this.a  = value.getPosition();
-        this.b  = value.getW();
-    }
-
-    public IdValueKey(DischargeTableValue value) {
-        this.id = value.getDischargeTable().getId();
-        this.a  = value.getQ();
-        this.b  = value.getW();
-    }
-
-    public IdValueKey(Range value) {
-        this.id = value.getRiver().getId();
-        this.a  = value.getA();
-        this.b  = value.getB();
-    }
-
-
-    public IdValueKey(int id, BigDecimal a, BigDecimal b) {
-        this.id = id;
-        this.a  = a;
-        this.b  = b;
-    }
-
-    private static final int hashCode(BigDecimal d) {
-        return d != null ? d.hashCode() : 0;
-    }
-
-    @Override
-    public int hashCode() {
-        return id | (hashCode(a) << 10) | (hashCode(b) << 20);
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        if (!(obj instanceof IdValueKey)) {
-            return false;
-        }
-
-        IdValueKey other = (IdValueKey) obj;
-
-        return !((id != other.id) 
-            || (a == null && other.a != null) 
-            || (a != null && other.a == null) 
-            || (a != null && !a.equals(other.a)) 
-            || (b == null && other.b != null) 
-            || (b != null && other.b == null) 
-            || (b != null && !b.equals(other.b)));
-    }
-}
-// vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 :
--- a/flys-backend/src/main/java/de/intevation/flys/importer/ImportRiver.java	Mon May 09 00:38:13 2011 +0000
+++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImportRiver.java	Mon May 09 10:04:23 2011 +0000
@@ -407,9 +407,12 @@
     public void storeGauges() {
         log.info("store gauges:");
         River river = getPeer();
+        Session session = ImporterSession.getInstance().getDatabaseSession();
         for (ImportGauge gauge: gauges) {
             log.info("\tgauge: " + gauge.getName());
             gauge.storeDependencies(river);
+            ImporterSession.getInstance().getDatabaseSession();
+            session.flush();
         }
     }
 
--- a/flys-backend/src/main/java/de/intevation/flys/importer/ImportWst.java	Mon May 09 00:38:13 2011 +0000
+++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImportWst.java	Mon May 09 10:04:23 2011 +0000
@@ -72,6 +72,8 @@
         for (ImportWstColumn column: columns) {
             column.storeDependencies(river);
         }
+        Session session = ImporterSession.getInstance().getDatabaseSession();
+        session.flush();
     }
 
     public Wst getPeer(River river) {
--- a/flys-backend/src/main/java/de/intevation/flys/importer/ImporterSession.java	Mon May 09 00:38:13 2011 +0000
+++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImporterSession.java	Mon May 09 10:04:23 2011 +0000
@@ -2,7 +2,7 @@
 
 import java.util.Iterator;
 import java.util.Map;
-import java.util.HashMap;
+import java.util.TreeMap;
 import java.util.LinkedHashMap;
 
 import java.math.BigDecimal;
@@ -27,8 +27,14 @@
 {
     private static Logger log = Logger.getLogger(ImporterSession.class);
 
-    public static final int MAX_WST_CACHE_SIZE = 20;
-    public static final int MAX_AT_CACHE_SIZE  = 20;
+    public static final int CACHE_SIZE = 25;
+
+    public static class Cache<K, V> extends LinkedHashMap<K, V> {
+        @Override
+        protected boolean removeEldestEntry(Map.Entry<K, V> eldest) {
+            return size() > CACHE_SIZE;
+        }
+    };
 
     private static final ThreadLocal<ImporterSession> SESSION =
         new ThreadLocal<ImporterSession>() {
@@ -40,12 +46,14 @@
 
     protected Session databaseSession;
 
-    protected Map<Integer, Map<ValueKey, WstColumnValue>> wstColumnValues;
+    protected Cache<Integer, Map<ValueKey, WstColumnValue>>
+        wstColumnValues;
 
-    protected Map<Integer, Map<ValueKey, DischargeTableValue>>
+    protected Cache<Integer, Map<ValueKey, DischargeTableValue>>
         dischargeTableValues;
 
-    protected Map<IdValueKey, Range> ranges;
+    protected Cache<Integer, Map<ValueKey, Range>>
+        ranges;
 
     public static ImporterSession getInstance() {
         return SESSION.get();
@@ -58,24 +66,12 @@
         //databaseSession.setFlushMode(FlushMode.MANUAL);
 
         wstColumnValues =
-            new LinkedHashMap<Integer, Map<ValueKey, WstColumnValue>>() {
-                @Override
-                protected boolean removeEldestEntry(
-                    Map.Entry<Integer, Map<ValueKey, WstColumnValue>> eldest
-                ) {
-                    return size() > MAX_WST_CACHE_SIZE;
-                }
-            };
+            new Cache<Integer, Map<ValueKey, WstColumnValue>>();
 
         dischargeTableValues =
-            new LinkedHashMap<Integer, Map<ValueKey, DischargeTableValue>>() {
-                @Override
-                protected boolean removeEldestEntry(
-                    Map.Entry<Integer, Map<ValueKey, DischargeTableValue>> eldest
-                ) {
-                    return size() > MAX_AT_CACHE_SIZE;
-                }
-            };
+            new Cache<Integer, Map<ValueKey, DischargeTableValue>>();
+
+        ranges = new Cache<Integer, Map<ValueKey, Range>>();
     }
 
     public Session getDatabaseSession() {
@@ -92,7 +88,8 @@
         Map<ValueKey, WstColumnValue> map = wstColumnValues.get(c);
 
         if (map == null) {
-            map = new HashMap<ValueKey, WstColumnValue>();
+            map = new TreeMap<ValueKey, WstColumnValue>(
+                ValueKey.EPSILON_COMPARATOR);
             wstColumnValues.put(c, map);
             Query query = databaseSession.createQuery(
                 "from WstColumnValue where wstColumn.id=:cid");
@@ -131,14 +128,15 @@
             dischargeTableValues.get(t);
 
         if (map == null) {
-            map = new HashMap<ValueKey, DischargeTableValue>();
+            map = new TreeMap<ValueKey, DischargeTableValue>(
+                ValueKey.EPSILON_COMPARATOR);
             dischargeTableValues.put(t, map);
             Query query = databaseSession.createQuery(
                 "from DischargeTableValue where dischargeTable.id=:tid");
             query.setParameter("tid", t);
             for (Iterator iter = query.iterate(); iter.hasNext();) {
                 DischargeTableValue dctv = (DischargeTableValue)iter.next();
-                map.put(new ValueKey(q, w), dctv);
+                map.put(new ValueKey(dctv.getQ(), dctv.getW()), dctv);
             }
         }
 
@@ -160,14 +158,26 @@
     }
 
     public Range getRange(River river, BigDecimal a, BigDecimal b) {
+        Integer r = river.getId();
 
-        if (ranges == null) {
-            loadRanges();
+        Map<ValueKey, Range> map = ranges.get(r);
+
+        if (map == null) {
+            map = new TreeMap<ValueKey, Range>(
+                ValueKey.EPSILON_COMPARATOR);
+            ranges.put(r, map);
+            Query query = databaseSession.createQuery(
+                "from Range where river.id=:rid");
+            query.setParameter("rid", r);
+            for (Iterator iter = query.iterate(); iter.hasNext();) {
+                Range range = (Range)iter.next();
+                map.put(new ValueKey(range.getA(), range.getB()), range);
+            }
         }
 
-        IdValueKey key = new IdValueKey(river.getId(), a, b);
+        ValueKey key = new ValueKey(a, b);
 
-        Range range = ranges.get(key);
+        Range range = map.get(key);
 
         if (range != null) {
             return range;
@@ -177,22 +187,9 @@
 
         databaseSession.save(range);
 
-        ranges.put(key, range);
+        map.put(key, range);
 
         return range;
     }
-
-    protected void loadRanges() {
-        log.info("load ranges");
-        ranges = new HashMap<IdValueKey, Range>();
-
-        Query query = databaseSession.createQuery("from Range");
-
-        for (Iterator iter = query.iterate(); iter.hasNext();) {
-            Range range = (Range)iter.next();
-            ranges.put(new IdValueKey(range), range);
-        }
-        log.info(ranges.size() + " values loaded");
-    }
 }
 // vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 :
--- a/flys-backend/src/main/java/de/intevation/flys/importer/ValueKey.java	Mon May 09 00:38:13 2011 +0000
+++ b/flys-backend/src/main/java/de/intevation/flys/importer/ValueKey.java	Mon May 09 10:04:23 2011 +0000
@@ -2,8 +2,32 @@
 
 import java.math.BigDecimal;
 
+import java.util.Comparator;
+
 public class ValueKey
 {
+    public static final double EPSILON = 1e-6;
+
+    public static final Comparator<ValueKey> EPSILON_COMPARATOR =
+        new Comparator<ValueKey>() 
+    {
+        public int compare(ValueKey x, ValueKey y) {
+            int cmp = ValueKey.compare(x.a, y.a);
+            if (cmp != 0) return cmp;
+            return ValueKey.compare(x.b, y.b);
+        }
+    };
+
+    public static int compare(BigDecimal a, BigDecimal b) {
+        if (a == null && b == null) return  0;
+        if (a != null && b == null) return +1;
+        if (a == null && b != null) return -1;
+
+        double diff = a.doubleValue() - b.doubleValue();
+        if (diff < -EPSILON) return -1;
+        return diff > EPSILON ? +1 : 0;
+    }
+
     protected BigDecimal a;
     protected BigDecimal b;
 

http://dive4elements.wald.intevation.org