# HG changeset patch
# User Sascha L. Teichmann <sascha.teichmann@intevation.de>
# Date 1304900145 0
# Node ID 04d449f7f0c9733fa616f0d397d8c7653419d0a5
# Parent  d50cd3a632e0620dcee7a81091316843a07355df
Importer: Change caching strategy not to cause OOM any more.


flys-backend/trunk@1855 c6561f87-3c4e-4783-a992-168aeb5c3f6f

diff -r d50cd3a632e0 -r 04d449f7f0c9 flys-backend/ChangeLog
--- a/flys-backend/ChangeLog	Sun May 08 22:41:07 2011 +0000
+++ b/flys-backend/ChangeLog	Mon May 09 00:15:45 2011 +0000
@@ -1,3 +1,21 @@
+2011-05-09	Sascha L. Teichmann	<sascha.teichmann@intevation.de>
+
+	* src/main/java/de/intevation/flys/importer/ImporterSession.java:
+	  Do not load _all_ values from discharge tables and
+	  wst columns. This is extremly slow and will lead
+	  to OOM if more rivers are imported. Now only the
+	  last 20 columns und discharge tables are cached.
+
+	* src/main/java/de/intevation/flys/importer/ValueKey.java:
+	  New. Key for caching discharge table values and wst
+	  column values.
+
+	* src/main/java/de/intevation/flys/importer/IdValueKey.java:
+	  Fixed bug in equals().
+
+	* src/main/java/de/intevation/flys/importer/ImportWstColumn.java:
+	  Removed too eloquent debug output.
+
 2011-05-09	Sascha L. Teichmann	<sascha.teichmann@intevation.de>
 
 	* src/main/java/de/intevation/flys/importer/IdValueKey.java:
diff -r d50cd3a632e0 -r 04d449f7f0c9 flys-backend/src/main/java/de/intevation/flys/importer/IdValueKey.java
--- a/flys-backend/src/main/java/de/intevation/flys/importer/IdValueKey.java	Sun May 08 22:41:07 2011 +0000
+++ b/flys-backend/src/main/java/de/intevation/flys/importer/IdValueKey.java	Mon May 09 00:15:45 2011 +0000
@@ -42,11 +42,12 @@
         return d != null ? d.hashCode() : 0;
     }
 
+    @Override
     public int hashCode() {
         return id | (hashCode(a) << 10) | (hashCode(b) << 20);
     }
 
-
+    @Override
     public boolean equals(Object obj) {
         if (!(obj instanceof IdValueKey)) {
             return false;
@@ -57,10 +58,10 @@
         return !((id != other.id) 
             || (a == null && other.a != null) 
             || (a != null && other.a == null) 
-            || (a != null && !a.equals(other)) 
+            || (a != null && !a.equals(other.a)) 
             || (b == null && other.b != null) 
             || (b != null && other.b == null) 
-            || (b != null && !b.equals(other)));
+            || (b != null && !b.equals(other.b)));
     }
 }
 // vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 :
diff -r d50cd3a632e0 -r 04d449f7f0c9 flys-backend/src/main/java/de/intevation/flys/importer/ImportWstColumn.java
--- a/flys-backend/src/main/java/de/intevation/flys/importer/ImportWstColumn.java	Sun May 08 22:41:07 2011 +0000
+++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImportWstColumn.java	Mon May 09 00:15:45 2011 +0000
@@ -91,12 +91,11 @@
     public void storeDependencies(River river) {
         log.info("store column '" + name + "'");
         WstColumn column = getPeer(river);
-        log.info("store q ranges");
+
         for (ImportWstColumnQRange columnQRange: columnQRanges) {
             columnQRange.getPeer(river);
         }
 
-        log.info("store w values");
         for (ImportWstColumnValue columnValue: columnValues) {
             columnValue.getPeer(river);
         }
diff -r d50cd3a632e0 -r 04d449f7f0c9 flys-backend/src/main/java/de/intevation/flys/importer/ImporterSession.java
--- a/flys-backend/src/main/java/de/intevation/flys/importer/ImporterSession.java	Sun May 08 22:41:07 2011 +0000
+++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImporterSession.java	Mon May 09 00:15:45 2011 +0000
@@ -3,6 +3,7 @@
 import java.util.Iterator;
 import java.util.Map;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 
 import java.math.BigDecimal;
 
@@ -26,6 +27,9 @@
 {
     private static Logger log = Logger.getLogger(ImporterSession.class);
 
+    public static final int MAX_WST_CACHE_SIZE = 20;
+    public static final int MAX_AT_CACHE_SIZE  = 20;
+
     private static final ThreadLocal<ImporterSession> SESSION =
         new ThreadLocal<ImporterSession>() {
             @Override
@@ -36,9 +40,10 @@
 
     protected Session databaseSession;
 
-    protected Map<IdValueKey, WstColumnValue> wstColumnValues;
+    protected Map<Integer, Map<ValueKey, WstColumnValue>> wstColumnValues;
 
-    protected Map<IdValueKey, DischargeTableValue> dischargeTableValues;
+    protected Map<Integer, Map<ValueKey, DischargeTableValue>>
+        dischargeTableValues;
 
     protected Map<IdValueKey, Range> ranges;
 
@@ -51,6 +56,26 @@
             SessionFactoryProvider.createSessionFactory();
         databaseSession = sessionFactory.openSession();
         databaseSession.setFlushMode(FlushMode.MANUAL);
+
+        wstColumnValues =
+            new LinkedHashMap<Integer, Map<ValueKey, WstColumnValue>>() {
+                @Override
+                protected boolean removeEldestEntry(
+                    Map.Entry<Integer, Map<ValueKey, WstColumnValue>> eldest
+                ) {
+                    return size() > MAX_WST_CACHE_SIZE;
+                }
+            };
+
+        dischargeTableValues =
+            new LinkedHashMap<Integer, Map<ValueKey, DischargeTableValue>>() {
+                @Override
+                protected boolean removeEldestEntry(
+                    Map.Entry<Integer, Map<ValueKey, DischargeTableValue>> eldest
+                ) {
+                    return size() > MAX_AT_CACHE_SIZE;
+                }
+            };
     }
 
     public Session getDatabaseSession() {
@@ -62,13 +87,25 @@
         BigDecimal position,
         BigDecimal w
     ) {
-        if (wstColumnValues == null) {
-            loadWstColumnValues();
+        Integer c = column.getId();
+
+        Map<ValueKey, WstColumnValue> map = wstColumnValues.get(c);
+
+        if (map == null) {
+            map = new HashMap<ValueKey, WstColumnValue>();
+            wstColumnValues.put(c, map);
+            Query query = databaseSession.createQuery(
+                "from WstColumnValue where wstColumn.id=:cid");
+            query.setParameter("cid", c);
+            for (Iterator iter = query.iterate(); iter.hasNext();) {
+                WstColumnValue wcv = (WstColumnValue)iter.next();
+                map.put(new ValueKey(wcv.getPosition(), wcv.getW()), wcv);
+            }
         }
 
-        IdValueKey key = new IdValueKey(column.getId(), position, w);
+        ValueKey key = new ValueKey(position, w);
 
-        WstColumnValue wcv = wstColumnValues.get(key);
+        WstColumnValue wcv = map.get(key);
 
         if (wcv != null) {
             return wcv;
@@ -78,62 +115,48 @@
 
         databaseSession.save(wcv);
 
-        wstColumnValues.put(key, wcv);
+        map.put(key, wcv);
 
         return wcv;
     }
 
-    protected void loadWstColumnValues() {
-        log.info("load wst column values");
-        wstColumnValues = new HashMap<IdValueKey, WstColumnValue>();
-
-        Query query = databaseSession.createQuery("from WstColumnValue");
-
-        for (Iterator iter = query.iterate(); iter.hasNext();) {
-            WstColumnValue wcv = (WstColumnValue)iter.next();
-            wstColumnValues.put(new IdValueKey(wcv), wcv);
-        }
-        log.info(wstColumnValues.size() + " values loaded");
-    }
-
     public DischargeTableValue getDischargeTableValue(
         DischargeTable table,
         BigDecimal     q,
         BigDecimal     w
     ) {
-        if (dischargeTableValues == null) {
-            loadDischargeTableValues();
-        }
+        Integer t = table.getId();
 
-        IdValueKey key = new IdValueKey(table.getId(), q, w);
+        Map<ValueKey, DischargeTableValue> map =
+            dischargeTableValues.get(t);
 
-        DischargeTableValue dtv = dischargeTableValues.get(key);
-
-        if (dtv != null) {
-            return dtv;
+        if (map == null) {
+            map = new HashMap<ValueKey, DischargeTableValue>();
+            dischargeTableValues.put(t, map);
+            Query query = databaseSession.createQuery(
+                "from DischargeTableValue where dischargeTable.id=:tid");
+            query.setParameter("tid", t);
+            for (Iterator iter = query.iterate(); iter.hasNext();) {
+                DischargeTableValue dctv = (DischargeTableValue)iter.next();
+                map.put(new ValueKey(q, w), dctv);
+            }
         }
 
-        dtv = new DischargeTableValue(table, q, w);
-
-        databaseSession.save(dtv);
-
-        dischargeTableValues.put(key, dtv);
-
-        return dtv;
-    }
+        ValueKey key = new ValueKey(q, w);
 
-    protected void loadDischargeTableValues() {
-        log.info("load discharge table values");
-
-        dischargeTableValues = new HashMap<IdValueKey, DischargeTableValue>();
+        DischargeTableValue dctv = map.get(key);
 
-        Query query = databaseSession.createQuery("from DischargeTableValue");
+        if (dctv != null) {
+            return dctv;
+        }
 
-        for (Iterator iter = query.iterate(); iter.hasNext();) {
-            DischargeTableValue dtv = (DischargeTableValue)iter.next();
-            dischargeTableValues.put(new IdValueKey(dtv), dtv);
-        }
-        log.info(dischargeTableValues.size() + " values loaded");
+        dctv = new DischargeTableValue(table, q, w);
+
+        databaseSession.save(dctv);
+
+        map.put(key, dctv);
+
+        return dctv;
     }
 
     public Range getRange(River river, BigDecimal a, BigDecimal b) {
diff -r d50cd3a632e0 -r 04d449f7f0c9 flys-backend/src/main/java/de/intevation/flys/importer/ValueKey.java
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/flys-backend/src/main/java/de/intevation/flys/importer/ValueKey.java	Mon May 09 00:15:45 2011 +0000
@@ -0,0 +1,39 @@
+package de.intevation.flys.importer;
+
+import java.math.BigDecimal;
+
+public class ValueKey
+{
+    protected BigDecimal a;
+    protected BigDecimal b;
+
+    public ValueKey() {
+    }
+
+    public ValueKey(BigDecimal a, BigDecimal b) {
+        this.a = a;
+        this.b = b;
+    }
+
+    @Override
+    public int hashCode() {
+        return ((a != null ? a.hashCode() : 0) << 16)
+              | (b != null ? b.hashCode() : 0);
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (!(other instanceof ValueKey)) {
+            return false;
+        }
+        ValueKey o = (ValueKey)other;
+        return !(
+               (a == null && o.a != null) 
+            || (a != null && o.a == null) 
+            || (a != null && !a.equals(o.a)) 
+            || (b == null && o.b != null) 
+            || (b != null && o.b == null)
+            || (b != null && !b.equals(o.b)));
+    }
+}
+// vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 :