Mercurial > dive4elements > river
changeset 501:04d449f7f0c9
Importer: Change caching strategy not to cause OOM any more.
flys-backend/trunk@1855 c6561f87-3c4e-4783-a992-168aeb5c3f6f
| author | Sascha L. Teichmann <sascha.teichmann@intevation.de> |
|---|---|
| date | Mon, 09 May 2011 00:15:45 +0000 |
| parents | d50cd3a632e0 |
| children | c6889097f81f |
| files | flys-backend/ChangeLog flys-backend/src/main/java/de/intevation/flys/importer/IdValueKey.java flys-backend/src/main/java/de/intevation/flys/importer/ImportWstColumn.java flys-backend/src/main/java/de/intevation/flys/importer/ImporterSession.java flys-backend/src/main/java/de/intevation/flys/importer/ValueKey.java |
| diffstat | 5 files changed, 131 insertions(+), 51 deletions(-) [+] |
line wrap: on
line diff
--- a/flys-backend/ChangeLog Sun May 08 22:41:07 2011 +0000 +++ b/flys-backend/ChangeLog Mon May 09 00:15:45 2011 +0000 @@ -1,3 +1,21 @@ +2011-05-09 Sascha L. Teichmann <sascha.teichmann@intevation.de> + + * src/main/java/de/intevation/flys/importer/ImporterSession.java: + Do not load _all_ values from discharge tables and + wst columns. This is extremly slow and will lead + to OOM if more rivers are imported. Now only the + last 20 columns und discharge tables are cached. + + * src/main/java/de/intevation/flys/importer/ValueKey.java: + New. Key for caching discharge table values and wst + column values. + + * src/main/java/de/intevation/flys/importer/IdValueKey.java: + Fixed bug in equals(). + + * src/main/java/de/intevation/flys/importer/ImportWstColumn.java: + Removed too eloquent debug output. + 2011-05-09 Sascha L. Teichmann <sascha.teichmann@intevation.de> * src/main/java/de/intevation/flys/importer/IdValueKey.java:
--- a/flys-backend/src/main/java/de/intevation/flys/importer/IdValueKey.java Sun May 08 22:41:07 2011 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/importer/IdValueKey.java Mon May 09 00:15:45 2011 +0000 @@ -42,11 +42,12 @@ return d != null ? d.hashCode() : 0; } + @Override public int hashCode() { return id | (hashCode(a) << 10) | (hashCode(b) << 20); } - + @Override public boolean equals(Object obj) { if (!(obj instanceof IdValueKey)) { return false; @@ -57,10 +58,10 @@ return !((id != other.id) || (a == null && other.a != null) || (a != null && other.a == null) - || (a != null && !a.equals(other)) + || (a != null && !a.equals(other.a)) || (b == null && other.b != null) || (b != null && other.b == null) - || (b != null && !b.equals(other))); + || (b != null && !b.equals(other.b))); } } // vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 :
--- a/flys-backend/src/main/java/de/intevation/flys/importer/ImportWstColumn.java Sun May 08 22:41:07 2011 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImportWstColumn.java Mon May 09 00:15:45 2011 +0000 @@ -91,12 +91,11 @@ public void storeDependencies(River river) { log.info("store column '" + name + "'"); WstColumn column = getPeer(river); - log.info("store q ranges"); + for (ImportWstColumnQRange columnQRange: columnQRanges) { columnQRange.getPeer(river); } - log.info("store w values"); for (ImportWstColumnValue columnValue: columnValues) { columnValue.getPeer(river); }
--- a/flys-backend/src/main/java/de/intevation/flys/importer/ImporterSession.java Sun May 08 22:41:07 2011 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImporterSession.java Mon May 09 00:15:45 2011 +0000 @@ -3,6 +3,7 @@ import java.util.Iterator; import java.util.Map; import java.util.HashMap; +import java.util.LinkedHashMap; import java.math.BigDecimal; @@ -26,6 +27,9 @@ { private static Logger log = Logger.getLogger(ImporterSession.class); + public static final int MAX_WST_CACHE_SIZE = 20; + public static final int MAX_AT_CACHE_SIZE = 20; + private static final ThreadLocal<ImporterSession> SESSION = new ThreadLocal<ImporterSession>() { @Override @@ -36,9 +40,10 @@ protected Session databaseSession; - protected Map<IdValueKey, WstColumnValue> wstColumnValues; + protected Map<Integer, Map<ValueKey, WstColumnValue>> wstColumnValues; - protected Map<IdValueKey, DischargeTableValue> dischargeTableValues; + protected Map<Integer, Map<ValueKey, DischargeTableValue>> + dischargeTableValues; protected Map<IdValueKey, Range> ranges; @@ -51,6 +56,26 @@ SessionFactoryProvider.createSessionFactory(); databaseSession = sessionFactory.openSession(); databaseSession.setFlushMode(FlushMode.MANUAL); + + wstColumnValues = + new LinkedHashMap<Integer, Map<ValueKey, WstColumnValue>>() { + @Override + protected boolean removeEldestEntry( + Map.Entry<Integer, Map<ValueKey, WstColumnValue>> eldest + ) { + return size() > MAX_WST_CACHE_SIZE; + } + }; + + dischargeTableValues = + new LinkedHashMap<Integer, Map<ValueKey, DischargeTableValue>>() { + @Override + protected boolean removeEldestEntry( + Map.Entry<Integer, Map<ValueKey, DischargeTableValue>> eldest + ) { + return size() > MAX_AT_CACHE_SIZE; + } + }; } public Session getDatabaseSession() { @@ -62,13 +87,25 @@ BigDecimal position, BigDecimal w ) { - if (wstColumnValues == null) { - loadWstColumnValues(); + Integer c = column.getId(); + + Map<ValueKey, WstColumnValue> map = wstColumnValues.get(c); + + if (map == null) { + map = new HashMap<ValueKey, WstColumnValue>(); + wstColumnValues.put(c, map); + Query query = databaseSession.createQuery( + "from WstColumnValue where wstColumn.id=:cid"); + query.setParameter("cid", c); + for (Iterator iter = query.iterate(); iter.hasNext();) { + WstColumnValue wcv = (WstColumnValue)iter.next(); + map.put(new ValueKey(wcv.getPosition(), wcv.getW()), wcv); + } } - IdValueKey key = new IdValueKey(column.getId(), position, w); + ValueKey key = new ValueKey(position, w); - WstColumnValue wcv = wstColumnValues.get(key); + WstColumnValue wcv = map.get(key); if (wcv != null) { return wcv; @@ -78,62 +115,48 @@ databaseSession.save(wcv); - wstColumnValues.put(key, wcv); + map.put(key, wcv); return wcv; } - protected void loadWstColumnValues() { - log.info("load wst column values"); - wstColumnValues = new HashMap<IdValueKey, WstColumnValue>(); - - Query query = databaseSession.createQuery("from WstColumnValue"); - - for (Iterator iter = query.iterate(); iter.hasNext();) { - WstColumnValue wcv = (WstColumnValue)iter.next(); - wstColumnValues.put(new IdValueKey(wcv), wcv); - } - log.info(wstColumnValues.size() + " values loaded"); - } - public DischargeTableValue getDischargeTableValue( DischargeTable table, BigDecimal q, BigDecimal w ) { - if (dischargeTableValues == null) { - loadDischargeTableValues(); - } + Integer t = table.getId(); - IdValueKey key = new IdValueKey(table.getId(), q, w); + Map<ValueKey, DischargeTableValue> map = + dischargeTableValues.get(t); - DischargeTableValue dtv = dischargeTableValues.get(key); - - if (dtv != null) { - return dtv; + if (map == null) { + map = new HashMap<ValueKey, DischargeTableValue>(); + dischargeTableValues.put(t, map); + Query query = databaseSession.createQuery( + "from DischargeTableValue where dischargeTable.id=:tid"); + query.setParameter("tid", t); + for (Iterator iter = query.iterate(); iter.hasNext();) { + DischargeTableValue dctv = (DischargeTableValue)iter.next(); + map.put(new ValueKey(q, w), dctv); + } } - dtv = new DischargeTableValue(table, q, w); - - databaseSession.save(dtv); - - dischargeTableValues.put(key, dtv); - - return dtv; - } + ValueKey key = new ValueKey(q, w); - protected void loadDischargeTableValues() { - log.info("load discharge table values"); - - dischargeTableValues = new HashMap<IdValueKey, DischargeTableValue>(); + DischargeTableValue dctv = map.get(key); - Query query = databaseSession.createQuery("from DischargeTableValue"); + if (dctv != null) { + return dctv; + } - for (Iterator iter = query.iterate(); iter.hasNext();) { - DischargeTableValue dtv = (DischargeTableValue)iter.next(); - dischargeTableValues.put(new IdValueKey(dtv), dtv); - } - log.info(dischargeTableValues.size() + " values loaded"); + dctv = new DischargeTableValue(table, q, w); + + databaseSession.save(dctv); + + map.put(key, dctv); + + return dctv; } public Range getRange(River river, BigDecimal a, BigDecimal b) {
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/importer/ValueKey.java Mon May 09 00:15:45 2011 +0000 @@ -0,0 +1,39 @@ +package de.intevation.flys.importer; + +import java.math.BigDecimal; + +public class ValueKey +{ + protected BigDecimal a; + protected BigDecimal b; + + public ValueKey() { + } + + public ValueKey(BigDecimal a, BigDecimal b) { + this.a = a; + this.b = b; + } + + @Override + public int hashCode() { + return ((a != null ? a.hashCode() : 0) << 16) + | (b != null ? b.hashCode() : 0); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof ValueKey)) { + return false; + } + ValueKey o = (ValueKey)other; + return !( + (a == null && o.a != null) + || (a != null && o.a == null) + || (a != null && !a.equals(o.a)) + || (b == null && o.b != null) + || (b != null && o.b == null) + || (b != null && !b.equals(o.b))); + } +} +// vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 :