Mercurial > dive4elements > river
changeset 503:bcc18293a547
Importer: Added unsharp lookup to avoid numerical problems.
flys-backend/trunk@1859 c6561f87-3c4e-4783-a992-168aeb5c3f6f
| author | Sascha L. Teichmann <sascha.teichmann@intevation.de> |
|---|---|
| date | Mon, 09 May 2011 10:04:23 +0000 |
| parents | c6889097f81f |
| children | ee693b8fbb55 |
| files | flys-backend/ChangeLog flys-backend/src/main/java/de/intevation/flys/importer/IdValueKey.java flys-backend/src/main/java/de/intevation/flys/importer/ImportRiver.java flys-backend/src/main/java/de/intevation/flys/importer/ImportWst.java flys-backend/src/main/java/de/intevation/flys/importer/ImporterSession.java flys-backend/src/main/java/de/intevation/flys/importer/ValueKey.java |
| diffstat | 6 files changed, 86 insertions(+), 110 deletions(-) [+] |
line wrap: on
line diff
--- a/flys-backend/ChangeLog Mon May 09 00:38:13 2011 +0000 +++ b/flys-backend/ChangeLog Mon May 09 10:04:23 2011 +0000 @@ -1,3 +1,20 @@ +2011-05-09 Sascha L. Teichmann <sascha.teichmann@intevation.de> + + * src/main/java/de/intevation/flys/importer/ValueKey.java: + Add some unsharp comparison (eps = 1e-6) to avoid + numerical problems. + + * src/main/java/de/intevation/flys/importer/IdValueKey.java: + Removed. Not need any longer. + + * src/main/java/de/intevation/flys/importer/ImporterSession.java: + Use unsharp lookup. + + * src/main/java/de/intevation/flys/importer/ImportWst.java, + src/main/java/de/intevation/flys/importer/ImportRiver.java: + Flush more often. Hopefully this reduces hibernate sync + problems?! + 2011-05-09 Sascha L. Teichmann <sascha.teichmann@intevation.de> * src/main/java/de/intevation/flys/importer/ImporterSession.java:
--- a/flys-backend/src/main/java/de/intevation/flys/importer/IdValueKey.java Mon May 09 00:38:13 2011 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,67 +0,0 @@ -package de.intevation.flys.importer; - -import de.intevation.flys.model.WstColumnValue; -import de.intevation.flys.model.DischargeTableValue; -import de.intevation.flys.model.Range; - -import java.math.BigDecimal; - -public class IdValueKey { - - protected int id; - protected BigDecimal a; - protected BigDecimal b; - - - public IdValueKey(WstColumnValue value) { - this.id = value.getWstColumn().getId(); - this.a = value.getPosition(); - this.b = value.getW(); - } - - public IdValueKey(DischargeTableValue value) { - this.id = value.getDischargeTable().getId(); - this.a = value.getQ(); - this.b = value.getW(); - } - - public IdValueKey(Range value) { - this.id = value.getRiver().getId(); - this.a = value.getA(); - this.b = value.getB(); - } - - - public IdValueKey(int id, BigDecimal a, BigDecimal b) { - this.id = id; - this.a = a; - this.b = b; - } - - private static final int hashCode(BigDecimal d) { - return d != null ? d.hashCode() : 0; - } - - @Override - public int hashCode() { - return id | (hashCode(a) << 10) | (hashCode(b) << 20); - } - - @Override - public boolean equals(Object obj) { - if (!(obj instanceof IdValueKey)) { - return false; - } - - IdValueKey other = (IdValueKey) obj; - - return !((id != other.id) - || (a == null && other.a != null) - || (a != null && other.a == null) - || (a != null && !a.equals(other.a)) - || (b == null && other.b != null) - || (b != null && other.b == null) - || (b != null && !b.equals(other.b))); - } -} -// vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 :
--- a/flys-backend/src/main/java/de/intevation/flys/importer/ImportRiver.java Mon May 09 00:38:13 2011 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImportRiver.java Mon May 09 10:04:23 2011 +0000 @@ -407,9 +407,12 @@ public void storeGauges() { log.info("store gauges:"); River river = getPeer(); + Session session = ImporterSession.getInstance().getDatabaseSession(); for (ImportGauge gauge: gauges) { log.info("\tgauge: " + gauge.getName()); gauge.storeDependencies(river); + ImporterSession.getInstance().getDatabaseSession(); + session.flush(); } }
--- a/flys-backend/src/main/java/de/intevation/flys/importer/ImportWst.java Mon May 09 00:38:13 2011 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImportWst.java Mon May 09 10:04:23 2011 +0000 @@ -72,6 +72,8 @@ for (ImportWstColumn column: columns) { column.storeDependencies(river); } + Session session = ImporterSession.getInstance().getDatabaseSession(); + session.flush(); } public Wst getPeer(River river) {
--- a/flys-backend/src/main/java/de/intevation/flys/importer/ImporterSession.java Mon May 09 00:38:13 2011 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImporterSession.java Mon May 09 10:04:23 2011 +0000 @@ -2,7 +2,7 @@ import java.util.Iterator; import java.util.Map; -import java.util.HashMap; +import java.util.TreeMap; import java.util.LinkedHashMap; import java.math.BigDecimal; @@ -27,8 +27,14 @@ { private static Logger log = Logger.getLogger(ImporterSession.class); - public static final int MAX_WST_CACHE_SIZE = 20; - public static final int MAX_AT_CACHE_SIZE = 20; + public static final int CACHE_SIZE = 25; + + public static class Cache<K, V> extends LinkedHashMap<K, V> { + @Override + protected boolean removeEldestEntry(Map.Entry<K, V> eldest) { + return size() > CACHE_SIZE; + } + }; private static final ThreadLocal<ImporterSession> SESSION = new ThreadLocal<ImporterSession>() { @@ -40,12 +46,14 @@ protected Session databaseSession; - protected Map<Integer, Map<ValueKey, WstColumnValue>> wstColumnValues; + protected Cache<Integer, Map<ValueKey, WstColumnValue>> + wstColumnValues; - protected Map<Integer, Map<ValueKey, DischargeTableValue>> + protected Cache<Integer, Map<ValueKey, DischargeTableValue>> dischargeTableValues; - protected Map<IdValueKey, Range> ranges; + protected Cache<Integer, Map<ValueKey, Range>> + ranges; public static ImporterSession getInstance() { return SESSION.get(); @@ -58,24 +66,12 @@ //databaseSession.setFlushMode(FlushMode.MANUAL); wstColumnValues = - new LinkedHashMap<Integer, Map<ValueKey, WstColumnValue>>() { - @Override - protected boolean removeEldestEntry( - Map.Entry<Integer, Map<ValueKey, WstColumnValue>> eldest - ) { - return size() > MAX_WST_CACHE_SIZE; - } - }; + new Cache<Integer, Map<ValueKey, WstColumnValue>>(); dischargeTableValues = - new LinkedHashMap<Integer, Map<ValueKey, DischargeTableValue>>() { - @Override - protected boolean removeEldestEntry( - Map.Entry<Integer, Map<ValueKey, DischargeTableValue>> eldest - ) { - return size() > MAX_AT_CACHE_SIZE; - } - }; + new Cache<Integer, Map<ValueKey, DischargeTableValue>>(); + + ranges = new Cache<Integer, Map<ValueKey, Range>>(); } public Session getDatabaseSession() { @@ -92,7 +88,8 @@ Map<ValueKey, WstColumnValue> map = wstColumnValues.get(c); if (map == null) { - map = new HashMap<ValueKey, WstColumnValue>(); + map = new TreeMap<ValueKey, WstColumnValue>( + ValueKey.EPSILON_COMPARATOR); wstColumnValues.put(c, map); Query query = databaseSession.createQuery( "from WstColumnValue where wstColumn.id=:cid"); @@ -131,14 +128,15 @@ dischargeTableValues.get(t); if (map == null) { - map = new HashMap<ValueKey, DischargeTableValue>(); + map = new TreeMap<ValueKey, DischargeTableValue>( + ValueKey.EPSILON_COMPARATOR); dischargeTableValues.put(t, map); Query query = databaseSession.createQuery( "from DischargeTableValue where dischargeTable.id=:tid"); query.setParameter("tid", t); for (Iterator iter = query.iterate(); iter.hasNext();) { DischargeTableValue dctv = (DischargeTableValue)iter.next(); - map.put(new ValueKey(q, w), dctv); + map.put(new ValueKey(dctv.getQ(), dctv.getW()), dctv); } } @@ -160,14 +158,26 @@ } public Range getRange(River river, BigDecimal a, BigDecimal b) { + Integer r = river.getId(); - if (ranges == null) { - loadRanges(); + Map<ValueKey, Range> map = ranges.get(r); + + if (map == null) { + map = new TreeMap<ValueKey, Range>( + ValueKey.EPSILON_COMPARATOR); + ranges.put(r, map); + Query query = databaseSession.createQuery( + "from Range where river.id=:rid"); + query.setParameter("rid", r); + for (Iterator iter = query.iterate(); iter.hasNext();) { + Range range = (Range)iter.next(); + map.put(new ValueKey(range.getA(), range.getB()), range); + } } - IdValueKey key = new IdValueKey(river.getId(), a, b); + ValueKey key = new ValueKey(a, b); - Range range = ranges.get(key); + Range range = map.get(key); if (range != null) { return range; @@ -177,22 +187,9 @@ databaseSession.save(range); - ranges.put(key, range); + map.put(key, range); return range; } - - protected void loadRanges() { - log.info("load ranges"); - ranges = new HashMap<IdValueKey, Range>(); - - Query query = databaseSession.createQuery("from Range"); - - for (Iterator iter = query.iterate(); iter.hasNext();) { - Range range = (Range)iter.next(); - ranges.put(new IdValueKey(range), range); - } - log.info(ranges.size() + " values loaded"); - } } // vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 :
--- a/flys-backend/src/main/java/de/intevation/flys/importer/ValueKey.java Mon May 09 00:38:13 2011 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/importer/ValueKey.java Mon May 09 10:04:23 2011 +0000 @@ -2,8 +2,32 @@ import java.math.BigDecimal; +import java.util.Comparator; + public class ValueKey { + public static final double EPSILON = 1e-6; + + public static final Comparator<ValueKey> EPSILON_COMPARATOR = + new Comparator<ValueKey>() + { + public int compare(ValueKey x, ValueKey y) { + int cmp = ValueKey.compare(x.a, y.a); + if (cmp != 0) return cmp; + return ValueKey.compare(x.b, y.b); + } + }; + + public static int compare(BigDecimal a, BigDecimal b) { + if (a == null && b == null) return 0; + if (a != null && b == null) return +1; + if (a == null && b != null) return -1; + + double diff = a.doubleValue() - b.doubleValue(); + if (diff < -EPSILON) return -1; + return diff > EPSILON ? +1 : 0; + } + protected BigDecimal a; protected BigDecimal b;