Mercurial > dive4elements > river
changeset 497:67fd63e4ef66
Importer: centralized caching
flys-backend/trunk@1851 c6561f87-3c4e-4783-a992-168aeb5c3f6f
line wrap: on
line diff
--- a/flys-backend/ChangeLog Sun May 08 11:20:39 2011 +0000 +++ b/flys-backend/ChangeLog Sun May 08 15:29:45 2011 +0000 @@ -1,3 +1,30 @@ +2011-05-08 Sascha L. Teichmann <sascha.teichmann@intevation.de> + + * src/main/java/de/intevation/flys/importer/ImporterSession.java: + New. Centralized caching in the thread local context. Importing + the Elbe leads to OOM because the column values of the + WST files where loaded separately for every file. + + * src/main/java/de/intevation/flys/importer/ImportPosition.java, + src/main/java/de/intevation/flys/importer/Importer.java, + src/main/java/de/intevation/flys/importer/ImportAnnotation.java, + src/main/java/de/intevation/flys/importer/ImportWstQRange.java, + src/main/java/de/intevation/flys/importer/ImportWst.java, + src/main/java/de/intevation/flys/importer/ImportMainValue.java, + src/main/java/de/intevation/flys/importer/ImportMainValueType.java, + src/main/java/de/intevation/flys/importer/ImportNamedMainValue.java, + src/main/java/de/intevation/flys/importer/ImportRiver.java, + src/main/java/de/intevation/flys/importer/ImportGauge.java, + src/main/java/de/intevation/flys/importer/ImportWstColumnValue.java, + src/main/java/de/intevation/flys/importer/ImportWstColumnQRange.java, + src/main/java/de/intevation/flys/importer/ImportWstColumn.java, + src/main/java/de/intevation/flys/importer/ImportRange.java, + src/main/java/de/intevation/flys/importer/ImportDischargeTableValue.java, + src/main/java/de/intevation/flys/importer/ImportDischargeTable.java, + src/main/java/de/intevation/flys/importer/ImportAttribute.java, + src/main/java/de/intevation/flys/importer/ImportTimeInterval.java: + Adjusted to use the new global context. + 2011-05-08 Sascha L. Teichmann <sascha.teichmann@intevation.de> * doc/schema/postgresql.sql: Dropped constraint which
--- a/flys-backend/src/main/java/de/intevation/flys/importer/ImportAnnotation.java Sun May 08 11:20:39 2011 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImportAnnotation.java Sun May 08 15:29:45 2011 +0000 @@ -79,7 +79,7 @@ Range r = range.getPeer(river); Attribute a = attribute.getPeer(); Position p = position.getPeer(); - Session session = Importer.sessionHolder.get(); + Session session = ImporterSession.getInstance().getDatabaseSession(); Query query = session.createQuery( "from Annotation where " + "range=:range and attribute=:attribute and position=:position");
--- a/flys-backend/src/main/java/de/intevation/flys/importer/ImportAttribute.java Sun May 08 11:20:39 2011 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImportAttribute.java Sun May 08 15:29:45 2011 +0000 @@ -47,7 +47,7 @@ public Attribute getPeer() { if (peer == null) { - Session session = Importer.sessionHolder.get(); + Session session = ImporterSession.getInstance().getDatabaseSession(); Query query = session.createQuery("from Attribute where value=:value"); query.setString("value", value); List<Attribute> attributes = query.list();
--- a/flys-backend/src/main/java/de/intevation/flys/importer/ImportDischargeTable.java Sun May 08 11:20:39 2011 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImportDischargeTable.java Sun May 08 15:29:45 2011 +0000 @@ -10,8 +10,6 @@ import de.intevation.flys.model.Gauge; import de.intevation.flys.model.TimeInterval; -import org.apache.log4j.Logger; - public class ImportDischargeTable { protected DischargeTable peer; @@ -72,7 +70,7 @@ TimeInterval ti = timeInterval != null ? timeInterval.getPeer() : null; - Session session = Importer.sessionHolder.get(); + Session session = ImporterSession.getInstance().getDatabaseSession(); Query query = session.createQuery( "from DischargeTable where " +
--- a/flys-backend/src/main/java/de/intevation/flys/importer/ImportDischargeTableValue.java Sun May 08 11:20:39 2011 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImportDischargeTableValue.java Sun May 08 15:29:45 2011 +0000 @@ -35,7 +35,7 @@ public DischargeTableValue getPeer(DischargeTable dischargeTable) { if (peer == null) { - Session session = Importer.sessionHolder.get(); + Session session = ImporterSession.getInstance().getDatabaseSession(); Query query = session.createQuery( "from DischargeTableValue where " +
--- a/flys-backend/src/main/java/de/intevation/flys/importer/ImportGauge.java Sun May 08 11:20:39 2011 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImportGauge.java Sun May 08 15:29:45 2011 +0000 @@ -226,7 +226,7 @@ public Gauge getPeer(River river) { if (peer == null) { - Session session = Importer.sessionHolder.get(); + Session session = ImporterSession.getInstance().getDatabaseSession(); Query query = session.createQuery( "from Gauge where name=:name " + "and river.id=:river");
--- a/flys-backend/src/main/java/de/intevation/flys/importer/ImportMainValue.java Sun May 08 11:20:39 2011 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImportMainValue.java Sun May 08 15:29:45 2011 +0000 @@ -59,7 +59,7 @@ public MainValue getPeer(River river) { if (peer == null) { - Session session = Importer.sessionHolder.get(); + Session session = ImporterSession.getInstance().getDatabaseSession(); Query query = session.createQuery("from MainValue where " + "gauge.id=:gauge_id and mainValue.id=:name_id " + "and value=:value");
--- a/flys-backend/src/main/java/de/intevation/flys/importer/ImportMainValueType.java Sun May 08 11:20:39 2011 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImportMainValueType.java Sun May 08 15:29:45 2011 +0000 @@ -47,7 +47,7 @@ public MainValueType getPeer() { if (peer == null) { - Session session = Importer.sessionHolder.get(); + Session session = ImporterSession.getInstance().getDatabaseSession(); Query query = session.createQuery("from MainValueType where name=:name"); query.setString("name", name); List<MainValueType> values = query.list();
--- a/flys-backend/src/main/java/de/intevation/flys/importer/ImportNamedMainValue.java Sun May 08 11:20:39 2011 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImportNamedMainValue.java Sun May 08 15:29:45 2011 +0000 @@ -45,7 +45,7 @@ public NamedMainValue getPeer() { if (peer == null) { MainValueType type = mainValueType.getPeer(); - Session session = Importer.sessionHolder.get(); + Session session = ImporterSession.getInstance().getDatabaseSession(); Query query = session.createQuery( "from NamedMainValue where " + "name=:name and type.id=:id");
--- a/flys-backend/src/main/java/de/intevation/flys/importer/ImportPosition.java Sun May 08 11:20:39 2011 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImportPosition.java Sun May 08 15:29:45 2011 +0000 @@ -35,7 +35,7 @@ public Position getPeer() { if (peer == null) { - Session session = Importer.sessionHolder.get(); + Session session = ImporterSession.getInstance().getDatabaseSession(); Query query = session.createQuery("from Position where value=:value"); query.setString("value", value); List<Position> positions = query.list();
--- a/flys-backend/src/main/java/de/intevation/flys/importer/ImportRange.java Sun May 08 11:20:39 2011 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImportRange.java Sun May 08 15:29:45 2011 +0000 @@ -67,7 +67,7 @@ public Range getPeer(River river) { if (peer == null) { - Session session = Importer.sessionHolder.get(); + Session session = ImporterSession.getInstance().getDatabaseSession(); Query query = session.createQuery( "from Range where a=:a and b=:b and river=:river"); query.setParameter("a", a);
--- a/flys-backend/src/main/java/de/intevation/flys/importer/ImportRiver.java Sun May 08 11:20:39 2011 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImportRiver.java Sun May 08 15:29:45 2011 +0000 @@ -415,7 +415,7 @@ public River getPeer() { if (peer == null) { - Session session = Importer.sessionHolder.get(); + Session session = ImporterSession.getInstance().getDatabaseSession(); Query query = session.createQuery("from River where name=:name"); query.setString("name", name); List<River> rivers = query.list();
--- a/flys-backend/src/main/java/de/intevation/flys/importer/ImportTimeInterval.java Sun May 08 11:20:39 2011 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImportTimeInterval.java Sun May 08 15:29:45 2011 +0000 @@ -45,7 +45,7 @@ public TimeInterval getPeer() { if (peer == null) { - Session session = Importer.sessionHolder.get(); + Session session = ImporterSession.getInstance().getDatabaseSession(); Query query = session.createQuery( "from TimeInterval where startTime=:a and stopTime=:b"); query.setParameter("a", startTime);
--- a/flys-backend/src/main/java/de/intevation/flys/importer/ImportWst.java Sun May 08 11:20:39 2011 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImportWst.java Sun May 08 15:29:45 2011 +0000 @@ -1,21 +1,15 @@ package de.intevation.flys.importer; import de.intevation.flys.model.Wst; -import de.intevation.flys.model.WstColumnValue; import de.intevation.flys.model.River; -import de.intevation.flys.backend.SessionFactoryProvider; - import org.apache.log4j.Logger; import org.hibernate.Session; -import org.hibernate.SessionFactory; import org.hibernate.Query; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; public class ImportWst { @@ -25,19 +19,13 @@ protected Integer kind; - //protected Map<String, ImportWstColumn> columns; protected List<ImportWstColumn> columns; - protected Map<WstColumnValueKey, WstColumnValue> wstCache; - protected Wst peer; public ImportWst() { kind = 0; columns = new ArrayList<ImportWstColumn>(); - - // This cache is used to avoid db queries for each WstColumnValue later - wstCache = getWstColumnValueCache(); } public ImportWst(String description) { @@ -64,7 +52,7 @@ public void setNumberColumns(int numColumns) { for (int i = 0; i < numColumns; ++i) { - columns.add(new ImportWstColumn(this, null, null, i, wstCache)); + columns.add(new ImportWstColumn(this, null, null, i)); } } @@ -84,43 +72,9 @@ } } - - public Map<WstColumnValueKey, WstColumnValue> getWstColumnValueCache() { - SessionFactory sessionFactory = - SessionFactoryProvider.createSessionFactory(); - - Session session = sessionFactory.openSession(); - - Importer.sessionHolder.set(session); - - try { - Query query = session.createQuery("from WstColumnValue"); - - List<WstColumnValue> values = query.list(); - Map<WstColumnValueKey, WstColumnValue> cache = - new HashMap<WstColumnValueKey, WstColumnValue>(); - - for (WstColumnValue value: values) { - cache.put( - new WstColumnValueKey(value), - value); - } - - logger.info("++++++++++++++++++++++++++++++++++++++++++++++++++++"); - logger.info(cache.size() + " WstColumnValue objects in WST cache."); - logger.info("++++++++++++++++++++++++++++++++++++++++++++++++++++"); - - return cache; - } - finally { - session.close(); - Importer.sessionHolder.remove(); - } - } - public Wst getPeer(River river) { if (peer == null) { - Session session = Importer.sessionHolder.get(); + Session session = ImporterSession.getInstance().getDatabaseSession(); Query query = session.createQuery( "from Wst where " + "river=:river and description=:description and kind=:kind");
--- a/flys-backend/src/main/java/de/intevation/flys/importer/ImportWstColumn.java Sun May 08 11:20:39 2011 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImportWstColumn.java Sun May 08 15:29:45 2011 +0000 @@ -2,7 +2,6 @@ import de.intevation.flys.model.Wst; import de.intevation.flys.model.WstColumn; -import de.intevation.flys.model.WstColumnValue; import de.intevation.flys.model.River; import org.hibernate.Session; @@ -10,7 +9,6 @@ import java.util.List; import java.util.ArrayList; -import java.util.Map; import java.math.BigDecimal; @@ -24,8 +22,6 @@ protected List<ImportWstColumnQRange> columnQRanges; protected List<ImportWstColumnValue> columnValues; - protected Map<WstColumnValueKey, WstColumnValue> wstCache; - protected WstColumn peer; public ImportWstColumn() { @@ -37,15 +33,13 @@ ImportWst wst, String name, String description, - Integer position, - Map<WstColumnValueKey, WstColumnValue> wstCache + Integer position ) { this(); this.wst = wst; this.name = name; this.description = description; this.position = position; - this.wstCache = wstCache; } public ImportWst getWst() { @@ -82,7 +76,7 @@ public void addColumnValue(BigDecimal position, BigDecimal w) { columnValues.add( - new ImportWstColumnValue(this, position, w, wstCache)); + new ImportWstColumnValue(this, position, w)); } public void addColumnQRange(ImportWstQRange columnQRange) { @@ -103,7 +97,7 @@ public WstColumn getPeer(River river) { if (peer == null) { Wst w = wst.getPeer(river); - Session session = Importer.sessionHolder.get(); + Session session = ImporterSession.getInstance().getDatabaseSession(); Query query = session.createQuery( "from WstColumn where " + "wst=:wst and name=:name and description=:description" +
--- a/flys-backend/src/main/java/de/intevation/flys/importer/ImportWstColumnQRange.java Sun May 08 11:20:39 2011 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImportWstColumnQRange.java Sun May 08 15:29:45 2011 +0000 @@ -48,7 +48,7 @@ if (peer == null) { WstColumn c = wstColumn.getPeer(river); WstQRange q = qRange.getPeer(river); - Session session = Importer.sessionHolder.get(); + Session session = ImporterSession.getInstance().getDatabaseSession(); Query query = session.createQuery( "from WstColumnQRange where " + "wstColumn=:c and wstQRange=:q");
--- a/flys-backend/src/main/java/de/intevation/flys/importer/ImportWstColumnValue.java Sun May 08 11:20:39 2011 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImportWstColumnValue.java Sun May 08 15:29:45 2011 +0000 @@ -8,11 +8,6 @@ import org.apache.log4j.Logger; -import org.hibernate.Session; - -import java.util.Map; - - public class ImportWstColumnValue { protected Logger logger = Logger.getLogger(ImportWstColumnValue.class); @@ -21,8 +16,6 @@ protected BigDecimal w; protected ImportWstColumn wstColumn; - protected Map<WstColumnValueKey, WstColumnValue> cache; - protected WstColumnValue peer; public ImportWstColumnValue() { @@ -31,13 +24,11 @@ public ImportWstColumnValue( ImportWstColumn wstColumn, BigDecimal position, - BigDecimal w, - Map<WstColumnValueKey, WstColumnValue> cache + BigDecimal w ) { this.wstColumn = wstColumn; this.position = position; this.w = w; - this.cache = cache; } public BigDecimal getPosition() { @@ -67,23 +58,8 @@ public WstColumnValue getPeer(River river) { if (peer == null) { WstColumn c = wstColumn.getPeer(river); - - // check the cache before we fire a db query - WstColumnValue value = cache.get(new WstColumnValueKey( - c.getId(), - position.doubleValue(), - w.doubleValue())); - - if (value != null) { - peer = value; - return peer; - } - - Session session = Importer.sessionHolder.get(); - peer = new WstColumnValue(c, position, w); - session.save(peer); - - cache.put(new WstColumnValueKey(peer), peer); + peer = ImporterSession.getInstance().getWstColumnValue( + c, position, w); } return peer;
--- a/flys-backend/src/main/java/de/intevation/flys/importer/ImportWstQRange.java Sun May 08 11:20:39 2011 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImportWstQRange.java Sun May 08 15:29:45 2011 +0000 @@ -48,7 +48,7 @@ public WstQRange getPeer(River river) { if (peer == null) { Range r = range.getPeer(river); - Session session = Importer.sessionHolder.get(); + Session session = ImporterSession.getInstance().getDatabaseSession(); Query query = session.createQuery( "from WstQRange where " + "range=:range and q=:q");
--- a/flys-backend/src/main/java/de/intevation/flys/importer/Importer.java Sun May 08 11:20:39 2011 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/importer/Importer.java Sun May 08 15:29:45 2011 +0000 @@ -9,13 +9,9 @@ import org.apache.log4j.Logger; -import org.hibernate.SessionFactory; -import org.hibernate.Session; import org.hibernate.Transaction; import org.hibernate.HibernateException; -import de.intevation.flys.backend.SessionFactoryProvider; - public class Importer { private static Logger log = Logger.getLogger(Importer.class); @@ -23,9 +19,6 @@ public static final boolean DRY_RUN = Boolean.getBoolean("flys.backend.importer.dry.run"); - public static final ThreadLocal<Session> sessionHolder = - new ThreadLocal<Session>(); - protected List<ImportRiver> rivers; public Importer() { @@ -46,29 +39,22 @@ public void writeRivers() { log.debug("write rivers started"); - Session session = sessionHolder.get(); - for (ImportRiver river: rivers) { log.debug("writing river '" + river.getName() + "'"); river.storeDependencies(); - session.flush(); + ImporterSession.getInstance().getDatabaseSession().flush(); } log.debug("write rivers finished"); } public void writeToDatabase() { - SessionFactory sessionFactory = - SessionFactoryProvider.createSessionFactory(); - - Session session = sessionFactory.openSession(); - - sessionHolder.set(session); Transaction tx = null; try { - tx = session.beginTransaction(); + tx = ImporterSession.getInstance() + .getDatabaseSession().beginTransaction(); try { writeRivers(); @@ -91,10 +77,6 @@ } throw re; } - finally { - session.close(); - sessionHolder.remove(); - } } public static void main(String [] args) {
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImporterSession.java Sun May 08 15:29:45 2011 +0000 @@ -0,0 +1,86 @@ +package de.intevation.flys.importer; + +import java.util.Iterator; +import java.util.Map; +import java.util.HashMap; + +import java.math.BigDecimal; + +import org.hibernate.SessionFactory; +import org.hibernate.Session; +import org.hibernate.Query; + +import de.intevation.flys.backend.SessionFactoryProvider; + +import de.intevation.flys.model.WstColumnValue; +import de.intevation.flys.model.WstColumn; + +public class ImporterSession +{ + private static final ThreadLocal<ImporterSession> SESSION = + new ThreadLocal<ImporterSession>() { + @Override + protected ImporterSession initialValue() { + return new ImporterSession(); + } + }; + + protected Session databaseSession; + + protected Map<WstColumnValueKey, WstColumnValue> wstColumnValues; + + public static ImporterSession getInstance() { + return SESSION.get(); + } + + public ImporterSession() { + SessionFactory sessionFactory = + SessionFactoryProvider.createSessionFactory(); + databaseSession = sessionFactory.openSession(); + } + + public Session getDatabaseSession() { + return databaseSession; + } + + public WstColumnValue getWstColumnValue( + WstColumn column, + BigDecimal position, + BigDecimal w + ) { + if (wstColumnValues == null) { + loadWstColumnValues(); + } + + WstColumnValueKey key = new WstColumnValueKey( + column.getId(), + position.doubleValue(), + w.doubleValue()); + + WstColumnValue wcv = wstColumnValues.get(key); + + if (wcv != null) { + return wcv; + } + + wcv = new WstColumnValue(column, position, w); + + databaseSession.save(wcv); + + wstColumnValues.put(key, wcv); + + return wcv; + } + + protected void loadWstColumnValues() { + wstColumnValues = new HashMap<WstColumnValueKey, WstColumnValue>(); + + Query query = databaseSession.createQuery("from WstColumnValue"); + + for (Iterator iter = query.iterate(); iter.hasNext();) { + WstColumnValue wcv = (WstColumnValue)iter.next(); + wstColumnValues.put(new WstColumnValueKey(wcv), wcv); + } + } +} +// vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 :