diff artifacts/src/main/java/org/dive4elements/river/utils/BatchLoader.java @ 6847:b0c88ac9f63b

Fixing Analysis: Significantly (8-10x) improved the performance of the generation of the overview data. The data is now prefetched in batches.
author Sascha L. Teichmann <teichmann@intevation.de>
date Sat, 17 Aug 2013 20:08:51 +0200
parents
children 0a5239a1e46e
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/artifacts/src/main/java/org/dive4elements/river/utils/BatchLoader.java	Sat Aug 17 20:08:51 2013 +0200
@@ -0,0 +1,116 @@
+/* Copyright (C) 2011, 2012, 2013 by Bundesanstalt für Gewässerkunde
+ * Software engineering by Intevation GmbH
+ *
+ * This file is Free Software under the GNU AGPL (>=v3)
+ * and comes with ABSOLUTELY NO WARRANTY! Check out the
+ * documentation coming with Dive4Elements River for details.
+ */
+
+package org.dive4elements.river.utils;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.log4j.Logger;
+import org.hibernate.SQLQuery;
+import org.hibernate.Session;
+
+/** To reduce the number of SQL queries send to the backend
+ *  (mainly by the fixings overviews) we execute them in batches of ids
+ *  and store the results in a small cache.
+ *  TODO: It currently relies on dynamic SQL.
+ *  Is there a way to use Hibernate with java.sql.Array
+ *  in cross database compatible manner?
+ */
+public abstract class BatchLoader<T> {
+
+    private static Logger log = Logger.getLogger(BatchLoader.class);
+
+    public static final int BATCH_SIZE = 100;
+
+    private Map<Integer, T> loaded;
+    private List<Integer>   rest;
+    private Session         session;
+    private String          sqlTemplate;
+
+    public BatchLoader(
+        List<Integer> columns,
+        Session       session,
+        String        sqlTemplate
+    ) {
+        rest             = new ArrayList<Integer>(columns.size());
+        loaded           = new HashMap<Integer, T>();
+        this.session     = session;
+        this.sqlTemplate = sqlTemplate;
+
+        // Insert in reverse order to minize searching.
+        for (int i = columns.size()-1; i >= 0; --i) {
+            rest.add(columns.get(i));
+        }
+    }
+
+    /** Searches for id and fill a batch to load containing the found id. */
+    private List<Integer> prepareBatch(int id) {
+        List<Integer> batch = new ArrayList<Integer>(BATCH_SIZE);
+
+        boolean found = false;
+
+        for (int i = rest.size()-1; batch.size() < BATCH_SIZE && i >= 0; --i) {
+            Integer cid = rest.get(i);
+            if (cid == id) {
+                found = true;
+                batch.add(cid);
+                rest.remove(i);
+            }
+            else if ((found && batch.size() < BATCH_SIZE)
+                 || (!found && batch.size() < BATCH_SIZE-1)) {
+                batch.add(cid);
+                rest.remove(i);
+            }
+        }
+
+        return batch;
+    }
+
+    /** Converts id to a list of comma separated ints. */
+    private static String idsAsString(List<Integer> ids) {
+        StringBuilder sb = new StringBuilder();
+        for (Iterator<Integer> i = ids.iterator(); i.hasNext();) {
+            sb.append(i.next());
+            if (i.hasNext()) {
+                sb.append(',');
+            }
+        }
+        return sb.toString();
+    }
+
+    /** Get data for id. */
+    public T get(int id) {
+        T already = loaded.get(id);
+        if (already != null) {
+            return already;
+        }
+
+        List<Integer> batch = prepareBatch(id);
+        if (batch.isEmpty()) {
+            return null;
+        }
+        String sql = sqlTemplate.replace("$IDS", idsAsString(batch));
+        if (log.isDebugEnabled()) {
+            log.debug(sql + " " + sql.length());
+        }
+        fill(session.createSQLQuery(sql));
+        return get(id);
+    }
+
+    /** Call this from fill() to store data in the cache. */
+    protected void cache(int key, T data) {
+        loaded.put(key, data);
+    }
+
+    /** Override this to fill the cache */
+    protected abstract void fill(SQLQuery query);
+}

http://dive4elements.wald.intevation.org