Mercurial > dive4elements > river

comparison artifacts/src/main/java/org/dive4elements/river/utils/BatchLoader.java @ 6847:b0c88ac9f63b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Fixing Analysis: Significantly (8-10x) improved the performance of the generation of the overview data. The data is now prefetched in batches.
author Sascha L. Teichmann <teichmann@intevation.de>
date Sat, 17 Aug 2013 20:08:51 +0200
parents
children 0a5239a1e46e
comparison
equal deleted inserted replaced
6846:b87b20c71407 6847:b0c88ac9f63b
1 /* Copyright (C) 2011, 2012, 2013 by Bundesanstalt für Gewässerkunde
2 * Software engineering by Intevation GmbH
3 *
4 * This file is Free Software under the GNU AGPL (>=v3)
5 * and comes with ABSOLUTELY NO WARRANTY! Check out the
6 * documentation coming with Dive4Elements River for details.
7 */
8
9 package org.dive4elements.river.utils;
10
11 import java.util.ArrayList;
12 import java.util.HashMap;
13 import java.util.Iterator;
14 import java.util.List;
15 import java.util.Map;
16
17 import org.apache.log4j.Logger;
18 import org.hibernate.SQLQuery;
19 import org.hibernate.Session;
20
21 /** To reduce the number of SQL queries send to the backend
22 * (mainly by the fixings overviews) we execute them in batches of ids
23 * and store the results in a small cache.
24 * TODO: It currently relies on dynamic SQL.
25 * Is there a way to use Hibernate with java.sql.Array
26 * in cross database compatible manner?
27 */
28 public abstract class BatchLoader<T> {
29
30 private static Logger log = Logger.getLogger(BatchLoader.class);
31
32 public static final int BATCH_SIZE = 100;
33
34 private Map<Integer, T> loaded;
35 private List<Integer> rest;
36 private Session session;
37 private String sqlTemplate;
38
39 public BatchLoader(
40 List<Integer> columns,
41 Session session,
42 String sqlTemplate
43 ) {
44 rest = new ArrayList<Integer>(columns.size());
45 loaded = new HashMap<Integer, T>();
46 this.session = session;
47 this.sqlTemplate = sqlTemplate;
48
49 // Insert in reverse order to minize searching.
50 for (int i = columns.size()-1; i >= 0; --i) {
51 rest.add(columns.get(i));
52 }
53 }
54
55 /** Searches for id and fill a batch to load containing the found id. */
56 private List<Integer> prepareBatch(int id) {
57 List<Integer> batch = new ArrayList<Integer>(BATCH_SIZE);
58
59 boolean found = false;
60
61 for (int i = rest.size()-1; batch.size() < BATCH_SIZE && i >= 0; --i) {
62 Integer cid = rest.get(i);
63 if (cid == id) {
64 found = true;
65 batch.add(cid);
66 rest.remove(i);
67 }
68 else if ((found && batch.size() < BATCH_SIZE)
69 || (!found && batch.size() < BATCH_SIZE-1)) {
70 batch.add(cid);
71 rest.remove(i);
72 }
73 }
74
75 return batch;
76 }
77
78 /** Converts id to a list of comma separated ints. */
79 private static String idsAsString(List<Integer> ids) {
80 StringBuilder sb = new StringBuilder();
81 for (Iterator<Integer> i = ids.iterator(); i.hasNext();) {
82 sb.append(i.next());
83 if (i.hasNext()) {
84 sb.append(',');
85 }
86 }
87 return sb.toString();
88 }
89
90 /** Get data for id. */
91 public T get(int id) {
92 T already = loaded.get(id);
93 if (already != null) {
94 return already;
95 }
96
97 List<Integer> batch = prepareBatch(id);
98 if (batch.isEmpty()) {
99 return null;
100 }
101 String sql = sqlTemplate.replace("$IDS", idsAsString(batch));
102 if (log.isDebugEnabled()) {
103 log.debug(sql + " " + sql.length());
104 }
105 fill(session.createSQLQuery(sql));
106 return get(id);
107 }
108
109 /** Call this from fill() to store data in the cache. */
110 protected void cache(int key, T data) {
111 loaded.put(key, data);
112 }
113
114 /** Override this to fill the cache */
115 protected abstract void fill(SQLQuery query);
116 }
http://dive4elements.wald.intevation.org