Mercurial > dive4elements > framework

comparison artifact-database/src/main/java/org/dive4elements/artifactdatabase/DatabaseCleaner.java @ 473:d0ac790a6c89 dive4elements-move

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Moved directories to org.dive4elements
author Sascha L. Teichmann <teichmann@intevation.de>
date Thu, 25 Apr 2013 10:57:18 +0200
parents artifact-database/src/main/java/de/intevation/artifactdatabase/DatabaseCleaner.java@f367be55dd35
children 415df0fc4fa1
comparison
equal deleted inserted replaced
472:783cc1b6b615 473:d0ac790a6c89
1 /*
2 * Copyright (c) 2010 by Intevation GmbH
3 *
4 * This program is free software under the LGPL (>=v2.1)
5 * Read the file LGPL.txt coming with the software for details
6 * or visit http://www.gnu.org/licenses/ if it does not exist.
7 */
8
9 package de.intevation.artifactdatabase;
10
11 import de.intevation.artifacts.common.utils.Config;
12 import de.intevation.artifacts.common.utils.StringUtils;
13
14 import de.intevation.artifacts.Artifact;
15
16 import de.intevation.artifactdatabase.db.SQL;
17 import de.intevation.artifactdatabase.db.DBConnection;
18
19 import java.sql.Connection;
20 import java.sql.PreparedStatement;
21 import java.sql.ResultSet;
22 import java.sql.SQLException;
23
24 import java.util.ArrayList;
25 import java.util.List;
26 import java.util.Set;
27 import java.util.Collections;
28
29 import javax.sql.DataSource;
30
31 import org.apache.log4j.Logger;
32
33 /**
34 * The database cleaner runs in background. It sleep for a configurable
35 * while and when it wakes up it removes outdated artifacts from the
36 * database. Outdated means that the the last access to the artifact
37 * is longer aga then the time to live of this artifact.<br>
38 * Before the artifact is finally removed from the system it is
39 * revived one last time an the #endOfLife() method of the artifact
40 * is called.<br>
41 * The artifact implementations may e.g. use this to remove some extrenal
42 * resources form the system.
43 *
44 * @author <a href="mailto:sascha.teichmann@intevation.de">Sascha L. Teichmann</a>
45 */
46 public class DatabaseCleaner
47 extends Thread
48 {
49 /**
50 * Implementors of this interface are able to create a
51 * living artifact from a given byte array.
52 */
53 public interface ArtifactReviver {
54
55 /**
56 * Called to revive an artifact from a given byte array.
57 * @param factoryName The name of the factory which
58 * created this artifact.
59 * @param bytes The bytes of the serialized artifact.
60 * @return The revived artfiact.
61 */
62 Artifact reviveArtifact(String factoryName, byte [] bytes);
63
64 void killedArtifacts(List<String> identifiers);
65 void killedCollections(List<String> identifiers);
66
67 } // interface ArtifactReviver
68
69 public interface LockedIdsProvider {
70 Set<Integer> getLockedIds();
71 } // interface LockedIdsProvider
72
73 private static Logger logger = Logger.getLogger(DatabaseCleaner.class);
74
75 /**
76 * Number of artifacts to be loaded at once. Used to
77 * mitigate the problem of a massive denial of service
78 * if too many artifacts have died since last cleanup.
79 */
80 public static final int MAX_ROWS = 50;
81
82 public static final Set<Integer> EMPTY_IDS = Collections.emptySet();
83
84 /**
85 * The SQL statement to select the outdated artifacts.
86 */
87 public String SQL_OUTDATED;
88
89 public String SQL_OUTDATED_COLLECTIONS;
90 public String SQL_DELETE_COLLECTION_ITEMS;
91 public String SQL_DELETE_COLLECTION;
92
93 /**
94 * The SQL statement to delete some artifacts from the database.
95 */
96 public String SQL_DELETE_ARTIFACT;
97
98 /**
99 * XPath to figure out how long the cleaner should sleep between
100 * cleanups. This is stored in the global configuration.
101 */
102 public static final String SLEEP_XPATH =
103 "/artifact-database/cleaner/sleep-time/text()";
104
105 /**
106 * Default nap time between cleanups: 5 minutes.
107 */
108 public static final long SLEEP_DEFAULT =
109 5 * 60 * 1000L; // 5 minutes
110
111 /**
112 * The configured nap time.
113 */
114 protected long sleepTime;
115
116 /**
117 * Internal locking mechanism to prevent some race conditions.
118 */
119 protected Object sleepLock = new Object();
120
121 /**
122 * A reference to the global context.
123 */
124 protected Object context;
125
126 /**
127 * A specialized Id filter which only delete some artifacts.
128 * This is used to prevent deletion of living artifacts.
129 */
130 protected LockedIdsProvider lockedIdsProvider;
131
132 /**
133 * The reviver used to bring the dead artifact on last
134 * time back to live to call endOfLife() on them.
135 */
136 protected ArtifactReviver reviver;
137
138 protected DBConnection dbConnection;
139
140 /**
141 * Default constructor.
142 */
143 public DatabaseCleaner() {
144 }
145
146 /**
147 * Constructor to create a cleaner with a given global context
148 * and a given reviver.
149 * @param context The global context of the artifact database
150 * @param reviver The reviver to awake artifact one last time.
151 */
152 public DatabaseCleaner(Object context, ArtifactReviver reviver, DBConfig config) {
153 setDaemon(true);
154 sleepTime = getSleepTime();
155 this.context = context;
156 this.reviver = reviver;
157 this.dbConnection = config.getDBConnection();
158 setupSQL(config.getSQL());
159 }
160
161 protected void setupSQL(SQL sql) {
162 SQL_OUTDATED = sql.get("artifacts.outdated");
163 SQL_OUTDATED_COLLECTIONS = sql.get("collections.outdated");
164 SQL_DELETE_COLLECTION_ITEMS = sql.get("delete.collection.items");
165 SQL_DELETE_COLLECTION = sql.get("delete.collection");
166 SQL_DELETE_ARTIFACT = sql.get("artifacts.delete");
167 }
168
169 /**
170 * Sets the filter that prevents deletion of living artifacts.
171 * Living artifacts are artifacts which are currently active
172 * inside the artifact database. Deleting them in this state
173 * would create severe internal problems.
174 */
175 public void setLockedIdsProvider(LockedIdsProvider lockedIdsProvider) {
176 this.lockedIdsProvider = lockedIdsProvider;
177 }
178
179 /**
180 * External hook to tell the cleaner to wake up before its
181 * regular nap time is over. This is the case when the artifact
182 * database finds an artifact which is already outdated.
183 */
184 public void wakeup() {
185 synchronized (sleepLock) {
186 sleepLock.notify();
187 }
188 }
189
190 /**
191 * Fetches the sleep time from the global configuration.
192 * @return the time to sleep between database cleanups in ms.
193 */
194 protected static long getSleepTime() {
195 String sleepTimeString = Config.getStringXPath(SLEEP_XPATH);
196
197 if (sleepTimeString == null) {
198 return SLEEP_DEFAULT;
199 }
200 try {
201 // sleep at least one second
202 return Math.max(Long.parseLong(sleepTimeString), 1000L);
203 }
204 catch (NumberFormatException nfe) {
205 logger.warn("Cleaner sleep time defaults to " + SLEEP_DEFAULT);
206 }
207 return SLEEP_DEFAULT;
208 }
209
210 private static class IdIdentifier {
211
212 int id;
213 String identifier;
214
215 private IdIdentifier(int id, String identifier) {
216 this.id = id;
217 this.identifier = identifier;
218 }
219 } // class IdIdentifier
220
221 private static final class IdData
222 extends IdIdentifier
223 {
224 byte [] data;
225 String factoryName;
226
227 public IdData(
228 int id,
229 String factoryName,
230 byte [] data,
231 String identifier
232 ) {
233 super(id, identifier);
234 this.factoryName = factoryName;
235 this.data = data;
236 }
237 } // class IdData
238
239 /**
240 * Cleaning is done in two phases. First we fetch a list of ids
241 * of artifacts. If there are artifacts the cleaning is done.
242 * Second we load the artifacts one by one one and call there
243 * endOfLife() method. In this loop we remove them from database, too.
244 * Each deletion is commited to ensure that a sudden failure
245 * of the artifact database server does delete artifacts twice
246 * or does not delete them at all. After this the first step
247 * is repeated.
248 */
249 protected void cleanup() {
250 logger.info("database cleanup");
251
252 Connection connection = null;
253 PreparedStatement fetchIds = null;
254 PreparedStatement stmnt = null;
255 ResultSet result = null;
256
257 DataSource dataSource = dbConnection.getDataSource();
258
259 Set<Integer> lockedIds = lockedIdsProvider != null
260 ? lockedIdsProvider.getLockedIds()
261 : EMPTY_IDS;
262
263 String questionMarks = lockedIds.isEmpty()
264 ? "-666" // XXX: A bit hackish.
265 : StringUtils.repeat('?', lockedIds.size(), ',');
266
267 List<String> deletedCollections = new ArrayList<String>();
268 List<String> deletedArtifacts = new ArrayList<String>();
269
270 try {
271 connection = dataSource.getConnection();
272 connection.setAutoCommit(false);
273
274 fetchIds = connection.prepareStatement(
275 SQL_OUTDATED.replace("$LOCKED_IDS$", questionMarks));
276
277 // some dbms like derby do not support LIMIT
278 // in SQL statements.
279 fetchIds.setMaxRows(MAX_ROWS);
280
281 // Fetch ids of outdated collections
282 stmnt = connection.prepareStatement(
283 SQL_OUTDATED_COLLECTIONS.replace(
284 "$LOCKED_IDS$", questionMarks));
285
286 // fill in the locked ids
287 int idx = 1;
288 for (Integer id: lockedIds) {
289 fetchIds.setInt(idx, id);
290 stmnt .setInt(idx, id);
291 ++idx;
292 }
293
294 ArrayList<IdIdentifier> cs = new ArrayList<IdIdentifier>();
295 result = stmnt.executeQuery();
296 while (result.next()) {
297 cs.add(new IdIdentifier(
298 result.getInt(1),
299 result.getString(2)));
300 }
301
302 result.close(); result = null;
303 stmnt.close(); stmnt = null;
304
305 // delete collection items
306 stmnt = connection.prepareStatement(SQL_DELETE_COLLECTION_ITEMS);
307
308 for (IdIdentifier id: cs) {
309 logger.debug("Mark collection for deletion: " + id.id);
310 stmnt.setInt(1, id.id);
311 stmnt.execute();
312 }
313
314 stmnt.close(); stmnt = null;
315
316 // delete collections
317 stmnt = connection.prepareStatement(SQL_DELETE_COLLECTION);
318
319 for (IdIdentifier id: cs) {
320 stmnt.setInt(1, id.id);
321 stmnt.execute();
322 deletedCollections.add(id.identifier);
323 }
324
325 stmnt.close(); stmnt = null;
326 connection.commit();
327
328 cs = null;
329
330 // remove artifacts
331 stmnt = connection.prepareStatement(SQL_DELETE_ARTIFACT);
332
333 for (;;) {
334 List<IdData> ids = new ArrayList<IdData>();
335
336 result = fetchIds.executeQuery();
337
338 while (result.next()) {
339 ids.add(new IdData(
340 result.getInt(1),
341 result.getString(2),
342 result.getBytes(3),
343 result.getString(4)));
344 }
345
346 result.close(); result = null;
347
348 if (ids.isEmpty()) {
349 break;
350 }
351
352 for (int i = ids.size()-1; i >= 0; --i) {
353 IdData idData = ids.get(i);
354 Artifact artifact = reviver.reviveArtifact(
355 idData.factoryName, idData.data);
356 idData.data = null;
357
358 logger.debug("Prepare Artifact (id="
359 + idData.id + ") for deletion.");
360
361 stmnt.setInt(1, idData.id);
362 stmnt.execute();
363 connection.commit();
364
365 try {
366 if (artifact != null) {
367 logger.debug("Call endOfLife for Artifact: "
368 + artifact.identifier());
369
370 artifact.endOfLife(context);
371 }
372 }
373 catch (Exception e) {
374 logger.error(e.getMessage(), e);
375 }
376
377 deletedArtifacts.add(idData.identifier);
378 } // for all fetched data
379 }
380 }
381 catch (SQLException sqle) {
382 logger.error(sqle.getLocalizedMessage(), sqle);
383 }
384 finally {
385 if (result != null) {
386 try { result.close(); }
387 catch (SQLException sqle) {}
388 }
389 if (stmnt != null) {
390 try { stmnt.close(); }
391 catch (SQLException sqle) {}
392 }
393 if (fetchIds != null) {
394 try { fetchIds.close(); }
395 catch (SQLException sqle) {}
396 }
397 if (connection != null) {
398 try { connection.close(); }
399 catch (SQLException sqle) {}
400 }
401 }
402
403 if (!deletedCollections.isEmpty()) {
404 reviver.killedCollections(deletedCollections);
405 }
406
407 if (!deletedArtifacts.isEmpty()) {
408 reviver.killedArtifacts(deletedArtifacts);
409 }
410
411 if (logger.isDebugEnabled()) {
412 logger.debug(
413 "collections removed: " + deletedCollections.size());
414 logger.debug(
415 "artifacts removed: " + deletedArtifacts.size());
416 }
417 }
418
419 /**
420 * The main code of the cleaner. It sleeps for the configured
421 * nap time, cleans up the database, sleeps again and so on.
422 */
423 @Override
424 public void run() {
425 logger.info("sleep time: " + sleepTime + "ms");
426 for (;;) {
427 cleanup();
428 long startTime = System.currentTimeMillis();
429
430 try {
431 synchronized (sleepLock) {
432 sleepLock.wait(sleepTime);
433 }
434 }
435 catch (InterruptedException ie) {
436 }
437
438 long stopTime = System.currentTimeMillis();
439
440 if (logger.isDebugEnabled()) {
441 logger.debug("Cleaner slept " + (stopTime - startTime) + "ms");
442 }
443 } // for (;;)
444 }
445 }
446 // vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 :
http://dive4elements.wald.intevation.org