ingo@100: /*
ingo@100: * Copyright (c) 2010 by Intevation GmbH
ingo@100: *
ingo@100: * This program is free software under the LGPL (>=v2.1)
ingo@100: * Read the file LGPL.txt coming with the software for details
ingo@100: * or visit http://www.gnu.org/licenses/ if it does not exist.
ingo@100: */
ingo@100:
sascha@30: package de.intevation.artifactdatabase;
sascha@30:
sascha@207: import de.intevation.artifacts.common.utils.Config;
sascha@207:
sascha@30: import de.intevation.artifacts.Artifact;
sascha@30:
sascha@30: import java.sql.Connection;
sascha@30: import java.sql.PreparedStatement;
sascha@30: import java.sql.ResultSet;
sascha@93: import java.sql.SQLException;
sascha@93:
sascha@93: import java.util.ArrayList;
sascha@93: import java.util.List;
sascha@230: import java.util.Set;
sascha@230: import java.util.Collections;
sascha@30:
sascha@30: import javax.sql.DataSource;
sascha@30:
sascha@30: import org.apache.log4j.Logger;
sascha@30:
sascha@30: /**
sascha@90: * The database cleaner runs in background. It sleep for a configurable
sascha@90: * while and when it wakes up it removes outdated artifacts from the
sascha@90: * database. Outdated means that the the last access to the artifact
sascha@90: * is longer aga then the time to live of this artifact.
sascha@90: * Before the artifact is finally removed from the system it is
sascha@90: * revived one last time an the #endOfLife() method of the artifact
sascha@90: * is called.
sascha@90: * The artifact implementations may e.g. use this to remove some extrenal
sascha@90: * resources form the system.
sascha@90: *
ingo@80: * @author Sascha L. Teichmann
sascha@30: */
sascha@30: public class DatabaseCleaner
sascha@30: extends Thread
sascha@30: {
sascha@90: /**
sascha@90: * Implementors of this interface are able to create a
sascha@90: * living artifact from a given byte array.
sascha@90: */
sascha@41: public interface ArtifactReviver {
sascha@41:
sascha@90: /**
sascha@90: * Called to revive an artifact from a given byte array.
sascha@90: * @param factoryName The name of the factory which
sascha@90: * created this artifact.
sascha@90: * @param bytes The bytes of the serialized artifact.
sascha@90: * @return The revived artfiact.
sascha@90: */
sascha@41: Artifact reviveArtifact(String factoryName, byte [] bytes);
sascha@41:
sascha@41: } // interface ArtifactReviver
sascha@41:
sascha@230: public interface LockedIdsProvider {
sascha@230: Set getLockedIds();
sascha@230: } // interface LockedIdsProvider
sascha@230:
sascha@30: private static Logger logger = Logger.getLogger(DatabaseCleaner.class);
sascha@30:
sascha@90: /**
sascha@90: * Number of artifacts to be loaded at once. Used to
sascha@90: * mitigate the problem of a massive denial of service
sascha@90: * if too many artifacts have died since last cleanup.
sascha@90: */
sascha@30: public static final int MAX_ROWS = 50;
sascha@30:
sascha@230: public static final Set EMPTY_IDS = Collections.emptySet();
sascha@230:
sascha@90: /**
sascha@90: * The SQL statement to select the outdated artifacts.
sascha@90: */
sascha@30: public static final String SQL_OUTDATED =
sascha@30: SQL.get("artifacts.outdated");
sascha@30:
sascha@232: public static final String SQL_OUTDATED_COLLECTIONS =
sascha@232: SQL.get("collections.outdated");
sascha@232:
sascha@232: public static final String SQL_DELETE_COLLECTION_ITEMS =
sascha@232: SQL.get("delete.collection.items");
sascha@232:
sascha@232: public static final String SQL_DELETE_COLLECTION =
sascha@232: SQL.get("delete.collection");
sascha@232:
sascha@90: /**
sascha@90: * The SQL statement to delete some artifacts from the database.
sascha@90: */
sascha@232: public static final String SQL_DELETE_ARTIFACT =
sascha@30: SQL.get("artifacts.delete");
sascha@30:
sascha@90: /**
sascha@90: * XPath to figure out how long the cleaner should sleep between
sascha@90: * cleanups. This is stored in the global configuration.
sascha@90: */
sascha@30: public static final String SLEEP_XPATH =
sascha@30: "/artifact-database/cleaner/sleep-time/text()";
sascha@30:
sascha@90: /**
sascha@90: * Default nap time between cleanups: 5 minutes.
sascha@90: */
sascha@30: public static final long SLEEP_DEFAULT =
sascha@30: 5 * 60 * 1000L; // 5 minutes
sascha@30:
sascha@90: /**
sascha@90: * The configured nap time.
sascha@90: */
sascha@30: protected long sleepTime;
sascha@30:
sascha@90: /**
sascha@90: * Internal locking mechanism to prevent some race conditions.
sascha@90: */
sascha@30: protected Object sleepLock = new Object();
sascha@30:
sascha@90: /**
sascha@90: * A reference to the global context.
sascha@90: */
sascha@30: protected Object context;
sascha@30:
sascha@90: /**
sascha@90: * A specialized Id filter which only delete some artifacts.
sascha@90: * This is used to prevent deletion of living artifacts.
sascha@90: */
sascha@230: protected LockedIdsProvider lockedIdsProvider;
sascha@32:
sascha@90: /**
sascha@90: * The reviver used to bring the dead artifact on last
sascha@90: * time back to live to call endOfLife() on them.
sascha@90: */
sascha@41: protected ArtifactReviver reviver;
sascha@41:
sascha@90: /**
sascha@90: * Default constructor.
sascha@90: */
sascha@30: public DatabaseCleaner() {
sascha@30: }
sascha@30:
sascha@90: /**
sascha@90: * Constructor to create a cleaner with a given global context
sascha@91: * and a given reviver.
sascha@90: * @param context The global context of the artifact database
sascha@90: * @param reviver The reviver to awake artifact one last time.
sascha@90: */
sascha@41: public DatabaseCleaner(Object context, ArtifactReviver reviver) {
sascha@30: setDaemon(true);
sascha@30: sleepTime = getSleepTime();
sascha@30: this.context = context;
sascha@41: this.reviver = reviver;
sascha@30: }
sascha@30:
sascha@90: /**
sascha@90: * Sets the filter that prevents deletion of living artifacts.
sascha@90: * Living artifacts are artifacts which are currently active
sascha@90: * inside the artifact database. Deleting them in this state
sascha@90: * would create severe internal problems.
sascha@90: * @param filter
sascha@90: */
sascha@230: public void setLockedIdsProvider(LockedIdsProvider lockedIdsProvider) {
sascha@230: this.lockedIdsProvider = lockedIdsProvider;
sascha@32: }
sascha@32:
sascha@90: /**
sascha@90: * External hook to tell the cleaner to wake up before its
sascha@90: * regular nap time is over. This is the case when the artifact
sascha@90: * database finds an artifact which is already outdated.
sascha@90: */
sascha@30: public void wakeup() {
sascha@30: synchronized (sleepLock) {
sascha@30: sleepLock.notify();
sascha@30: }
sascha@30: }
sascha@30:
sascha@90: /**
sascha@90: * Fetches the sleep time from the global configuration.
sascha@90: * @return the time to sleep between database cleanups in ms.
sascha@90: */
sascha@30: protected static long getSleepTime() {
sascha@30: String sleepTimeString = Config.getStringXPath(SLEEP_XPATH);
sascha@30:
sascha@30: if (sleepTimeString == null) {
sascha@30: return SLEEP_DEFAULT;
sascha@30: }
sascha@30: try {
sascha@30: // sleep at least one second
sascha@30: return Math.max(Long.parseLong(sleepTimeString), 1000L);
sascha@30: }
sascha@30: catch (NumberFormatException nfe) {
sascha@30: logger.warn("Cleaner sleep time defaults to " + SLEEP_DEFAULT);
sascha@30: }
sascha@30: return SLEEP_DEFAULT;
sascha@30: }
sascha@30:
sascha@230: private static final class IdData {
sascha@230:
sascha@230: int id;
sascha@30: byte [] data;
sascha@41: String factoryName;
sascha@30:
sascha@41: public IdData(int id, String factoryName, byte [] data) {
sascha@230: this.id = id;
sascha@41: this.factoryName = factoryName;
sascha@41: this.data = data;
sascha@30: }
sascha@30: } // class IdData
sascha@30:
sascha@30: /**
sascha@30: * Cleaning is done in two phases. First we fetch a list of ids
sascha@30: * of artifacts. If there are artifacts the cleaning is done.
sascha@30: * Second we load the artifacts one by one one and call there
sascha@30: * endOfLife() method. In this loop we remove them from database, too.
sascha@30: * Each deletion is commited to ensure that a sudden failure
sascha@30: * of the artifact database server does delete artifacts twice
sascha@30: * or does not delete them at all. After this the first step
sascha@30: * is repeated.
sascha@30: */
sascha@30: protected void cleanup() {
sascha@30: logger.info("database cleanup");
sascha@30:
sascha@232: Connection connection = null;
sascha@232: PreparedStatement fetchIds = null;
sascha@232: PreparedStatement stmnt = null;
sascha@232: ResultSet result = null;
sascha@30:
sascha@232: int removedCollections = 0;
sascha@232: int removedArtifacts = 0;
sascha@30:
sascha@30: DataSource dataSource = DBConnection.getDataSource();
sascha@231:
sascha@231: Set lockedIds = lockedIdsProvider != null
sascha@231: ? lockedIdsProvider.getLockedIds()
sascha@231: : EMPTY_IDS;
sascha@231:
sascha@238: String questionMarks = lockedIds.isEmpty()
sascha@241: ? "-666" // XXX: A bit hackish.
sascha@238: : StringUtils.repeat('?', lockedIds.size(), ',');
sascha@231:
sascha@30: try {
sascha@30: connection = dataSource.getConnection();
sascha@30: connection.setAutoCommit(false);
sascha@231:
sascha@231: fetchIds = connection.prepareStatement(
sascha@231: SQL_OUTDATED.replace("$LOCKED_IDS$", questionMarks));
sascha@231:
sascha@30: // some dbms like derby do not support LIMIT
sascha@30: // in SQL statements.
sascha@30: fetchIds.setMaxRows(MAX_ROWS);
sascha@30:
sascha@232: // Fetch ids of outdated collections
sascha@232: stmnt = connection.prepareStatement(
sascha@232: SQL_OUTDATED_COLLECTIONS.replace(
sascha@232: "$LOCKED_IDS$", questionMarks));
sascha@232:
sascha@232: // fill in the locked ids
sascha@232: int idx = 1;
sascha@232: for (Integer id: lockedIds) {
sascha@232: fetchIds.setInt(idx, id);
sascha@232: stmnt .setInt(idx, id);
sascha@232: ++idx;
sascha@232: }
sascha@232:
sascha@232: ArrayList cs = new ArrayList();
sascha@232: result = stmnt.executeQuery();
sascha@232: while (result.next()) {
sascha@232: cs.add(result.getInt(1));
sascha@232: }
sascha@232:
sascha@232: result.close(); result = null;
sascha@232: stmnt.close(); stmnt = null;
sascha@232:
sascha@232: // delete collection items
sascha@232: stmnt = connection.prepareStatement(SQL_DELETE_COLLECTION_ITEMS);
sascha@232:
sascha@232: for (Integer id: cs) {
sascha@232: stmnt.setInt(1, id);
sascha@232: stmnt.execute();
sascha@232: }
sascha@232:
sascha@232: stmnt.close(); stmnt = null;
sascha@232:
sascha@232: // delete collections
sascha@232: stmnt = connection.prepareStatement(SQL_DELETE_COLLECTION);
sascha@232:
sascha@232: for (Integer id: cs) {
sascha@232: stmnt.setInt(1, id);
sascha@232: stmnt.execute();
sascha@232: }
sascha@232:
sascha@232: stmnt.close(); stmnt = null;
sascha@232: connection.commit();
sascha@232:
sascha@232: removedCollections = cs.size(); cs = null;
sascha@232:
sascha@232: // remove artifacts
sascha@232: stmnt = connection.prepareStatement(SQL_DELETE_ARTIFACT);
sascha@232:
sascha@30: for (;;) {
sascha@230: List ids = new ArrayList();
sascha@230:
sascha@30: result = fetchIds.executeQuery();
sascha@30:
sascha@30: while (result.next()) {
sascha@231: ids.add(new IdData(
sascha@231: result.getInt(1),
sascha@231: result.getString(2),
sascha@231: result.getBytes(3)));
sascha@30: }
sascha@30:
sascha@30: result.close(); result = null;
sascha@30:
sascha@30: if (ids.isEmpty()) {
sascha@30: break;
sascha@30: }
sascha@30:
sascha@30: for (int i = ids.size()-1; i >= 0; --i) {
sascha@230: IdData idData = ids.get(i);
sascha@41: Artifact artifact = reviver.reviveArtifact(
sascha@41: idData.factoryName, idData.data);
sascha@30: idData.data = null;
sascha@30:
sascha@232: stmnt.setInt(1, idData.id);
sascha@232: stmnt.execute();
sascha@30: connection.commit();
sascha@30:
sascha@30: try {
sascha@30: if (artifact != null) {
sascha@30: artifact.endOfLife(context);
sascha@30: }
sascha@30: }
sascha@30: catch (Exception e) {
sascha@30: logger.error(e.getLocalizedMessage(), e);
sascha@30: }
sascha@30: } // for all fetched data
sascha@30:
sascha@30: removedArtifacts += ids.size();
sascha@30: }
sascha@30: }
sascha@30: catch (SQLException sqle) {
sascha@30: logger.error(sqle.getLocalizedMessage(), sqle);
sascha@30: }
sascha@30: finally {
sascha@30: if (result != null) {
sascha@30: try { result.close(); }
sascha@30: catch (SQLException sqle) {}
sascha@30: }
sascha@232: if (stmnt != null) {
sascha@232: try { stmnt.close(); }
sascha@232: catch (SQLException sqle) {}
sascha@232: }
sascha@30: if (fetchIds != null) {
sascha@30: try { fetchIds.close(); }
sascha@30: catch (SQLException sqle) {}
sascha@30: }
sascha@30: if (connection != null) {
sascha@30: try { connection.close(); }
sascha@30: catch (SQLException sqle) {}
sascha@30: }
sascha@30: }
sascha@30:
sascha@232: logger.info("collections removed: " + removedCollections);
sascha@232: logger.info("artifacts removed: " + removedArtifacts);
sascha@30: }
sascha@30:
sascha@90: /**
sascha@90: * The main code of the cleaner. It sleeps for the configured
sascha@90: * nap time, cleans up the database, sleeps again and so on.
sascha@90: */
sascha@90: @Override
sascha@30: public void run() {
sascha@30: logger.info("sleep time: " + sleepTime + "ms");
sascha@30: for (;;) {
sascha@30: cleanup();
sascha@48: long startTime = System.currentTimeMillis();
sascha@48:
sascha@30: try {
sascha@30: synchronized (sleepLock) {
sascha@30: sleepLock.wait(sleepTime);
sascha@30: }
sascha@30: }
sascha@30: catch (InterruptedException ie) {
sascha@30: }
sascha@48:
sascha@48: long stopTime = System.currentTimeMillis();
sascha@48:
sascha@48: if (logger.isDebugEnabled()) {
sascha@48: logger.debug("Cleaner slept " + (stopTime - startTime) + "ms");
sascha@48: }
sascha@30: } // for (;;)
sascha@30: }
sascha@30: }
sascha@90: // vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 :