ingo@100: /* ingo@100: * Copyright (c) 2010 by Intevation GmbH ingo@100: * ingo@100: * This program is free software under the LGPL (>=v2.1) ingo@100: * Read the file LGPL.txt coming with the software for details ingo@100: * or visit http://www.gnu.org/licenses/ if it does not exist. ingo@100: */ ingo@100: sascha@30: package de.intevation.artifactdatabase; sascha@30: sascha@207: import de.intevation.artifacts.common.utils.Config; sascha@301: import de.intevation.artifacts.common.utils.StringUtils; sascha@207: sascha@30: import de.intevation.artifacts.Artifact; sascha@30: sascha@305: import de.intevation.artifactdatabase.db.SQL; sascha@305: import de.intevation.artifactdatabase.db.DBConnection; sascha@305: sascha@30: import java.sql.Connection; sascha@30: import java.sql.PreparedStatement; sascha@30: import java.sql.ResultSet; sascha@93: import java.sql.SQLException; sascha@93: sascha@93: import java.util.ArrayList; sascha@93: import java.util.List; sascha@230: import java.util.Set; sascha@230: import java.util.Collections; sascha@30: sascha@30: import javax.sql.DataSource; sascha@30: sascha@30: import org.apache.log4j.Logger; sascha@30: sascha@30: /** sascha@90: * The database cleaner runs in background. It sleep for a configurable sascha@90: * while and when it wakes up it removes outdated artifacts from the sascha@90: * database. Outdated means that the the last access to the artifact sascha@90: * is longer aga then the time to live of this artifact.
sascha@90: * Before the artifact is finally removed from the system it is sascha@90: * revived one last time an the #endOfLife() method of the artifact sascha@90: * is called.
sascha@90: * The artifact implementations may e.g. use this to remove some extrenal sascha@90: * resources form the system. sascha@90: * ingo@80: * @author Sascha L. Teichmann sascha@30: */ sascha@30: public class DatabaseCleaner sascha@30: extends Thread sascha@30: { sascha@90: /** sascha@90: * Implementors of this interface are able to create a sascha@90: * living artifact from a given byte array. sascha@90: */ sascha@41: public interface ArtifactReviver { sascha@41: sascha@90: /** sascha@90: * Called to revive an artifact from a given byte array. sascha@90: * @param factoryName The name of the factory which sascha@90: * created this artifact. sascha@90: * @param bytes The bytes of the serialized artifact. sascha@90: * @return The revived artfiact. sascha@90: */ sascha@41: Artifact reviveArtifact(String factoryName, byte [] bytes); sascha@41: sascha@314: void killedArtifacts(List identifiers); sascha@314: void killedCollections(List identifiers); sascha@314: sascha@41: } // interface ArtifactReviver sascha@41: sascha@230: public interface LockedIdsProvider { sascha@230: Set getLockedIds(); sascha@230: } // interface LockedIdsProvider sascha@230: sascha@30: private static Logger logger = Logger.getLogger(DatabaseCleaner.class); sascha@30: sascha@90: /** sascha@90: * Number of artifacts to be loaded at once. Used to sascha@90: * mitigate the problem of a massive denial of service sascha@90: * if too many artifacts have died since last cleanup. sascha@90: */ sascha@30: public static final int MAX_ROWS = 50; sascha@30: sascha@230: public static final Set EMPTY_IDS = Collections.emptySet(); sascha@230: sascha@90: /** sascha@90: * The SQL statement to select the outdated artifacts. sascha@90: */ sascha@305: public String SQL_OUTDATED; sascha@30: sascha@305: public String SQL_OUTDATED_COLLECTIONS; sascha@305: public String SQL_DELETE_COLLECTION_ITEMS; sascha@305: public String SQL_DELETE_COLLECTION; sascha@232: sascha@90: /** sascha@90: * The SQL statement to delete some artifacts from the database. sascha@90: */ sascha@305: public String SQL_DELETE_ARTIFACT; sascha@30: sascha@90: /** sascha@90: * XPath to figure out how long the cleaner should sleep between sascha@90: * cleanups. This is stored in the global configuration. sascha@90: */ sascha@30: public static final String SLEEP_XPATH = sascha@30: "/artifact-database/cleaner/sleep-time/text()"; sascha@30: sascha@90: /** sascha@90: * Default nap time between cleanups: 5 minutes. sascha@90: */ sascha@30: public static final long SLEEP_DEFAULT = sascha@30: 5 * 60 * 1000L; // 5 minutes sascha@30: sascha@90: /** sascha@90: * The configured nap time. sascha@90: */ sascha@30: protected long sleepTime; sascha@30: sascha@90: /** sascha@90: * Internal locking mechanism to prevent some race conditions. sascha@90: */ sascha@30: protected Object sleepLock = new Object(); sascha@30: sascha@90: /** sascha@90: * A reference to the global context. sascha@90: */ sascha@30: protected Object context; sascha@30: sascha@90: /** sascha@90: * A specialized Id filter which only delete some artifacts. sascha@90: * This is used to prevent deletion of living artifacts. sascha@90: */ sascha@230: protected LockedIdsProvider lockedIdsProvider; sascha@32: sascha@90: /** sascha@90: * The reviver used to bring the dead artifact on last sascha@90: * time back to live to call endOfLife() on them. sascha@90: */ sascha@41: protected ArtifactReviver reviver; sascha@41: sascha@305: protected DBConnection dbConnection; sascha@305: sascha@90: /** sascha@90: * Default constructor. sascha@90: */ sascha@30: public DatabaseCleaner() { sascha@30: } sascha@30: sascha@90: /** sascha@90: * Constructor to create a cleaner with a given global context sascha@91: * and a given reviver. sascha@90: * @param context The global context of the artifact database sascha@90: * @param reviver The reviver to awake artifact one last time. sascha@90: */ sascha@305: public DatabaseCleaner(Object context, ArtifactReviver reviver, DBConfig config) { sascha@30: setDaemon(true); sascha@30: sleepTime = getSleepTime(); sascha@30: this.context = context; sascha@41: this.reviver = reviver; sascha@305: this.dbConnection = config.getDBConnection(); sascha@305: setupSQL(config.getSQL()); sascha@305: } sascha@305: sascha@305: protected void setupSQL(SQL sql) { sascha@305: SQL_OUTDATED = sql.get("artifacts.outdated"); sascha@305: SQL_OUTDATED_COLLECTIONS = sql.get("collections.outdated"); sascha@305: SQL_DELETE_COLLECTION_ITEMS = sql.get("delete.collection.items"); sascha@305: SQL_DELETE_COLLECTION = sql.get("delete.collection"); sascha@305: SQL_DELETE_ARTIFACT = sql.get("artifacts.delete"); sascha@30: } sascha@30: sascha@90: /** sascha@90: * Sets the filter that prevents deletion of living artifacts. sascha@90: * Living artifacts are artifacts which are currently active sascha@90: * inside the artifact database. Deleting them in this state sascha@90: * would create severe internal problems. sascha@90: * @param filter sascha@90: */ sascha@230: public void setLockedIdsProvider(LockedIdsProvider lockedIdsProvider) { sascha@230: this.lockedIdsProvider = lockedIdsProvider; sascha@32: } sascha@32: sascha@90: /** sascha@90: * External hook to tell the cleaner to wake up before its sascha@90: * regular nap time is over. This is the case when the artifact sascha@90: * database finds an artifact which is already outdated. sascha@90: */ sascha@30: public void wakeup() { sascha@30: synchronized (sleepLock) { sascha@30: sleepLock.notify(); sascha@30: } sascha@30: } sascha@30: sascha@90: /** sascha@90: * Fetches the sleep time from the global configuration. sascha@90: * @return the time to sleep between database cleanups in ms. sascha@90: */ sascha@30: protected static long getSleepTime() { sascha@30: String sleepTimeString = Config.getStringXPath(SLEEP_XPATH); sascha@30: sascha@30: if (sleepTimeString == null) { sascha@30: return SLEEP_DEFAULT; sascha@30: } sascha@30: try { sascha@30: // sleep at least one second sascha@30: return Math.max(Long.parseLong(sleepTimeString), 1000L); sascha@30: } sascha@30: catch (NumberFormatException nfe) { sascha@30: logger.warn("Cleaner sleep time defaults to " + SLEEP_DEFAULT); sascha@30: } sascha@30: return SLEEP_DEFAULT; sascha@30: } sascha@30: sascha@314: private static class IdIdentifier { sascha@230: sascha@230: int id; sascha@314: String identifier; sascha@314: sascha@314: private IdIdentifier(int id, String identifier) { sascha@314: this.id = id; sascha@314: this.identifier = identifier; sascha@314: } sascha@314: } // class IdIdentifier sascha@314: sascha@314: private static final class IdData sascha@314: extends IdIdentifier sascha@314: { sascha@30: byte [] data; sascha@41: String factoryName; sascha@30: sascha@314: public IdData( sascha@314: int id, sascha@314: String factoryName, sascha@314: byte [] data, sascha@314: String identifier sascha@314: ) { sascha@314: super(id, identifier); sascha@41: this.factoryName = factoryName; sascha@41: this.data = data; sascha@30: } sascha@30: } // class IdData sascha@30: sascha@30: /** sascha@30: * Cleaning is done in two phases. First we fetch a list of ids sascha@30: * of artifacts. If there are artifacts the cleaning is done. sascha@30: * Second we load the artifacts one by one one and call there sascha@30: * endOfLife() method. In this loop we remove them from database, too. sascha@30: * Each deletion is commited to ensure that a sudden failure sascha@30: * of the artifact database server does delete artifacts twice sascha@30: * or does not delete them at all. After this the first step sascha@30: * is repeated. sascha@30: */ sascha@30: protected void cleanup() { sascha@30: logger.info("database cleanup"); sascha@30: sascha@232: Connection connection = null; sascha@232: PreparedStatement fetchIds = null; sascha@232: PreparedStatement stmnt = null; sascha@232: ResultSet result = null; sascha@30: sascha@305: DataSource dataSource = dbConnection.getDataSource(); sascha@231: sascha@231: Set lockedIds = lockedIdsProvider != null sascha@231: ? lockedIdsProvider.getLockedIds() sascha@231: : EMPTY_IDS; sascha@231: sascha@238: String questionMarks = lockedIds.isEmpty() sascha@241: ? "-666" // XXX: A bit hackish. sascha@238: : StringUtils.repeat('?', lockedIds.size(), ','); sascha@231: sascha@314: List deletedCollections = new ArrayList(); sascha@314: List deletedArtifacts = new ArrayList(); sascha@314: sascha@30: try { sascha@30: connection = dataSource.getConnection(); sascha@30: connection.setAutoCommit(false); sascha@231: sascha@231: fetchIds = connection.prepareStatement( sascha@231: SQL_OUTDATED.replace("$LOCKED_IDS$", questionMarks)); sascha@231: sascha@30: // some dbms like derby do not support LIMIT sascha@30: // in SQL statements. sascha@30: fetchIds.setMaxRows(MAX_ROWS); sascha@30: sascha@232: // Fetch ids of outdated collections sascha@232: stmnt = connection.prepareStatement( sascha@232: SQL_OUTDATED_COLLECTIONS.replace( sascha@232: "$LOCKED_IDS$", questionMarks)); sascha@232: sascha@232: // fill in the locked ids sascha@232: int idx = 1; sascha@232: for (Integer id: lockedIds) { sascha@232: fetchIds.setInt(idx, id); sascha@232: stmnt .setInt(idx, id); sascha@232: ++idx; sascha@232: } sascha@232: sascha@314: ArrayList cs = new ArrayList(); sascha@232: result = stmnt.executeQuery(); sascha@232: while (result.next()) { sascha@314: cs.add(new IdIdentifier( sascha@314: result.getInt(1), sascha@314: result.getString(2))); sascha@232: } sascha@232: sascha@232: result.close(); result = null; sascha@232: stmnt.close(); stmnt = null; sascha@232: sascha@232: // delete collection items sascha@232: stmnt = connection.prepareStatement(SQL_DELETE_COLLECTION_ITEMS); sascha@232: sascha@314: for (IdIdentifier id: cs) { sascha@314: stmnt.setInt(1, id.id); sascha@232: stmnt.execute(); sascha@232: } sascha@232: sascha@232: stmnt.close(); stmnt = null; sascha@232: sascha@232: // delete collections sascha@232: stmnt = connection.prepareStatement(SQL_DELETE_COLLECTION); sascha@232: sascha@314: for (IdIdentifier id: cs) { sascha@314: stmnt.setInt(1, id.id); sascha@232: stmnt.execute(); sascha@314: deletedCollections.add(id.identifier); sascha@232: } sascha@232: sascha@232: stmnt.close(); stmnt = null; sascha@232: connection.commit(); sascha@232: sascha@314: cs = null; sascha@232: sascha@232: // remove artifacts sascha@232: stmnt = connection.prepareStatement(SQL_DELETE_ARTIFACT); sascha@232: sascha@30: for (;;) { sascha@230: List ids = new ArrayList(); sascha@230: sascha@30: result = fetchIds.executeQuery(); sascha@30: sascha@30: while (result.next()) { sascha@231: ids.add(new IdData( sascha@231: result.getInt(1), sascha@231: result.getString(2), sascha@314: result.getBytes(3), sascha@314: result.getString(4))); sascha@30: } sascha@30: sascha@30: result.close(); result = null; sascha@30: sascha@30: if (ids.isEmpty()) { sascha@30: break; sascha@30: } sascha@30: sascha@30: for (int i = ids.size()-1; i >= 0; --i) { sascha@230: IdData idData = ids.get(i); sascha@41: Artifact artifact = reviver.reviveArtifact( sascha@41: idData.factoryName, idData.data); sascha@30: idData.data = null; sascha@30: sascha@232: stmnt.setInt(1, idData.id); sascha@232: stmnt.execute(); sascha@30: connection.commit(); sascha@30: sascha@30: try { sascha@30: if (artifact != null) { sascha@30: artifact.endOfLife(context); sascha@30: } sascha@30: } sascha@30: catch (Exception e) { sascha@30: logger.error(e.getLocalizedMessage(), e); sascha@30: } sascha@314: sascha@314: deletedArtifacts.add(idData.identifier); sascha@30: } // for all fetched data sascha@30: } sascha@30: } sascha@30: catch (SQLException sqle) { sascha@30: logger.error(sqle.getLocalizedMessage(), sqle); sascha@30: } sascha@30: finally { sascha@30: if (result != null) { sascha@30: try { result.close(); } sascha@30: catch (SQLException sqle) {} sascha@30: } sascha@232: if (stmnt != null) { sascha@232: try { stmnt.close(); } sascha@232: catch (SQLException sqle) {} sascha@232: } sascha@30: if (fetchIds != null) { sascha@30: try { fetchIds.close(); } sascha@30: catch (SQLException sqle) {} sascha@30: } sascha@30: if (connection != null) { sascha@30: try { connection.close(); } sascha@30: catch (SQLException sqle) {} sascha@30: } sascha@30: } sascha@30: sascha@315: if (!deletedCollections.isEmpty()) { sascha@315: reviver.killedCollections(deletedCollections); sascha@315: } sascha@315: sascha@315: if (!deletedArtifacts.isEmpty()) { sascha@315: reviver.killedArtifacts(deletedArtifacts); sascha@315: } sascha@314: sascha@314: if (logger.isDebugEnabled()) { sascha@314: logger.debug( sascha@314: "collections removed: " + deletedCollections.size()); sascha@314: logger.debug( sascha@314: "artifacts removed: " + deletedArtifacts.size()); sascha@314: } sascha@30: } sascha@30: sascha@90: /** sascha@90: * The main code of the cleaner. It sleeps for the configured sascha@90: * nap time, cleans up the database, sleeps again and so on. sascha@90: */ sascha@90: @Override sascha@30: public void run() { sascha@30: logger.info("sleep time: " + sleepTime + "ms"); sascha@30: for (;;) { sascha@30: cleanup(); sascha@48: long startTime = System.currentTimeMillis(); sascha@48: sascha@30: try { sascha@30: synchronized (sleepLock) { sascha@30: sleepLock.wait(sleepTime); sascha@30: } sascha@30: } sascha@30: catch (InterruptedException ie) { sascha@30: } sascha@48: sascha@48: long stopTime = System.currentTimeMillis(); sascha@48: sascha@48: if (logger.isDebugEnabled()) { sascha@48: logger.debug("Cleaner slept " + (stopTime - startTime) + "ms"); sascha@48: } sascha@30: } // for (;;) sascha@30: } sascha@30: } sascha@90: // vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 :