ingo@100: /*
ingo@100: * Copyright (c) 2010 by Intevation GmbH
ingo@100: *
ingo@100: * This program is free software under the LGPL (>=v2.1)
ingo@100: * Read the file LGPL.txt coming with the software for details
ingo@100: * or visit http://www.gnu.org/licenses/ if it does not exist.
ingo@100: */
ingo@100:
teichmann@475: package org.dive4elements.artifactdatabase;
sascha@30:
teichmann@475: import org.dive4elements.artifacts.common.utils.Config;
teichmann@475: import org.dive4elements.artifacts.common.utils.StringUtils;
sascha@207:
teichmann@475: import org.dive4elements.artifacts.Artifact;
sascha@30:
teichmann@475: import org.dive4elements.artifactdatabase.db.SQL;
teichmann@541: import org.dive4elements.artifactdatabase.db.SQLExecutor;
sascha@305:
sascha@30: import java.sql.PreparedStatement;
sascha@30: import java.sql.ResultSet;
sascha@93: import java.sql.SQLException;
sascha@93:
sascha@93: import java.util.ArrayList;
sascha@93: import java.util.List;
sascha@230: import java.util.Set;
teichmann@542: import java.util.HashSet;
sascha@230: import java.util.Collections;
sascha@30:
tom@570: import org.apache.logging.log4j.Logger;
tom@570: import org.apache.logging.log4j.LogManager;
sascha@30:
sascha@30: /**
sascha@90: * The database cleaner runs in background. It sleep for a configurable
sascha@90: * while and when it wakes up it removes outdated artifacts from the
sascha@90: * database. Outdated means that the the last access to the artifact
sascha@90: * is longer aga then the time to live of this artifact.
sascha@90: * Before the artifact is finally removed from the system it is
sascha@90: * revived one last time an the #endOfLife() method of the artifact
sascha@90: * is called.
sascha@90: * The artifact implementations may e.g. use this to remove some extrenal
sascha@90: * resources form the system.
sascha@90: *
ingo@80: * @author Sascha L. Teichmann
sascha@30: */
sascha@30: public class DatabaseCleaner
sascha@30: extends Thread
sascha@30: {
sascha@90: /**
sascha@90: * Implementors of this interface are able to create a
sascha@90: * living artifact from a given byte array.
sascha@90: */
sascha@41: public interface ArtifactReviver {
sascha@41:
sascha@90: /**
sascha@90: * Called to revive an artifact from a given byte array.
sascha@90: * @param factoryName The name of the factory which
sascha@90: * created this artifact.
sascha@90: * @param bytes The bytes of the serialized artifact.
sascha@90: * @return The revived artfiact.
sascha@90: */
sascha@41: Artifact reviveArtifact(String factoryName, byte [] bytes);
sascha@41:
sascha@314: void killedArtifacts(List identifiers);
sascha@314: void killedCollections(List identifiers);
sascha@314:
sascha@41: } // interface ArtifactReviver
sascha@41:
sascha@230: public interface LockedIdsProvider {
sascha@230: Set getLockedIds();
sascha@230: } // interface LockedIdsProvider
sascha@230:
tom@570: private static Logger logger = LogManager.getLogger(DatabaseCleaner.class);
sascha@30:
sascha@90: /**
sascha@90: * Number of artifacts to be loaded at once. Used to
sascha@90: * mitigate the problem of a massive denial of service
sascha@90: * if too many artifacts have died since last cleanup.
sascha@90: */
sascha@30: public static final int MAX_ROWS = 50;
sascha@30:
sascha@230: public static final Set EMPTY_IDS = Collections.emptySet();
sascha@230:
sascha@90: /**
sascha@90: * The SQL statement to select the outdated artifacts.
sascha@90: */
sascha@305: public String SQL_OUTDATED;
sascha@30:
sascha@305: public String SQL_OUTDATED_COLLECTIONS;
sascha@305: public String SQL_DELETE_COLLECTION_ITEMS;
sascha@305: public String SQL_DELETE_COLLECTION;
teichmann@542: public String SQL_COLLECTION_ITEMS_ARTIFACT_IDS;
sascha@232:
sascha@90: /**
sascha@90: * The SQL statement to delete some artifacts from the database.
sascha@90: */
sascha@305: public String SQL_DELETE_ARTIFACT;
sascha@30:
sascha@90: /**
sascha@90: * XPath to figure out how long the cleaner should sleep between
sascha@90: * cleanups. This is stored in the global configuration.
sascha@90: */
sascha@30: public static final String SLEEP_XPATH =
sascha@30: "/artifact-database/cleaner/sleep-time/text()";
sascha@30:
sascha@90: /**
sascha@90: * Default nap time between cleanups: 5 minutes.
sascha@90: */
sascha@30: public static final long SLEEP_DEFAULT =
sascha@30: 5 * 60 * 1000L; // 5 minutes
sascha@30:
sascha@90: /**
sascha@90: * The configured nap time.
sascha@90: */
sascha@30: protected long sleepTime;
sascha@30:
sascha@90: /**
sascha@90: * Internal locking mechanism to prevent some race conditions.
sascha@90: */
sascha@30: protected Object sleepLock = new Object();
sascha@30:
sascha@90: /**
sascha@90: * A reference to the global context.
sascha@90: */
sascha@30: protected Object context;
sascha@30:
sascha@90: /**
sascha@90: * A specialized Id filter which only delete some artifacts.
sascha@90: * This is used to prevent deletion of living artifacts.
sascha@90: */
sascha@230: protected LockedIdsProvider lockedIdsProvider;
sascha@32:
sascha@90: /**
sascha@90: * The reviver used to bring the dead artifact on last
sascha@90: * time back to live to call endOfLife() on them.
sascha@90: */
sascha@41: protected ArtifactReviver reviver;
sascha@41:
teichmann@541: protected SQLExecutor sqlExecutor;
sascha@305:
sascha@90: /**
sascha@90: * Default constructor.
sascha@90: */
sascha@30: public DatabaseCleaner() {
sascha@30: }
sascha@30:
sascha@90: /**
sascha@90: * Constructor to create a cleaner with a given global context
sascha@91: * and a given reviver.
sascha@90: * @param context The global context of the artifact database
sascha@90: * @param reviver The reviver to awake artifact one last time.
sascha@90: */
teichmann@541: public DatabaseCleaner(
teichmann@541: Object context,
teichmann@541: ArtifactReviver reviver,
teichmann@541: SQLExecutor sqlExecutor,
teichmann@541: DBConfig config
teichmann@541: ) {
sascha@30: setDaemon(true);
sascha@30: sleepTime = getSleepTime();
sascha@30: this.context = context;
sascha@41: this.reviver = reviver;
teichmann@541: this.sqlExecutor = sqlExecutor;
sascha@305: setupSQL(config.getSQL());
sascha@305: }
sascha@305:
sascha@305: protected void setupSQL(SQL sql) {
teichmann@542: SQL_OUTDATED = sql.get("artifacts.outdated");
teichmann@542: SQL_OUTDATED_COLLECTIONS = sql.get("collections.outdated");
teichmann@542: SQL_DELETE_COLLECTION_ITEMS = sql.get("delete.collection.items");
teichmann@542: SQL_DELETE_COLLECTION = sql.get("delete.collection");
teichmann@542: SQL_DELETE_ARTIFACT = sql.get("artifacts.delete");
teichmann@542: SQL_COLLECTION_ITEMS_ARTIFACT_IDS = sql.get("collection.items.artifact.id");
sascha@30: }
sascha@30:
sascha@90: /**
sascha@90: * Sets the filter that prevents deletion of living artifacts.
sascha@90: * Living artifacts are artifacts which are currently active
sascha@90: * inside the artifact database. Deleting them in this state
sascha@90: * would create severe internal problems.
sascha@90: */
sascha@230: public void setLockedIdsProvider(LockedIdsProvider lockedIdsProvider) {
sascha@230: this.lockedIdsProvider = lockedIdsProvider;
sascha@32: }
sascha@32:
sascha@90: /**
sascha@90: * External hook to tell the cleaner to wake up before its
sascha@90: * regular nap time is over. This is the case when the artifact
sascha@90: * database finds an artifact which is already outdated.
sascha@90: */
sascha@30: public void wakeup() {
sascha@30: synchronized (sleepLock) {
sascha@30: sleepLock.notify();
sascha@30: }
sascha@30: }
sascha@30:
sascha@90: /**
sascha@90: * Fetches the sleep time from the global configuration.
sascha@90: * @return the time to sleep between database cleanups in ms.
sascha@90: */
sascha@30: protected static long getSleepTime() {
sascha@30: String sleepTimeString = Config.getStringXPath(SLEEP_XPATH);
sascha@30:
sascha@30: if (sleepTimeString == null) {
sascha@30: return SLEEP_DEFAULT;
sascha@30: }
sascha@30: try {
sascha@30: // sleep at least one second
sascha@30: return Math.max(Long.parseLong(sleepTimeString), 1000L);
sascha@30: }
sascha@30: catch (NumberFormatException nfe) {
sascha@30: logger.warn("Cleaner sleep time defaults to " + SLEEP_DEFAULT);
sascha@30: }
sascha@30: return SLEEP_DEFAULT;
sascha@30: }
sascha@30:
sascha@314: private static class IdIdentifier {
sascha@230:
sascha@230: int id;
sascha@314: String identifier;
sascha@314:
sascha@314: private IdIdentifier(int id, String identifier) {
sascha@314: this.id = id;
sascha@314: this.identifier = identifier;
sascha@314: }
sascha@314: } // class IdIdentifier
sascha@314:
sascha@394: private static final class IdData
sascha@314: extends IdIdentifier
sascha@314: {
sascha@30: byte [] data;
sascha@41: String factoryName;
sascha@30:
sascha@314: public IdData(
sascha@314: int id,
sascha@394: String factoryName,
sascha@314: byte [] data,
sascha@314: String identifier
sascha@314: ) {
sascha@314: super(id, identifier);
sascha@41: this.factoryName = factoryName;
sascha@41: this.data = data;
sascha@30: }
sascha@30: } // class IdData
sascha@30:
sascha@30: /**
sascha@30: * Cleaning is done in two phases. First we fetch a list of ids
sascha@30: * of artifacts. If there are artifacts the cleaning is done.
sascha@30: * Second we load the artifacts one by one one and call there
sascha@30: * endOfLife() method. In this loop we remove them from database, too.
sascha@30: * Each deletion is commited to ensure that a sudden failure
sascha@30: * of the artifact database server does delete artifacts twice
sascha@30: * or does not delete them at all. After this the first step
sascha@30: * is repeated.
sascha@30: */
sascha@30: protected void cleanup() {
sascha@30: logger.info("database cleanup");
sascha@30:
teichmann@541: final Set lockedIds = lockedIdsProvider != null
sascha@231: ? lockedIdsProvider.getLockedIds()
sascha@231: : EMPTY_IDS;
sascha@231:
teichmann@541: final String questionMarks = lockedIds.isEmpty()
sascha@241: ? "-666" // XXX: A bit hackish.
sascha@238: : StringUtils.repeat('?', lockedIds.size(), ',');
sascha@231:
teichmann@541: final List deletedCollections = new ArrayList();
teichmann@541: final List deletedArtifacts = new ArrayList();
sascha@232:
teichmann@541: SQLExecutor.Instance exec = sqlExecutor.new Instance() {
sascha@30:
teichmann@541: @Override
teichmann@541: public boolean doIt() throws SQLException {
sascha@30:
teichmann@542: PreparedStatement collectionItems = null;
teichmann@542: PreparedStatement fetchIds = null;
teichmann@542: PreparedStatement stmnt = null;
teichmann@542: ResultSet result = null;
teichmann@542:
teichmann@542: HashSet collectionItemsIds =
teichmann@542: new HashSet();
sascha@30:
teichmann@541: try {
teichmann@542: collectionItems = conn.prepareStatement(
teichmann@542: SQL_COLLECTION_ITEMS_ARTIFACT_IDS);
teichmann@542:
teichmann@542: result = collectionItems.executeQuery();
teichmann@542:
teichmann@542: while (result.next()) {
teichmann@542: collectionItemsIds.add(result.getInt(1));
teichmann@542: }
teichmann@542: result.close(); result = null;
teichmann@542:
teichmann@541: fetchIds = conn.prepareStatement(
teichmann@541: SQL_OUTDATED.replace("$LOCKED_IDS$", questionMarks));
ingo@393:
teichmann@541: // Fetch ids of outdated collections
teichmann@541: stmnt = conn.prepareStatement(
teichmann@541: SQL_OUTDATED_COLLECTIONS.replace(
teichmann@541: "$LOCKED_IDS$", questionMarks));
teichmann@541:
teichmann@541: // fill in the locked ids
teichmann@541: int idx = 1;
teichmann@541: for (Integer id: lockedIds) {
teichmann@541: fetchIds.setInt(idx, id);
teichmann@541: stmnt .setInt(idx, id);
teichmann@541: ++idx;
sascha@30: }
sascha@314:
teichmann@541: ArrayList cs = new ArrayList();
teichmann@541: result = stmnt.executeQuery();
teichmann@541: while (result.next()) {
teichmann@541: cs.add(new IdIdentifier(
teichmann@541: result.getInt(1),
teichmann@541: result.getString(2)));
teichmann@541: }
teichmann@541:
teichmann@541: result.close(); result = null;
teichmann@541: stmnt.close(); stmnt = null;
teichmann@541:
teichmann@541: // delete collection items
teichmann@541: stmnt = conn.prepareStatement(SQL_DELETE_COLLECTION_ITEMS);
teichmann@541:
teichmann@541: for (IdIdentifier id: cs) {
teichmann@541: logger.debug("Mark collection for deletion: " + id.id);
teichmann@541: stmnt.setInt(1, id.id);
teichmann@541: stmnt.execute();
teichmann@541: }
teichmann@541:
teichmann@541: stmnt.close(); stmnt = null;
teichmann@541:
teichmann@541: // delete collections
teichmann@541: stmnt = conn.prepareStatement(SQL_DELETE_COLLECTION);
teichmann@541:
teichmann@541: for (IdIdentifier id: cs) {
teichmann@541: stmnt.setInt(1, id.id);
teichmann@541: stmnt.execute();
teichmann@541: deletedCollections.add(id.identifier);
teichmann@541: }
teichmann@541:
teichmann@541: stmnt.close(); stmnt = null;
teichmann@541: conn.commit();
teichmann@541:
teichmann@541: cs = null;
teichmann@541:
teichmann@541: // remove artifacts
teichmann@541: stmnt = conn.prepareStatement(SQL_DELETE_ARTIFACT);
teichmann@541:
teichmann@541: for (;;) {
teichmann@541: List ids = new ArrayList();
teichmann@541:
teichmann@541: result = fetchIds.executeQuery();
teichmann@541:
teichmann@542: int total = 0;
teichmann@542:
teichmann@541: while (result.next()) {
teichmann@542: total++;
teichmann@542: int id = result.getInt(1);
teichmann@542: if (!collectionItemsIds.contains(id)) {
teichmann@542: ids.add(new IdData(
teichmann@542: id,
teichmann@542: result.getString(2),
teichmann@542: result.getBytes(3),
teichmann@542: result.getString(4)));
teichmann@542: }
teichmann@541: }
teichmann@541:
teichmann@541: result.close(); result = null;
teichmann@541:
teichmann@542: if (total == 0) {
teichmann@542: break;
teichmann@542: }
teichmann@542:
teichmann@541: if (ids.isEmpty()) {
andre@543: break;
teichmann@541: }
teichmann@541:
teichmann@541: for (int i = ids.size()-1; i >= 0; --i) {
teichmann@541: IdData idData = ids.get(i);
teichmann@541: Artifact artifact = reviver.reviveArtifact(
teichmann@541: idData.factoryName, idData.data);
teichmann@541: idData.data = null;
teichmann@541:
teichmann@541: logger.debug("Prepare Artifact (id="
teichmann@541: + idData.id + ") for deletion.");
teichmann@541:
teichmann@541: stmnt.setInt(1, idData.id);
teichmann@541: stmnt.execute();
teichmann@541: conn.commit();
teichmann@541:
teichmann@541: try {
teichmann@541: if (artifact != null) {
teichmann@541: logger.debug("Call endOfLife for Artifact: "
teichmann@541: + artifact.identifier());
teichmann@541:
teichmann@541: artifact.endOfLife(context);
teichmann@541: }
teichmann@541: }
teichmann@541: catch (Exception e) {
teichmann@541: logger.error(e.getMessage(), e);
teichmann@541: }
teichmann@541:
teichmann@541: deletedArtifacts.add(idData.identifier);
teichmann@541: } // for all fetched data
teichmann@541: }
teichmann@541: }
teichmann@541: finally {
teichmann@541: if (result != null) {
teichmann@541: try { result.close(); }
teichmann@541: catch (SQLException sqle) {}
teichmann@541: }
teichmann@541: if (stmnt != null) {
teichmann@541: try { stmnt.close(); }
teichmann@541: catch (SQLException sqle) {}
teichmann@541: }
teichmann@541: if (fetchIds != null) {
teichmann@541: try { fetchIds.close(); }
teichmann@541: catch (SQLException sqle) {}
teichmann@541: }
teichmann@542: if (collectionItems != null) {
teichmann@542: try { collectionItems.close(); }
teichmann@542: catch (SQLException sqle) {}
teichmann@542: }
teichmann@541: }
teichmann@541: return true;
sascha@30: }
teichmann@541: };
teichmann@541:
teichmann@541: if (!exec.runWriteNoRollback()) {
teichmann@541: logger.error("Deleting artifacts failed.");
sascha@30: }
sascha@30:
sascha@315: if (!deletedCollections.isEmpty()) {
sascha@315: reviver.killedCollections(deletedCollections);
sascha@315: }
sascha@315:
sascha@315: if (!deletedArtifacts.isEmpty()) {
sascha@315: reviver.killedArtifacts(deletedArtifacts);
sascha@315: }
sascha@314:
sascha@314: if (logger.isDebugEnabled()) {
sascha@314: logger.debug(
sascha@314: "collections removed: " + deletedCollections.size());
sascha@314: logger.debug(
sascha@314: "artifacts removed: " + deletedArtifacts.size());
sascha@314: }
sascha@30: }
sascha@30:
sascha@90: /**
sascha@90: * The main code of the cleaner. It sleeps for the configured
sascha@90: * nap time, cleans up the database, sleeps again and so on.
sascha@90: */
sascha@90: @Override
sascha@30: public void run() {
sascha@30: logger.info("sleep time: " + sleepTime + "ms");
sascha@30: for (;;) {
sascha@30: cleanup();
sascha@48: long startTime = System.currentTimeMillis();
sascha@48:
sascha@30: try {
sascha@30: synchronized (sleepLock) {
sascha@30: sleepLock.wait(sleepTime);
sascha@30: }
sascha@30: }
sascha@30: catch (InterruptedException ie) {
sascha@30: }
sascha@48:
sascha@48: long stopTime = System.currentTimeMillis();
sascha@48:
sascha@48: if (logger.isDebugEnabled()) {
sascha@48: logger.debug("Cleaner slept " + (stopTime - startTime) + "ms");
sascha@48: }
sascha@30: } // for (;;)
sascha@30: }
sascha@30: }
sascha@90: // vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 :