changeset 7471:fff862f4ef76

Experimental caching of datacage recommendations. The respective hook is called a lot and running the datacage over and over again when loading data can be expensive. So the generated recommendations are cached for some time. Hopefully this improves the overall speed of loading data from the datacage.
author Sascha L. Teichmann <teichmann@intevation.de>
date Wed, 30 Oct 2013 15:26:21 +0100
parents e590599031d8
children 9cf5fd49837a
files artifacts/doc/conf/cache.xml artifacts/src/main/java/org/dive4elements/river/artifacts/CollectionMonitor.java
diffstat 2 files changed, 80 insertions(+), 16 deletions(-) [+]
line wrap: on
line diff
--- a/artifacts/doc/conf/cache.xml	Wed Oct 30 15:02:16 2013 +0100
+++ b/artifacts/doc/conf/cache.xml	Wed Oct 30 15:26:21 2013 +0100
@@ -76,6 +76,16 @@
            memoryStoreEvictionPolicy="LFU"
            />
 
+    <!-- This one is used to cache the recommendation nodes generated from datacage -->
+    <cache name="recommendations"
+           maxElementsInMemory="150"
+           eternal="false"
+           timeToLiveSeconds="1800"
+           memoryStoreEvictionPolicy="LRU"
+           overflowToDisk="false"
+           diskPersistent="false"
+       />
+
     <!-- This one is used for the cross section next neighbor lookup -->
     <cache name="cross-section-kms"
            maxElementsInMemory="50"
@@ -149,7 +159,8 @@
     <cache name="official-lines"
            maxElementsInMemory="2"
            timeToLiveSeconds="14400"
-           />
+       />
+
 
     <!-- This one is used for the cross section lookup
          Because of lazy fetching and relatively big amount of data, disabled
--- a/artifacts/src/main/java/org/dive4elements/river/artifacts/CollectionMonitor.java	Wed Oct 30 15:02:16 2013 +0100
+++ b/artifacts/src/main/java/org/dive4elements/river/artifacts/CollectionMonitor.java	Wed Oct 30 15:26:21 2013 +0100
@@ -12,6 +12,8 @@
 import java.util.List;
 import java.util.Map;
 
+import net.sf.ehcache.Cache;
+
 import org.w3c.dom.Document;
 import org.w3c.dom.Element;
 import org.w3c.dom.Node;
@@ -22,15 +24,19 @@
 import org.dive4elements.artifacts.CallContext;
 import org.dive4elements.artifacts.Hook;
 
+import org.dive4elements.artifacts.common.utils.XMLUtils;
 import org.dive4elements.artifacts.common.utils.XMLUtils.ElementCreator;
 
 import org.dive4elements.artifactdatabase.state.Output;
 
+import org.dive4elements.river.artifacts.cache.CacheFactory;
 import org.dive4elements.river.artifacts.datacage.Recommendations;
 
 /** Monitors collection changes. */
 public class CollectionMonitor implements Hook {
 
+    public static final String CACHE_NAME = "recommendations";
+
     @Override
     public void setup(Node cfg) {
     }
@@ -55,23 +61,70 @@
 
         Element result = (Element)results.item(0);
 
+        result.appendChild(getRecommendedElement(flys, context, doc));
+    }
+
+    protected Element getRecommendedElement(
+        D4EArtifact artifact,
+        CallContext context,
+        Document    doc
+    ) {
+        String [] outs = extractOutputNames(artifact, context);
+
+        Element recommendations = null;
+
+        Cache cache = CacheFactory.getCache(CACHE_NAME);
+
+        if (cache != null) {
+            String key = generateCacheKey(artifact, outs);
+
+            net.sf.ehcache.Element ce = cache.get(key);
+            if (ce != null) { // Found in cache.
+                Element e = (Element)ce.getValue();
+                // Sync to avoid thread issues with XML DOM docs.
+                synchronized (e.getOwnerDocument()) {
+                    recommendations = (Element)doc.importNode(e, true);
+                }
+            } else { // Not found in cache -> generate it.
+                Element r = createElement(XMLUtils.newDocument());
+
+                Recommendations.getInstance().recommend(
+                    artifact, null, outs,
+                    getNoneUserSpecificParameters(artifact, context), r);
+
+                recommendations = (Element)doc.importNode(r, true);
+
+                cache.put(new net.sf.ehcache.Element(key, r));
+            }
+        } else { // No cache configured -> append directly.
+
+            recommendations = createElement(doc);
+
+            Recommendations.getInstance().recommend(
+                artifact, null, outs,
+                getNoneUserSpecificParameters(artifact, context),
+                recommendations);
+        }
+
+        return recommendations;
+    }
+
+    private static final Element createElement(Document doc) {
         ElementCreator creator = new ElementCreator(
             doc,
             ArtifactNamespaceContext.NAMESPACE_URI,
             ArtifactNamespaceContext.NAMESPACE_PREFIX);
 
-        Element recommended = creator.create("recommended-artifacts");
-
-        String[] outs              = extractOutputNames(flys, context);
-        Map<String, Object> params = getNoneUserSpecificParameters(flys, context);
+        return creator.create("recommended-artifacts");
+    }
 
-        Recommendations rec = Recommendations.getInstance();
-
-        // TODO For newer official-lines recommendations we actually
-        // need user-id (null here).
-        rec.recommend(flys, null, outs, params, recommended);
-
-        result.appendChild(recommended);
+    private static final String generateCacheKey(D4EArtifact artifact, String [] outs) {
+        StringBuilder sb = new StringBuilder(artifact.hash());
+        // XXX: The hash really should be unique enough.
+        for (String out: outs) {
+            sb.append(';').append(out);
+        }
+        return sb.toString();
     }
 
 
@@ -79,7 +132,7 @@
      * Get outputnames from current state (only the ones for which
      * facets exist).
      */
-    public static String[] extractOutputNames(
+    private static final String [] extractOutputNames(
         D4EArtifact flys,
         CallContext  context)
     {
@@ -104,11 +157,11 @@
     /**
      * Creates Map from Strings "recommended" to "true".
      */
-    protected Map<String, Object> getNoneUserSpecificParameters(
+    private static final Map<String, Object> getNoneUserSpecificParameters(
         D4EArtifact flys,
-        CallContext  context)
+        CallContext context)
     {
-        Map<String, Object> params = new HashMap<String, Object>(1);
+        Map<String, Object> params = new HashMap<String, Object>();
         params.put("recommended", "true");
 
         return params;

http://dive4elements.wald.intevation.org