diff artifacts/src/main/java/org/dive4elements/river/artifacts/model/fixings/Fitting.java @ 9415:9744ce3c3853

Rework of fixanalysis computation and dWt and WQ facets. Got rid of strange remapping and bitshifting code by explicitely saving the column information and using it in the facets. The facets also put the valid station range into their xml-metadata
author gernotbelger
date Thu, 16 Aug 2018 16:27:53 +0200
parents ddcd52d239cd
children f51e23eb036a
line wrap: on
line diff
--- a/artifacts/src/main/java/org/dive4elements/river/artifacts/model/fixings/Fitting.java	Thu Aug 16 15:47:10 2018 +0200
+++ b/artifacts/src/main/java/org/dive4elements/river/artifacts/model/fixings/Fitting.java	Thu Aug 16 16:27:53 2018 +0200
@@ -8,9 +8,8 @@
 
 package org.dive4elements.river.artifacts.model.fixings;
 
-import gnu.trove.TDoubleArrayList;
-
 import java.util.ArrayList;
+import java.util.Date;
 import java.util.List;
 
 import org.apache.commons.math.MathException;
@@ -18,7 +17,6 @@
 import org.apache.commons.math.optimization.general.LevenbergMarquardtOptimizer;
 import org.apache.commons.math.stat.descriptive.moment.StandardDeviation;
 import org.apache.log4j.Logger;
-
 import org.dive4elements.river.artifacts.math.GrubbsOutlier;
 import org.dive4elements.river.artifacts.math.fitting.Function;
 
@@ -30,79 +28,86 @@
         QWD create(double q, double w, double deltaW, boolean isOutlier);
     }
 
+    private static final class FittingData {
+        public final FixingColumnWithData event;
+        public final double w;
+        public final double q;
+        public final boolean isInterpolated;
+
+        public FittingData(final FixingColumnWithData event, final double w, final double q, final boolean isInterpolated) {
+            this.event = event;
+            this.w = w;
+            this.q = q;
+            this.isInterpolated = isInterpolated;
+        }
+    }
+
     private final double chiSqr;
 
     private final double[] parameters;
 
     private final double standardDeviation;
 
-    private final List<QWD> qwds;
+    private final double maxQ;
 
-    public Fitting(final double[] parameters, final double standardDeviation, final double chiSqr, final List<QWD> qwds) {
+    public Fitting(final double[] parameters, final double standardDeviation, final double chiSqr, final double maxQ) {
         this.parameters = parameters;
         this.standardDeviation = standardDeviation;
         this.chiSqr = chiSqr;
-        this.qwds = qwds;
+        this.maxQ = maxQ;
     }
 
     public double getChiSquare() {
-        return chiSqr;
-    }
-
-    /**
-     * Returns all referenced and outliers as one array.
-     */
-    public QWD[] getFixingsArray() {
-        return qwds.toArray(new QWD[qwds.size()]);
+        return this.chiSqr;
     }
 
     public double getMaxQ() {
-        double maxQ = -Double.MAX_VALUE;
-
-        for (QWD qw : qwds) {
-            final double q = qw.getQ();
-            if (!qw.isOutlier() && q > maxQ)
-                maxQ = q;
-        }
-
-        return maxQ;
+        return this.maxQ;
     }
 
     public double[] getParameters() {
-        return parameters;
+        return this.parameters;
     }
 
     public double getStandardDeviation() {
-        return standardDeviation;
+        return this.standardDeviation;
     }
 
-    public static Fitting fit(final Function function, final QWDFactory qwdFactory, final boolean checkOutliers, final double[] qs, final double[] ws) {
+    public static Fitting fit(final FixResultColumns resultColumns, final double km, final Function function, final boolean checkOutliers,
+            final List<FixingColumnWithData> eventColumns) {
 
-        final TDoubleArrayList xs = new TDoubleArrayList(qs.length);
-        final TDoubleArrayList ys = new TDoubleArrayList(ws.length);
+        final int numEvents = eventColumns.size();
+        final List<FittingData> data = new ArrayList<>(numEvents);
 
-        for (int i = 0; i < qs.length; ++i) {
-            if (!Double.isNaN(qs[i]) && !Double.isNaN(ws[i])) {
-                xs.add(qs[i]);
-                ys.add(ws[i]);
+        final double[] wTemp = new double[1];
+
+        for (final FixingColumnWithData event : eventColumns) {
+
+            final double q = event.getQ(km);
+            if (!Double.isNaN(q)) {
+                final boolean isInterpolated = !event.getW(km, wTemp, 0);
+                final double w = wTemp[0];
+
+                if (!Double.isNaN(w))
+                    data.add(new FittingData(event, w, q, isInterpolated));
             }
         }
 
-        if (xs.size() < 2) {
-            log.warn("Too less points.");
+        if (data.size() < 2) {
+            log.warn("Not enough data for fitting.");
             return null;
         }
 
-        final List<Double> inputs = new ArrayList<>(xs.size());
-        final List<QWD> qwds = new ArrayList<>(xs.size());
-        final List<QWD> outliers = new ArrayList<>(xs.size());
+        final List<FittingData> outliers = new ArrayList<>(data.size());
 
         org.dive4elements.river.artifacts.math.Function instance = null;
         LevenbergMarquardtOptimizer lmo = null;
         double[] parameters = null;
 
         for (;;) {
+
             parameters = null;
+
             for (double tolerance = 1e-10; tolerance < 1e-1; tolerance *= 10d) {
 
                 lmo = new LevenbergMarquardtOptimizer();
@@ -110,17 +115,16 @@
                 lmo.setOrthoTolerance(tolerance);
                 lmo.setParRelativeTolerance(tolerance);
 
-                CurveFitter cf = new CurveFitter(lmo);
+                try {
+                    final CurveFitter cf = new CurveFitter(lmo);
 
-                for (int i = 0, N = xs.size(); i < N; ++i) {
-                    cf.addObservedPoint(xs.getQuick(i), ys.getQuick(i));
-                }
+                    for (final FittingData fittingData : data)
+                        cf.addObservedPoint(fittingData.q, fittingData.w);
 
-                try {
                     parameters = cf.fit(function, function.getInitialGuess());
                     break;
                 }
-                catch (MathException me) {
+                catch (final MathException me) {
                     if (log.isDebugEnabled()) {
                         log.debug("tolerance " + tolerance + " + failed.", me);
                     }
@@ -137,20 +141,21 @@
                 return null;
             }
 
-            // This is the paraterized function for a given km.
+            // This is the parameterized function for a given km.
             instance = function.instantiate(parameters);
 
             if (!checkOutliers)
                 break;
 
-            inputs.clear();
+            /* find the outlier */
+            final List<Double> inputs = new ArrayList<>(data.size());
+            for (final FittingData fittingData : data) {
 
-            for (int i = 0, N = xs.size(); i < N; ++i) {
-                double y = instance.value(xs.getQuick(i));
-                if (Double.isNaN(y)) {
+                double y = instance.value(fittingData.q);
+                if (Double.isNaN(y))
                     y = Double.MAX_VALUE;
-                }
-                inputs.add(Double.valueOf(ys.getQuick(i) - y));
+
+                inputs.add(Double.valueOf(fittingData.w - y));
             }
 
             final Integer outlier = GrubbsOutlier.findOutlier(inputs);
@@ -158,41 +163,59 @@
                 break;
 
             final int idx = outlier.intValue();
-            outliers.add(qwdFactory.create(xs.getQuick(idx), ys.getQuick(idx), Double.NaN, true));
-            xs.remove(idx);
-            ys.remove(idx);
-        }
-        
-        for (QWD outlier : outliers) {
-            
-            final double w = outlier.getW();
-            final double q = outlier.getQ();
-            
-            final double dw = (w - instance.value(q)) * 100.0;
 
-            outlier.setDeltaW(dw);
-            
-            qwds.add(outlier);
+            // outliers.add(qwdFactory.create(xs.getQuick(idx), ys.getQuick(idx), Double.NaN, true));
+            final FittingData removed = data.remove(idx);
+            outliers.add(removed);
         }
 
-        final StandardDeviation stdDev = new StandardDeviation();
-
-        for (int i = 0; i < xs.size(); ++i) {
-
-            final QWD qwd = qwdFactory.create(xs.getQuick(i), ys.getQuick(i), Double.NaN, false);
+        /* now build result data */
+        final List<QWD> qwds = new ArrayList<>(data.size());
 
-            double dw = (qwd.getW() - instance.value(qwd.getQ())) * 100.0;
-            qwd.setDeltaW(dw);
-            
+        /* calculate dW of outliers against the resulting function and add them to results */
+        for (final FittingData outlier : outliers) {
+            final QWD qwd = createQWD(outlier, instance, true);
             qwds.add(qwd);
-            
-            stdDev.increment(dw);
+            resultColumns.addQWD(outlier.event, km, qwd);
+        }
+
+        /*
+         * calculate dW of used values against the resulting function and add them to results , also calculate standard *
+         * deviation
+         */
+        final StandardDeviation stdDev = new StandardDeviation();
+        double maxQ = -Double.MAX_VALUE;
+
+        for (final FittingData fittingData : data) {
+
+            final QWD qwd = createQWD(fittingData, instance, false);
+            qwds.add(qwd);
+            resultColumns.addQWD(fittingData.event, km, qwd);
+
+            stdDev.increment(qwd.getDeltaW());
+
+            final double q = qwd.getQ();
+            if (q > maxQ)
+                maxQ = q;
         }
 
         final double standardDeviation = stdDev.getResult();
 
         final double chiSqr = lmo.getChiSquare();
 
-        return new Fitting(parameters, standardDeviation, chiSqr, qwds);
+        return new Fitting(parameters, standardDeviation, chiSqr, maxQ);
+    }
+
+    private static QWD createQWD(final FittingData data, final org.dive4elements.river.artifacts.math.Function function, final boolean isOutlier) {
+
+        final FixingColumnWithData event = data.event;
+        final Date date = event.getDate();
+        final boolean isInterpolated = data.isInterpolated;
+
+        final double w = data.w;
+        final double q = data.q;
+        final double dw = (w - function.value(q)) * 100.0;
+
+        return new QWD(q, w, date, isInterpolated, dw, isOutlier);
     }
 }
\ No newline at end of file

http://dive4elements.wald.intevation.org