Mercurial > dive4elements > river

annotate flys-artifacts/src/main/java/de/intevation/flys/artifacts/math/GrubbsOutlier.java @ 5682:b536c0e7abd7

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Added old calculation and additional data to dc config for bed difference charts.
author Raimund Renkert <rrenkert@intevation.de>
date Fri, 12 Apr 2013 14:51:29 +0200
parents a7d080347ac3
children
rev   line source
4794
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
1 package de.intevation.flys.artifacts.math;
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
2
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
3 import java.util.List;
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
4
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
5 import org.apache.commons.math.MathException;
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
6
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
7 import org.apache.commons.math.distribution.TDistributionImpl;
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
8
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
9 import org.apache.commons.math.stat.descriptive.moment.Mean;
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
10 import org.apache.commons.math.stat.descriptive.moment.StandardDeviation;
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
11
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
12 import org.apache.log4j.Logger;
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
13
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
14 public class GrubbsOutlier
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
15 {
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
16 public static final double EPSILON = 1e-5;
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
17
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
18 public static final double DEFAULT_ALPHA = 0.05;
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
19
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
20 private static Logger log = Logger.getLogger(GrubbsOutlier.class);
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
21
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
22 protected GrubbsOutlier() {
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
23 }
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
24
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
25 public static Integer findOutlier(List<Double> values) {
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
26 return findOutlier(values, DEFAULT_ALPHA, null);
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
27 }
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
28
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
29 public static Integer findOutlier(
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
30 List<Double> values,
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
31 double alpha,
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
32 double[] stdDevResult
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
33 ) {
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
34 boolean debug = log.isDebugEnabled();
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
35
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
36 if (debug) {
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
37 log.debug("outliers significance: " + alpha);
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
38 }
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
39
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
40 alpha = 1d - alpha;
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
41
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
42 int N = values.size();
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
43
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
44 if (debug) {
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
45 log.debug("Values to check: " + N);
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
46 }
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
47
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
48 if (N < 3) {
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
49 return null;
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
50 }
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
51
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
52 Mean mean = new Mean();
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
53 StandardDeviation std = new StandardDeviation();
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
54
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
55 for (Double value: values) {
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
56 double v = value.doubleValue();
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
57 mean.increment(v);
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
58 std .increment(v);
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
59 }
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
60
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
61 double m = mean.getResult();
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
62 double s = std.getResult();
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
63
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
64 if (debug) {
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
65 log.debug("mean: " + m);
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
66 log.debug("std dev: " + s);
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
67 }
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
68
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
69 double maxZ = -Double.MAX_VALUE;
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
70 int iv = -1;
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
71 for (int i = N-1; i >= 0; --i) {
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
72 double v = values.get(i).doubleValue();
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
73 double z = Math.abs(v - m);
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
74 if (z > maxZ) {
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
75 maxZ = z;
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
76 iv = i;
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
77 }
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
78 }
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
79
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
80 if (Math.abs(s) < EPSILON) {
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
81 return null;
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
82 }
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
83
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
84 maxZ /= s;
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
85
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
86 TDistributionImpl tdist = new TDistributionImpl(N-2);
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
87
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
88 double t;
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
89
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
90 try {
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
91 t = tdist.inverseCumulativeProbability(alpha/(N+N));
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
92 }
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
93 catch (MathException me) {
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
94 log.error(me);
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
95 return null;
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
96 }
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
97
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
98 t *= t;
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
99
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
100 double za = ((N-1)/Math.sqrt(N))*Math.sqrt(t/(N-2d+t));
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
101
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
102 if (debug) {
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
103 log.debug("max: " + maxZ + " crit: " + za);
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
104 }
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
105 if (stdDevResult != null) {
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
106 stdDevResult[0] = std.getResult();
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
107 }
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
108 return maxZ > za
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
109 ? Integer.valueOf(iv)
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
110 : null;
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
111 }
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
112 }
a7d080347ac3 MINFO: Allow two methods for outlier test in SQ relation.
Raimund Renkert <rrenkert@intevation.de>
parents:
diff changeset
113 // vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 :
http://dive4elements.wald.intevation.org