Mercurial > dive4elements > river

comparison artifacts/src/main/java/org/dive4elements/river/artifacts/model/fixings/FixCalculation.java @ 9415:9744ce3c3853

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Rework of fixanalysis computation and dWt and WQ facets. Got rid of strange remapping and bitshifting code by explicitely saving the column information and using it in the facets. The facets also put the valid station range into their xml-metadata
author gernotbelger
date Thu, 16 Aug 2018 16:27:53 +0200
parents 0274c7444b2d
children 2b83d3a96703
comparison
equal deleted inserted replaced
9414:096f151a0a9f 9415:9744ce3c3853
7 */ 7 */
8 8
9 package org.dive4elements.river.artifacts.model.fixings; 9 package org.dive4elements.river.artifacts.model.fixings;
10 10
11 import java.util.ArrayList; 11 import java.util.ArrayList;
12 import java.util.Date;
13 import java.util.HashMap;
14 import java.util.List; 12 import java.util.List;
15 import java.util.Map;
16 13
17 import org.apache.log4j.Logger; 14 import org.apache.log4j.Logger;
18 import org.dive4elements.artifacts.common.utils.StringUtils; 15 import org.dive4elements.artifacts.common.utils.StringUtils;
19 import org.dive4elements.river.artifacts.access.FixAccess; 16 import org.dive4elements.river.artifacts.access.FixAccess;
20 import org.dive4elements.river.artifacts.math.fitting.Function; 17 import org.dive4elements.river.artifacts.math.fitting.Function;
21 import org.dive4elements.river.artifacts.math.fitting.FunctionFactory; 18 import org.dive4elements.river.artifacts.math.fitting.FunctionFactory;
22 import org.dive4elements.river.artifacts.model.Calculation; 19 import org.dive4elements.river.artifacts.model.Calculation;
23 import org.dive4elements.river.artifacts.model.CalculationResult; 20 import org.dive4elements.river.artifacts.model.CalculationResult;
24 import org.dive4elements.river.artifacts.model.FixingsColumn;
25 import org.dive4elements.river.artifacts.model.FixingsColumnFactory;
26 import org.dive4elements.river.artifacts.model.FixingsOverview;
27 import org.dive4elements.river.artifacts.model.FixingsOverview.Fixing;
28 import org.dive4elements.river.artifacts.model.FixingsOverview.Fixing.Filter;
29 import org.dive4elements.river.artifacts.model.FixingsOverview.IdsFilter;
30 import org.dive4elements.river.artifacts.model.FixingsOverviewFactory;
31 import org.dive4elements.river.artifacts.model.Parameters; 21 import org.dive4elements.river.artifacts.model.Parameters;
22 import org.dive4elements.river.artifacts.model.fixings.FixingsOverview.IdsFilter;
32 import org.dive4elements.river.utils.DoubleUtil; 23 import org.dive4elements.river.utils.DoubleUtil;
33 import org.dive4elements.river.utils.KMIndex;
34 24
35 /** Calculation base class for fix. */ 25 /** Calculation base class for fix. */
36 public abstract class FixCalculation extends Calculation { 26 public abstract class FixCalculation extends Calculation {
27
28 private static final long serialVersionUID = 1L;
29
37 private static Logger log = Logger.getLogger(FixCalculation.class); 30 private static Logger log = Logger.getLogger(FixCalculation.class);
38 31
39 public static final double EPSILON = 1e-4; 32 private static final String[] STANDARD_COLUMNS = { "km", "chi_sqr", "max_q", "std-dev" };
40
41 public static final String[] STANDARD_COLUMNS = { "km", "chi_sqr", "max_q", "std-dev" };
42 33
43 protected static class FitResult { 34 protected static class FitResult {
44 35
45 private final Parameters parameters; 36 private final Parameters parameters;
46 private final KMIndex<QWD[]> fixings; 37
47 38 private final FixResultColumns resultColumns;
48 public FitResult(final Parameters parameters, final KMIndex<QWD[]> fixings) { 39
40 public FitResult(final Parameters parameters, final FixResultColumns resultColumns) {
49 this.parameters = parameters; 41 this.parameters = parameters;
50 this.fixings = fixings; 42 this.resultColumns = resultColumns;
51 } 43 }
52 44
53 public Parameters getParameters() { 45 public Parameters getParameters() {
54 return this.parameters; 46 return this.parameters;
55 } 47 }
56 48
57 public KMIndex<QWD[]> getFixings() { 49 public FixResultColumns getResultColumns() {
58 return this.fixings; 50 return this.resultColumns;
59 } 51 }
60 } 52 }
61
62 /**
63 * Helper class to bundle the meta information of a column
64 * and the real data.
65 */
66 protected static class Column {
67
68 protected Fixing.Column meta;
69 protected FixingsColumn data;
70 protected int index;
71
72 public Column() {
73 }
74
75 public Column(final Fixing.Column meta, final FixingsColumn data, final int index) {
76 this.meta = meta;
77 this.data = data;
78 this.index = index;
79 }
80
81 public Date getDate() {
82 return this.meta.getStartTime();
83 }
84
85 public String getDescription() {
86 return this.meta.getDescription();
87 }
88
89 public int getIndex() {
90 return this.index;
91 }
92
93 public int getId() {
94 return this.meta.getId();
95 }
96
97 public boolean getQW(final double km, final double[] qs, final double[] ws, final int index) {
98 qs[index] = this.data.getQ(km);
99 return this.data.getW(km, ws, index);
100 }
101
102 public boolean getQW(final double km, final double[] wq) {
103 this.data.getW(km, wq, 0);
104 if (Double.isNaN(wq[0]))
105 return false;
106 wq[1] = this.data.getQ(km);
107 return !Double.isNaN(wq[1]);
108 }
109 } // class Column
110
111 /**
112 * Helper class to find the data belonging to meta info more quickly.
113 */
114 protected static class ColumnCache {
115
116 protected Map<Integer, Column> columns;
117
118 public ColumnCache() {
119 this.columns = new HashMap<>();
120 }
121
122 public Column getColumn(final Fixing.Column meta) {
123 final Integer key = meta.getId();
124 Column column = this.columns.get(key);
125 if (column == null) {
126 final FixingsColumn data = FixingsColumnFactory.getInstance().getColumnData(meta);
127 if (data != null) {
128 column = new Column(meta, data, this.columns.size());
129 this.columns.put(key, column);
130 }
131 }
132 return column;
133 }
134 } // class ColumnCache
135 53
136 protected String river; 54 protected String river;
137 protected double from; 55 protected double from;
138 protected double to; 56 protected double to;
139 protected double step; 57 protected double step;
218 136
219 /** 137 /**
220 * Create filter to accept only the chosen events. 138 * Create filter to accept only the chosen events.
221 * This factored out out to be overwritten. 139 * This factored out out to be overwritten.
222 */ 140 */
223 protected Filter createFilter() { 141 protected FixingColumnFilter createFilter() {
224 return new IdsFilter(this.events); 142 return new IdsFilter(this.events);
225 } 143 }
226 144
227 protected List<Column> getEventColumns(final FixingsOverview overview, final ColumnCache cc) { 145 protected List<FixingColumnWithData> getEventColumns(final FixingsOverview overview, final ColumnCache cc) {
228 final Filter filter = createFilter(); 146 final FixingColumnFilter filter = createFilter();
229 147
230 final List<Fixing.Column> metas = overview.filter(null, filter); 148 final List<FixingColumn> metas = overview.filter(null, filter);
231 149
232 final List<Column> columns = new ArrayList<>(metas.size()); 150 final List<FixingColumnWithData> columns = new ArrayList<>(metas.size());
233 151
234 for (final Fixing.Column meta : metas) { 152 for (final FixingColumn meta : metas) {
235 153
236 final Column data = cc.getColumn(meta); 154 final FixingColumnWithData data = cc.getColumn(meta);
237 if (data == null) { 155 if (data == null) {
238 addProblem("fix.cannot.load.data"); 156 addProblem("fix.cannot.load.data");
239 } else { 157 } else {
240 columns.add(data); 158 columns.add(data);
241 } 159 }
243 161
244 return columns; 162 return columns;
245 } 163 }
246 164
247 // Fit a function to the given points from fixation. 165 // Fit a function to the given points from fixation.
248 protected FitResult doFitting(final FixingsOverview overview, final ColumnCache cc, final Function func) { 166 protected final FitResult doFitting(final FixingsOverview overview, final ColumnCache cc, final Function func) {
249 final boolean debug = log.isDebugEnabled(); 167 final boolean debug = log.isDebugEnabled();
250 168
251 final List<Column> eventColumns = getEventColumns(overview, cc); 169 final FixResultColumns resultColumns = new FixResultColumns();
170
171 final List<FixingColumnWithData> eventColumns = getEventColumns(overview, cc);
252 172
253 if (eventColumns.size() < 2) { 173 if (eventColumns.size() < 2) {
254 addProblem("fix.too.less.data.columns"); 174 addProblem("fix.too.less.data.columns");
255 return null; 175 return null;
256 } 176 }
257 177
258 final double[] qs = new double[eventColumns.size()];
259 final double[] ws = new double[qs.length];
260 final boolean[] interpolated = new boolean[ws.length];
261
262 final Fitting.QWDFactory qwdFactory = new Fitting.QWDFactory() {
263 @Override
264 public QWD create(final double q, final double w, double deltaW, boolean isOutlier) {
265 // Check all the event columns for close match
266 // and take the description and the date from meta.
267 for (int i = 0; i < qs.length; ++i) {
268 if (Math.abs(qs[i] - q) < EPSILON && Math.abs(ws[i] - w) < EPSILON) {
269 final Column column = eventColumns.get(i);
270 return new QWD(qs[i], ws[i], column.getDescription(), column.getDate(), interpolated[i], deltaW, column.getId(), isOutlier); // Use database id here
271 }
272 }
273 log.warn("cannot find column for (" + q + ", " + w + ")");
274 return new QWD(q, w, isOutlier);
275 }
276 };
277
278 final String[] parameterNames = func.getParameterNames(); 178 final String[] parameterNames = func.getParameterNames();
279 179
280 final Parameters results = new Parameters(StringUtils.join(STANDARD_COLUMNS, parameterNames)); 180 final Parameters results = new Parameters(StringUtils.join(STANDARD_COLUMNS, parameterNames));
281 181
282 boolean invalid = false; 182 boolean invalid = false;
284 final double[] kms = DoubleUtil.explode(this.from, this.to, this.step / 1000.0); 184 final double[] kms = DoubleUtil.explode(this.from, this.to, this.step / 1000.0);
285 185
286 if (debug) { 186 if (debug) {
287 log.debug("number of kms: " + kms.length); 187 log.debug("number of kms: " + kms.length);
288 } 188 }
289
290 final KMIndex<QWD[]> fixings = new KMIndex<>();
291 189
292 final int kmIndex = results.columnIndex("km"); 190 final int kmIndex = results.columnIndex("km");
293 final int chiSqrIndex = results.columnIndex("chi_sqr"); 191 final int chiSqrIndex = results.columnIndex("chi_sqr");
294 final int maxQIndex = results.columnIndex("max_q"); 192 final int maxQIndex = results.columnIndex("max_q");
295 final int stdDevIndex = results.columnIndex("std-dev"); 193 final int stdDevIndex = results.columnIndex("std-dev");
297 195
298 int numFailed = 0; 196 int numFailed = 0;
299 197
300 for (final double km : kms) { 198 for (final double km : kms) {
301 199
302 // Fill Qs and Ws from event columns. 200 final Fitting fitting = Fitting.fit(resultColumns, km, func, this.preprocessing, eventColumns);
303 for (int j = 0; j < ws.length; ++j)
304 interpolated[j] = !eventColumns.get(j).getQW(km, qs, ws, j);
305
306 final Fitting fitting = Fitting.fit(func, qwdFactory, this.preprocessing, qs, ws);
307 if (fitting == null) { 201 if (fitting == null) {
308 log.debug("Fitting for km: " + km + " failed"); 202 log.debug("Fitting for km: " + km + " failed");
309 ++numFailed; 203 ++numFailed;
310 addProblem(km, "fix.fitting.failed"); 204 addProblem(km, "fix.fitting.failed");
311 continue; 205 continue;
312 } 206 }
313
314 final QWD[] fixingsArray = fitting.getFixingsArray();
315 fixings.add(km, fixingsArray);
316 207
317 final int row = results.newRow(); 208 final int row = results.newRow();
318 final double[] values = fitting.getParameters(); 209 final double[] values = fitting.getParameters();
319 210
320 results.set(row, kmIndex, km); 211 results.set(row, kmIndex, km);
336 if (invalid) { 227 if (invalid) {
337 addProblem("fix.invalid.values"); 228 addProblem("fix.invalid.values");
338 results.removeNaNs(); 229 results.removeNaNs();
339 } 230 }
340 231
341 fixings.sort(); 232 resultColumns.sortAll();
342 233
343 return new FitResult(results, fixings); 234 return new FitResult(results, resultColumns);
344 } 235 }
345 236
346 public CalculationResult calculate() { 237 public CalculationResult calculate() {
347 final FixingsOverview overview = FixingsOverviewFactory.getOverview(this.river); 238 final FixingsOverview overview = FixingsOverviewFactory.getOverview(this.river);
348 239
354 245
355 if (func == null) { 246 if (func == null) {
356 addProblem("fix.invalid.function.name"); 247 addProblem("fix.invalid.function.name");
357 } 248 }
358 249
359 if (hasProblems()) { 250 if (hasProblems())
360 return new CalculationResult(this); 251 return new CalculationResult(this);
361 } 252
362 final CalculationResult result = innerCalculate(overview, func); 253 return innerCalculate(overview, func);
363
364 if (result != null) {
365 // Workaraound to deal with same dates in data set
366 final Object o = result.getData();
367 if (o instanceof FixResult) {
368 final FixResult fr = (FixResult) o;
369 fr.makeEventsDatesUnique();
370 fr.remapEventIndicesToRank();
371 }
372 }
373
374 return result;
375 } 254 }
376 255
377 protected abstract CalculationResult innerCalculate(FixingsOverview overview, Function function); 256 protected abstract CalculationResult innerCalculate(FixingsOverview overview, Function function);
378 } 257 }
http://dive4elements.wald.intevation.org