Mercurial > dive4elements > river

comparison artifacts/src/main/java/org/dive4elements/river/artifacts/model/fixings/FixCalculation.java @ 9099:850ce16034e9

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
2.3.4.1.10 Berechnung mit Start-km > End-km
author gernotbelger
date Mon, 28 May 2018 13:22:45 +0200
parents 6650485c2c9b
children 202fd59b4f21
comparison
equal deleted inserted replaced
9098:32dd7e761e4e 9099:850ce16034e9
6 * documentation coming with Dive4Elements River for details. 6 * documentation coming with Dive4Elements River for details.
7 */ 7 */
8 8
9 package org.dive4elements.river.artifacts.model.fixings; 9 package org.dive4elements.river.artifacts.model.fixings;
10 10
11 import org.dive4elements.artifacts.common.utils.StringUtils;
12
13 import org.dive4elements.river.artifacts.access.FixAccess;
14
15 import org.dive4elements.river.artifacts.math.fitting.Function;
16 import org.dive4elements.river.artifacts.math.fitting.FunctionFactory;
17
18 import org.dive4elements.river.artifacts.model.Calculation;
19 import org.dive4elements.river.artifacts.model.CalculationResult;
20 import org.dive4elements.river.artifacts.model.FixingsColumn;
21 import org.dive4elements.river.artifacts.model.FixingsColumnFactory;
22
23 import org.dive4elements.river.artifacts.model.FixingsOverview.Fixing.Filter;
24
25 import org.dive4elements.river.artifacts.model.FixingsOverview.Fixing;
26 import org.dive4elements.river.artifacts.model.FixingsOverview.IdsFilter;
27
28 import org.dive4elements.river.artifacts.model.FixingsOverview;
29 import org.dive4elements.river.artifacts.model.FixingsOverviewFactory;
30 import org.dive4elements.river.artifacts.model.Parameters;
31
32 import org.dive4elements.river.utils.DoubleUtil;
33 import org.dive4elements.river.utils.KMIndex;
34
35 import java.util.ArrayList; 11 import java.util.ArrayList;
36 import java.util.Date; 12 import java.util.Date;
37 import java.util.HashMap; 13 import java.util.HashMap;
38 import java.util.List; 14 import java.util.List;
39 import java.util.Map; 15 import java.util.Map;
40 16
41 import org.apache.log4j.Logger; 17 import org.apache.log4j.Logger;
18 import org.dive4elements.artifacts.common.utils.StringUtils;
19 import org.dive4elements.river.artifacts.access.FixAccess;
20 import org.dive4elements.river.artifacts.math.fitting.Function;
21 import org.dive4elements.river.artifacts.math.fitting.FunctionFactory;
22 import org.dive4elements.river.artifacts.model.Calculation;
23 import org.dive4elements.river.artifacts.model.CalculationResult;
24 import org.dive4elements.river.artifacts.model.FixingsColumn;
25 import org.dive4elements.river.artifacts.model.FixingsColumnFactory;
26 import org.dive4elements.river.artifacts.model.FixingsOverview;
27 import org.dive4elements.river.artifacts.model.FixingsOverview.Fixing;
28 import org.dive4elements.river.artifacts.model.FixingsOverview.Fixing.Filter;
29 import org.dive4elements.river.artifacts.model.FixingsOverview.IdsFilter;
30 import org.dive4elements.river.artifacts.model.FixingsOverviewFactory;
31 import org.dive4elements.river.artifacts.model.Parameters;
32 import org.dive4elements.river.utils.DoubleUtil;
33 import org.dive4elements.river.utils.KMIndex;
42 34
43 /** Calculation base class for fix. */ 35 /** Calculation base class for fix. */
44 public abstract class FixCalculation 36 public abstract class FixCalculation extends Calculation {
45 extends Calculation
46 {
47 private static Logger log = Logger.getLogger(FixCalculation.class); 37 private static Logger log = Logger.getLogger(FixCalculation.class);
48 38
49 public static final double EPSILON = 1e-4; 39 public static final double EPSILON = 1e-4;
50 40
51 public static final String [] STANDARD_COLUMNS = { 41 public static final String[] STANDARD_COLUMNS = { "km", "chi_sqr", "max_q", "std-dev" };
52 "km", "chi_sqr", "max_q", "std-dev"
53 };
54 42
55 protected static class FitResult { 43 protected static class FitResult {
56 44
57 protected Parameters parameters; 45 protected Parameters parameters;
58 protected KMIndex<QWD []> referenced; 46 protected KMIndex<QWD[]> referenced;
59 protected KMIndex<QWI []> outliers; 47 protected KMIndex<QWI[]> outliers;
60 48
61 public FitResult() { 49 public FitResult() {
62 } 50 }
63 51
64 public FitResult( 52 public FitResult(final Parameters parameters, final KMIndex<QWD[]> referenced, final KMIndex<QWI[]> outliers) {
65 Parameters parameters,
66 KMIndex<QWD []> referenced,
67 KMIndex<QWI []> outliers
68 ) {
69 this.parameters = parameters; 53 this.parameters = parameters;
70 this.referenced = referenced; 54 this.referenced = referenced;
71 this.outliers = outliers; 55 this.outliers = outliers;
72 } 56 }
73 57
74 public Parameters getParameters() { 58 public Parameters getParameters() {
75 return parameters; 59 return this.parameters;
76 } 60 }
77 61
78 public KMIndex<QWD []> getReferenced() { 62 public KMIndex<QWD[]> getReferenced() {
79 return referenced; 63 return this.referenced;
80 } 64 }
81 65
82 public KMIndex<QWI []> getOutliers() { 66 public KMIndex<QWI[]> getOutliers() {
83 return outliers; 67 return this.outliers;
84 } 68 }
85 } // class FitResult 69 } // class FitResult
86 70
87 /** Helper class to bundle the meta information of a column 71 /**
88 * and the real data. 72 * Helper class to bundle the meta information of a column
73 * and the real data.
89 */ 74 */
90 protected static class Column { 75 protected static class Column {
91 76
92 protected Fixing.Column meta; 77 protected Fixing.Column meta;
93 protected FixingsColumn data; 78 protected FixingsColumn data;
94 protected int index; 79 protected int index;
95 80
96 public Column() { 81 public Column() {
97 } 82 }
98 83
99 public Column(Fixing.Column meta, FixingsColumn data, int index) { 84 public Column(final Fixing.Column meta, final FixingsColumn data, final int index) {
100 this.meta = meta; 85 this.meta = meta;
101 this.data = data; 86 this.data = data;
102 this.index = index; 87 this.index = index;
103 } 88 }
104 89
105 public Date getDate() { 90 public Date getDate() {
106 return meta.getStartTime(); 91 return this.meta.getStartTime();
107 } 92 }
108 93
109 public String getDescription() { 94 public String getDescription() {
110 return meta.getDescription(); 95 return this.meta.getDescription();
111 } 96 }
112 97
113 public int getIndex() { 98 public int getIndex() {
114 return index; 99 return this.index;
115 } 100 }
116 101
117 public int getId() { 102 public int getId() {
118 return meta.getId(); 103 return this.meta.getId();
119 } 104 }
120 105
121 public boolean getQW( 106 public boolean getQW(final double km, final double[] qs, final double[] ws, final int index) {
122 double km, 107 qs[index] = this.data.getQ(km);
123 double [] qs, 108 return this.data.getW(km, ws, index);
124 double [] ws, 109 }
125 int index 110
126 ) { 111 public boolean getQW(final double km, final double[] wq) {
127 qs[index] = data.getQ(km); 112 this.data.getW(km, wq, 0);
128 return data.getW(km, ws, index); 113 if (Double.isNaN(wq[0]))
129 } 114 return false;
130 115 wq[1] = this.data.getQ(km);
131 public boolean getQW(double km, double [] wq) {
132 data.getW(km, wq, 0);
133 if (Double.isNaN(wq[0])) return false;
134 wq[1] = data.getQ(km);
135 return !Double.isNaN(wq[1]); 116 return !Double.isNaN(wq[1]);
136 } 117 }
137 } // class Column 118 } // class Column
138 119
139 /** 120 /**
142 protected static class ColumnCache { 123 protected static class ColumnCache {
143 124
144 protected Map<Integer, Column> columns; 125 protected Map<Integer, Column> columns;
145 126
146 public ColumnCache() { 127 public ColumnCache() {
147 columns = new HashMap<Integer, Column>(); 128 this.columns = new HashMap<>();
148 } 129 }
149 130
150 public Column getColumn(Fixing.Column meta) { 131 public Column getColumn(final Fixing.Column meta) {
151 Integer key = meta.getId(); 132 final Integer key = meta.getId();
152 Column column = columns.get(key); 133 Column column = this.columns.get(key);
153 if (column == null) { 134 if (column == null) {
154 FixingsColumn data = FixingsColumnFactory 135 final FixingsColumn data = FixingsColumnFactory.getInstance().getColumnData(meta);
155 .getInstance()
156 .getColumnData(meta);
157 if (data != null) { 136 if (data != null) {
158 column = new Column(meta, data, columns.size()); 137 column = new Column(meta, data, this.columns.size());
159 columns.put(key, column); 138 this.columns.put(key, column);
160 } 139 }
161 } 140 }
162 return column; 141 return column;
163 } 142 }
164 } // class ColumnCache 143 } // class ColumnCache
165 144
166 145 protected String river;
167 protected String river; 146 protected double from;
168 protected double from; 147 protected double to;
169 protected double to; 148 protected double step;
170 protected double step;
171 protected boolean preprocessing; 149 protected boolean preprocessing;
172 protected String function; 150 protected String function;
173 protected int [] events; 151 protected int[] events;
174 protected int qSectorStart; 152 protected int qSectorStart;
175 protected int qSectorEnd; 153 protected int qSectorEnd;
176 154
177 public FixCalculation() { 155 public FixCalculation() {
178 } 156 }
179 157
180 public FixCalculation(FixAccess access) { 158 public FixCalculation(final FixAccess access) {
181 String river = access.getRiverName(); 159 final String river = access.getRiverName();
182 Double from = access.getFrom(); 160 final Double from = access.getLowerKm();
183 Double to = access.getTo(); 161 final Double to = access.getUpperKm();
184 Double step = access.getStep(); 162 final Double step = access.getStep();
185 String function = access.getFunction(); 163 final String function = access.getFunction();
186 int [] events = access.getEvents(); 164 final int[] events = access.getEvents();
187 Integer qSectorStart = access.getQSectorStart(); 165 final Integer qSectorStart = access.getQSectorStart();
188 Integer qSectorEnd = access.getQSectorEnd(); 166 final Integer qSectorEnd = access.getQSectorEnd();
189 Boolean preprocessing = access.getPreprocessing(); 167 final Boolean preprocessing = access.getPreprocessing();
190 168
191 if (river == null) { 169 if (river == null) {
192 addProblem("fix.missing.river"); 170 addProblem("fix.missing.river");
193 } 171 }
194 172
223 if (preprocessing == null) { 201 if (preprocessing == null) {
224 addProblem("fix.missing.preprocessing"); 202 addProblem("fix.missing.preprocessing");
225 } 203 }
226 204
227 if (!hasProblems()) { 205 if (!hasProblems()) {
228 this.river = river; 206 this.river = river;
229 this.from = from; 207 this.from = from;
230 this.to = to; 208 this.to = to;
231 this.step = step; 209 this.step = step;
232 this.function = function; 210 this.function = function;
233 this.events = events; 211 this.events = events;
234 this.qSectorStart = qSectorStart; 212 this.qSectorStart = qSectorStart;
235 this.qSectorEnd = qSectorEnd; 213 this.qSectorEnd = qSectorEnd;
236 this.preprocessing = preprocessing; 214 this.preprocessing = preprocessing;
237 } 215 }
238 } 216 }
239 217
240 protected static String toString( 218 protected static String toString(final String[] parameterNames, final double[] values) {
241 String [] parameterNames, 219 final StringBuilder sb = new StringBuilder();
242 double [] values
243 ) {
244 StringBuilder sb = new StringBuilder();
245 for (int i = 0; i < parameterNames.length; ++i) { 220 for (int i = 0; i < parameterNames.length; ++i) {
246 if (i > 0) sb.append(", "); 221 if (i > 0)
222 sb.append(", ");
247 sb.append(parameterNames[i]).append(": ").append(values[i]); 223 sb.append(parameterNames[i]).append(": ").append(values[i]);
248 } 224 }
249 return sb.toString(); 225 return sb.toString();
250 } 226 }
251 227
252 228 /**
253 /** Create filter to accept only the chosen events. 229 * Create filter to accept only the chosen events.
254 * This factored out out to be overwritten. 230 * This factored out out to be overwritten.
255 */ 231 */
256 protected Filter createFilter() { 232 protected Filter createFilter() {
257 return new IdsFilter(events); 233 return new IdsFilter(this.events);
258 } 234 }
259 235
260 protected List<Column> getEventColumns( 236 protected List<Column> getEventColumns(final FixingsOverview overview, final ColumnCache cc) {
261 FixingsOverview overview, 237 final FixingsColumnFactory fcf = FixingsColumnFactory.getInstance();
262 ColumnCache cc 238
263 ) { 239 final Filter filter = createFilter();
264 FixingsColumnFactory fcf = FixingsColumnFactory.getInstance(); 240
265 241 final List<Fixing.Column> metas = overview.filter(null, filter);
266 Filter filter = createFilter(); 242
267 243 final List<Column> columns = new ArrayList<>(metas.size());
268 List<Fixing.Column> metas = overview.filter(null, filter); 244
269 245 for (final Fixing.Column meta : metas) {
270 List<Column> columns = new ArrayList<Column>(metas.size()); 246
271 247 final Column data = cc.getColumn(meta);
272 for (Fixing.Column meta: metas) {
273
274 Column data = cc.getColumn(meta);
275 if (data == null) { 248 if (data == null) {
276 addProblem("fix.cannot.load.data"); 249 addProblem("fix.cannot.load.data");
277 } 250 } else {
278 else {
279 columns.add(data); 251 columns.add(data);
280 } 252 }
281 } 253 }
282 254
283 return columns; 255 return columns;
284 } 256 }
285 257
286 // Fit a function to the given points from fixation. 258 // Fit a function to the given points from fixation.
287 protected FitResult doFitting( 259 protected FitResult doFitting(final FixingsOverview overview, final ColumnCache cc, final Function func) {
288 FixingsOverview overview, 260 final boolean debug = log.isDebugEnabled();
289 ColumnCache cc,
290 Function func
291 ) {
292 boolean debug = log.isDebugEnabled();
293 261
294 final List<Column> eventColumns = getEventColumns(overview, cc); 262 final List<Column> eventColumns = getEventColumns(overview, cc);
295 263
296 if (eventColumns.size() < 2) { 264 if (eventColumns.size() < 2) {
297 addProblem("fix.too.less.data.columns"); 265 addProblem("fix.too.less.data.columns");
298 return null; 266 return null;
299 } 267 }
300 268
301 final double [] qs = new double[eventColumns.size()]; 269 final double[] qs = new double[eventColumns.size()];
302 final double [] ws = new double[qs.length]; 270 final double[] ws = new double[qs.length];
303 final boolean [] interpolated = new boolean[ws.length]; 271 final boolean[] interpolated = new boolean[ws.length];
304 272
305 Fitting.QWDFactory qwdFactory = new Fitting.QWDFactory() { 273 final Fitting.QWDFactory qwdFactory = new Fitting.QWDFactory() {
306 @Override 274 @Override
307 public QWD create(double q, double w) { 275 public QWD create(final double q, final double w) {
308 // Check all the event columns for close match 276 // Check all the event columns for close match
309 // and take the description and the date from meta. 277 // and take the description and the date from meta.
310 for (int i = 0; i < qs.length; ++i) { 278 for (int i = 0; i < qs.length; ++i) {
311 if (Math.abs(qs[i]-q) < EPSILON 279 if (Math.abs(qs[i] - q) < EPSILON && Math.abs(ws[i] - w) < EPSILON) {
312 && Math.abs(ws[i]-w) < EPSILON) { 280 final Column column = eventColumns.get(i);
313 Column column = eventColumns.get(i); 281 return new QWD(qs[i], ws[i], column.getDescription(), column.getDate(), interpolated[i], 0d, column.getId()); // Use database id here
314 return new QWD(
315 qs[i], ws[i],
316 column.getDescription(),
317 column.getDate(),
318 interpolated[i],
319 0d,
320 column.getId()); // Use database id here
321 } 282 }
322 } 283 }
323 log.warn("cannot find column for (" + q + ", " + w + ")"); 284 log.warn("cannot find column for (" + q + ", " + w + ")");
324 return new QWD(q, w); 285 return new QWD(q, w);
325 } 286 }
326 }; 287 };
327 288
328 Fitting fitting = new Fitting(func, qwdFactory, preprocessing); 289 final Fitting fitting = new Fitting(func, qwdFactory, this.preprocessing);
329 290
330 String [] parameterNames = func.getParameterNames(); 291 final String[] parameterNames = func.getParameterNames();
331 292
332 Parameters results = 293 final Parameters results = new Parameters(StringUtils.join(STANDARD_COLUMNS, parameterNames));
333 new Parameters(
334 StringUtils.join(STANDARD_COLUMNS, parameterNames));
335 294
336 boolean invalid = false; 295 boolean invalid = false;
337 296
338 double [] kms = DoubleUtil.explode(from, to, step / 1000.0); 297 final double[] kms = DoubleUtil.explode(this.from, this.to, this.step / 1000.0);
339 298
340 if (debug) { 299 if (debug) {
341 log.debug("number of kms: " + kms.length); 300 log.debug("number of kms: " + kms.length);
342 } 301 }
343 302
344 KMIndex<QWI []> outliers = new KMIndex<QWI []>(); 303 final KMIndex<QWI[]> outliers = new KMIndex<>();
345 KMIndex<QWD []> referenced = new KMIndex<QWD []>(kms.length); 304 final KMIndex<QWD[]> referenced = new KMIndex<>(kms.length);
346 305
347 int kmIndex = results.columnIndex("km"); 306 final int kmIndex = results.columnIndex("km");
348 int chiSqrIndex = results.columnIndex("chi_sqr"); 307 final int chiSqrIndex = results.columnIndex("chi_sqr");
349 int maxQIndex = results.columnIndex("max_q"); 308 final int maxQIndex = results.columnIndex("max_q");
350 int stdDevIndex = results.columnIndex("std-dev"); 309 final int stdDevIndex = results.columnIndex("std-dev");
351 int [] parameterIndices = results.columnIndices(parameterNames); 310 final int[] parameterIndices = results.columnIndices(parameterNames);
352 311
353 int numFailed = 0; 312 int numFailed = 0;
354 313
355 for (int i = 0; i < kms.length; ++i) { 314 for (final double km2 : kms) {
356 double km = kms[i]; 315 final double km = km2;
357 316
358 // Fill Qs and Ws from event columns. 317 // Fill Qs and Ws from event columns.
359 for (int j = 0; j < ws.length; ++j) { 318 for (int j = 0; j < ws.length; ++j) {
360 interpolated[j] = !eventColumns.get(j).getQW(km, qs, ws, j); 319 interpolated[j] = !eventColumns.get(j).getQW(km, qs, ws, j);
361 } 320 }
367 ++numFailed; 326 ++numFailed;
368 addProblem(km, "fix.fitting.failed"); 327 addProblem(km, "fix.fitting.failed");
369 continue; 328 continue;
370 } 329 }
371 330
372 QWD [] refs = fitting.referencedToArray(); 331 final QWD[] refs = fitting.referencedToArray();
373 332
374 referenced.add(km, refs); 333 referenced.add(km, refs);
375 334
376 if (fitting.hasOutliers()) { 335 if (fitting.hasOutliers()) {
377 outliers.add(km, fitting.outliersToArray()); 336 outliers.add(km, fitting.outliersToArray());
378 } 337 }
379 338
380 int row = results.newRow(); 339 final int row = results.newRow();
381 double [] values = fitting.getParameters(); 340 final double[] values = fitting.getParameters();
382 341
383 results.set(row, kmIndex, km); 342 results.set(row, kmIndex, km);
384 results.set(row, chiSqrIndex, fitting.getChiSquare()); 343 results.set(row, chiSqrIndex, fitting.getChiSquare());
385 results.set(row, stdDevIndex, fitting.getStandardDeviation()); 344 results.set(row, stdDevIndex, fitting.getStandardDeviation());
386 results.set(row, maxQIndex, fitting.getMaxQ()); 345 results.set(row, maxQIndex, fitting.getMaxQ());
387 invalid |= results.set(row, parameterIndices, values); 346 invalid |= results.set(row, parameterIndices, values);
388 347
389 if (debug) { 348 if (debug) {
390 log.debug("km: "+km+" " + toString(parameterNames, values)); 349 log.debug("km: " + km + " " + toString(parameterNames, values));
391 } 350 }
392 } 351 }
393 352
394 if (debug) { 353 if (debug) {
395 log.debug("success: " + (kms.length - numFailed)); 354 log.debug("success: " + (kms.length - numFailed));
402 } 361 }
403 362
404 outliers.sort(); 363 outliers.sort();
405 referenced.sort(); 364 referenced.sort();
406 365
407 return new FitResult( 366 return new FitResult(results, referenced, outliers);
408 results,
409 referenced,
410 outliers);
411 } 367 }
412 368
413 public CalculationResult calculate() { 369 public CalculationResult calculate() {
414 FixingsOverview overview = 370 final FixingsOverview overview = FixingsOverviewFactory.getOverview(this.river);
415 FixingsOverviewFactory.getOverview(river);
416 371
417 if (overview == null) { 372 if (overview == null) {
418 addProblem("fix.no.overview.available"); 373 addProblem("fix.no.overview.available");
419 } 374 }
420 375
421 Function func = FunctionFactory.getInstance() 376 final Function func = FunctionFactory.getInstance().getFunction(this.function);
422 .getFunction(function);
423 377
424 if (func == null) { 378 if (func == null) {
425 addProblem("fix.invalid.function.name"); 379 addProblem("fix.invalid.function.name");
426 } 380 }
427 381
428 if (hasProblems()) { 382 if (hasProblems()) {
429 return new CalculationResult(this); 383 return new CalculationResult(this);
430 } 384 }
431 CalculationResult result = innerCalculate(overview, func); 385 final CalculationResult result = innerCalculate(overview, func);
432 386
433 if (result != null) { 387 if (result != null) {
434 // Workaraound to deal with same dates in data set 388 // Workaraound to deal with same dates in data set
435 Object o = result.getData(); 389 final Object o = result.getData();
436 if (o instanceof FixResult) { 390 if (o instanceof FixResult) {
437 FixResult fr = (FixResult)o; 391 final FixResult fr = (FixResult) o;
438 fr.makeReferenceEventsDatesUnique(); 392 fr.makeReferenceEventsDatesUnique();
439 fr.remapReferenceIndicesToRank(); 393 fr.remapReferenceIndicesToRank();
440 } 394 }
441 } 395 }
442 396
443 return result; 397 return result;
444 } 398 }
445 399
446 protected abstract CalculationResult innerCalculate( 400 protected abstract CalculationResult innerCalculate(FixingsOverview overview, Function function);
447 FixingsOverview overview,
448 Function function
449 );
450 } 401 }
451 // vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 : 402 // vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 :
http://dive4elements.wald.intevation.org