Mercurial > dive4elements > river

annotate backend/src/main/java/org/dive4elements/river/importer/ImportFlowVelocityModel.java @ 9650:a2a42a6bac6b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Importer (s/u-info) extensions: outer try/catch for parse and log of line no, catching parsing exception if not enough value fields, parsing error and warning log messages with line number, detecting and rejecting duplicate data series, better differentiation between error and warning log messages
author mschaefer
date Mon, 23 Mar 2020 14:57:03 +0100
parents 392bbcd8a88b
children
rev   line source
5844
4dd33b86dc61 Added header to river backend.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5838
diff changeset
1 /* Copyright (C) 2011, 2012, 2013 by Bundesanstalt für Gewässerkunde
4dd33b86dc61 Added header to river backend.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5838
diff changeset
2 * Software engineering by Intevation GmbH
4dd33b86dc61 Added header to river backend.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5838
diff changeset
3 *
5992
4c3ccf2b0304 Removed trailing whitespace.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5844
diff changeset
4 * This file is Free Software under the GNU AGPL (>=v3)
5844
4dd33b86dc61 Added header to river backend.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5838
diff changeset
5 * and comes with ABSOLUTELY NO WARRANTY! Check out the
5992
4c3ccf2b0304 Removed trailing whitespace.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5844
diff changeset
6 * documentation coming with Dive4Elements River for details.
5844
4dd33b86dc61 Added header to river backend.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5838
diff changeset
7 */
4dd33b86dc61 Added header to river backend.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5838
diff changeset
8
5829
18619c1e7c2a Repaired internal references.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5828
diff changeset
9 package org.dive4elements.river.importer;
2827
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
10
2828
ac13e466a55e Added a parser for flow velocity model data and adjusted the db relation schema (missing q column).
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2827
diff changeset
11 import java.util.ArrayList;
2827
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
12 import java.util.List;
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
13
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
14 import org.apache.log4j.Logger;
8986
392bbcd8a88b Database inserts accelerated by suppressing unnecessary database queries for new data series
mschaefer
parents: 5992
diff changeset
15 import org.dive4elements.river.importer.common.StoreMode;
5829
18619c1e7c2a Repaired internal references.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5828
diff changeset
16 import org.dive4elements.river.model.DischargeZone;
18619c1e7c2a Repaired internal references.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5828
diff changeset
17 import org.dive4elements.river.model.FlowVelocityModel;
18619c1e7c2a Repaired internal references.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5828
diff changeset
18 import org.dive4elements.river.model.River;
8986
392bbcd8a88b Database inserts accelerated by suppressing unnecessary database queries for new data series
mschaefer
parents: 5992
diff changeset
19 import org.hibernate.Query;
392bbcd8a88b Database inserts accelerated by suppressing unnecessary database queries for new data series
mschaefer
parents: 5992
diff changeset
20 import org.hibernate.Session;
2827
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
21
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
22
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
23 public class ImportFlowVelocityModel {
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
24
3943
a5b003595d6c Store minfo values into database only if their peer has been successfully stored.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2833
diff changeset
25 private static final Logger log = Logger
8986
392bbcd8a88b Database inserts accelerated by suppressing unnecessary database queries for new data series
mschaefer
parents: 5992
diff changeset
26 .getLogger(ImportFlowVelocityModel.class);
2827
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
27
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
28 private String description;
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
29
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
30 private ImportDischargeZone dischargeZone;
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
31
8986
392bbcd8a88b Database inserts accelerated by suppressing unnecessary database queries for new data series
mschaefer
parents: 5992
diff changeset
32 private final List<ImportFlowVelocityModelValue> values;
392bbcd8a88b Database inserts accelerated by suppressing unnecessary database queries for new data series
mschaefer
parents: 5992
diff changeset
33
392bbcd8a88b Database inserts accelerated by suppressing unnecessary database queries for new data series
mschaefer
parents: 5992
diff changeset
34 protected StoreMode storeMode;
2827
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
35
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
36 private FlowVelocityModel peer;
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
37
2828
ac13e466a55e Added a parser for flow velocity model data and adjusted the db relation schema (missing q column).
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2827
diff changeset
38 public ImportFlowVelocityModel() {
8986
392bbcd8a88b Database inserts accelerated by suppressing unnecessary database queries for new data series
mschaefer
parents: 5992
diff changeset
39 this.values = new ArrayList<>();
392bbcd8a88b Database inserts accelerated by suppressing unnecessary database queries for new data series
mschaefer
parents: 5992
diff changeset
40 this.storeMode = StoreMode.NONE;
2828
ac13e466a55e Added a parser for flow velocity model data and adjusted the db relation schema (missing q column).
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2827
diff changeset
41 }
ac13e466a55e Added a parser for flow velocity model data and adjusted the db relation schema (missing q column).
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2827
diff changeset
42
8986
392bbcd8a88b Database inserts accelerated by suppressing unnecessary database queries for new data series
mschaefer
parents: 5992
diff changeset
43 public ImportFlowVelocityModel(final String description) {
5689
a3f7da9bf0d1 Set description when importing flow velocity models
Tom Gottfried <tom.gottfried@intevation.de>
parents: 5446
diff changeset
44 this();
a3f7da9bf0d1 Set description when importing flow velocity models
Tom Gottfried <tom.gottfried@intevation.de>
parents: 5446
diff changeset
45 this.description = description;
a3f7da9bf0d1 Set description when importing flow velocity models
Tom Gottfried <tom.gottfried@intevation.de>
parents: 5446
diff changeset
46 }
a3f7da9bf0d1 Set description when importing flow velocity models
Tom Gottfried <tom.gottfried@intevation.de>
parents: 5446
diff changeset
47
8986
392bbcd8a88b Database inserts accelerated by suppressing unnecessary database queries for new data series
mschaefer
parents: 5992
diff changeset
48 public ImportFlowVelocityModel(final ImportDischargeZone dischargeZone,
392bbcd8a88b Database inserts accelerated by suppressing unnecessary database queries for new data series
mschaefer
parents: 5992
diff changeset
49 final String description) {
2828
ac13e466a55e Added a parser for flow velocity model data and adjusted the db relation schema (missing q column).
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2827
diff changeset
50 this();
2827
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
51 this.dischargeZone = dischargeZone;
3943
a5b003595d6c Store minfo values into database only if their peer has been successfully stored.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2833
diff changeset
52 this.description = description;
2827
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
53 }
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
54
8986
392bbcd8a88b Database inserts accelerated by suppressing unnecessary database queries for new data series
mschaefer
parents: 5992
diff changeset
55 public void setDischargeZone(final ImportDischargeZone dischargeZone) {
2828
ac13e466a55e Added a parser for flow velocity model data and adjusted the db relation schema (missing q column).
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2827
diff changeset
56 this.dischargeZone = dischargeZone;
ac13e466a55e Added a parser for flow velocity model data and adjusted the db relation schema (missing q column).
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2827
diff changeset
57 }
ac13e466a55e Added a parser for flow velocity model data and adjusted the db relation schema (missing q column).
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2827
diff changeset
58
8986
392bbcd8a88b Database inserts accelerated by suppressing unnecessary database queries for new data series
mschaefer
parents: 5992
diff changeset
59 public void setDescription(final String description) {
2828
ac13e466a55e Added a parser for flow velocity model data and adjusted the db relation schema (missing q column).
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2827
diff changeset
60 this.description = description;
ac13e466a55e Added a parser for flow velocity model data and adjusted the db relation schema (missing q column).
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2827
diff changeset
61 }
ac13e466a55e Added a parser for flow velocity model data and adjusted the db relation schema (missing q column).
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2827
diff changeset
62
8986
392bbcd8a88b Database inserts accelerated by suppressing unnecessary database queries for new data series
mschaefer
parents: 5992
diff changeset
63 public void addValue(final ImportFlowVelocityModelValue value) {
2828
ac13e466a55e Added a parser for flow velocity model data and adjusted the db relation schema (missing q column).
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2827
diff changeset
64 this.values.add(value);
ac13e466a55e Added a parser for flow velocity model data and adjusted the db relation schema (missing q column).
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2827
diff changeset
65 }
ac13e466a55e Added a parser for flow velocity model data and adjusted the db relation schema (missing q column).
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2827
diff changeset
66
8986
392bbcd8a88b Database inserts accelerated by suppressing unnecessary database queries for new data series
mschaefer
parents: 5992
diff changeset
67 public void storeDependencies(final River river) {
2827
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
68 log.debug("store dependencies");
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
69
8986
392bbcd8a88b Database inserts accelerated by suppressing unnecessary database queries for new data series
mschaefer
parents: 5992
diff changeset
70 if (this.dischargeZone == null) {
2828
ac13e466a55e Added a parser for flow velocity model data and adjusted the db relation schema (missing q column).
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2827
diff changeset
71 log.warn("skip flow velocity model: No discharge zone specified.");
ac13e466a55e Added a parser for flow velocity model data and adjusted the db relation schema (missing q column).
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2827
diff changeset
72 return;
ac13e466a55e Added a parser for flow velocity model data and adjusted the db relation schema (missing q column).
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2827
diff changeset
73 }
ac13e466a55e Added a parser for flow velocity model data and adjusted the db relation schema (missing q column).
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2827
diff changeset
74
8986
392bbcd8a88b Database inserts accelerated by suppressing unnecessary database queries for new data series
mschaefer
parents: 5992
diff changeset
75 this.dischargeZone.storeDependencies(river);
2827
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
76
8986
392bbcd8a88b Database inserts accelerated by suppressing unnecessary database queries for new data series
mschaefer
parents: 5992
diff changeset
77 final FlowVelocityModel peer = getPeer(river);
2827
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
78
3943
a5b003595d6c Store minfo values into database only if their peer has been successfully stored.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2833
diff changeset
79 if (peer != null) {
a5b003595d6c Store minfo values into database only if their peer has been successfully stored.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2833
diff changeset
80 int i = 0;
2833
5b54a648f702 Finished flow velocity data import: finished parsing meta data of model files and repaired broken HQL statements.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2832
diff changeset
81
8986
392bbcd8a88b Database inserts accelerated by suppressing unnecessary database queries for new data series
mschaefer
parents: 5992
diff changeset
82 for (final ImportFlowVelocityModelValue value : this.values) {
392bbcd8a88b Database inserts accelerated by suppressing unnecessary database queries for new data series
mschaefer
parents: 5992
diff changeset
83 value.storeDependencies(peer, this.storeMode);
3943
a5b003595d6c Store minfo values into database only if their peer has been successfully stored.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2833
diff changeset
84 i++;
a5b003595d6c Store minfo values into database only if their peer has been successfully stored.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2833
diff changeset
85 }
a5b003595d6c Store minfo values into database only if their peer has been successfully stored.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2833
diff changeset
86
a5b003595d6c Store minfo values into database only if their peer has been successfully stored.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2833
diff changeset
87 log.info("stored " + i + " flow velocity model values.");
2827
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
88 }
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
89 }
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
90
8986
392bbcd8a88b Database inserts accelerated by suppressing unnecessary database queries for new data series
mschaefer
parents: 5992
diff changeset
91 public FlowVelocityModel getPeer(final River river) {
392bbcd8a88b Database inserts accelerated by suppressing unnecessary database queries for new data series
mschaefer
parents: 5992
diff changeset
92 if (this.peer == null) {
392bbcd8a88b Database inserts accelerated by suppressing unnecessary database queries for new data series
mschaefer
parents: 5992
diff changeset
93 final Session session = ImporterSession.getInstance()
392bbcd8a88b Database inserts accelerated by suppressing unnecessary database queries for new data series
mschaefer
parents: 5992
diff changeset
94 .getDatabaseSession();
2827
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
95
8986
392bbcd8a88b Database inserts accelerated by suppressing unnecessary database queries for new data series
mschaefer
parents: 5992
diff changeset
96 final DischargeZone zone = this.dischargeZone.getPeer(river);
2827
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
97
8986
392bbcd8a88b Database inserts accelerated by suppressing unnecessary database queries for new data series
mschaefer
parents: 5992
diff changeset
98 final Query query = session.createQuery("from FlowVelocityModel where "
392bbcd8a88b Database inserts accelerated by suppressing unnecessary database queries for new data series
mschaefer
parents: 5992
diff changeset
99 + " dischargeZone=:dischargeZone");
2827
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
100
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
101 query.setParameter("dischargeZone", zone);
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
102
8986
392bbcd8a88b Database inserts accelerated by suppressing unnecessary database queries for new data series
mschaefer
parents: 5992
diff changeset
103 final List<FlowVelocityModel> model = query.list();
2827
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
104
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
105 if (model.isEmpty()) {
8986
392bbcd8a88b Database inserts accelerated by suppressing unnecessary database queries for new data series
mschaefer
parents: 5992
diff changeset
106 this.peer = new FlowVelocityModel(zone, this.description);
392bbcd8a88b Database inserts accelerated by suppressing unnecessary database queries for new data series
mschaefer
parents: 5992
diff changeset
107 session.save(this.peer);
392bbcd8a88b Database inserts accelerated by suppressing unnecessary database queries for new data series
mschaefer
parents: 5992
diff changeset
108 this.storeMode = StoreMode.INSERT;
2827
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
109 }
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
110 else {
8986
392bbcd8a88b Database inserts accelerated by suppressing unnecessary database queries for new data series
mschaefer
parents: 5992
diff changeset
111 this.peer = model.get(0);
392bbcd8a88b Database inserts accelerated by suppressing unnecessary database queries for new data series
mschaefer
parents: 5992
diff changeset
112 this.storeMode = StoreMode.UPDATE;
2827
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
113 }
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
114 }
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
115
8986
392bbcd8a88b Database inserts accelerated by suppressing unnecessary database queries for new data series
mschaefer
parents: 5992
diff changeset
116 return this.peer;
2827
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
117 }
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
118 }
85b25e74594f Added temp classes used during the import process of flow velocity data.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
119 // vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 :
http://dive4elements.wald.intevation.org