Mercurial > dive4elements > river

changeset 9656:31549fdfaf4f

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Importer (s/u-info) extensions: flow-depth: uniform formatting of from-to series names, warning instead of cancelling in case of missing column values, detecting, logging and skipping columns with wrong unit, better counting of inserted/updated values for each column
author mschaefer
date Mon, 23 Mar 2020 15:21:39 +0100
parents 1f57381b3bb5
children a79881a892c9
files backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/FlowDepthColumnSeriesImport.java backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/FlowDepthKmLineImport.java backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/FlowDepthSeriesImport.java backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/FlowDepthColumnParser.java backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/FlowDepthParser.java
diffstat 5 files changed, 60 insertions(+), 34 deletions(-) [+]
line wrap: on
line diff
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/FlowDepthColumnSeriesImport.java	Mon Mar 23 15:16:35 2020 +0100
+++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/FlowDepthColumnSeriesImport.java	Mon Mar 23 15:21:39 2020 +0100
@@ -18,7 +18,7 @@
 import org.dive4elements.river.model.River;
 import org.dive4elements.river.model.sinfo.FlowDepthColumn;
 import org.dive4elements.river.model.sinfo.FlowDepthValue;
-import org.hibernate.SQLQuery;
+import org.hibernate.Query;
 import org.hibernate.Session;
 
 /**
@@ -55,7 +55,11 @@
     public FlowDepthColumnSeriesImport(final String filename, final FlowDepthSeriesImport parent, final String colName, final File relativeFilePath) {
         super(filename);
         this.parent = parent;
-        this.colName = colName;
+        final String[] items = (colName == null) ? new String[] {} : colName.split("\\-");
+        if (items.length == 2)
+            this.colName = items[0].trim() + " - " + items[1].trim();
+        else
+            this.colName = colName;
         this.relativeFilePath = relativeFilePath;
     }
 
@@ -91,20 +95,22 @@
         return log;
     }
 
-    @Override
-    public List<FlowDepthColumn> querySeriesItem(final Session session, final River river) {
-        /*
-         * final Query query = session.createQuery("FROM FlowDepthColumn WHERE (FlowDepth=:parent) AND lower(name)=:colname");
-         * query.setParameter("parent", this.parent.getPeer(river));
-         * query.setParameter("colname", this.colName.toLowerCase());
-         */
-        // FIXME the normal query raises a null pointer exception
-        final SQLQuery query = session.createSQLQuery("SELECT * FROM flow_depth_column WHERE (flow_depth_id=:parent) AND (lower(name)=:colname)");
-        query.setParameter("parent", this.parent.getPeer(river).getId());
+    private List<FlowDepthColumn> querySeriesItem(final Session session, final River river) {
+        final Query query = session.createQuery("FROM FlowDepthColumn WHERE flowDepth=:parent AND lower(name)=:colname");
+        query.setParameter("parent", this.parent.getPeer(river));
         query.setParameter("colname", this.colName.toLowerCase());
         return query.list();
     }
 
+    @Override
+    public List<FlowDepthColumn> querySeriesItem(final Session session, final River river, final boolean doQueryParent) {
+        final Query query = session.createQuery("FROM FlowDepthColumn c INNER JOIN c.flowDepth s"
+                + " WHERE s.river=:river AND lower(s.filename)=:filename AND lower(c.name)=:colname");
+        query.setParameter("river", river);
+        query.setParameter("filename", this.filename.toLowerCase());
+        query.setParameter("colname", this.colName.toLowerCase());
+        return query.list();
+    }
 
     @Override
     public FlowDepthColumn createSeriesItem(final River river) {
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/FlowDepthKmLineImport.java	Mon Mar 23 15:16:35 2020 +0100
+++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/FlowDepthKmLineImport.java	Mon Mar 23 15:21:39 2020 +0100
@@ -46,7 +46,7 @@
 
     @Override
     protected FlowDepthValue queryValueItem(final Session session, final FlowDepthColumn parent) {
-        final Query query = session.createQuery("FROM FlowDepthValue WHERE (FlowDepthColumn=:parent)"
+        final Query query = session.createQuery("FROM FlowDepthValue WHERE (flowDepthColumn=:parent)"
                 + " AND (station BETWEEN (:station-0.0001) AND (:station+0.0001))");
         query.setParameter("parent", parent);
         query.setParameter("station", this.station);
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/FlowDepthSeriesImport.java	Mon Mar 23 15:16:35 2020 +0100
+++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/FlowDepthSeriesImport.java	Mon Mar 23 15:21:39 2020 +0100
@@ -132,7 +132,7 @@
         return this.peer;
     }
 
-    private List<FlowDepth> querySeriesItem(final Session session, final River river) {
+    public List<FlowDepth> querySeriesItem(final Session session, final River river) {
         final Query query = session.createQuery("FROM FlowDepth WHERE river=:river AND lower(filename)=:filename");
         query.setParameter("river", river);
         query.setParameter("filename", this.filename.toLowerCase());
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/FlowDepthColumnParser.java	Mon Mar 23 15:16:35 2020 +0100
+++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/FlowDepthColumnParser.java	Mon Mar 23 15:21:39 2020 +0100
@@ -44,8 +44,7 @@
     /***** CONSTRUCTORS *****/
 
     public FlowDepthColumnParser(final File importPath, final File rootRelativePath, final ImportRiver river, final FlowDepthSeriesImport parent,
-            final int colIndex,
-            final String colName) {
+            final int colIndex, final String colName) {
         super(importPath, rootRelativePath, river);
         this.parent = parent;
         this.colIndex = colIndex;
@@ -62,7 +61,7 @@
 
     @Override
     protected void logStartInfo() {
-        getLog().info(String.format("Start parsing column %d '%s':;'%s'", this.colIndex, this.colName, this.rootRelativePath));
+        getLog().info(String.format("Start parsing column %d '%s':;'%s'", this.colIndex + 1, this.colName, this.rootRelativePath));
     }
 
     @Override
@@ -78,7 +77,11 @@
 
     @Override
     protected FlowDepthKmLineImport createKmLineImport(final Double km, final String[] values) {
-        final double tkheight = parseDoubleWithNull(values[this.colIndex]).doubleValue();
-        return new FlowDepthKmLineImport(km, tkheight);
+        final Number value = parseDoubleCheckNull(values, this.colIndex);
+        if ((value == null) || Double.isNaN(value.doubleValue())) {
+            logLineWarning("Column %d: " + INVALID_VALUE_ERROR_FORMAT, this.colIndex + 1, "depth");
+            return null;
+        }
+        return new FlowDepthKmLineImport(km, value.doubleValue());
     }
 }
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/FlowDepthParser.java	Mon Mar 23 15:16:35 2020 +0100
+++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/FlowDepthParser.java	Mon Mar 23 15:21:39 2020 +0100
@@ -12,7 +12,6 @@
 
 import java.io.File;
 import java.io.FileInputStream;
-import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.LineNumberReader;
 import java.util.ArrayList;
@@ -23,13 +22,16 @@
 import org.apache.log4j.Logger;
 import org.dive4elements.river.importer.Config;
 import org.dive4elements.river.importer.ImportRiver;
+import org.dive4elements.river.importer.ImporterSession;
 import org.dive4elements.river.importer.common.AbstractParser;
 import org.dive4elements.river.importer.common.ParsingState;
 import org.dive4elements.river.importer.sinfo.importitem.FlowDepthColumnSeriesImport;
 import org.dive4elements.river.importer.sinfo.importitem.FlowDepthKmLineImport;
 import org.dive4elements.river.importer.sinfo.importitem.FlowDepthSeriesImport;
+import org.dive4elements.river.model.sinfo.FlowDepth;
 import org.dive4elements.river.model.sinfo.FlowDepthColumn;
 import org.dive4elements.river.model.sinfo.FlowDepthValue;
+import org.hibernate.Session;
 
 /**
  * Reads and parses the header of a flow depth file and handles the parse and store of the columns
@@ -53,7 +55,7 @@
 
     private static final Pattern COLUMN_TITLE = Pattern.compile("Flie((.)|(ss))tiefe\\s*\\((.+?)\\)\\s*\\[m\\].*", Pattern.CASE_INSENSITIVE);
 
-    private final FlowDepthSeriesImport tkhGroup;
+    private final FlowDepthSeriesImport flowdepthGroup;
 
     private final List<FlowDepthColumnParser> colParsers;
 
@@ -62,8 +64,8 @@
 
     public FlowDepthParser(final File importPath, final File rootRelativePath, final ImportRiver river) {
         super(importPath, rootRelativePath, river);
-        this.tkhGroup = new FlowDepthSeriesImport(importPath.getName().replaceAll("\\.csv", ""));
-        this.seriesHeader = new FlowDepthColumnSeriesImport(this.tkhGroup.getFilename(), this.tkhGroup, null, null);
+        this.flowdepthGroup = new FlowDepthSeriesImport(importPath.getName().replaceAll("\\.csv", ""));
+        this.seriesHeader = new FlowDepthColumnSeriesImport(this.flowdepthGroup.getFilename(), this.flowdepthGroup, null, null);
         this.colParsers = new ArrayList<>();
     }
 
@@ -94,7 +96,7 @@
     }
 
     @Override
-    public void parse() throws IOException {
+    public void parse() throws Exception {
         getLog().info("Start parsing:;'" + this.rootRelativePath + "'");
         // this.seriesHeader = createSeriesImport(this.importPath.getName().replaceAll("\\.csv", ""));
         this.metaPatternsMatched.clear();
@@ -106,7 +108,7 @@
                 this.in = new LineNumberReader(new InputStreamReader(new FileInputStream(this.importPath), ENCODING));
             }
             catch (final Exception e) {
-                logError("Could not open (" + e.getMessage() + ")");
+                logError("Could not open (%s)", e.getMessage());
                 this.headerParsingState = ParsingState.STOP;
             }
             this.currentLine = null;
@@ -118,6 +120,8 @@
                 if (this.currentLine.isEmpty())
                     continue;
                 handleMetaLine();
+                if (this.headerParsingState == ParsingState.DONE)
+                    checkMetaData();
             }
         }
         finally {
@@ -152,7 +156,7 @@
         final Matcher m = META_YEAR.matcher(this.currentLine);
         if (m.matches()) {
             this.metaPatternsMatched.add(META_YEAR);
-            this.tkhGroup.setYear(Integer.parseInt(m.group(1)));
+            this.flowdepthGroup.setYear(Integer.parseInt(m.group(1)));
             return true;
         }
         return false;
@@ -167,7 +171,7 @@
         final Matcher m = META_SOUNDING.matcher(this.currentLine);
         if (m.matches()) {
             this.metaPatternsMatched.add(META_SOUNDING);
-            this.tkhGroup.setSounding_info(parseMetaInfo(m.group(1).trim()));
+            this.flowdepthGroup.setSounding_info(parseMetaInfo(m.group(1).trim()));
             return true;
         }
         return false;
@@ -177,7 +181,7 @@
         final Matcher m = META_EVALUATOR.matcher(this.currentLine);
         if (m.matches()) {
             this.metaPatternsMatched.add(META_EVALUATOR);
-            this.tkhGroup.setEvaluation_by(parseMetaInfo(m.group(1).trim()));
+            this.flowdepthGroup.setEvaluation_by(parseMetaInfo(m.group(1).trim()));
             return true;
         }
         return false;
@@ -187,27 +191,40 @@
     protected boolean handleMetaColumnTitles() {
         if (!super.handleMetaColumnTitles())
             return false;
-        this.tkhGroup.setKmrange_info(this.seriesHeader.getKmrange_info());
-        this.tkhGroup.setNotes(this.seriesHeader.getNotes());
+        this.flowdepthGroup.setKmrange_info(this.seriesHeader.getKmrange_info());
+        this.flowdepthGroup.setNotes(this.seriesHeader.getNotes());
         for (int i = 1; i <= this.columnTitles.size() - 1; i++) {
             final Matcher m = COLUMN_TITLE.matcher(this.columnTitles.get(i));
             if (m.matches())
-                this.colParsers.add(new FlowDepthColumnParser(this.importPath, this.rootRelativePath, this.river, this.tkhGroup, i, m.group(4).trim()));
+                this.colParsers.add(new FlowDepthColumnParser(this.importPath, this.rootRelativePath, this.river, this.flowdepthGroup, i, m.group(4).trim()));
             else
-                logWarning("No title found in column " + i + ", skipped");
+                logLineWarning("Invalid title/unit in column %d (%s)", i + 1, this.columnTitles.get(i));
         }
         return true;
     }
 
+    /**
+     * Checks the existence of the active series in the database
+     */
+    @Override
+    protected boolean checkSeriesExistsAlready() {
+        if (!checkRiverExists())
+            return false;
+        final Session session = ImporterSession.getInstance().getDatabaseSession();
+        final List<FlowDepth> rows = this.flowdepthGroup.querySeriesItem(session, this.river.getPeer());
+        return !rows.isEmpty();
+    }
+
+
     @Override
     public void store() {
         if (this.headerParsingState != ParsingState.STOP) {
-            this.tkhGroup.getPeer(this.river.getPeer());
+            this.flowdepthGroup.getPeer(this.river.getPeer());
             for (final FlowDepthColumnParser colParser : this.colParsers)
                 colParser.store();
         }
         else
-            logWarning("Severe parsing errors, not storing series '" + this.tkhGroup.getFilename() + "'");
+            logWarning("Severe parsing errors, not storing series '%s'", this.flowdepthGroup.getFilename());
     }
 
     @Override
http://dive4elements.wald.intevation.org