Mercurial > dive4elements > river

changeset 9655:1f57381b3bb5

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Importer (s/u-info) extensions: uniform formatting of from-to series names, warning instead of cancelling in case of missing tkh column values, detecting, logging and skipping columns with wrong unit
author mschaefer
date Mon, 23 Mar 2020 15:16:35 +0100
parents 8a2a777a8372
children 31549fdfaf4f
files backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/TkhColumnSeriesImport.java backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/TkhKmLineImport.java backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/TkhSeriesImport.java backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/TkhColumnParser.java backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/TkhParser.java
diffstat 5 files changed, 51 insertions(+), 23 deletions(-) [+]
line wrap: on
line diff
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/TkhColumnSeriesImport.java	Mon Mar 23 15:10:09 2020 +0100
+++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/TkhColumnSeriesImport.java	Mon Mar 23 15:16:35 2020 +0100
@@ -55,7 +55,11 @@
     public TkhColumnSeriesImport(final String filename, final TkhSeriesImport parent, final String colName, final File relativeFilePath) {
         super(filename);
         this.parent = parent;
-        this.colName = colName;
+        final String[] items = (colName == null) ? new String[] {} : colName.split("\\-");
+        if (items.length == 2)
+            this.colName = items[0].trim() + " - " + items[1].trim();
+        else
+            this.colName = colName;
         this.relativeFilePath = relativeFilePath;
     }
 
@@ -91,14 +95,22 @@
         return log;
     }
 
-    @Override
-    public List<TkhColumn> querySeriesItem(final Session session, final River river) {
+    private List<TkhColumn> querySeriesItem(final Session session, final River river) {
         final Query query = session.createQuery("FROM TkhColumn WHERE tkh=:parent AND lower(name)=:colname");
         query.setParameter("parent", this.parent.getPeer(river));
         query.setParameter("colname", this.colName.toLowerCase());
         return query.list();
     }
 
+    @Override
+    public List<TkhColumn> querySeriesItem(final Session session, final River river, final boolean doQueryParent) {
+        final Query query = session.createQuery("FROM TkhColumn c INNER JOIN c.tkh s"
+                + " WHERE s.river=:river AND lower(s.filename)=:filename AND lower(c.name)=:colname");
+        query.setParameter("river", river);
+        query.setParameter("filename", this.filename.toLowerCase());
+        query.setParameter("colname", this.colName.toLowerCase());
+        return query.list();
+    }
 
     @Override
     public TkhColumn createSeriesItem(final River river) {
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/TkhKmLineImport.java	Mon Mar 23 15:10:09 2020 +0100
+++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/TkhKmLineImport.java	Mon Mar 23 15:16:35 2020 +0100
@@ -15,7 +15,7 @@
 import org.dive4elements.river.importer.common.AbstractKmLineImport;
 import org.dive4elements.river.model.sinfo.TkhColumn;
 import org.dive4elements.river.model.sinfo.TkhValue;
-import org.hibernate.SQLQuery;
+import org.hibernate.Query;
 import org.hibernate.Session;
 
 /**
@@ -46,15 +46,10 @@
 
     @Override
     protected TkhValue queryValueItem(final Session session, final TkhColumn parent) {
-        /*
-         * final Query query = session.createQuery("FROM TkhValue WHERE (TkhColumn=:parent)"
-         * + " AND (station BETWEEN (:station-0.0001) AND (:station+0.0001))");
-         * query.setParameter("parent", parent);
-         * query.setParameter("station", this.station);
-         */
-        final SQLQuery query = session.createSQLQuery("SELECT * FROM tkh_values WHERE (tkh_column_id=:parent)"
+        final Query query = session.createQuery("FROM TkhValue WHERE (tkhColumn=:parent)"
                 + " AND (station BETWEEN (:station-0.0001) AND (:station+0.0001))");
-        query.setParameter("parent", parent.getId()).setParameter("station", this.station);
+        query.setParameter("parent", parent);
+        query.setParameter("station", this.station);
         final List rows = query.list();
         if (!rows.isEmpty())
             return (TkhValue) rows.get(0);
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/TkhSeriesImport.java	Mon Mar 23 15:10:09 2020 +0100
+++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/TkhSeriesImport.java	Mon Mar 23 15:16:35 2020 +0100
@@ -144,7 +144,7 @@
         return this.peer;
     }
 
-    private List<Tkh> querySeriesItem(final Session session, final River river) {
+    public List<Tkh> querySeriesItem(final Session session, final River river) {
         final Query query = session.createQuery("FROM Tkh WHERE river=:river AND lower(filename)=:filename");
         query.setParameter("river", river);
         query.setParameter("filename", this.filename.toLowerCase());
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/TkhColumnParser.java	Mon Mar 23 15:10:09 2020 +0100
+++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/TkhColumnParser.java	Mon Mar 23 15:16:35 2020 +0100
@@ -61,7 +61,7 @@
 
     @Override
     protected void logStartInfo() {
-        getLog().info(String.format("Start parsing column %d '%s':;'%s'", this.colIndex, this.colName, this.rootRelativePath));
+        getLog().info(String.format("Start parsing column %d '%s':;'%s'", this.colIndex + 1, this.colName, this.rootRelativePath));
     }
 
     @Override
@@ -77,8 +77,13 @@
 
     @Override
     protected TkhKmLineImport createKmLineImport(final Double km, final String[] values) {
+        final Number value = parseDoubleCheckNull(values, this.colIndex);
+        if ((value == null) || Double.isNaN(value.doubleValue())) {
+            logLineWarning("Column %d: " + INVALID_VALUE_ERROR_FORMAT, this.colIndex + 1, "TKH");
+            return null;
+        }
         // convert from cm to m
-        final double tkheight = parseDoubleWithNull(values[this.colIndex]).doubleValue() / 100.0;
-        return new TkhKmLineImport(km, tkheight);
+        final double tkh = value.doubleValue() / 100.0;
+        return new TkhKmLineImport(km, tkh);
     }
 }
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/TkhParser.java	Mon Mar 23 15:10:09 2020 +0100
+++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/TkhParser.java	Mon Mar 23 15:16:35 2020 +0100
@@ -12,7 +12,6 @@
 
 import java.io.File;
 import java.io.FileInputStream;
-import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.LineNumberReader;
 import java.text.DateFormat;
@@ -26,6 +25,7 @@
 import org.apache.log4j.Logger;
 import org.dive4elements.river.importer.Config;
 import org.dive4elements.river.importer.ImportRiver;
+import org.dive4elements.river.importer.ImporterSession;
 import org.dive4elements.river.importer.common.AbstractParser;
 import org.dive4elements.river.importer.common.ParsingState;
 import org.dive4elements.river.importer.sinfo.importitem.TkhColumnSeriesImport;
@@ -34,6 +34,7 @@
 import org.dive4elements.river.model.sinfo.Tkh;
 import org.dive4elements.river.model.sinfo.TkhColumn;
 import org.dive4elements.river.model.sinfo.TkhValue;
+import org.hibernate.Session;
 
 /**
  * Reads and parses the header of a TKH file and handles the parse and store of the columns
@@ -131,7 +132,7 @@
     }
 
     @Override
-    public void parse() throws IOException {
+    public void parse() throws Exception {
         getLog().info("Start parsing:;'" + this.rootRelativePath + "'");
         // this.seriesHeader = createSeriesImport(this.importPath.getName().replaceAll("\\.csv", ""));
         this.metaPatternsMatched.clear();
@@ -143,7 +144,7 @@
                 this.in = new LineNumberReader(new InputStreamReader(new FileInputStream(this.importPath), ENCODING));
             }
             catch (final Exception e) {
-                logError("Could not open (" + e.getMessage() + ")");
+                logError("Could not open (%s)", e.getMessage());
                 this.headerParsingState = ParsingState.STOP;
             }
             this.currentLine = null;
@@ -155,6 +156,8 @@
                 if (this.currentLine.isEmpty())
                     continue;
                 handleMetaLine();
+                if (this.headerParsingState == ParsingState.DONE)
+                    checkMetaData();
             }
         }
         finally {
@@ -194,12 +197,12 @@
                     day = dateFormat.parse(m.group(1));
             }
             catch (final Exception e) {
-                logError("Invalid date in line " + this.in.getLineNumber());
+                logLineWarning("Invalid date");
             }
             this.metaPatternsMatched.add(META_DATUM);
             this.tkhGroup.setDay(day);
             if (day == null)
-                logWarning("No date specified");
+                logLineWarning("No date specified");
             return true;
         }
         return false;
@@ -241,11 +244,24 @@
             if (m.matches())
                 this.colParsers.add(new TkhColumnParser(this.importPath, this.rootRelativePath, this.river, this.tkhGroup, i, m.group(7).trim()));
             else
-                logWarning("No title found in column " + i + ", skipped");
+                logLineWarning("Invalid title in column %d (%s)", i + 1, this.columnTitles.get(i));
         }
         return true;
     }
 
+    /**
+     * Checks the existence of the active series in the database
+     */
+    @Override
+    protected boolean checkSeriesExistsAlready() {
+        if (!checkRiverExists())
+            return false;
+        final Session session = ImporterSession.getInstance().getDatabaseSession();
+        final List<Tkh> rows = this.tkhGroup.querySeriesItem(session, this.river.getPeer());
+        return !rows.isEmpty();
+    }
+
+
     @Override
     public void store() {
         if (this.headerParsingState != ParsingState.STOP) {
@@ -254,7 +270,7 @@
                 colParser.store();
         }
         else
-            logWarning("Severe parsing errors, not storing series '" + this.tkhGroup.getFilename() + "'");
+            logWarning("Severe parsing errors, not storing series '%s'", this.tkhGroup.getFilename());
     }
 
     @Override
http://dive4elements.wald.intevation.org