changeset 1204:22858e7cca79

Integrated PRF parsing into importer. Needs testing! flys-backend/trunk@2309 c6561f87-3c4e-4783-a992-168aeb5c3f6f
author Sascha L. Teichmann <sascha.teichmann@intevation.de>
date Thu, 07 Jul 2011 22:25:38 +0000
parents 3c01bef43a98
children 5f1506fc7636
files flys-backend/ChangeLog flys-backend/src/main/java/de/intevation/flys/importer/ImportCrossSection.java flys-backend/src/main/java/de/intevation/flys/importer/ImportCrossSectionLine.java flys-backend/src/main/java/de/intevation/flys/importer/ImportRiver.java flys-backend/src/main/java/de/intevation/flys/importer/ImportTimeInterval.java flys-backend/src/main/java/de/intevation/flys/importer/PRFParser.java flys-backend/src/main/java/de/intevation/flys/importer/XY.java flys-backend/src/main/java/de/intevation/flys/model/CrossSection.java flys-backend/src/main/java/de/intevation/flys/model/CrossSectionLine.java flys-backend/src/main/java/de/intevation/flys/model/CrossSectionPoint.java
diffstat 10 files changed, 406 insertions(+), 63 deletions(-) [+]
line wrap: on
line diff
--- a/flys-backend/ChangeLog	Thu Jul 07 15:59:24 2011 +0000
+++ b/flys-backend/ChangeLog	Thu Jul 07 22:25:38 2011 +0000
@@ -1,3 +1,27 @@
+2011-07-08	Sascha L. Teichmann	<sascha.teichmann@intevation.de>
+
+	Parse all PRFs in all subfolders of a river and store them
+	as cross sections into the database. Needs testing!
+
+	* src/main/java/de/intevation/flys/importer/ImportCrossSection.java,
+	  src/main/java/de/intevation/flys/importer/ImportCrossSectionLine.java:
+	  New. Importer models for cross sections.
+
+	* src/main/java/de/intevation/flys/importer/XY.java:
+	  New. Made top level class from inner PRFParser.XY.
+
+	* src/main/java/de/intevation/flys/importer/PRFParser.java:
+	  Moved out XY class. Renamed callback.
+
+	* src/main/java/de/intevation/flys/model/CrossSection.java,
+	  src/main/java/de/intevation/flys/model/CrossSectionLine.java,
+	  src/main/java/de/intevation/flys/model/CrossSectionPoint.java,
+	  src/main/java/de/intevation/flys/importer/ImportTimeInterval.java:
+	  Added convinience constructors.
+
+	* src/main/java/de/intevation/flys/importer/ImportRiver.java:
+	  Parse and store cross sections into database.
+
 2011-07-07	Sascha L. Teichmann	<sascha.teichmann@intevation.de>
 
 	* doc/schema/postgresql.sql: Introduced a new table cross_section_line
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImportCrossSection.java	Thu Jul 07 22:25:38 2011 +0000
@@ -0,0 +1,109 @@
+package de.intevation.flys.importer;
+
+import de.intevation.flys.model.River;
+import de.intevation.flys.model.CrossSection;
+import de.intevation.flys.model.TimeInterval;
+
+import org.hibernate.Session;
+import org.hibernate.Query;
+
+import java.util.List;
+
+public class ImportCrossSection
+{
+    protected ImportRiver                  river;
+    protected String                       description;
+    protected ImportTimeInterval           timeInterval;
+    protected List<ImportCrossSectionLine> lines;
+
+    protected CrossSection peer;
+
+    public ImportCrossSection() {
+    }
+
+    public ImportCrossSection(
+        ImportRiver                  river,
+        String                       description,
+        ImportTimeInterval           timeInterval,
+        List<ImportCrossSectionLine> lines
+    ) {
+        this.river        = river;
+        this.description  = description;
+        this.timeInterval = timeInterval;
+        this.lines        = lines;
+        wireWithLines();
+    }
+
+    public void wireWithLines() {
+        for (ImportCrossSectionLine line: lines) {
+            line.setCrossSection(this);
+        }
+    }
+
+    public ImportRiver getRiver() {
+        return river;
+    }
+
+    public void setRiver(ImportRiver river) {
+        this.river = river;
+    }
+
+    public String getDescription() {
+        return description;
+    }
+
+    public void setDescription(String description) {
+        this.description = description;
+    }
+
+    public ImportTimeInterval getTimeInterval() {
+        return timeInterval;
+    }
+
+    public void setTimeInterval(ImportTimeInterval timeInterval) {
+        this.timeInterval = timeInterval;
+    }
+
+    public void storeDependencies() {
+
+        getPeer();
+
+        for (ImportCrossSectionLine line: lines) {
+            line.storeDependencies();
+        }
+    }
+
+    public CrossSection getPeer() {
+
+        if (peer == null) {
+            River r = river.getPeer();
+            TimeInterval t = timeInterval != null
+                ? timeInterval.getPeer()
+                : null;
+
+            Session session =
+                ImporterSession.getInstance().getDatabaseSession();
+
+            Query query = session.createQuery(
+                "from CrossSection where " +
+                "river=:r and "            +
+                "timeInterval=:t and "     +
+                "description=:d");
+
+            query.setParameter("r", r);
+            query.setParameter("t", t);
+            query.setParameter("d", description);
+
+            List<CrossSection> crossSections = query.list();
+            if (crossSections.isEmpty()) {
+                peer = new CrossSection(r, t, description);
+                session.save(peer);
+            }
+            else {
+                peer = crossSections.get(0);
+            }
+        }
+        return peer;
+    }
+}
+// vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 :
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImportCrossSectionLine.java	Thu Jul 07 22:25:38 2011 +0000
@@ -0,0 +1,117 @@
+package de.intevation.flys.importer;
+
+import de.intevation.flys.model.CrossSection;
+import de.intevation.flys.model.CrossSectionPoint;
+import de.intevation.flys.model.CrossSectionLine;
+
+import org.hibernate.Session;
+import org.hibernate.Query;
+
+import java.math.BigDecimal;
+
+import java.util.List;
+import java.util.Comparator;
+import java.util.Map;
+import java.util.TreeMap;
+
+public class ImportCrossSectionLine
+{
+    public static final Comparator<CrossSectionPoint> INDEX_CMP =
+        new Comparator<CrossSectionPoint>() {
+            public int compare(CrossSectionPoint a, CrossSectionPoint b) {
+                return a.getColPos().compareTo(b.getColPos());
+            }
+        };
+
+    protected BigDecimal         km;
+    protected ImportCrossSection crossSection;
+    protected List<XY>           points;
+
+    protected CrossSectionLine peer;
+
+    public ImportCrossSectionLine() {
+    }
+
+    public ImportCrossSectionLine(BigDecimal km, List<XY> points) {
+        this.km     = km;
+        this.points = points;
+    }
+
+    public ImportCrossSection getCrossSection() {
+        return crossSection;
+    }
+
+    public void setCrossSection(ImportCrossSection crossSection) {
+        this.crossSection = crossSection;
+    }
+
+    public BigDecimal getKm() {
+        return km;
+    }
+
+    public void setKm(BigDecimal km) {
+        this.km = km;
+    }
+
+    public void storeDependencies() {
+        storePoints();
+    }
+
+    protected void storePoints() {
+        CrossSectionLine csl = getPeer();
+
+        Map<CrossSectionPoint, CrossSectionPoint> map =
+            new TreeMap<CrossSectionPoint, CrossSectionPoint>(INDEX_CMP);
+
+        // build index for faster collision lookup
+        for (CrossSectionPoint point: csl.getPoints()) {
+            map.put(point, point);
+        }
+
+        Session session =
+            ImporterSession.getInstance().getDatabaseSession();
+
+        CrossSectionPoint key = new CrossSectionPoint();
+
+        for (XY xy: points) {
+            key.setColPos(xy.getIndex());
+            CrossSectionPoint csp = map.get(key);
+            if (csp == null) { // create new
+                csp = new CrossSectionPoint(
+                    csl, key.getColPos(),
+                    new BigDecimal(xy.getX()),
+                    new BigDecimal(xy.getY()));
+            }
+            else { // update old
+                csp.setX(new BigDecimal(xy.getX()));
+                csp.setY(new BigDecimal(xy.getY()));
+            }
+            session.save(csp);
+        }
+    }
+
+    public CrossSectionLine getPeer() {
+        if (peer == null) {
+            CrossSection cs = crossSection.getPeer();
+
+            Session session =
+                ImporterSession.getInstance().getDatabaseSession();
+
+            Query query = session.createQuery(
+                "from CrossSectionLine where crossSection=:cs and km=:km");
+            query.setParameter("cs", cs);
+            query.setParameter("km", km);
+
+            List<CrossSectionLine> lines = query.list();
+            if (lines.isEmpty()) {
+                peer = new CrossSectionLine(cs, km);
+                session.save(peer);
+            }
+            else {
+                peer = lines.get(0);
+            }
+        }
+        return peer;
+    }
+}
+// vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 :
--- a/flys-backend/src/main/java/de/intevation/flys/importer/ImportRiver.java	Thu Jul 07 15:59:24 2011 +0000
+++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImportRiver.java	Thu Jul 07 22:25:38 2011 +0000
@@ -1,7 +1,12 @@
 package de.intevation.flys.importer;
 
+import java.math.BigDecimal;
+
 import java.util.List;
+import java.util.Map;
 import java.util.ArrayList;
+import java.util.Date;
+import java.util.Calendar;
 
 import java.io.File;
 import java.io.IOException;
@@ -16,6 +21,7 @@
 import org.hibernate.Query;
 
 public class ImportRiver
+implements   PRFParser.Callback
 {
     private static Logger log = Logger.getLogger(ImportRiver.class);
 
@@ -48,6 +54,8 @@
 
     protected List<ImportAnnotation> annotations;
 
+    protected List<ImportCrossSection> crossSections;
+
     protected List<ImportWst> extraWsts;
 
     protected List<ImportWst> fixations;
@@ -65,6 +73,7 @@
     protected River peer;
 
     public ImportRiver() {
+        crossSections   = new ArrayList<ImportCrossSection>();
         extraWsts       = new ArrayList<ImportWst>();
         fixations       = new ArrayList<ImportWst>();
         officialLines   = new ArrayList<ImportWst>();
@@ -120,6 +129,7 @@
     public void parseDependencies() throws IOException {
         parseGauges();
         parseAnnotations();
+        parsePRFs();
         parseWst();
         parseExtraWsts();
         parseFixations();
@@ -345,8 +355,50 @@
         annotations = aparser.getAnnotations();
     }
 
+    public void parsePRFs() {
+        log.info("looking for PRF files");
+        PRFParser parser = new PRFParser();
+        File riverDir = wstFile
+            .getParentFile()  // Basisdaten
+            .getParentFile()  // Hydrologie
+            .getParentFile(); // <river>
+        parser.parsePRFs(riverDir, this);
+    }
+
+    public static Date yearToDate(int year) {
+        Calendar cal = Calendar.getInstance();
+        cal.set(year, 5, 15, 12, 0, 0);
+        long ms = cal.getTimeInMillis();
+        cal.setTimeInMillis(ms - ms%1000);
+        return cal.getTime();
+    }
+
+    @Override
+    public void prfParsed(PRFParser parser) {
+        log.debug("callback from PRF parser");
+
+        String  description = parser.getDescription();
+        Integer year        = parser.getYear();
+        ImportTimeInterval ti = year != null
+            ? new ImportTimeInterval(yearToDate(year))
+            : null;
+
+        List<ImportCrossSectionLine> lines =
+            new ArrayList<ImportCrossSectionLine>();
+
+        for (Map.Entry<Double, List<XY>> entry: parser.getData().entrySet()) {
+            BigDecimal km     = new BigDecimal(entry.getKey());
+            List<XY>   points = entry.getValue();
+            lines.add(new ImportCrossSectionLine(km, points));
+        }
+
+        crossSections.add(new ImportCrossSection(
+            this, description, ti, lines));
+    }
+
     public void storeDependencies() {
         storeAnnotations();
+        storeCrossSections();
         storeGauges();
         storeWst();
         storeExtraWsts();
@@ -356,6 +408,13 @@
         storeFloodProtection();
     }
 
+    public void storeCrossSections() {
+        log.info("store cross sections");
+        for (ImportCrossSection crossSection: crossSections) {
+            crossSection.storeDependencies();
+        }
+    }
+
     public void storeWst() {
         River river = getPeer();
         wst.storeDependencies(river);
--- a/flys-backend/src/main/java/de/intevation/flys/importer/ImportTimeInterval.java	Thu Jul 07 15:59:24 2011 +0000
+++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImportTimeInterval.java	Thu Jul 07 22:25:38 2011 +0000
@@ -22,6 +22,10 @@
     public ImportTimeInterval() {
     }
 
+    public ImportTimeInterval(Date startTime) {
+        this.startTime = startTime;
+    }
+
     public ImportTimeInterval(Date startTime, Date stopTime) {
         this.startTime = startTime;
         this.stopTime  = stopTime;
--- a/flys-backend/src/main/java/de/intevation/flys/importer/PRFParser.java	Thu Jul 07 15:59:24 2011 +0000
+++ b/flys-backend/src/main/java/de/intevation/flys/importer/PRFParser.java	Thu Jul 07 22:25:38 2011 +0000
@@ -39,61 +39,9 @@
     public static final int MIN_YEAR = 1800;
     public static final int MAX_YEAR = 2100;
 
-    public static final double X_EPSILON = 1e-4;
-
-    public interface PRFCallback {
-        void found(PRFParser parser);
-    } // interface PRFParser
-
-    public static final class XY
-    implements Comparable<XY>
-    {
-        protected double x;
-        protected double y;
-        protected int    index;
-
-        public XY() {
-        }
-
-        public XY(double x, double y, int index) {
-            this.x     = x;
-            this.y     = y;
-            this.index = index;
-        }
-
-        @Override
-        public int compareTo(XY other) {
-            if (x + X_EPSILON < other.x) return -1;
-            if (x > other.x + X_EPSILON) return +1;
-            if (index < other.index)     return -1;
-            if (index > other.index)     return +1;
-            return 0;
-        }
-
-        public double getX() {
-            return x;
-        }
-
-        public void setX(double x) {
-            this.x = x;
-        }
-
-        public double getY() {
-            return y;
-        }
-
-        public void setY(double y) {
-            this.y = y;
-        }
-
-        public int getIndex() {
-            return index;
-        }
-
-        public void setIndex(int index) {
-            this.index = index;
-        }
-    } // class XY
+    public interface Callback {
+        void prfParsed(PRFParser parser);
+    } // interface Parser
 
     public static class DataFormat {
 
@@ -432,9 +380,7 @@
         description = null;
     }
 
-    public static void parsePRFs(File root, PRFCallback callback) {
-
-        PRFParser parser = new PRFParser();
+    public void parsePRFs(File root, Callback callback) {
 
         Stack<File> stack = new Stack<File>();
         stack.push(root);
@@ -452,11 +398,11 @@
             else if (file.isFile()
                 && file.getName().toLowerCase().endsWith(".prf")
             ) {
-                parser.reset();
-                boolean success = parser.parse(file);
+                reset();
+                boolean success = parse(file);
                 log.info("parsing " + (success ? "succeeded" : "failed"));
                 if (success && callback != null) {
-                    callback.found(parser);
+                    callback.prfParsed(this);
                 }
             }
         }
@@ -464,8 +410,10 @@
 
     public static void main(String [] args) {
 
+        PRFParser parser = new PRFParser();
+
         for (String arg: args) {
-            parsePRFs(new File(arg), null);
+            parser.parsePRFs(new File(arg), null);
         }
     }
 }
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/flys-backend/src/main/java/de/intevation/flys/importer/XY.java	Thu Jul 07 22:25:38 2011 +0000
@@ -0,0 +1,54 @@
+package de.intevation.flys.importer;
+
+public class XY
+implements   Comparable<XY>
+{
+    public static final double X_EPSILON = 1e-4;
+
+    protected double x;
+    protected double y;
+    protected int    index;
+
+    public XY() {
+    }
+
+    public XY(double x, double y, int index) {
+        this.x     = x;
+        this.y     = y;
+        this.index = index;
+    }
+
+    @Override
+    public int compareTo(XY other) {
+        if (x + X_EPSILON < other.x) return -1;
+        if (x > other.x + X_EPSILON) return +1;
+        if (index < other.index)     return -1;
+        if (index > other.index)     return +1;
+        return 0;
+    }
+
+    public double getX() {
+        return x;
+    }
+
+    public void setX(double x) {
+        this.x = x;
+    }
+
+    public double getY() {
+        return y;
+    }
+
+    public void setY(double y) {
+        this.y = y;
+    }
+
+    public int getIndex() {
+        return index;
+    }
+
+    public void setIndex(int index) {
+        this.index = index;
+    }
+}
+// vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 :
--- a/flys-backend/src/main/java/de/intevation/flys/model/CrossSection.java	Thu Jul 07 15:59:24 2011 +0000
+++ b/flys-backend/src/main/java/de/intevation/flys/model/CrossSection.java	Thu Jul 07 22:25:38 2011 +0000
@@ -29,6 +29,16 @@
     public CrossSection() {
     }
 
+    public CrossSection(
+        River        river,
+        TimeInterval timeInterval,
+        String       description
+    ) {
+        this.river        = river;
+        this.timeInterval = timeInterval;
+        this.description  = description;
+    }
+
     @Id
     @SequenceGenerator(
         name           = "SEQUENCE_CROSS_SECTIONS_ID_SEQ",
--- a/flys-backend/src/main/java/de/intevation/flys/model/CrossSectionLine.java	Thu Jul 07 15:59:24 2011 +0000
+++ b/flys-backend/src/main/java/de/intevation/flys/model/CrossSectionLine.java	Thu Jul 07 22:25:38 2011 +0000
@@ -24,12 +24,18 @@
 {
     private Integer                 id;
     private BigDecimal              km;
-    private List<CrossSectionPoint> points;
     private CrossSection            crossSection;
 
+    private List<CrossSectionPoint> points;
+
     public CrossSectionLine() {
     }
 
+    public CrossSectionLine(CrossSection crossSection, BigDecimal km) {
+        this.crossSection = crossSection;
+        this.km           = km;
+    }
+
     @Id
     @SequenceGenerator(
         name           = "SEQUENCE_CROSS_SECTION_LINES_ID_SEQ",
--- a/flys-backend/src/main/java/de/intevation/flys/model/CrossSectionPoint.java	Thu Jul 07 15:59:24 2011 +0000
+++ b/flys-backend/src/main/java/de/intevation/flys/model/CrossSectionPoint.java	Thu Jul 07 22:25:38 2011 +0000
@@ -28,6 +28,18 @@
     public CrossSectionPoint() {
     }
 
+    public CrossSectionPoint(
+        CrossSectionLine crossSectionLine,
+        Integer          colPos,
+        BigDecimal       x,
+        BigDecimal       y
+    ) {
+        this.crossSectionLine = crossSectionLine;
+        this.colPos           = colPos;
+        this.x                = x;
+        this.y                = y;
+    }
+
     @Id
     @SequenceGenerator(
         name           = "SEQUENCE_CROSS_SECTION_POINTS_ID_SEQ",

http://dive4elements.wald.intevation.org