# HG changeset patch # User Sascha L. Teichmann # Date 1310077538 0 # Node ID 22858e7cca7927da0bf800b3f4c6e2b48df2732d # Parent 3c01bef43a98e2ad81e3a5f5115e90a848051990 Integrated PRF parsing into importer. Needs testing! flys-backend/trunk@2309 c6561f87-3c4e-4783-a992-168aeb5c3f6f diff -r 3c01bef43a98 -r 22858e7cca79 flys-backend/ChangeLog --- a/flys-backend/ChangeLog Thu Jul 07 15:59:24 2011 +0000 +++ b/flys-backend/ChangeLog Thu Jul 07 22:25:38 2011 +0000 @@ -1,3 +1,27 @@ +2011-07-08 Sascha L. Teichmann + + Parse all PRFs in all subfolders of a river and store them + as cross sections into the database. Needs testing! + + * src/main/java/de/intevation/flys/importer/ImportCrossSection.java, + src/main/java/de/intevation/flys/importer/ImportCrossSectionLine.java: + New. Importer models for cross sections. + + * src/main/java/de/intevation/flys/importer/XY.java: + New. Made top level class from inner PRFParser.XY. + + * src/main/java/de/intevation/flys/importer/PRFParser.java: + Moved out XY class. Renamed callback. + + * src/main/java/de/intevation/flys/model/CrossSection.java, + src/main/java/de/intevation/flys/model/CrossSectionLine.java, + src/main/java/de/intevation/flys/model/CrossSectionPoint.java, + src/main/java/de/intevation/flys/importer/ImportTimeInterval.java: + Added convinience constructors. + + * src/main/java/de/intevation/flys/importer/ImportRiver.java: + Parse and store cross sections into database. + 2011-07-07 Sascha L. Teichmann * doc/schema/postgresql.sql: Introduced a new table cross_section_line diff -r 3c01bef43a98 -r 22858e7cca79 flys-backend/src/main/java/de/intevation/flys/importer/ImportCrossSection.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImportCrossSection.java Thu Jul 07 22:25:38 2011 +0000 @@ -0,0 +1,109 @@ +package de.intevation.flys.importer; + +import de.intevation.flys.model.River; +import de.intevation.flys.model.CrossSection; +import de.intevation.flys.model.TimeInterval; + +import org.hibernate.Session; +import org.hibernate.Query; + +import java.util.List; + +public class ImportCrossSection +{ + protected ImportRiver river; + protected String description; + protected ImportTimeInterval timeInterval; + protected List lines; + + protected CrossSection peer; + + public ImportCrossSection() { + } + + public ImportCrossSection( + ImportRiver river, + String description, + ImportTimeInterval timeInterval, + List lines + ) { + this.river = river; + this.description = description; + this.timeInterval = timeInterval; + this.lines = lines; + wireWithLines(); + } + + public void wireWithLines() { + for (ImportCrossSectionLine line: lines) { + line.setCrossSection(this); + } + } + + public ImportRiver getRiver() { + return river; + } + + public void setRiver(ImportRiver river) { + this.river = river; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public ImportTimeInterval getTimeInterval() { + return timeInterval; + } + + public void setTimeInterval(ImportTimeInterval timeInterval) { + this.timeInterval = timeInterval; + } + + public void storeDependencies() { + + getPeer(); + + for (ImportCrossSectionLine line: lines) { + line.storeDependencies(); + } + } + + public CrossSection getPeer() { + + if (peer == null) { + River r = river.getPeer(); + TimeInterval t = timeInterval != null + ? timeInterval.getPeer() + : null; + + Session session = + ImporterSession.getInstance().getDatabaseSession(); + + Query query = session.createQuery( + "from CrossSection where " + + "river=:r and " + + "timeInterval=:t and " + + "description=:d"); + + query.setParameter("r", r); + query.setParameter("t", t); + query.setParameter("d", description); + + List crossSections = query.list(); + if (crossSections.isEmpty()) { + peer = new CrossSection(r, t, description); + session.save(peer); + } + else { + peer = crossSections.get(0); + } + } + return peer; + } +} +// vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 : diff -r 3c01bef43a98 -r 22858e7cca79 flys-backend/src/main/java/de/intevation/flys/importer/ImportCrossSectionLine.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImportCrossSectionLine.java Thu Jul 07 22:25:38 2011 +0000 @@ -0,0 +1,117 @@ +package de.intevation.flys.importer; + +import de.intevation.flys.model.CrossSection; +import de.intevation.flys.model.CrossSectionPoint; +import de.intevation.flys.model.CrossSectionLine; + +import org.hibernate.Session; +import org.hibernate.Query; + +import java.math.BigDecimal; + +import java.util.List; +import java.util.Comparator; +import java.util.Map; +import java.util.TreeMap; + +public class ImportCrossSectionLine +{ + public static final Comparator INDEX_CMP = + new Comparator() { + public int compare(CrossSectionPoint a, CrossSectionPoint b) { + return a.getColPos().compareTo(b.getColPos()); + } + }; + + protected BigDecimal km; + protected ImportCrossSection crossSection; + protected List points; + + protected CrossSectionLine peer; + + public ImportCrossSectionLine() { + } + + public ImportCrossSectionLine(BigDecimal km, List points) { + this.km = km; + this.points = points; + } + + public ImportCrossSection getCrossSection() { + return crossSection; + } + + public void setCrossSection(ImportCrossSection crossSection) { + this.crossSection = crossSection; + } + + public BigDecimal getKm() { + return km; + } + + public void setKm(BigDecimal km) { + this.km = km; + } + + public void storeDependencies() { + storePoints(); + } + + protected void storePoints() { + CrossSectionLine csl = getPeer(); + + Map map = + new TreeMap(INDEX_CMP); + + // build index for faster collision lookup + for (CrossSectionPoint point: csl.getPoints()) { + map.put(point, point); + } + + Session session = + ImporterSession.getInstance().getDatabaseSession(); + + CrossSectionPoint key = new CrossSectionPoint(); + + for (XY xy: points) { + key.setColPos(xy.getIndex()); + CrossSectionPoint csp = map.get(key); + if (csp == null) { // create new + csp = new CrossSectionPoint( + csl, key.getColPos(), + new BigDecimal(xy.getX()), + new BigDecimal(xy.getY())); + } + else { // update old + csp.setX(new BigDecimal(xy.getX())); + csp.setY(new BigDecimal(xy.getY())); + } + session.save(csp); + } + } + + public CrossSectionLine getPeer() { + if (peer == null) { + CrossSection cs = crossSection.getPeer(); + + Session session = + ImporterSession.getInstance().getDatabaseSession(); + + Query query = session.createQuery( + "from CrossSectionLine where crossSection=:cs and km=:km"); + query.setParameter("cs", cs); + query.setParameter("km", km); + + List lines = query.list(); + if (lines.isEmpty()) { + peer = new CrossSectionLine(cs, km); + session.save(peer); + } + else { + peer = lines.get(0); + } + } + return peer; + } +} +// vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 : diff -r 3c01bef43a98 -r 22858e7cca79 flys-backend/src/main/java/de/intevation/flys/importer/ImportRiver.java --- a/flys-backend/src/main/java/de/intevation/flys/importer/ImportRiver.java Thu Jul 07 15:59:24 2011 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImportRiver.java Thu Jul 07 22:25:38 2011 +0000 @@ -1,7 +1,12 @@ package de.intevation.flys.importer; +import java.math.BigDecimal; + import java.util.List; +import java.util.Map; import java.util.ArrayList; +import java.util.Date; +import java.util.Calendar; import java.io.File; import java.io.IOException; @@ -16,6 +21,7 @@ import org.hibernate.Query; public class ImportRiver +implements PRFParser.Callback { private static Logger log = Logger.getLogger(ImportRiver.class); @@ -48,6 +54,8 @@ protected List annotations; + protected List crossSections; + protected List extraWsts; protected List fixations; @@ -65,6 +73,7 @@ protected River peer; public ImportRiver() { + crossSections = new ArrayList(); extraWsts = new ArrayList(); fixations = new ArrayList(); officialLines = new ArrayList(); @@ -120,6 +129,7 @@ public void parseDependencies() throws IOException { parseGauges(); parseAnnotations(); + parsePRFs(); parseWst(); parseExtraWsts(); parseFixations(); @@ -345,8 +355,50 @@ annotations = aparser.getAnnotations(); } + public void parsePRFs() { + log.info("looking for PRF files"); + PRFParser parser = new PRFParser(); + File riverDir = wstFile + .getParentFile() // Basisdaten + .getParentFile() // Hydrologie + .getParentFile(); // + parser.parsePRFs(riverDir, this); + } + + public static Date yearToDate(int year) { + Calendar cal = Calendar.getInstance(); + cal.set(year, 5, 15, 12, 0, 0); + long ms = cal.getTimeInMillis(); + cal.setTimeInMillis(ms - ms%1000); + return cal.getTime(); + } + + @Override + public void prfParsed(PRFParser parser) { + log.debug("callback from PRF parser"); + + String description = parser.getDescription(); + Integer year = parser.getYear(); + ImportTimeInterval ti = year != null + ? new ImportTimeInterval(yearToDate(year)) + : null; + + List lines = + new ArrayList(); + + for (Map.Entry> entry: parser.getData().entrySet()) { + BigDecimal km = new BigDecimal(entry.getKey()); + List points = entry.getValue(); + lines.add(new ImportCrossSectionLine(km, points)); + } + + crossSections.add(new ImportCrossSection( + this, description, ti, lines)); + } + public void storeDependencies() { storeAnnotations(); + storeCrossSections(); storeGauges(); storeWst(); storeExtraWsts(); @@ -356,6 +408,13 @@ storeFloodProtection(); } + public void storeCrossSections() { + log.info("store cross sections"); + for (ImportCrossSection crossSection: crossSections) { + crossSection.storeDependencies(); + } + } + public void storeWst() { River river = getPeer(); wst.storeDependencies(river); diff -r 3c01bef43a98 -r 22858e7cca79 flys-backend/src/main/java/de/intevation/flys/importer/ImportTimeInterval.java --- a/flys-backend/src/main/java/de/intevation/flys/importer/ImportTimeInterval.java Thu Jul 07 15:59:24 2011 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/importer/ImportTimeInterval.java Thu Jul 07 22:25:38 2011 +0000 @@ -22,6 +22,10 @@ public ImportTimeInterval() { } + public ImportTimeInterval(Date startTime) { + this.startTime = startTime; + } + public ImportTimeInterval(Date startTime, Date stopTime) { this.startTime = startTime; this.stopTime = stopTime; diff -r 3c01bef43a98 -r 22858e7cca79 flys-backend/src/main/java/de/intevation/flys/importer/PRFParser.java --- a/flys-backend/src/main/java/de/intevation/flys/importer/PRFParser.java Thu Jul 07 15:59:24 2011 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/importer/PRFParser.java Thu Jul 07 22:25:38 2011 +0000 @@ -39,61 +39,9 @@ public static final int MIN_YEAR = 1800; public static final int MAX_YEAR = 2100; - public static final double X_EPSILON = 1e-4; - - public interface PRFCallback { - void found(PRFParser parser); - } // interface PRFParser - - public static final class XY - implements Comparable - { - protected double x; - protected double y; - protected int index; - - public XY() { - } - - public XY(double x, double y, int index) { - this.x = x; - this.y = y; - this.index = index; - } - - @Override - public int compareTo(XY other) { - if (x + X_EPSILON < other.x) return -1; - if (x > other.x + X_EPSILON) return +1; - if (index < other.index) return -1; - if (index > other.index) return +1; - return 0; - } - - public double getX() { - return x; - } - - public void setX(double x) { - this.x = x; - } - - public double getY() { - return y; - } - - public void setY(double y) { - this.y = y; - } - - public int getIndex() { - return index; - } - - public void setIndex(int index) { - this.index = index; - } - } // class XY + public interface Callback { + void prfParsed(PRFParser parser); + } // interface Parser public static class DataFormat { @@ -432,9 +380,7 @@ description = null; } - public static void parsePRFs(File root, PRFCallback callback) { - - PRFParser parser = new PRFParser(); + public void parsePRFs(File root, Callback callback) { Stack stack = new Stack(); stack.push(root); @@ -452,11 +398,11 @@ else if (file.isFile() && file.getName().toLowerCase().endsWith(".prf") ) { - parser.reset(); - boolean success = parser.parse(file); + reset(); + boolean success = parse(file); log.info("parsing " + (success ? "succeeded" : "failed")); if (success && callback != null) { - callback.found(parser); + callback.prfParsed(this); } } } @@ -464,8 +410,10 @@ public static void main(String [] args) { + PRFParser parser = new PRFParser(); + for (String arg: args) { - parsePRFs(new File(arg), null); + parser.parsePRFs(new File(arg), null); } } } diff -r 3c01bef43a98 -r 22858e7cca79 flys-backend/src/main/java/de/intevation/flys/importer/XY.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/importer/XY.java Thu Jul 07 22:25:38 2011 +0000 @@ -0,0 +1,54 @@ +package de.intevation.flys.importer; + +public class XY +implements Comparable +{ + public static final double X_EPSILON = 1e-4; + + protected double x; + protected double y; + protected int index; + + public XY() { + } + + public XY(double x, double y, int index) { + this.x = x; + this.y = y; + this.index = index; + } + + @Override + public int compareTo(XY other) { + if (x + X_EPSILON < other.x) return -1; + if (x > other.x + X_EPSILON) return +1; + if (index < other.index) return -1; + if (index > other.index) return +1; + return 0; + } + + public double getX() { + return x; + } + + public void setX(double x) { + this.x = x; + } + + public double getY() { + return y; + } + + public void setY(double y) { + this.y = y; + } + + public int getIndex() { + return index; + } + + public void setIndex(int index) { + this.index = index; + } +} +// vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 : diff -r 3c01bef43a98 -r 22858e7cca79 flys-backend/src/main/java/de/intevation/flys/model/CrossSection.java --- a/flys-backend/src/main/java/de/intevation/flys/model/CrossSection.java Thu Jul 07 15:59:24 2011 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/model/CrossSection.java Thu Jul 07 22:25:38 2011 +0000 @@ -29,6 +29,16 @@ public CrossSection() { } + public CrossSection( + River river, + TimeInterval timeInterval, + String description + ) { + this.river = river; + this.timeInterval = timeInterval; + this.description = description; + } + @Id @SequenceGenerator( name = "SEQUENCE_CROSS_SECTIONS_ID_SEQ", diff -r 3c01bef43a98 -r 22858e7cca79 flys-backend/src/main/java/de/intevation/flys/model/CrossSectionLine.java --- a/flys-backend/src/main/java/de/intevation/flys/model/CrossSectionLine.java Thu Jul 07 15:59:24 2011 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/model/CrossSectionLine.java Thu Jul 07 22:25:38 2011 +0000 @@ -24,12 +24,18 @@ { private Integer id; private BigDecimal km; - private List points; private CrossSection crossSection; + private List points; + public CrossSectionLine() { } + public CrossSectionLine(CrossSection crossSection, BigDecimal km) { + this.crossSection = crossSection; + this.km = km; + } + @Id @SequenceGenerator( name = "SEQUENCE_CROSS_SECTION_LINES_ID_SEQ", diff -r 3c01bef43a98 -r 22858e7cca79 flys-backend/src/main/java/de/intevation/flys/model/CrossSectionPoint.java --- a/flys-backend/src/main/java/de/intevation/flys/model/CrossSectionPoint.java Thu Jul 07 15:59:24 2011 +0000 +++ b/flys-backend/src/main/java/de/intevation/flys/model/CrossSectionPoint.java Thu Jul 07 22:25:38 2011 +0000 @@ -28,6 +28,18 @@ public CrossSectionPoint() { } + public CrossSectionPoint( + CrossSectionLine crossSectionLine, + Integer colPos, + BigDecimal x, + BigDecimal y + ) { + this.crossSectionLine = crossSectionLine; + this.colPos = colPos; + this.x = x; + this.y = y; + } + @Id @SequenceGenerator( name = "SEQUENCE_CROSS_SECTION_POINTS_ID_SEQ",