teichmann@5844: /* Copyright (C) 2011, 2012, 2013 by Bundesanstalt für Gewässerkunde teichmann@5844: * Software engineering by Intevation GmbH teichmann@5844: * teichmann@5992: * This file is Free Software under the GNU AGPL (>=v3) teichmann@5844: * and comes with ABSOLUTELY NO WARRANTY! Check out the teichmann@5992: * documentation coming with Dive4Elements River for details. teichmann@5844: */ teichmann@5844: teichmann@5829: package org.dive4elements.river.importer; teichmann@5446: mschaefer@8971: import java.io.File; mschaefer@8971: import java.io.FilenameFilter; mschaefer@8971: import java.io.IOException; mschaefer@8971: import java.util.ArrayList; mschaefer@8971: import java.util.Calendar; mschaefer@8971: import java.util.Date; mschaefer@8971: import java.util.HashSet; mschaefer@8971: import java.util.Iterator; mschaefer@8971: import java.util.List; mschaefer@8971: import java.util.Map; mschaefer@8971: import java.util.Set; teichmann@5446: mschaefer@8971: import org.apache.log4j.Logger; teichmann@5829: import org.dive4elements.artifacts.common.utils.FileTools; mschaefer@8971: import org.dive4elements.artifacts.common.utils.FileTools.HashedFile; mschaefer@8971: import org.dive4elements.river.backend.utils.DouglasPeuker; mschaefer@8971: import org.dive4elements.river.importer.parsers.AbstractSedimentLoadParser; teichmann@5829: import org.dive4elements.river.importer.parsers.AnnotationClassifier; teichmann@5829: import org.dive4elements.river.importer.parsers.AnnotationsParser; tom@8557: import org.dive4elements.river.importer.parsers.BedHeightParser; teichmann@5829: import org.dive4elements.river.importer.parsers.CrossSectionParser; teichmann@5829: import org.dive4elements.river.importer.parsers.DA50Parser; teichmann@5829: import org.dive4elements.river.importer.parsers.DA66Parser; teichmann@5829: import org.dive4elements.river.importer.parsers.FlowVelocityMeasurementParser; teichmann@5829: import org.dive4elements.river.importer.parsers.FlowVelocityModelParser; teichmann@5829: import org.dive4elements.river.importer.parsers.HYKParser; teichmann@5829: import org.dive4elements.river.importer.parsers.MeasurementStationsParser; teichmann@5829: import org.dive4elements.river.importer.parsers.MorphologicalWidthParser; teichmann@6335: import org.dive4elements.river.importer.parsers.OfficialLinesConfigParser; teichmann@5829: import org.dive4elements.river.importer.parsers.PRFParser; teichmann@5829: import org.dive4elements.river.importer.parsers.PegelGltParser; rrenkert@7841: import org.dive4elements.river.importer.parsers.PorosityParser; teichmann@5829: import org.dive4elements.river.importer.parsers.SQRelationParser; teichmann@5829: import org.dive4elements.river.importer.parsers.SedimentDensityParser; teichmann@8025: import org.dive4elements.river.importer.parsers.SedimentLoadLSParser; tom@8056: import org.dive4elements.river.importer.parsers.SedimentLoadParser; mschaefer@8971: import org.dive4elements.river.importer.parsers.W80CSVParser; teichmann@5829: import org.dive4elements.river.importer.parsers.W80Parser; teichmann@5829: import org.dive4elements.river.importer.parsers.WaterlevelDifferencesParser; teichmann@5829: import org.dive4elements.river.importer.parsers.WaterlevelParser; teichmann@5829: import org.dive4elements.river.importer.parsers.WstParser; mschaefer@8971: import org.dive4elements.river.importer.sinfo.SInfoImporter; mschaefer@8971: import org.dive4elements.river.importer.uinfo.UInfoImporter; teichmann@5829: import org.dive4elements.river.model.River; teichmann@5829: import org.dive4elements.river.model.Unit; teichmann@5446: import org.hibernate.Query; teichmann@5446: import org.hibernate.Session; teichmann@5446: felix@4706: /** Import all river-related data (files) that can be found. */ sascha@177: public class ImportRiver sascha@177: { teichmann@8200: /** Private log. */ sascha@184: private static Logger log = Logger.getLogger(ImportRiver.class); sascha@184: sascha@184: public static final String PEGEL_GLT = "PEGEL.GLT"; sascha@184: sascha@483: public static final String FIXATIONS = "Fixierungen"; sascha@483: sascha@482: public static final String EXTRA_LONGITUDINALS = mschaefer@8971: "Zus.Laengsschnitte"; sascha@482: sascha@484: public static final String [] OFFICIAL_LINES_FOLDERS = { mschaefer@8971: "Basisdaten", mschaefer@8971: "Fixierungen" }; sascha@484: sascha@484: public static final String OFFICIAL_LINES = mschaefer@8971: "Amtl_Linien.wst"; sascha@484: teichmann@6335: public static final String OFFICIAL_LINES_CONFIG = mschaefer@8971: "Amtl_Linien.config"; teichmann@6335: sascha@490: public static final String FLOOD_WATER = "HW-Marken"; sascha@490: sascha@494: public static final String FLOOD_PROTECTION = mschaefer@8971: "HW-Schutzanlagen"; sascha@494: ingo@2806: public static final String MINFO_DIR = "Morphologie"; ingo@2806: ingo@2806: public static final String BED_HEIGHT_DIR = "Sohlhoehen"; ingo@2806: ingo@2806: public static final String BED_HEIGHT_SINGLE_DIR = "Einzeljahre"; ingo@2806: ingo@2815: public static final String SEDIMENT_DENSITY_DIR = "Sedimentdichte"; ingo@2815: rrenkert@7841: public static final String POROSITY_DIR = "Porositaet"; rrenkert@7841: tom@8856: public static final String MORPHOLOGICAL_WIDTH_DIR = mschaefer@8971: "morphologische_Breite"; ingo@2821: tom@8856: public static final String FLOW_VELOCITY_DIR = mschaefer@8971: "Geschwindigkeit_Schubspannung"; ingo@2826: ingo@2826: public static final String FLOW_VELOCITY_MODEL = "Modellrechnungen"; ingo@2826: ingo@2826: public static final String FLOW_VELOCITY_MEASUREMENTS = "v-Messungen"; ingo@2826: tom@8032: public static final String SEDIMENT_LOAD_DIR = "Fracht"; ingo@2839: tom@8032: public static final String SEDIMENT_LOAD_LS_DIR = "Laengsschnitte"; ingo@2839: tom@8056: public static final String SEDIMENT_LOAD_MS_DIR = "Messstellen"; tom@8056: tom@8032: public static final String SEDIMENT_LOAD_SINGLE_DIR = "Einzeljahre"; ingo@2839: tom@8032: public static final String SEDIMENT_LOAD_EPOCH_DIR = "Epochen"; tom@8032: tom@8856: public static final String SEDIMENT_LOAD_OFF_EPOCH_DIR = mschaefer@8971: "amtliche Epochen"; felix@6724: ingo@2844: public static final String MINFO_FIXATIONS_DIR = "Fixierungsanalyse"; ingo@2844: ingo@2844: public static final String MINFO_WATERLEVELS_DIR = "Wasserspiegellagen"; ingo@2844: tom@8856: public static final String MINFO_WATERLEVEL_DIFF_DIR = mschaefer@8971: "Wasserspiegeldifferenzen"; ingo@2851: ingo@4193: public static final String MINFO_BASE_DIR = "Basisdaten"; ingo@4193: tom@8856: public static final String MINFO_CORE_DATA_FILE = mschaefer@8971: "Stammdaten_Messstellen.csv"; ingo@4193: tom@8856: public static final String MINFO_SQ_DIR = mschaefer@8971: "Feststofftransport-Abfluss-Beziehung"; ingo@3328: sascha@177: protected String name; sascha@177: rrenkert@7751: protected String modelUuid; rrenkert@7751: felix@5020: protected Long officialNumber; sascha@177: mschaefer@8971: /** mschaefer@8971: * Path of the river/Hydrologie/Basisdaten/river.wst file from which all other file paths are derived mschaefer@8971: */ felix@5020: protected File wstFile; felix@5020: felix@5020: protected File bbInfoFile; sascha@177: sascha@184: protected List gauges; sascha@184: sascha@186: protected List annotations; sascha@186: sascha@1220: protected List hyks; sascha@1220: sascha@1204: protected List crossSections; sascha@1204: sascha@482: protected List extraWsts; sascha@482: sascha@483: protected List fixations; sascha@483: sascha@484: protected List officialLines; sascha@484: sascha@490: protected List floodWater; sascha@490: sascha@494: protected List floodProtection; sascha@494: felix@5261: /** Wst-structures from waterlevel-csv files. */ felix@5261: protected List waterlevels; felix@5261: felix@5227: /** Wst-structures from waterlevel-difference-csv files. */ felix@5227: protected List waterlevelDifferences; felix@5227: tom@8559: protected List bedHeights; ingo@2806: ingo@2815: protected List sedimentDensities; ingo@2815: rrenkert@7841: protected List porosities; rrenkert@7841: ingo@2821: protected List morphologicalWidths; ingo@2821: ingo@2827: protected List flowVelocityModels; ingo@2827: ingo@2832: protected List flowVelocityMeasurements; ingo@2832: tom@8032: protected List sedimentLoadLSs; ingo@2839: tom@8056: protected List sedimentLoads; tom@8056: ingo@4193: protected List measurementStations; ingo@4193: ingo@3328: protected List sqRelations; ingo@3328: sascha@201: protected ImportWst wst; sascha@201: ingo@2347: protected ImportUnit wstUnit; ingo@2347: sascha@765: protected AnnotationClassifier annotationClassifier; sascha@765: felix@5017: /** Database-mapped River instance. */ sascha@188: protected River peer; sascha@188: mschaefer@8971: /** mschaefer@8971: * Importer for a river's S-INFO files. mschaefer@8971: */ mschaefer@8971: private final SInfoImporter sinfoImporter; mschaefer@8971: mschaefer@8971: /** mschaefer@8971: * Importer for a river's U-INFO files. mschaefer@8971: */ mschaefer@8971: private final UInfoImporter uinfoImporter; felix@4778: felix@4778: /** Callback-implementation for CrossSectionParsers. */ teichmann@5083: private class ImportRiverCrossSectionParserCallback teichmann@5083: implements CrossSectionParser.Callback { felix@4778: mschaefer@8971: private final Set files = new HashSet<>(); mschaefer@8971: private final String type; felix@4778: felix@4778: /** felix@4778: * Create new Callback, given type which is used for logging felix@4778: * purposes only. felix@4778: */ mschaefer@8971: public ImportRiverCrossSectionParserCallback(final String type) { felix@4778: this.type = type; felix@4778: } felix@4778: felix@4778: felix@4778: /** Accept file if not duplicate. */ teichmann@5084: @Override mschaefer@8971: public boolean accept(final File file) { mschaefer@8971: final HashedFile hf = new HashedFile(file); mschaefer@8971: final boolean success = this.files.add(hf); felix@4778: if (!success) { mschaefer@8971: log.warn(this.type + " file '" + file mschaefer@8971: + "' seems to be a duplicate."); felix@4778: } felix@4778: return success; felix@4778: } felix@4778: felix@4778: felix@4778: /** Add crosssection. */ teichmann@5084: @Override mschaefer@8971: public void parsed(final CrossSectionParser parser) { mschaefer@8971: log.debug("callback from " + this.type + " parser"); teichmann@5084: mschaefer@8971: final String description = parser.getDescription(); mschaefer@8971: final Integer year = parser.getYear(); mschaefer@8971: final ImportTimeInterval ti = year != null mschaefer@8971: ? new ImportTimeInterval(yearToDate(year)) mschaefer@8971: : null; teichmann@5084: mschaefer@8971: final Map> data = parser.getData(); teichmann@5084: mschaefer@8971: final List lines = mschaefer@8971: new ArrayList<>(data.size()); teichmann@5084: mschaefer@8971: final Double simplificationEpsilon = mschaefer@8971: Config.INSTANCE.getCrossSectionSimplificationEpsilon(); mschaefer@8971: mschaefer@8971: long numReadPoints = 0L; mschaefer@8971: long numRemainingPoints = 0L; mschaefer@8971: mschaefer@8971: for (final Map.Entry> entry: data.entrySet()) { mschaefer@8971: final Double km = entry.getKey(); mschaefer@8971: List points = entry.getValue(); mschaefer@8971: numReadPoints += points.size(); mschaefer@8971: if (simplificationEpsilon != null) { mschaefer@8971: points = DouglasPeuker.simplify( mschaefer@8971: points, simplificationEpsilon); mschaefer@8971: } mschaefer@8971: numRemainingPoints += points.size(); mschaefer@8971: lines.add(new ImportCrossSectionLine(km, points)); mschaefer@8971: } mschaefer@8971: mschaefer@8971: ImportRiver.this.addCrossSections(description, ti, lines); mschaefer@8971: mschaefer@8971: if (simplificationEpsilon != null) { mschaefer@8971: final double percent = numReadPoints > 0L mschaefer@8971: ? ((double)numRemainingPoints/numReadPoints)*100d mschaefer@8971: : 0d; mschaefer@8971: mschaefer@8971: log.info(String.format( mschaefer@8971: "Number of points in cross section: %d / %d (%.2f%%)", mschaefer@8971: numReadPoints, numRemainingPoints, percent)); mschaefer@8971: } felix@4778: } teichmann@5149: } // ImportRiverCrossSectionParserCallback felix@4778: felix@4778: teichmann@5084: private void addCrossSections( mschaefer@8971: final String description, mschaefer@8971: final ImportTimeInterval ti, mschaefer@8971: final List lines mschaefer@8971: ) { mschaefer@8971: this.crossSections.add( mschaefer@8971: new ImportCrossSection(this, description, ti, lines)); teichmann@5084: } teichmann@5084: teichmann@5084: sascha@177: public ImportRiver() { mschaefer@8971: this.hyks = new ArrayList<>(); mschaefer@8971: this.crossSections = new ArrayList<>(); mschaefer@8971: this.extraWsts = new ArrayList<>(); mschaefer@8971: this.fixations = new ArrayList<>(); mschaefer@8971: this.officialLines = new ArrayList<>(); mschaefer@8971: this.floodWater = new ArrayList<>(); mschaefer@8971: this.waterlevels = new ArrayList<>(); mschaefer@8971: this.waterlevelDifferences = new ArrayList<>(); mschaefer@8971: this.floodProtection = new ArrayList<>(); mschaefer@8971: this.sedimentDensities = new ArrayList<>(); mschaefer@8971: this.porosities = new ArrayList<>(); mschaefer@8971: this.morphologicalWidths = new ArrayList<>(); mschaefer@8971: this.flowVelocityModels = new ArrayList<>(); mschaefer@8971: this.flowVelocityMeasurements = mschaefer@8971: new ArrayList<>(); mschaefer@8971: this.sedimentLoadLSs = new ArrayList<>(); mschaefer@8971: this.sedimentLoads = new ArrayList<>(); mschaefer@8971: this.measurementStations = new ArrayList<>(); mschaefer@8971: this.sqRelations = new ArrayList<>(); mschaefer@8971: this.sinfoImporter = new SInfoImporter(); mschaefer@8971: this.uinfoImporter = new UInfoImporter(); sascha@177: } sascha@177: sascha@766: public ImportRiver( mschaefer@8971: final String name, mschaefer@8971: final String modelUuid, mschaefer@8971: final File wstFile, mschaefer@8971: final File bbInfoFile, mschaefer@8971: final AnnotationClassifier annotationClassifier mschaefer@8971: ) { sascha@482: this(); sascha@766: this.name = name; rrenkert@7751: this.modelUuid = modelUuid; sascha@766: this.wstFile = wstFile; sascha@766: this.bbInfoFile = bbInfoFile; sascha@766: this.annotationClassifier = annotationClassifier; sascha@177: } sascha@177: sascha@177: public String getName() { mschaefer@8971: return this.name; sascha@177: } sascha@177: mschaefer@8971: public void setName(final String name) { sascha@177: this.name = name; sascha@177: } sascha@177: rrenkert@7751: public String getModelUuid() { mschaefer@8971: return this.modelUuid; rrenkert@7751: } rrenkert@7751: mschaefer@8971: public void setModelUuid(final String modelUuid) { rrenkert@7751: this.modelUuid = modelUuid; rrenkert@7751: } rrenkert@7751: felix@5020: public Long getOfficialNumber() { felix@5020: return this.officialNumber; felix@5020: } felix@5020: mschaefer@8971: public void setOfficialNumber(final Long officialNumber) { felix@5020: this.officialNumber = officialNumber; felix@5020: } felix@5020: sascha@177: public File getWstFile() { mschaefer@8971: return this.wstFile; sascha@177: } sascha@177: mschaefer@8971: public void setWstFile(final File wstFile) { sascha@177: this.wstFile = wstFile; sascha@177: } sascha@177: sascha@177: public File getBBInfo() { mschaefer@8971: return this.bbInfoFile; sascha@177: } sascha@177: mschaefer@8971: public void setBBInfo(final File bbInfoFile) { sascha@177: this.bbInfoFile = bbInfoFile; sascha@177: } sascha@184: sascha@201: public ImportWst getWst() { mschaefer@8971: return this.wst; sascha@201: } sascha@201: mschaefer@8971: public void setWst(final ImportWst wst) { sascha@201: this.wst = wst; sascha@201: } sascha@201: mschaefer@8971: private File getMinfoDir() { mschaefer@8971: return new File(getRiverDir(), MINFO_DIR); mschaefer@8971: } mschaefer@8971: mschaefer@8971: private File getRiverDir() { mschaefer@8971: return this.wstFile.getParentFile().getParentFile().getParentFile(); ingo@2806: } ingo@2806: mschaefer@8988: public AnnotationClassifier getAnnotationClassifier() { mschaefer@8988: return this.annotationClassifier; mschaefer@8988: } mschaefer@8988: sascha@186: public void parseDependencies() throws IOException { mschaefer@8971: log.info("Root dir is '" + getRiverDir() + "'"); sascha@186: parseGauges(); sascha@186: parseAnnotations(); sascha@1204: parsePRFs(); felix@4712: parseDA66s(); felix@4767: parseDA50s(); felix@4767: parseW80s(); felix@5810: parseW80CSVs(); sascha@1220: parseHYKs(); sascha@197: parseWst(); sascha@482: parseExtraWsts(); sascha@483: parseFixations(); sascha@484: parseOfficialLines(); sascha@490: parseFloodWater(); sascha@494: parseFloodProtection(); tom@8056: parseMeasurementStations(); ingo@2806: parseBedHeight(); ingo@2815: parseSedimentDensity(); rrenkert@7841: parsePorosity(); ingo@2821: parseMorphologicalWidth(); ingo@2826: parseFlowVelocity(); tom@8032: parseSedimentLoadLS(); tom@8056: parseSedimentLoad(); ingo@2844: parseWaterlevels(); ingo@2851: parseWaterlevelDifferences(); ingo@3328: parseSQRelation(); mschaefer@8971: this.sinfoImporter.setup(getRiverDir(), this); mschaefer@8971: this.sinfoImporter.parse(); mschaefer@8971: this.uinfoImporter.setup(getRiverDir(), this); mschaefer@8971: this.uinfoImporter.parse(); sascha@494: } sascha@494: sascha@494: public void parseFloodProtection() throws IOException { sascha@1223: if (Config.INSTANCE.skipFloodProtection()) { sascha@1223: log.info("skip parsing flood protection"); sascha@1223: return; sascha@1223: } sascha@1223: sascha@494: log.info("Parse flood protection wst file"); sascha@494: mschaefer@8971: final File riverDir = this.wstFile.getParentFile().getParentFile(); sascha@494: mschaefer@8971: final File dir = FileTools.repair(new File(riverDir, FLOOD_PROTECTION)); sascha@494: sascha@494: if (!dir.isDirectory() || !dir.canRead()) { sascha@494: log.info("no directory '" + dir + "' found"); sascha@494: return; sascha@494: } sascha@494: mschaefer@8971: final File [] files = dir.listFiles(); sascha@494: sascha@494: if (files == null) { sascha@494: log.warn("cannot read '" + dir + "'"); sascha@494: return; sascha@494: } sascha@494: mschaefer@8971: for (final File file: files) { sascha@494: if (!file.isFile() || !file.canRead()) { sascha@494: continue; sascha@494: } mschaefer@8971: final String name = file.getName().toLowerCase(); sascha@494: if (!(name.endsWith(".zus") || name.endsWith(".wst"))) { sascha@494: continue; sascha@494: } sascha@494: log.info("found file '" + file.getName() + "'"); teichmann@7339: try { mschaefer@8971: final WstParser wstParser = new WstParser(); teichmann@7339: wstParser.parse(file); mschaefer@8971: final ImportWst iw = wstParser.getWst(); teichmann@7339: iw.setKind(5); tom@8856: iw.setDescription( mschaefer@8971: FLOOD_PROTECTION + "/" + iw.getDescription()); mschaefer@8971: this.floodProtection.add(iw); teichmann@7339: } mschaefer@8971: catch (final WstParser.ParseException e) { tom@7343: log.error(e.getMessage()); teichmann@7339: } sascha@494: } sascha@490: } sascha@490: felix@5027: public void storeOfficialNumber() { felix@5027: if (Config.INSTANCE.skipBWASTR()) { felix@5027: log.info("skip storing official number."); felix@5027: return; felix@5027: } mschaefer@8971: getPeer().setOfficialNumber(this.officialNumber); felix@5027: } ingo@2806: ingo@2806: public void parseBedHeight() throws IOException { mschaefer@8971: final File minfoDir = getMinfoDir(); mschaefer@8971: final File bedHeightDir = new File(minfoDir, BED_HEIGHT_DIR); mschaefer@8971: final File singlesDir = new File(bedHeightDir, BED_HEIGHT_SINGLE_DIR); ingo@2806: tom@8559: if (Config.INSTANCE.skipBedHeight()) { tom@8762: log.info("skip parsing bed heights."); ingo@2810: } ingo@2810: else { tom@8762: log.info("Parse bed heights."); tom@8559: parseBedHeights(singlesDir); ingo@2810: } ingo@2806: } ingo@2806: ingo@2806: ingo@2815: protected void parseSedimentDensity() throws IOException { ingo@2815: if (Config.INSTANCE.skipSedimentDensity()) { ingo@2815: log.info("skip parsing sediment density."); ingo@2815: return; ingo@2815: } ingo@2815: ingo@3327: log.debug("Parse sediment density"); ingo@3327: mschaefer@8971: final File minfoDir = getMinfoDir(); mschaefer@8971: final File sediment = new File(minfoDir, SEDIMENT_DENSITY_DIR); ingo@2815: mschaefer@8971: final File[] files = sediment.listFiles(); ingo@2815: ingo@2815: if (files == null) { sascha@3660: log.warn("Cannot read directory '" + sediment + "'"); ingo@2815: return; ingo@2815: } ingo@2815: mschaefer@8971: final SedimentDensityParser parser = new SedimentDensityParser(); ingo@2815: mschaefer@8971: for (final File file: files) { ingo@2815: parser.parse(file); ingo@2815: } ingo@2815: mschaefer@8971: this.sedimentDensities = parser.getSedimentDensities(); ingo@2815: mschaefer@8971: log.info("Parsed " + this.sedimentDensities.size() mschaefer@8971: + " sediment densities."); ingo@2815: } ingo@2815: rrenkert@7841: protected void parsePorosity() throws IOException { rrenkert@7841: if (Config.INSTANCE.skipPorosity()) { rrenkert@7841: log.info("skip parsing porosity."); rrenkert@7841: return; rrenkert@7841: } rrenkert@7841: rrenkert@7841: log.debug("Parse porosity"); rrenkert@7841: mschaefer@8971: final File minfoDir = getMinfoDir(); mschaefer@8971: final File porosity = new File(minfoDir, POROSITY_DIR); rrenkert@7841: mschaefer@8971: final File[] files = porosity.listFiles(); rrenkert@7841: rrenkert@7841: if (files == null) { rrenkert@7841: log.warn("Cannot read directory '" + porosity + "'"); rrenkert@7841: return; rrenkert@7841: } rrenkert@7841: mschaefer@8971: final PorosityParser parser = new PorosityParser(); rrenkert@7841: mschaefer@8971: for (final File file: files) { rrenkert@7841: parser.parse(file); rrenkert@7841: } rrenkert@7841: mschaefer@8971: this.porosities = parser.getPorosities(); rrenkert@7841: mschaefer@8971: log.info("Parsed " + this.porosities.size() + " porosities."); rrenkert@7841: } ingo@2815: ingo@2821: protected void parseMorphologicalWidth() throws IOException { ingo@2821: if (Config.INSTANCE.skipMorphologicalWidth()) { ingo@2821: log.info("skip parsing morphological width."); ingo@2821: return; ingo@2821: } ingo@2821: ingo@3327: log.debug("Parse morphological width"); ingo@3327: mschaefer@8971: final File minfoDir = getMinfoDir(); mschaefer@8971: final File morphDir = new File(minfoDir, MORPHOLOGICAL_WIDTH_DIR); ingo@2821: mschaefer@8971: final File[] files = morphDir.listFiles(); ingo@2821: ingo@2821: if (files == null) { sascha@3660: log.warn("Cannot read directory '" + morphDir + "'"); ingo@2821: return; ingo@2821: } ingo@2821: mschaefer@8971: final MorphologicalWidthParser parser = new MorphologicalWidthParser(); ingo@2821: mschaefer@8971: for (final File file: files) { ingo@2821: parser.parse(file); ingo@2821: } ingo@2821: mschaefer@8971: this.morphologicalWidths = parser.getMorphologicalWidths(); ingo@2821: mschaefer@8971: log.info("Parsed " + this.morphologicalWidths.size() mschaefer@8971: + " morph. widths files."); ingo@2821: } ingo@2821: ingo@2821: ingo@2826: protected void parseFlowVelocity() throws IOException { ingo@2826: if (Config.INSTANCE.skipFlowVelocity()) { ingo@2826: log.info("skip parsing flow velocity"); ingo@2826: return; ingo@2826: } ingo@2826: ingo@3328: log.debug("Parse flow velocity"); ingo@3328: mschaefer@8971: final File minfoDir = getMinfoDir(); mschaefer@8971: final File flowDir = new File(minfoDir, FLOW_VELOCITY_DIR); mschaefer@8971: final File modelDir = new File(flowDir, FLOW_VELOCITY_MODEL); mschaefer@8971: final File measureDir = new File(flowDir, FLOW_VELOCITY_MEASUREMENTS); ingo@2826: mschaefer@8971: final File[] modelFiles = modelDir.listFiles(); mschaefer@8971: final File[] measureFiles = measureDir.listFiles(); ingo@2826: ingo@2826: if (modelFiles == null) { sascha@3660: log.warn("Cannot read directory '" + modelDir + "'"); ingo@2826: } ingo@2826: else { mschaefer@8971: final FlowVelocityModelParser parser = new FlowVelocityModelParser(); ingo@2828: mschaefer@8971: for (final File model: modelFiles) { ingo@2826: log.debug("Parse file '" + model + "'"); ingo@2828: parser.parse(model); ingo@2826: } ingo@2828: mschaefer@8971: this.flowVelocityModels = parser.getModels(); ingo@2826: } ingo@2826: ingo@2826: if (measureFiles == null) { sascha@3660: log.warn("Cannot read directory '" + measureDir + "'"); ingo@2826: } ingo@2826: else { mschaefer@8971: final FlowVelocityMeasurementParser parser = mschaefer@8971: new FlowVelocityMeasurementParser(); ingo@2832: mschaefer@8971: for (final File measurement: measureFiles) { ingo@2826: log.debug("Parse file '" + measurement + "'"); ingo@2832: parser.parse(measurement); ingo@2826: } ingo@2832: mschaefer@8971: this.flowVelocityMeasurements = parser.getMeasurements(); ingo@2826: } ingo@2826: } ingo@2826: ingo@2826: tom@8056: private void parseSedimentLoadFiles( mschaefer@8971: final File[] files, mschaefer@8971: final AbstractSedimentLoadParser parser mschaefer@8971: ) throws IOException { mschaefer@8971: for (final File file: files) { mschaefer@8971: if (file.isDirectory()) { mschaefer@8971: for (final File child: file.listFiles()) { mschaefer@8971: parser.parse(child); mschaefer@8971: } mschaefer@8971: } mschaefer@8971: else { mschaefer@8971: parser.parse(file); mschaefer@8971: } mschaefer@8971: } felix@6724: } felix@6724: tom@8056: tom@8056: private void parseSedimentLoadDir( mschaefer@8971: final File sedimentLoadDir, mschaefer@8971: final AbstractSedimentLoadParser parser mschaefer@8971: ) throws IOException { tom@8056: mschaefer@8971: final File[] sedimentLoadSubDirs = { mschaefer@8971: new File(sedimentLoadDir, mschaefer@8971: SEDIMENT_LOAD_SINGLE_DIR), mschaefer@8971: new File(sedimentLoadDir, mschaefer@8971: SEDIMENT_LOAD_EPOCH_DIR), mschaefer@8971: new File(sedimentLoadDir, mschaefer@8971: SEDIMENT_LOAD_OFF_EPOCH_DIR), tom@8056: }; tom@8056: mschaefer@8971: for (final File subDir : sedimentLoadSubDirs) { mschaefer@8971: final File[] files = subDir.listFiles(); tom@8056: tom@8056: if (files == null || files.length == 0) { tom@8056: log.warn("Cannot read directory '" + subDir + "'"); tom@8056: } tom@8056: else { tom@8056: parseSedimentLoadFiles(files, parser); tom@8056: } tom@8056: } tom@8056: } tom@8056: tom@8056: tom@8032: protected void parseSedimentLoadLS() throws IOException { tom@8032: if (Config.INSTANCE.skipSedimentLoadLS()) { tom@8032: log.info("skip parsing sediment load longitudinal section data"); ingo@2839: return; ingo@2839: } ingo@2839: tom@8032: log.debug("Parse sediment load longitudinal section data"); ingo@3327: mschaefer@8971: final SedimentLoadLSParser parser = new SedimentLoadLSParser(); tom@8056: mschaefer@8971: final File minfoDir = getMinfoDir(); mschaefer@8971: final File sedimentLoadDir = new File(minfoDir, SEDIMENT_LOAD_DIR); mschaefer@8971: final File sedimentLoadLSDir = new File(sedimentLoadDir, mschaefer@8971: SEDIMENT_LOAD_LS_DIR); ingo@2839: tom@8056: parseSedimentLoadDir(sedimentLoadLSDir, parser); ingo@2839: mschaefer@8971: this.sedimentLoadLSs = parser.getSedimentLoadLSs(); tom@8056: } ingo@2839: ingo@2840: tom@8056: protected void parseSedimentLoad() throws IOException { tom@8056: if (Config.INSTANCE.skipSedimentLoad()) { tom@8856: log.info( mschaefer@8971: "skip parsing sediment load data at measurement stations"); tom@8056: return; ingo@2839: } ingo@2839: tom@8056: log.debug("Parse sediment load data at measurement stations"); felix@6724: mschaefer@8971: final SedimentLoadParser parser = new SedimentLoadParser(getPeer()); ingo@2840: mschaefer@8971: final File minfoDir = getMinfoDir(); mschaefer@8971: final File sedimentLoadDir = new File(minfoDir, SEDIMENT_LOAD_DIR); mschaefer@8971: final File sedimentLoadMSDir = new File(sedimentLoadDir, mschaefer@8971: SEDIMENT_LOAD_MS_DIR); tom@8056: tom@8056: parseSedimentLoadDir(sedimentLoadMSDir, parser); tom@8056: mschaefer@8971: this.sedimentLoads = parser.getSedimentLoads(); ingo@2839: } ingo@2839: ingo@2839: ingo@2844: protected void parseWaterlevels() throws IOException { ingo@2844: if (Config.INSTANCE.skipWaterlevels()) { ingo@2844: log.info("skip parsing waterlevels"); ingo@2844: return; ingo@2844: } ingo@2844: ingo@2851: log.info("Parse waterlevels"); ingo@2851: mschaefer@8971: final File minfo = getMinfoDir(); mschaefer@8971: final File fixDir = new File(minfo, MINFO_FIXATIONS_DIR); mschaefer@8971: final File wspDir = new File(fixDir, MINFO_WATERLEVELS_DIR); ingo@2844: mschaefer@8971: final File[] files = wspDir.listFiles(); ingo@2844: ingo@2844: if (files == null) { felix@5262: log.warn("Cannot read directory for wl '" + wspDir + "'"); ingo@2844: return; ingo@2844: } ingo@2844: mschaefer@8971: final WaterlevelParser parser = new WaterlevelParser(); ingo@2844: mschaefer@8971: for (final File file: files) { ingo@2844: parser.parse(file); ingo@2844: } ingo@2844: felix@5050: // The parsed ImportWaterlevels are converted to felix@5050: // 'fixation'-wsts now. mschaefer@8971: for(final ImportWst iw: parser.getWaterlevels()) { felix@5227: iw.setDescription("CSV/" + iw.getDescription()); felix@6283: iw.setKind(7); mschaefer@8971: this.waterlevels.add(iw); felix@5050: } ingo@2851: } ingo@2844: ingo@4193: protected void parseMeasurementStations() throws IOException { ingo@4193: if (Config.INSTANCE.skipMeasurementStations()) { ingo@4193: log.info("skip parsing measurement stations"); ingo@4193: return; ingo@4193: } ingo@4193: ingo@4193: log.info("Parse measurement stations"); ingo@4193: mschaefer@8971: final File minfo = getMinfoDir(); mschaefer@8971: final File minfoBaseDir = new File(minfo, MINFO_BASE_DIR); mschaefer@8971: final File coredataFile = new File(minfoBaseDir, MINFO_CORE_DATA_FILE); ingo@4193: ingo@4193: if (coredataFile == null || !coredataFile.exists()) { tom@8856: log.warn("No core data file '" mschaefer@8971: + coredataFile.getAbsolutePath() + "' found"); ingo@4193: return; ingo@4193: } ingo@4193: mschaefer@8971: final MeasurementStationsParser parser = new MeasurementStationsParser(); ingo@4193: try { ingo@4193: parser.parse(coredataFile); mschaefer@8971: this.measurementStations = parser.getMeasurementStations(); ingo@4193: mschaefer@8971: log.info("Successfully parsed " + this.measurementStations.size() mschaefer@8971: + " measurement stations."); ingo@4193: } mschaefer@8971: catch (final IOException ioe) { ingo@4193: log.error("unable to parse file '" + coredataFile.getName() + mschaefer@8971: ": " + ioe.getMessage()); ingo@4193: } ingo@4193: } ingo@4193: ingo@2851: ingo@2851: protected void parseWaterlevelDifferences() throws IOException { ingo@2851: if (Config.INSTANCE.skipWaterlevelDifferences()) { ingo@2851: log.info("skip parsing waterlevel differences"); ingo@2851: return; ingo@2851: } ingo@2851: ingo@2851: log.info("Parse waterlevel differences"); ingo@2851: mschaefer@8971: final File minfo = getMinfoDir(); mschaefer@8971: final File fixDir = new File(minfo, MINFO_FIXATIONS_DIR); mschaefer@8971: final File diffDir = new File(fixDir, MINFO_WATERLEVEL_DIFF_DIR); ingo@2851: mschaefer@8971: final File[] files = diffDir.listFiles(); ingo@2851: ingo@2851: if (files == null) { ingo@2851: log.warn("Cannot read directory '" + diffDir + "'"); ingo@2851: return; ingo@2851: } ingo@2851: mschaefer@8971: final WaterlevelDifferencesParser parser = new WaterlevelDifferencesParser(); ingo@2851: mschaefer@8971: for (final File file: files) { ingo@2851: parser.parse(file); ingo@2851: } ingo@2851: felix@5232: // WaterlevelDifferences become Wsts now. mschaefer@8971: for(final ImportWst iw: parser.getDifferences()) { felix@5232: iw.setDescription("CSV/" + iw.getDescription()); felix@6283: iw.setKind(6); mschaefer@8971: this.waterlevelDifferences.add(iw); felix@5232: } ingo@2844: } ingo@2844: ingo@2844: ingo@3328: protected void parseSQRelation() throws IOException { ingo@3328: if (Config.INSTANCE.skipSQRelation()) { ingo@3329: log.info("skip parsing sq relation"); ingo@3328: return; ingo@3328: } ingo@3328: ingo@3328: log.info("Parse sq relations"); ingo@3328: mschaefer@8971: final File minfo = getMinfoDir(); mschaefer@8971: final File sqDir = new File(minfo, MINFO_SQ_DIR); ingo@3328: mschaefer@8971: final File[] files = sqDir.listFiles(); ingo@3328: ingo@3328: if (files == null) { ingo@3328: log.warn("Cannot read directory '" + sqDir + "'"); ingo@3328: return; ingo@3328: } ingo@3328: mschaefer@8971: final SQRelationParser parser = new SQRelationParser(getPeer()); ingo@3328: mschaefer@8971: for (final File file: files) { ingo@3328: parser.parse(file); ingo@3328: } ingo@3328: mschaefer@8971: this.sqRelations = parser.getSQRelations(); ingo@3328: mschaefer@8971: log.debug("Parsed " + this.sqRelations.size() + " SQ relations."); ingo@3328: } ingo@3328: ingo@3328: mschaefer@8971: protected void parseBedHeights(final File dir) throws IOException { ingo@2806: log.debug("Parse bed height singles"); ingo@2806: mschaefer@8971: final File[] files = dir.listFiles(new FilenameFilter() { mschaefer@8971: @Override mschaefer@8971: public boolean accept(final File dir, final String name) { mschaefer@8971: return name.toLowerCase().endsWith(".csv"); mschaefer@8971: } mschaefer@8971: }); ingo@2806: ingo@2806: if (files == null) { sascha@3660: log.warn("Cannot read directory '" + dir + "'"); ingo@2806: return; ingo@2806: } ingo@2806: mschaefer@8971: final BedHeightParser parser = new BedHeightParser(); ingo@2806: mschaefer@8971: for (final File file: files) { ingo@2806: parser.parse(file); ingo@2806: } ingo@2806: mschaefer@8971: this.bedHeights = parser.getBedHeights(); ingo@2806: } ingo@2806: sascha@490: public void parseFloodWater() throws IOException { sascha@1223: if (Config.INSTANCE.skipFloodWater()) { sascha@1223: log.info("skip parsing flod water"); sascha@1223: return; sascha@1223: } sascha@1223: sascha@490: log.info("Parse flood water wst file"); sascha@490: mschaefer@8971: final File riverDir = this.wstFile.getParentFile().getParentFile(); sascha@490: mschaefer@8971: final File dir = FileTools.repair(new File(riverDir, FLOOD_WATER)); sascha@490: sascha@490: if (!dir.isDirectory() || !dir.canRead()) { sascha@490: log.info("no directory '" + dir + "' found"); sascha@490: return; sascha@490: } sascha@490: mschaefer@8971: final File [] files = dir.listFiles(); sascha@490: sascha@490: if (files == null) { sascha@490: log.warn("cannot read '" + dir + "'"); sascha@490: return; sascha@490: } sascha@490: mschaefer@8971: for (final File file: files) { sascha@490: if (!file.isFile() || !file.canRead()) { sascha@490: continue; sascha@490: } mschaefer@8971: final String name = file.getName().toLowerCase(); sascha@490: if (!(name.endsWith(".zus") || name.endsWith(".wst"))) { sascha@490: continue; sascha@490: } sascha@490: log.info("found file '" + file.getName() + "'"); teichmann@7339: try { mschaefer@8971: final WstParser wstParser = new WstParser(); teichmann@7339: wstParser.parse(file); mschaefer@8971: final ImportWst iw = wstParser.getWst(); teichmann@7339: iw.setKind(4); teichmann@7339: iw.setDescription(FLOOD_WATER + "/" + iw.getDescription()); mschaefer@8971: this.floodWater.add(iw); teichmann@7339: } mschaefer@8971: catch (final WstParser.ParseException e) { tom@7343: log.error(e.getMessage()); teichmann@7339: } sascha@490: } sascha@484: } sascha@484: sascha@484: public void parseOfficialLines() throws IOException { sascha@1223: if (Config.INSTANCE.skipOfficialLines()) { sascha@1223: log.info("skip parsing official lines"); sascha@1223: return; sascha@1223: } sascha@1223: sascha@484: log.info("Parse official wst files"); sascha@484: mschaefer@8971: final File riverDir = this.wstFile.getParentFile().getParentFile(); sascha@484: mschaefer@8971: for (final String folder: OFFICIAL_LINES_FOLDERS) { mschaefer@8971: final File dir = FileTools.repair(new File(riverDir, folder)); sascha@484: sascha@484: if (!dir.isDirectory() || !dir.canRead()) { sascha@484: log.info("no directory '" + folder + "' found"); sascha@484: continue; sascha@484: } sascha@484: mschaefer@8971: final File file = FileTools.repair(new File(dir, OFFICIAL_LINES)); sascha@484: if (!file.isFile() || !file.canRead()) { sascha@484: log.warn("no official lines wst file found"); sascha@484: continue; sascha@484: } sascha@484: log.debug("Found WST file: " + file); sascha@484: mschaefer@8971: final ImportWst iw = new ImportWst( mschaefer@8971: ImportOfficialWstColumn.COLUMN_FACTORY); teichmann@6337: mschaefer@8971: final WstParser wstParser = new WstParser(iw); teichmann@7339: try { teichmann@7339: wstParser.parse(file); teichmann@7339: } mschaefer@8971: catch (final WstParser.ParseException e) { tom@7343: log.error(e.getMessage()); teichmann@7339: continue; teichmann@7339: } teichmann@7339: sascha@484: iw.setKind(3); sascha@484: iw.setDescription(folder + "/" + iw.getDescription()); teichmann@6335: mschaefer@8971: final File configFile = FileTools.repair( mschaefer@8971: new File(dir, OFFICIAL_LINES_CONFIG)); teichmann@6335: if (!configFile.isFile() || !configFile.canRead()) { teichmann@6335: log.warn("no config file for official lines found"); teichmann@6335: } teichmann@6335: else { mschaefer@8971: final OfficialLinesConfigParser olcp = mschaefer@8971: new OfficialLinesConfigParser(); teichmann@6335: try { teichmann@6335: olcp.parse(configFile); teichmann@6335: } mschaefer@8971: catch (final IOException ioe) { teichmann@6335: log.warn("Error reading offical lines config", ioe); teichmann@6335: } mschaefer@8971: final List mainValueNames = olcp.getMainValueNames(); teichmann@6335: if (mainValueNames.isEmpty()) { tom@8856: log.warn( mschaefer@8971: "config file for offical lines contains no entries"); teichmann@6335: } teichmann@6335: else { teichmann@6345: // Join as much as possible. mschaefer@8971: final Iterator wi = iw.getColumns().iterator(); mschaefer@8971: final Iterator si = olcp.getMainValueNames().iterator(); teichmann@6345: while (wi.hasNext() && si.hasNext()) { mschaefer@8971: final ImportOfficialWstColumn wc = mschaefer@8971: (ImportOfficialWstColumn)wi.next(); mschaefer@8971: final String name = si.next(); mschaefer@8971: final ImportOfficialLine iol = mschaefer@8971: new ImportOfficialLine(name, wc); teichmann@6345: wc.setOfficialLine(iol); teichmann@6345: } teichmann@6335: } teichmann@6335: } teichmann@6335: mschaefer@8971: this.officialLines.add(iw); sascha@484: } // for all folders sascha@484: sascha@483: } sascha@483: sascha@483: public void parseFixations() throws IOException { sascha@1223: if (Config.INSTANCE.skipFixations()) { sascha@1223: log.info("skip parsing fixations"); sascha@1223: return; sascha@1223: } sascha@1223: sascha@483: log.info("Parse fixation wst files"); sascha@483: mschaefer@8971: final File riverDir = this.wstFile.getParentFile().getParentFile(); sascha@483: mschaefer@8971: final File fixDir = FileTools.repair( mschaefer@8971: new File(riverDir, FIXATIONS)); sascha@483: sascha@483: if (!fixDir.isDirectory() || !fixDir.canRead()) { sascha@483: log.info("no fixation wst file directory found"); sascha@483: return; sascha@483: } sascha@483: mschaefer@8971: final File [] files = fixDir.listFiles(); sascha@483: sascha@483: if (files == null) { sascha@483: log.warn("cannot read fixations wst file directory"); sascha@483: return; sascha@483: } sascha@483: mschaefer@8971: for (final File file: files) { sascha@483: if (!file.isFile() || !file.canRead()) { sascha@483: continue; sascha@483: } mschaefer@8971: final String name = file.getName().toLowerCase(); sascha@483: if (!name.endsWith(".wst")) { sascha@483: continue; sascha@483: } sascha@483: log.debug("Found WST file: " + file); sascha@483: teichmann@7339: try { mschaefer@8971: final WstParser wstParser = new WstParser(); teichmann@7339: wstParser.parse(file); mschaefer@8971: final ImportWst iw = wstParser.getWst(); teichmann@7339: iw.setKind(2); teichmann@7339: iw.setDescription(FIXATIONS+ "/" + iw.getDescription()); mschaefer@8971: this.fixations.add(iw); teichmann@7339: } mschaefer@8971: catch (final WstParser.ParseException e) { tom@7343: log.error(e.getMessage()); teichmann@7339: } sascha@483: } sascha@482: } sascha@482: sascha@482: public void parseExtraWsts() throws IOException { sascha@1223: if (Config.INSTANCE.skipExtraWsts()) { sascha@1223: log.info("skip parsing extra WST files"); sascha@1223: return; sascha@1223: } sascha@1223: sascha@482: log.info("Parse extra longitudinal wst files"); sascha@482: mschaefer@8971: final File riverDir = this.wstFile.getParentFile().getParentFile(); sascha@482: mschaefer@8971: final File extraDir = FileTools.repair( mschaefer@8971: new File(riverDir, EXTRA_LONGITUDINALS)); sascha@482: sascha@482: if (!extraDir.isDirectory() || !extraDir.canRead()) { sascha@482: log.info("no extra longitudinal wst file directory found"); sascha@482: return; sascha@482: } sascha@482: mschaefer@8971: final File [] files = extraDir.listFiles(); sascha@482: sascha@482: if (files == null) { sascha@482: log.warn("cannot read extra longitudinal wst file directory"); sascha@482: return; sascha@482: } sascha@482: mschaefer@8971: for (final File file: files) { sascha@482: if (!file.isFile() || !file.canRead()) { sascha@482: continue; sascha@482: } mschaefer@8971: final String name = file.getName().toLowerCase(); sascha@482: if (!(name.endsWith(".zus") || name.endsWith(".wst"))) { sascha@482: continue; sascha@482: } sascha@482: log.debug("Found WST file: " + file); sascha@482: teichmann@7339: try { mschaefer@8971: final WstParser wstParser = new WstParser(); teichmann@7339: wstParser.parse(file); mschaefer@8971: final ImportWst iw = wstParser.getWst(); teichmann@7339: iw.setKind(1); tom@8856: iw.setDescription( mschaefer@8971: EXTRA_LONGITUDINALS + "/" + iw.getDescription()); mschaefer@8971: this.extraWsts.add(iw); teichmann@7339: } mschaefer@8971: catch (final WstParser.ParseException e) { tom@7343: log.error(e.getMessage()); teichmann@7339: } sascha@482: } sascha@482: sascha@197: } sascha@197: sascha@197: public void parseWst() throws IOException { sascha@1223: if (Config.INSTANCE.skipWst()) { sascha@1223: log.info("skip parsing WST file"); sascha@1223: return; sascha@1223: } sascha@1223: mschaefer@8971: final WstParser wstParser = new WstParser(); teichmann@7339: try { mschaefer@8971: wstParser.parse(this.wstFile); mschaefer@8971: this.wst = wstParser.getWst(); mschaefer@8971: this.wst.setKmUp(this.wst.guessWaterLevelIncreasing()); teichmann@7339: } mschaefer@8971: catch (final WstParser.ParseException e) { tom@7343: log.error(e.getMessage()); teichmann@7339: } sascha@186: } sascha@186: sascha@184: public void parseGauges() throws IOException { sascha@1223: if (Config.INSTANCE.skipGauges()) { sascha@1223: log.info("skip parsing gauges"); sascha@1223: return; sascha@1223: } sascha@1223: mschaefer@8971: File gltFile = new File(this.wstFile.getParentFile(), PEGEL_GLT); sascha@184: gltFile = FileTools.repair(gltFile); sascha@184: sascha@184: if (!gltFile.isFile() || !gltFile.canRead()) { sascha@184: log.warn("cannot read gauges from '" + gltFile + "'"); sascha@184: return; sascha@184: } sascha@184: mschaefer@8971: final PegelGltParser pgltp = new PegelGltParser(); sascha@184: pgltp.parse(gltFile); sascha@184: mschaefer@8971: this.gauges = pgltp.getGauges(); sascha@189: mschaefer@8971: for (final ImportGauge gauge: this.gauges) { sascha@189: gauge.parseDependencies(); sascha@189: } sascha@184: } sascha@186: sascha@186: public void parseAnnotations() throws IOException { sascha@1223: if (Config.INSTANCE.skipAnnotations()) { sascha@1223: log.info("skip parsing annotations"); sascha@1223: return; sascha@1223: } sascha@1223: mschaefer@8971: final File riverDir = this.wstFile.getParentFile().getParentFile(); mschaefer@8971: final AnnotationsParser aparser = mschaefer@8971: new AnnotationsParser(this.annotationClassifier); sascha@186: aparser.parse(riverDir); sascha@186: mschaefer@8971: this.annotations = aparser.getAnnotations(); sascha@186: } sascha@188: sascha@1220: public void parseHYKs() { sascha@1223: if (Config.INSTANCE.skipHYKs()) { sascha@1223: log.info("skip parsing HYK files"); sascha@1223: return; sascha@1223: } sascha@1223: sascha@1220: log.info("looking for HYK files"); mschaefer@8971: final HYKParser parser = new HYKParser(); mschaefer@8971: final File riverDir = this.wstFile mschaefer@8971: .getParentFile() // Basisdaten mschaefer@8971: .getParentFile() // Hydrologie mschaefer@8971: .getParentFile(); // sascha@1220: sascha@1220: parser.parseHYKs(riverDir, new HYKParser.Callback() { sascha@1220: mschaefer@8971: Set hfs = new HashSet<>(); sascha@1220: sascha@1220: @Override mschaefer@8971: public boolean hykAccept(final File file) { mschaefer@8971: final HashedFile hf = new HashedFile(file); mschaefer@8971: final boolean success = this.hfs.add(hf); sascha@1220: if (!success) { tom@8856: log.warn("HYK file '" + file mschaefer@8971: + "' seems to be a duplicate."); sascha@1220: } sascha@1220: return success; sascha@1220: } sascha@1220: sascha@1220: @Override mschaefer@8971: public void hykParsed(final HYKParser parser) { sascha@1220: log.debug("callback from HYK parser"); mschaefer@8971: final ImportHYK hyk = parser.getHYK(); sascha@1220: hyk.setRiver(ImportRiver.this); mschaefer@8971: ImportRiver.this.hyks.add(hyk); sascha@1220: } sascha@1220: }); sascha@1220: } sascha@1220: felix@4712: felix@4767: /** Create a W80 Parser and parse w80 files found. */ felix@4767: public void parseW80s() { felix@4767: if (Config.INSTANCE.skipW80s()) { felix@4767: log.info("skip parsing W80s"); felix@4781: return; felix@4767: } mschaefer@8971: final W80Parser parser = new W80Parser(); mschaefer@8971: final File riverDir = this.wstFile mschaefer@8971: .getParentFile() // Basisdaten mschaefer@8971: .getParentFile() // Hydrologie mschaefer@8971: .getParentFile(); // felix@4767: mschaefer@8971: final ImportRiverCrossSectionParserCallback w80Callback = mschaefer@8971: new ImportRiverCrossSectionParserCallback("w80"); felix@4779: parser.parseW80s(riverDir, w80Callback); felix@4767: } felix@4767: felix@5810: /** Create a W80 Parser and parse w80 files found. */ felix@5810: public void parseW80CSVs() { felix@5810: if (Config.INSTANCE.skipW80CSVs()) { felix@5810: log.info("skip parsing W80 csvs"); felix@5810: return; felix@5810: } mschaefer@8971: final W80CSVParser parser = new W80CSVParser(); mschaefer@8971: final File riverDir = this.wstFile mschaefer@8971: .getParentFile() // Basisdaten mschaefer@8971: .getParentFile() // Hydrologie mschaefer@8971: .getParentFile(); // felix@5810: felix@5810: // Construct the Cross-Section-Data path. mschaefer@8971: final File csDir = new File(riverDir.getPath() mschaefer@8971: + File.separator + "Geodaesie" mschaefer@8971: + File.separator + "Querprofile" mschaefer@8971: + File.separator + "QP-Daten"); felix@5810: mschaefer@8971: final ImportRiverCrossSectionParserCallback w80CSVCallback = mschaefer@8971: new ImportRiverCrossSectionParserCallback("w80-csv"); felix@5810: parser.parseW80CSVs(csDir, w80CSVCallback); felix@5810: } felix@5810: felix@4782: felix@4782: /** felix@4782: * Create and use a DA50Parser, parse the files found, add the felix@4782: * ross-sections found. felix@4782: */ felix@4767: public void parseDA50s() { felix@4767: if (Config.INSTANCE.skipDA50s()) { felix@4767: log.info("skip parsing DA50s"); felix@4781: return; felix@4767: } mschaefer@8971: final DA50Parser parser = new DA50Parser(); mschaefer@8971: final File riverDir = this.wstFile mschaefer@8971: .getParentFile() // Basisdaten mschaefer@8971: .getParentFile() // Hydrologie mschaefer@8971: .getParentFile(); // felix@4782: mschaefer@8971: final ImportRiverCrossSectionParserCallback da50Callback = mschaefer@8971: new ImportRiverCrossSectionParserCallback("da50"); teichmann@5084: felix@4782: parser.parseDA50s(riverDir, da50Callback); felix@4767: } felix@4767: felix@4724: felix@4712: /** Create a DA66 Parser and parse the da66 files found. */ tom@8856: // TODO this is a copy of parsePRFs, extract interfaces tom@8856: //(e.g. CrossSectionParser). felix@4712: public void parseDA66s() { felix@4712: if (Config.INSTANCE.skipDA66s()) { felix@4712: log.info("skip parsing DA66s"); felix@4712: return; felix@4712: } felix@4712: felix@4712: log.info("looking for DA66 files"); mschaefer@8971: final DA66Parser parser = new DA66Parser(); mschaefer@8971: final File riverDir = this.wstFile mschaefer@8971: .getParentFile() // Basisdaten mschaefer@8971: .getParentFile() // Hydrologie mschaefer@8971: .getParentFile(); // felix@4712: mschaefer@8971: final ImportRiverCrossSectionParserCallback da66Callback = mschaefer@8971: new ImportRiverCrossSectionParserCallback("da66"); teichmann@5084: felix@4779: parser.parseDA66s(riverDir, da66Callback); felix@4712: } felix@4712: felix@4712: /** Create a PRFParser and let it parse the prf files found. */ sascha@1204: public void parsePRFs() { sascha@1223: if (Config.INSTANCE.skipPRFs()) { sascha@1223: log.info("skip parsing PRFs"); sascha@1223: return; sascha@1223: } sascha@1223: sascha@1204: log.info("looking for PRF files"); mschaefer@8971: final PRFParser parser = new PRFParser(); mschaefer@8971: final File riverDir = this.wstFile mschaefer@8971: .getParentFile() // Basisdaten mschaefer@8971: .getParentFile() // Hydrologie mschaefer@8971: .getParentFile(); // sascha@1206: mschaefer@8971: final ImportRiverCrossSectionParserCallback prfCallback = mschaefer@8971: new ImportRiverCrossSectionParserCallback("prf"); felix@4779: parser.parsePRFs(riverDir, prfCallback); sascha@1204: } sascha@1204: mschaefer@8971: public static Date yearToDate(final int year) { mschaefer@8971: final Calendar cal = Calendar.getInstance(); sascha@1204: cal.set(year, 5, 15, 12, 0, 0); mschaefer@8971: final long ms = cal.getTimeInMillis(); sascha@1204: cal.setTimeInMillis(ms - ms%1000); sascha@1204: return cal.getTime(); sascha@1204: } sascha@1204: sascha@190: public void storeDependencies() { tom@6823: /* test whether river is already in database. tom@6823: * Otherwise it makes no sense to skip waterlevel model WST-file tom@6823: * because the altitude reference is taken from there. */ mschaefer@8971: final Session session = ImporterSession.getInstance().getDatabaseSession(); mschaefer@8971: final Query query = session.createQuery("from River where name=:name"); mschaefer@8971: query.setString("name", this.name); mschaefer@8971: final List rivers = query.list(); tom@6823: if (rivers.isEmpty() && Config.INSTANCE.skipWst()){ tom@8856: log.error("River not yet in database. " mschaefer@8971: + "You cannot skip importing waterlevel model."); tom@6823: return; tom@6823: } tom@6823: ingo@2347: storeWstUnit(); sascha@190: storeAnnotations(); sascha@1221: storeHYKs(); sascha@1204: storeCrossSections(); sascha@190: storeGauges(); sascha@201: storeWst(); sascha@482: storeExtraWsts(); sascha@483: storeFixations(); sascha@484: storeOfficialLines(); sascha@490: storeFloodWater(); sascha@494: storeFloodProtection(); tom@8056: storeMeasurementStations(); ingo@2806: storeBedHeight(); ingo@2815: storeSedimentDensity(); rrenkert@7841: storePorosity(); ingo@2821: storeMorphologicalWidth(); ingo@2826: storeFlowVelocity(); tom@8032: storeSedimentLoadLS(); tom@8056: storeSedimentLoad(); felix@5261: storeWaterlevels(); ingo@2851: storeWaterlevelDifferences(); ingo@3328: storeSQRelations(); felix@5080: storeOfficialNumber(); mschaefer@8971: this.sinfoImporter.store(); mschaefer@8971: this.uinfoImporter.store(); sascha@201: } sascha@201: ingo@2347: public void storeWstUnit() { mschaefer@8971: if (this.wst == null) { tom@8856: log.warn("No unit given. " mschaefer@8971: + "Waterlevel-model WST-file has to be imported already."); ingo@2347: } ingo@2347: else { mschaefer@8971: this.wstUnit = this.wst.getUnit(); ingo@2347: } ingo@2347: } ingo@2347: sascha@1221: public void storeHYKs() { sascha@1223: if (!Config.INSTANCE.skipHYKs()) { sascha@1223: log.info("store HYKs"); sascha@1223: getPeer(); mschaefer@8971: for (final ImportHYK hyk: this.hyks) { sascha@1223: hyk.storeDependencies(); sascha@1223: } sascha@1221: } sascha@1221: } sascha@1221: sascha@1204: public void storeCrossSections() { felix@5810: if (!Config.INSTANCE.skipPRFs() mschaefer@8971: || !Config.INSTANCE.skipDA66s() mschaefer@8971: || !Config.INSTANCE.skipDA50s() mschaefer@8971: || !Config.INSTANCE.skipW80s() mschaefer@8971: || !Config.INSTANCE.skipW80CSVs()) { sascha@1223: log.info("store cross sections"); sascha@1223: getPeer(); mschaefer@8971: for (final ImportCrossSection crossSection: this.crossSections) { sascha@1223: crossSection.storeDependencies(); sascha@1223: } sascha@1204: } sascha@1204: } sascha@1204: sascha@201: public void storeWst() { mschaefer@8971: if (this.wst != null && !Config.INSTANCE.skipWst()) { mschaefer@8971: final River river = getPeer(); mschaefer@8971: this.wst.storeDependencies(river); teichmann@7183: teichmann@7183: // The flow direction of the main wst and the corresponding teichmann@7183: // waterlevels determine if the river is 'km_up'. mschaefer@8971: final Session session = ImporterSession.getInstance() mschaefer@8971: .getDatabaseSession(); mschaefer@8971: river.setKmUp(this.wst.getKmUp()); teichmann@7183: session.save(river); sascha@483: } sascha@483: } sascha@483: sascha@1223: public void storeFixations() { felix@5262: if (!Config.INSTANCE.skipFixations()) { felix@5262: log.info("store fixation wsts"); mschaefer@8971: final River river = getPeer(); mschaefer@8971: for (final ImportWst fWst: this.fixations) { felix@5261: log.debug("Fixation name: " + fWst.getDescription()); felix@5261: fWst.storeDependencies(river); sascha@1223: } sascha@1223: } sascha@1223: } sascha@1223: felix@5232: felix@5261: /** Store wsts from waterlevel-csv files. */ felix@5261: public void storeWaterlevels() { felix@5261: if (!Config.INSTANCE.skipWaterlevels()) felix@5262: mschaefer@8971: log.info("store waterlevel wsts from csv"); mschaefer@8971: final River river = getPeer(); mschaefer@8971: for (final ImportWst wWst: this.waterlevels) { felix@5261: log.debug("Waterlevel name: " + wWst.getDescription()); felix@5261: wWst.storeDependencies(river); felix@5261: } felix@5261: } felix@5262: felix@5261: felix@5232: /** Store wsts from waterleveldifference-csv files. */ felix@5232: public void storeWaterlevelDifferences() { felix@5232: if (!Config.INSTANCE.skipWaterlevelDifferences()) teichmann@5280: mschaefer@8971: log.info("store waterleveldifferences wsts from csv"); mschaefer@8971: final River river = getPeer(); mschaefer@8971: for (final ImportWst dWst: this.waterlevelDifferences) { felix@5232: log.debug("water.diff.: name " + dWst.getDescription()); felix@5232: dWst.storeDependencies(river); felix@5232: } felix@5232: } teichmann@5280: felix@5232: sascha@482: public void storeExtraWsts() { sascha@1223: if (!Config.INSTANCE.skipExtraWsts()) { sascha@1223: log.info("store extra wsts"); mschaefer@8971: final River river = getPeer(); mschaefer@8971: for (final ImportWst wst: this.extraWsts) { sascha@1223: log.debug("name: " + wst.getDescription()); sascha@1223: wst.storeDependencies(river); sascha@1223: } sascha@482: } sascha@482: } sascha@482: sascha@484: public void storeOfficialLines() { mschaefer@8971: if (Config.INSTANCE.skipOfficialLines() || this.officialLines.isEmpty()) { teichmann@6347: return; teichmann@6347: } teichmann@6347: teichmann@6347: log.info("store official lines wsts"); mschaefer@8971: final River river = getPeer(); mschaefer@8971: for (final ImportWst wst: this.officialLines) { teichmann@6347: log.debug("name: " + wst.getDescription()); teichmann@6347: wst.storeDependencies(river); teichmann@6347: teichmann@6347: // Store the official lines after the columns are store. mschaefer@8971: for (final ImportWstColumn wc: wst.getColumns()) { mschaefer@8971: final ImportOfficialWstColumn owc = (ImportOfficialWstColumn)wc; mschaefer@8971: final ImportOfficialLine ioc = owc.getOfficialLine(); teichmann@6347: if (ioc != null) { teichmann@6347: if (ioc.getPeer(river) == null) { tom@8856: log.warn("Cannot store official line: " mschaefer@8971: + ioc.getName()); teichmann@6347: } teichmann@6347: } sascha@1223: } sascha@484: } sascha@484: } sascha@484: sascha@490: public void storeFloodWater() { sascha@1223: if (!Config.INSTANCE.skipFloodWater()) { sascha@1223: log.info("store flood water wsts"); mschaefer@8971: final River river = getPeer(); mschaefer@8971: for (final ImportWst wst: this.floodWater) { sascha@1223: log.debug("name: " + wst.getDescription()); sascha@1223: wst.storeDependencies(river); sascha@1223: } sascha@490: } sascha@490: } sascha@490: felix@5262: sascha@494: public void storeFloodProtection() { sascha@1223: if (!Config.INSTANCE.skipFloodProtection()) { sascha@1223: log.info("store flood protection wsts"); mschaefer@8971: final River river = getPeer(); mschaefer@8971: for (final ImportWst wst: this.floodProtection) { sascha@1223: log.debug("name: " + wst.getDescription()); sascha@1223: wst.storeDependencies(river); sascha@1223: } sascha@494: } sascha@494: } sascha@494: ingo@2806: ingo@2806: public void storeBedHeight() { tom@8559: if (!Config.INSTANCE.skipBedHeight()) { tom@8559: log.info("store bed heights"); mschaefer@8971: final River river = getPeer(); ingo@2810: mschaefer@8971: if (this.bedHeights != null) { mschaefer@8971: for (final ImportBedHeight tmp: this.bedHeights) { mschaefer@8971: final ImportBedHeight single = tmp; ingo@2810: mschaefer@8971: final String desc = single.getDescription(); ingo@2810: tom@8559: log.debug("name: " + desc); tom@8559: tom@8559: single.storeDependencies(river); tom@8559: } ingo@2806: } tom@8559: else { tom@8559: log.info("No bed heights to store."); tom@8559: } ingo@2810: } ingo@2810: } ingo@2806: ingo@2810: ingo@2815: public void storeSedimentDensity() { ingo@2815: if (!Config.INSTANCE.skipSedimentDensity()) { ingo@2815: log.info("store sediment density"); ingo@2815: mschaefer@8971: final River river = getPeer(); ingo@2817: mschaefer@8971: for (final ImportSedimentDensity density: this.sedimentDensities) { mschaefer@8971: final String desc = density.getDescription(); ingo@2817: ingo@2817: log.debug("name: " + desc); ingo@2817: teichmann@7252: density.storeDependencies(river); ingo@2817: } ingo@2815: } ingo@2815: } ingo@2815: rrenkert@7841: public void storePorosity() { rrenkert@7841: if (!Config.INSTANCE.skipPorosity()) { rrenkert@7841: log.info("store porosity"); rrenkert@7841: mschaefer@8971: final River river = getPeer(); rrenkert@7841: mschaefer@8971: for (final ImportPorosity porosity: this.porosities) { mschaefer@8971: final String desc = porosity.getDescription(); rrenkert@7841: rrenkert@7841: log.debug("name: " + desc); rrenkert@7841: rrenkert@7841: porosity.storeDependencies(river); rrenkert@7841: } rrenkert@7841: } rrenkert@7841: } rrenkert@7841: ingo@2821: public void storeMorphologicalWidth() { ingo@2821: if (!Config.INSTANCE.skipMorphologicalWidth()) { ingo@2821: log.info("store morphological width"); ingo@2821: mschaefer@8971: final River river = getPeer(); ingo@2821: mschaefer@8971: for (final ImportMorphWidth width: this.morphologicalWidths) { teichmann@7252: width.storeDependencies(river); ingo@2821: } ingo@2821: } ingo@2821: } ingo@2821: ingo@2826: public void storeFlowVelocity() { ingo@2826: if (!Config.INSTANCE.skipFlowVelocity()) { ingo@2826: log.info("store flow velocity"); ingo@2826: mschaefer@8971: final River river = getPeer(); ingo@2827: mschaefer@8971: for (final ImportFlowVelocityModel flowVelocityModel: this.flowVelocityModels mschaefer@8971: ) { teichmann@7252: flowVelocityModel.storeDependencies(river); ingo@2827: } ingo@2832: mschaefer@8971: for (final ImportFlowVelocityMeasurement m: this.flowVelocityMeasurements) { teichmann@7252: m.storeDependencies(river); ingo@2832: } ingo@2826: } ingo@2826: } ingo@2826: ingo@2839: tom@8032: public void storeSedimentLoadLS() { tom@8032: if (!Config.INSTANCE.skipSedimentLoadLS()) { tom@8032: log.info("store sediment load longitudinal section data"); ingo@2839: mschaefer@8971: final River river = getPeer(); ingo@2839: mschaefer@8971: for (final ImportSedimentLoadLS sedimentLoadLS: this.sedimentLoadLSs) { tom@8032: sedimentLoadLS.storeDependencies(river); ingo@2839: } ingo@2839: } ingo@2839: } ingo@2839: ingo@2844: tom@8056: public void storeSedimentLoad() { tom@8056: if (!Config.INSTANCE.skipSedimentLoad()) { tom@8056: log.info("store sediment load data at measurement stations"); tom@8056: mschaefer@8971: for (final ImportSedimentLoad sedimentLoad: this.sedimentLoads) { tom@8056: sedimentLoad.storeDependencies(); tom@8056: } tom@8056: } tom@8056: } tom@8056: tom@8056: ingo@4193: public void storeMeasurementStations() { ingo@4193: if (!Config.INSTANCE.skipMeasurementStations()) { ingo@4193: log.info("store measurement stations"); ingo@4193: mschaefer@8971: final River river = getPeer(); ingo@4193: ingo@4193: int count = 0; ingo@4193: mschaefer@8971: for (final ImportMeasurementStation station: this.measurementStations) { mschaefer@8971: final boolean success = station.storeDependencies(river); teichmann@7252: if (success) { teichmann@7252: count++; teichmann@7252: } ingo@4193: } ingo@4193: ingo@4193: log.info("stored " + count + " measurement stations."); ingo@4193: } ingo@4193: } ingo@4193: ingo@4193: ingo@3328: public void storeSQRelations() { ingo@3328: if (!Config.INSTANCE.skipSQRelation()) { ingo@3328: log.info("store sq relations"); ingo@3328: ingo@3329: int count = 0; ingo@3329: mschaefer@8971: for (final ImportSQRelation sqRelation: this.sqRelations) { tom@8641: sqRelation.storeDependencies(); teichmann@7252: count++; ingo@3329: } ingo@3329: ingo@3329: log.info("stored " + count + " sq relations."); ingo@3328: } ingo@3328: } ingo@3328: ingo@3328: sascha@188: public void storeAnnotations() { sascha@1223: if (!Config.INSTANCE.skipAnnotations()) { mschaefer@8971: final River river = getPeer(); mschaefer@8971: for (final ImportAnnotation annotation: this.annotations) { sascha@1223: annotation.getPeer(river); sascha@1223: } sascha@188: } sascha@188: } sascha@188: sascha@190: public void storeGauges() { sascha@1223: if (!Config.INSTANCE.skipGauges()) { sascha@1223: log.info("store gauges:"); mschaefer@8971: final River river = getPeer(); mschaefer@8971: final Session session = ImporterSession.getInstance() mschaefer@8971: .getDatabaseSession(); mschaefer@8971: for (final ImportGauge gauge: this.gauges) { sascha@1223: log.info("\tgauge: " + gauge.getName()); sascha@1223: gauge.storeDependencies(river); sascha@1223: ImporterSession.getInstance().getDatabaseSession(); sascha@1223: session.flush(); sascha@1223: } sascha@191: } sascha@190: } sascha@190: sascha@188: public River getPeer() { mschaefer@8971: if (this.peer == null) { mschaefer@8971: final Session session = ImporterSession.getInstance() mschaefer@8971: .getDatabaseSession(); mschaefer@8971: final Query query = session.createQuery("from River where name=:name"); ingo@2347: felix@5027: Unit u = null; mschaefer@8971: if (this.wstUnit != null) { mschaefer@8971: u = this.wstUnit.getPeer(); felix@5027: } ingo@2347: mschaefer@8971: query.setString("name", this.name); mschaefer@8971: final List rivers = query.list(); sascha@188: if (rivers.isEmpty()) { mschaefer@8971: log.info("Store new river '" + this.name + "'"); mschaefer@8971: this.peer = new River(this.name, u, this.modelUuid); felix@5027: if (!Config.INSTANCE.skipBWASTR()) { mschaefer@8971: this.peer.setOfficialNumber(this.officialNumber); felix@5027: } mschaefer@8971: session.save(this.peer); sascha@188: } sascha@188: else { mschaefer@8971: this.peer = rivers.get(0); sascha@188: } sascha@188: } mschaefer@8971: return this.peer; sascha@188: } sascha@177: } sascha@177: // vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 :