Mercurial > dive4elements > river
changeset 8988:ae76f618d990
Checks added for missing import directory
line wrap: on
line diff
--- a/backend/src/main/java/org/dive4elements/river/importer/ImportRiver.java Sun Apr 08 18:08:35 2018 +0200 +++ b/backend/src/main/java/org/dive4elements/river/importer/ImportRiver.java Sun Apr 08 18:09:32 2018 +0200 @@ -391,6 +391,10 @@ return this.wstFile.getParentFile().getParentFile().getParentFile(); } + public AnnotationClassifier getAnnotationClassifier() { + return this.annotationClassifier; + } + public void parseDependencies() throws IOException { log.info("Root dir is '" + getRiverDir() + "'"); parseGauges();
--- a/backend/src/main/java/org/dive4elements/river/importer/common/AbstractKmLineImport.java Sun Apr 08 18:08:35 2018 +0200 +++ b/backend/src/main/java/org/dive4elements/river/importer/common/AbstractKmLineImport.java Sun Apr 08 18:09:32 2018 +0200 @@ -29,10 +29,12 @@ protected StoreMode storeMode; + /***** CONSTRUCTOR *****/ public AbstractKmLineImport(final double km) { this.station = km; + this.storeMode = StoreMode.NONE; } @@ -52,10 +54,8 @@ * it */ protected KMTUPLE getPeer(final SERIES parent, final StoreMode parentStoreMode) { - if (this.peer != null) { - this.storeMode = StoreMode.NONE; + if (this.peer != null) return this.peer; - } final Session session = ImporterSession.getInstance().getDatabaseSession(); KMTUPLE value = null; if (parentStoreMode != StoreMode.INSERT) {
--- a/backend/src/main/java/org/dive4elements/river/importer/common/AbstractSeriesImport.java Sun Apr 08 18:08:35 2018 +0200 +++ b/backend/src/main/java/org/dive4elements/river/importer/common/AbstractSeriesImport.java Sun Apr 08 18:09:32 2018 +0200 @@ -45,7 +45,7 @@ protected Map<StoreMode, Integer> valueStoreCount; - protected StoreMode seriesStoreMode; + protected StoreMode storeMode; /***** CONSTRUCTORS *****/ @@ -55,6 +55,7 @@ this.valueStoreCount = new EnumMap<>(StoreMode.class); for (final StoreMode mode : StoreMode.values()) this.valueStoreCount.put(mode, Integer.valueOf(0)); + this.storeMode = StoreMode.NONE; } public AbstractSeriesImport(final String filename) { @@ -118,11 +119,11 @@ final SERIES peer = getPeer(river); if (peer != null) { for (final KMLINE value : this.values) { - incrementValueStoreCount(value.store(peer, this.seriesStoreMode)); + incrementValueStoreCount(value.store(peer, this.storeMode)); } } ImporterSession.getInstance().getDatabaseSession().flush(); - return this.seriesStoreMode; + return this.storeMode; } /** @@ -139,21 +140,19 @@ /** * Gets the model object of the data series, inserting it into the database if not already existing */ - public SERIES getPeer(final River river) { - if (this.peer != null) { - this.seriesStoreMode = StoreMode.NONE; + protected SERIES getPeer(final River river) { + if (this.peer != null) return this.peer; - } final Session session = ImporterSession.getInstance().getDatabaseSession(); final List<SERIES> rows = querySeriesItem(session, river); if (rows.isEmpty()) { getLog().info("Create new database instance"); this.peer = createSeriesItem(river); session.save(this.peer); - this.seriesStoreMode = StoreMode.INSERT; + this.storeMode = StoreMode.INSERT; } else { this.peer = rows.get(0); - this.seriesStoreMode = StoreMode.UPDATE; + this.storeMode = StoreMode.UPDATE; } return this.peer; }
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/BedMobilityParser.java Sun Apr 08 18:08:35 2018 +0200 +++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/BedMobilityParser.java Sun Apr 08 18:09:32 2018 +0200 @@ -66,7 +66,9 @@ */ public static List<BedMobilityParser> createParsers(final File importDir, final File relativeDir, final ImportRiver river) { final List<BedMobilityParser> parsers = new ArrayList<>(); - parsers.add(new BedMobilityParser(new File(importDir, IMPORT_FILENAME), new File(relativeDir, IMPORT_FILENAME), river)); + final File importFile = new File(importDir, IMPORT_FILENAME); + if (importFile.exists()) + parsers.add(new BedMobilityParser(importFile, new File(relativeDir, IMPORT_FILENAME), river)); return parsers; }
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/ChannelParser.java Sun Apr 08 18:08:35 2018 +0200 +++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/ChannelParser.java Sun Apr 08 18:09:32 2018 +0200 @@ -79,7 +79,9 @@ */ public static List<ChannelParser> createParsers(final File importDir, final File relativeDir, final ImportRiver river) { final List<ChannelParser> parsers = new ArrayList<>(); - parsers.add(new ChannelParser(new File(importDir, IMPORT_FILENAME), new File(relativeDir, IMPORT_FILENAME), river)); + final File importFile = new File(importDir, IMPORT_FILENAME); + if (importFile.exists()) + parsers.add(new ChannelParser(importFile, new File(relativeDir, IMPORT_FILENAME), river)); return parsers; }
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/CollisionParser.java Sun Apr 08 18:08:35 2018 +0200 +++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/CollisionParser.java Sun Apr 08 18:09:32 2018 +0200 @@ -100,8 +100,10 @@ */ public static List<CollisionParser> createParsers(final File importDir, final File relativeDir, final ImportRiver river) { final List<CollisionParser> parsers = new ArrayList<>(); - for (final File file : listFiles(importDir, ".csv")) - parsers.add(new CollisionParser(file, new File(relativeDir, file.getName()), river)); + if (importDir.exists()) { + for (final File file : listFiles(importDir, ".csv")) + parsers.add(new CollisionParser(file, new File(relativeDir, file.getName()), river)); + } return parsers; }
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/DailyDischargeParser.java Sun Apr 08 18:08:35 2018 +0200 +++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/DailyDischargeParser.java Sun Apr 08 18:09:32 2018 +0200 @@ -79,14 +79,16 @@ */ public static List<DailyDischargeParser> createParsers(final File importDir, final File relativeDir, final ImportRiver river) { final List<DailyDischargeParser> parsers = new ArrayList<>(); - final File[] files = importDir.listFiles(new FilenameFilter() { - @Override - public boolean accept(final File dir, final String name) { - return IMPORT_FILENAME.matcher(name).matches(); - } - }); - for (final File file : files) - parsers.add(new DailyDischargeParser(file, new File(relativeDir, file.getName()), river)); + if (importDir.exists()) { + final File[] files = importDir.listFiles(new FilenameFilter() { + @Override + public boolean accept(final File dir, final String name) { + return IMPORT_FILENAME.matcher(name).matches(); + } + }); + for (final File file : files) + parsers.add(new DailyDischargeParser(file, new File(relativeDir, file.getName()), river)); + } return parsers; }
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/DepthEvolutionParser.java Sun Apr 08 18:08:35 2018 +0200 +++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/DepthEvolutionParser.java Sun Apr 08 18:09:32 2018 +0200 @@ -77,8 +77,9 @@ */ public static List<DepthEvolutionParser> createParsers(final File importDir, final File relativeDir, final ImportRiver river) { final List<DepthEvolutionParser> parsers = new ArrayList<>(); - for (final File file : listFiles(importDir, ".csv")) - parsers.add(new DepthEvolutionParser(file, new File(relativeDir, file.getName()), river)); + if (importDir.exists()) + for (final File file : listFiles(importDir, ".csv")) + parsers.add(new DepthEvolutionParser(file, new File(relativeDir, file.getName()), river)); return parsers; }
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/FlowDepthParser.java Sun Apr 08 18:08:35 2018 +0200 +++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/FlowDepthParser.java Sun Apr 08 18:09:32 2018 +0200 @@ -87,8 +87,9 @@ */ public static List<FlowDepthParser> createParsers(final File importDir, final File relativeDir, final ImportRiver river) { final List<FlowDepthParser> parsers = new ArrayList<>(); - for (final File file : listFiles(importDir, ".csv")) - parsers.add(new FlowDepthParser(file, new File(relativeDir, file.getName()), river)); + if (importDir.exists()) + for (final File file : listFiles(importDir, ".csv")) + parsers.add(new FlowDepthParser(file, new File(relativeDir, file.getName()), river)); return parsers; }
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/InfrastructureParser.java Sun Apr 08 18:08:35 2018 +0200 +++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/InfrastructureParser.java Sun Apr 08 18:09:32 2018 +0200 @@ -26,7 +26,6 @@ import org.dive4elements.river.importer.common.ParsingState; import org.dive4elements.river.importer.sinfo.importitem.InfrastructureKmLineImport; import org.dive4elements.river.importer.sinfo.importitem.InfrastructureSeriesImport; -import org.dive4elements.river.model.AnnotationType; import org.dive4elements.river.model.sinfo.Infrastructure; import org.dive4elements.river.model.sinfo.InfrastructureValue; @@ -66,8 +65,6 @@ private final HashMap<String, ImportAttribute> bankAttributes; - private final HashMap<String, ImportAnnotationType> types; - /***** CONSTRUCTORS *****/ @@ -79,9 +76,6 @@ this.bankAttributes.put("links", new ImportAttribute(DB_BANK_LEFT)); this.bankAttributes.put("rechts", new ImportAttribute(DB_BANK_RIGHT)); this.bankAttributes.put("", new ImportAttribute(DB_BANK_NULL)); - this.types = new HashMap<>(); - for (final AnnotationType type : AnnotationType.getTypes()) - this.types.put(type.getName().trim().toLowerCase(), new ImportAnnotationType(type.getName())); } @@ -104,8 +98,9 @@ */ public static List<InfrastructureParser> createParsers(final File importDir, final File relativeDir, final ImportRiver river) { final List<InfrastructureParser> parsers = new ArrayList<>(); - for (final File file : listFiles(importDir, ".csv")) - parsers.add(new InfrastructureParser(file, new File(relativeDir, file.getName()), river)); + if (importDir.exists()) + for (final File file : listFiles(importDir, ".csv")) + parsers.add(new InfrastructureParser(file, new File(relativeDir, file.getName()), river)); return parsers; } @@ -137,12 +132,15 @@ final Matcher m = META_TYPE.matcher(this.currentLine); if (m.matches()) { this.metaPatternsMatched.add(META_TYPE); - if (this.types.containsKey(m.group(1).trim().toLowerCase())) - this.seriesHeader.setType(this.types.get(m.group(1).trim().toLowerCase())); + if (this.river.getAnnotationClassifier() != null) { + final ImportAnnotationType type = this.river.getAnnotationClassifier().classifyDescription(m.group(1).trim(), + this.river.getAnnotationClassifier().getDefaultType()); + this.seriesHeader.setType(type); + log.info(String.format("Type name in file: '%s', will be assigned to database name '%s'", m.group(1).trim(), type.getName())); + } else { - final ImportAnnotationType type = new ImportAnnotationType(m.group(1).trim()); - this.types.put(m.group(1).trim().toLowerCase(), type); - this.seriesHeader.setType(type); + log.error("No annotation types file configured, cannot process type '" + m.group(1).trim() + "'"); + this.headerParsingState = ParsingState.STOP; } return true; }
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/SelectedAdditionalParser.java Sun Apr 08 18:08:35 2018 +0200 +++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/SelectedAdditionalParser.java Sun Apr 08 18:09:32 2018 +0200 @@ -17,6 +17,7 @@ import java.io.LineNumberReader; import java.io.Serializable; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import org.apache.log4j.Logger; @@ -43,9 +44,7 @@ private static final Logger log = Logger.getLogger(SelectedAdditionalParser.class); - private static final String IMPORT_Q_FILENAME = "Mit_Abflussdaten.txt"; - - private static final String IMPORT_W_FILENAME = "Ohne_Abflussdaten.txt"; + private static final String IMPORT_FILENAME = "Zus_Laengsschnitte.txt"; private enum SelectionType { WITH_Q("Q", "with discharge"), // @@ -66,6 +65,13 @@ public String getLogText() { return this.logText; } + + public static SelectionType parse(final String path) { + if (path.toLowerCase().endsWith(".wst")) + return WITH_Q; + else + return WITHOUT_Q; + } } private final File importPath; @@ -74,19 +80,16 @@ private final ImportRiver river; - private final SelectionType selectionType; - - private final List<String> links; + private final HashMap<String, SelectionType> links; /***** CONSTRUCTORS *****/ - public SelectedAdditionalParser(final File importPath, final File rootRelativePath, final ImportRiver river, final SelectionType selectionType) { + public SelectedAdditionalParser(final File importPath, final File rootRelativePath, final ImportRiver river) { this.importPath = importPath; this.rootRelativePath = rootRelativePath; this.river = river; - this.selectionType = selectionType; - this.links = new ArrayList<>(); + this.links = new HashMap<>(); } @@ -104,10 +107,9 @@ */ public static List<SelectedAdditionalParser> createParsers(final File importDir, final File relativeDir, final ImportRiver river) { final List<SelectedAdditionalParser> parsers = new ArrayList<>(); - parsers.add(new SelectedAdditionalParser(new File(importDir, IMPORT_Q_FILENAME), new File(relativeDir, IMPORT_Q_FILENAME), - river, SelectionType.WITH_Q)); - parsers.add(new SelectedAdditionalParser(new File(importDir, IMPORT_W_FILENAME), new File(relativeDir, IMPORT_W_FILENAME), - river, SelectionType.WITHOUT_Q)); + final File importFile = new File(importDir, IMPORT_FILENAME); + if (importFile.exists()) + parsers.add(new SelectedAdditionalParser(importFile, new File(relativeDir, IMPORT_FILENAME), river)); return parsers; } @@ -123,8 +125,8 @@ line = in.readLine(); if (line == null) break; - if (!line.trim().isEmpty() && !line.trim().startsWith(AbstractParser.START_META_CHAR)) - this.links.add(line.trim()); + if (!line.trim().isEmpty() && !line.trim().startsWith(AbstractParser.START_META_CHAR) && !this.links.containsKey(line.trim())) + this.links.put(line.trim(), SelectionType.parse(line.trim())); } log.info("Number of file links found: " + this.links.size()); } @@ -138,17 +140,16 @@ public void store() { final Session session = ImporterSession.getInstance().getDatabaseSession(); final SQLQuery reset = session.createSQLQuery("UPDATE wsts SET sinfo_selection = NULL WHERE (river_id=:river_id) AND (kind=1)" - + " AND (sinfo_selection=:seltype)"); + + " AND (sinfo_selection IS NOT NULL)"); reset.setParameter("river_id", this.river.getPeer().getId()); - reset.setParameter("seltype", this.selectionType.getKey()); reset.executeUpdate(); final Query query = session.createQuery("FROM Wst WHERE (river=:river) AND (kind=1) AND (lower(description) LIKE :path)"); query.setParameter("river", this.river.getPeer()); int count = 0; - for (final String wstfile : this.links) { - count += updateWst(session, query, this.river.getPeer(), wstfile, this.selectionType); + for (final String wstfile : this.links.keySet()) { + count += updateWst(session, query, this.river.getPeer(), wstfile, this.links.get(wstfile)); } - log.info("Updated " + count + " wsts for selected additionals " + this.selectionType.getLogText()); + log.info("Updated " + count + " wsts for selected additionals"); } private int updateWst(final Session session, final Query query, final River river, final String path, final SelectionType selectionType) {
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/TkhParser.java Sun Apr 08 18:08:35 2018 +0200 +++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/TkhParser.java Sun Apr 08 18:09:32 2018 +0200 @@ -87,8 +87,9 @@ */ public static List<TkhParser> createParsers(final File importDir, final File relativeDir, final ImportRiver river) { final List<TkhParser> parsers = new ArrayList<>(); - for (final File file : listFiles(importDir, ".csv")) - parsers.add(new TkhParser(file, new File(relativeDir, file.getName()), river)); + if (importDir.exists()) + for (final File file : listFiles(importDir, ".csv")) + parsers.add(new TkhParser(file, new File(relativeDir, file.getName()), river)); return parsers; }
--- a/backend/src/main/java/org/dive4elements/river/importer/uinfo/parsers/SalixParser.java Sun Apr 08 18:08:35 2018 +0200 +++ b/backend/src/main/java/org/dive4elements/river/importer/uinfo/parsers/SalixParser.java Sun Apr 08 18:09:32 2018 +0200 @@ -87,7 +87,9 @@ */ public static List<SalixParser> createParsers(final File importDir, final File relativeDir, final ImportRiver river) { final List<SalixParser> parsers = new ArrayList<>(); - parsers.add(new SalixParser(new File(importDir, IMPORT_FILENAME), new File(relativeDir, IMPORT_FILENAME), river)); + final File importFile = new File(importDir, IMPORT_FILENAME); + if (importFile.exists()) + parsers.add(new SalixParser(importFile, new File(relativeDir, IMPORT_FILENAME), river)); return parsers; }