teichmann@5844: /* Copyright (C) 2011, 2012, 2013 by Bundesanstalt für Gewässerkunde teichmann@5844: * Software engineering by Intevation GmbH teichmann@5844: * teichmann@5992: * This file is Free Software under the GNU AGPL (>=v3) teichmann@5844: * and comes with ABSOLUTELY NO WARRANTY! Check out the teichmann@5992: * documentation coming with Dive4Elements River for details. teichmann@5844: */ teichmann@5844: teichmann@5829: package org.dive4elements.river.importer; sascha@177: sascha@177: import java.io.File; sascha@177: import java.io.IOException; mschaefer@8970: import java.net.MalformedURLException; mschaefer@8970: import java.sql.SQLException; felix@5027: import java.util.HashMap; sascha@180: import java.util.List; felix@5027: import java.util.Map; sascha@180: sascha@177: import org.apache.log4j.Logger; mschaefer@8970: import org.apache.log4j.PropertyConfigurator; mschaefer@8970: import org.dive4elements.artifacts.common.utils.XMLUtils; mschaefer@8970: import org.dive4elements.river.backend.utils.StringUtil; mschaefer@8970: import org.dive4elements.river.importer.parsers.AnnotationClassifier; mschaefer@8970: import org.dive4elements.river.importer.parsers.BundesWasserStrassenParser; mschaefer@8970: import org.dive4elements.river.importer.parsers.InfoGewParser; mschaefer@8970: import org.hibernate.HibernateException; sascha@180: import org.hibernate.Transaction; mschaefer@8970: import org.slf4j.bridge.SLF4JBridgeHandler; sascha@765: import org.w3c.dom.Document; sascha@765: felix@5016: /** Data Importer. Further processing happens per-river. */ sascha@177: public class Importer sascha@177: { mschaefer@9011: static final String VERSION = "3.3.0"; mschaefer@9011: teichmann@8200: /** Private log. */ tom@8843: private static final Logger log = Logger.getLogger(Importer.class); sascha@177: tom@8843: private static final String BWASTR_ID_CSV_FILE = "BWASTR_ID.csv"; felix@5027: sascha@180: protected List rivers; sascha@180: sascha@180: public Importer() { sascha@180: } sascha@180: mschaefer@8970: public Importer(final List rivers) { sascha@180: this.rivers = rivers; sascha@180: } sascha@180: sascha@180: public List getRivers() { mschaefer@8970: return this.rivers; sascha@180: } sascha@180: mschaefer@8970: public void setRivers(final List rivers) { sascha@180: this.rivers = rivers; sascha@180: } sascha@180: felix@5027: /** Write rivers and their dependencies/dependants to db. */ sascha@180: public void writeRivers() { sascha@182: log.debug("write rivers started"); sascha@188: mschaefer@8970: for (final ImportRiver river: this.rivers) { sascha@188: log.debug("writing river '" + river.getName() + "'"); sascha@190: river.storeDependencies(); sascha@497: ImporterSession.getInstance().getDatabaseSession().flush(); sascha@180: } sascha@188: sascha@182: log.debug("write rivers finished"); sascha@180: } sascha@180: sascha@180: public void writeToDatabase() { sascha@183: sascha@180: Transaction tx = null; sascha@180: sascha@180: try { mschaefer@8970: tx = ImporterSession.getInstance().getDatabaseSession().beginTransaction(); sascha@180: sascha@188: try { sascha@188: writeRivers(); sascha@188: } mschaefer@8970: catch (final HibernateException he) { sascha@188: Throwable t = he.getCause(); sascha@188: while (t instanceof SQLException) { mschaefer@8970: final SQLException sqle = (SQLException) t; sascha@188: log.error("SQL exeception chain:", sqle); sascha@188: t = sqle.getNextException(); sascha@188: } sascha@188: throw he; sascha@188: } sascha@180: sascha@180: tx.commit(); sascha@180: } mschaefer@8970: catch (final RuntimeException re) { sascha@180: if (tx != null) { sascha@180: tx.rollback(); sascha@180: } sascha@180: throw re; sascha@180: } sascha@180: } sascha@180: sascha@765: public static AnnotationClassifier getAnnotationClassifier() { mschaefer@8970: final String annotationTypes = Config.INSTANCE.getAnnotationTypes(); sascha@765: sascha@765: if (annotationTypes == null) { sascha@766: log.info("no annotation types file configured."); sascha@765: return null; sascha@765: } sascha@765: mschaefer@8970: final File file = new File(annotationTypes); sascha@765: sascha@766: log.info("use annotation types file '" + file + "'"); sascha@766: sascha@765: if (!(file.isFile() && file.canRead())) { sascha@765: log.warn("annotation type file '" + file + "' is not readable."); sascha@765: return null; sascha@765: } sascha@765: mschaefer@8970: final Document rules = XMLUtils.parseDocument(file, false, null); sascha@765: sascha@765: if (rules == null) { sascha@765: log.warn("cannot parse annotation types file."); sascha@765: return null; sascha@765: } sascha@765: sascha@765: return new AnnotationClassifier(rules); sascha@765: } sascha@765: felix@4707: felix@4707: /** Starting point for importing river data. */ mschaefer@8970: public static void main(final String [] args) { sascha@177: mschaefer@8970: configureLogging(); sascha@177: mschaefer@9011: log.info("IMPORTER version " + VERSION); mschaefer@8970: log.info("START parsing rivers..."); mschaefer@8970: mschaefer@8970: final InfoGewParser infoGewParser = new InfoGewParser(getAnnotationClassifier()); ingo@2806: felix@5027: File bwastrFile = null; felix@5027: mschaefer@8970: // Main parsing loop for all river gew file paths in args mschaefer@8970: // FIXME: Multiple rivers lead to reparsing the already parsed rivers again in InfoGewParser.parse... mschaefer@8970: for (final String gew : args) { mschaefer@8970: log.info("Parsing info gew file: " + gew); mschaefer@8970: final File gewFile = new File(gew); felix@5027: if (bwastrFile == null) { mschaefer@8970: bwastrFile = new File(gewFile.getParentFile(), BWASTR_ID_CSV_FILE); felix@5027: } sascha@177: try { felix@5027: infoGewParser.parse(gewFile); sascha@177: } mschaefer@8970: catch (final IOException ioe) { mschaefer@8970: log.error("error parsing gew: " + gew, ioe); aheinecke@5106: System.exit(1); sascha@177: } sascha@177: } sascha@180: mschaefer@8970: // Parse a single river gew file specified in the flys.backend.importer.infogew.file property mschaefer@8970: // (seems to be an alternative to the args way) mschaefer@8970: final String gew = Config.INSTANCE.getInfoGewFile(); ingo@2804: if (gew != null && gew.length() > 0) { mschaefer@8970: log.info("Parsing info gew file: " + gew); mschaefer@8970: final File gewFile = new File(gew); felix@5027: if (bwastrFile == null) { mschaefer@8970: bwastrFile = new File(gewFile.getParentFile(), BWASTR_ID_CSV_FILE); felix@5027: } ingo@2804: try { felix@5027: infoGewParser.parse(gewFile); ingo@2804: } mschaefer@8970: catch (final IOException ioe) { mschaefer@8970: log.error("error parsing gew: " + gew, ioe); aheinecke@5106: System.exit(1); ingo@2804: } ingo@2804: } ingo@2804: felix@5027: // Look for official numbers. mschaefer@8970: final BundesWasserStrassenParser bwastrIdParser = new BundesWasserStrassenParser(); felix@5027: felix@5027: // Read bwastFile (river-dir + BWASTR_ID_CSV_FILE). felix@5027: if (!Config.INSTANCE.skipBWASTR()) { felix@5027: try{ felix@5027: bwastrIdParser.parse(bwastrFile); mschaefer@8970: final HashMap map = bwastrIdParser.getMap(); felix@5027: felix@5027: // Now link rivers with official numbers. mschaefer@8970: for(final ImportRiver river: infoGewParser.getRivers()) { mschaefer@8970: for(final Map.Entry entry: map.entrySet()) { mschaefer@8970: if (StringUtil.containsIgnoreCase(river.getName(), entry.getKey())) { felix@5027: river.setOfficialNumber(entry.getValue()); mschaefer@8970: log.debug(river.getName() + " is mapped to bwastr " + entry.getValue()); felix@5027: } felix@5027: } felix@5027: } mschaefer@8970: } mschaefer@8970: catch (final IOException ioe) { mschaefer@8970: log.warn("BWASTR-file could not be loaded: " + ioe.getMessage()); felix@5027: } felix@5027: } felix@5027: else { mschaefer@8970: log.debug("Skip reading BWASTR_ID.csv"); felix@5027: } felix@5027: mschaefer@8970: // Write all parsed objects to the database sascha@1223: if (!Config.INSTANCE.dryRun()) { sascha@199: new Importer(infoGewParser.getRivers()).writeToDatabase(); sascha@199: } felix@4708: else { felix@4708: log.info("Dry run, not writing to database."); felix@4708: } sascha@177: } mschaefer@8970: mschaefer@8970: /** mschaefer@8970: * Tries to load the Log4j configuration from the property 'log4j.configuration'. mschaefer@8970: */ mschaefer@8970: private static final void configureLogging() { mschaefer@8970: final String configPath = System.getProperty("log4j.configuration"); mschaefer@8970: try { mschaefer@8970: final File propFile = new File(configPath); mschaefer@8970: if (propFile.isFile() && propFile.canRead()) { mschaefer@8970: try { mschaefer@8970: PropertyConfigurator.configure(propFile.toURI().toURL()); mschaefer@8970: SLF4JBridgeHandler.install(); mschaefer@8970: } mschaefer@8970: catch (final MalformedURLException mue) { mschaefer@8970: mue.printStackTrace(System.err); mschaefer@8970: } mschaefer@8970: } mschaefer@8970: } mschaefer@8970: catch (final Exception e) { mschaefer@8970: e.printStackTrace(System.err); mschaefer@8970: } mschaefer@8970: } mschaefer@8970: sascha@177: } sascha@177: // vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 :