aheinecke@5167: #!/bin/bash
aheinecke@5167: # Import script for rivers
aheinecke@5167: #
aheinecke@5167: # Authors:
aheinecke@5167: # Andre Heinecke <aheinecke@intevation.de>
aheinecke@5167: #
aheinecke@5167: # Copyright:
aheinecke@5168: # Copyright (C) 2013 Intevation GmbH
aheinecke@5167: #
aheinecke@5167: # This program is free software; you can redistribute it and/or
aheinecke@5167: # modify it under the terms of the GNU General Public License
aheinecke@5167: # as published by the Free Software Foundation; either version 2
aheinecke@5167: # of the License, or (at your option) any later version.
aheinecke@5167: #
aheinecke@5167: # This program is distributed in the hope that it will be useful,
aheinecke@5167: # but WITHOUT ANY WARRANTY; without even the implied warranty of
aheinecke@5167: # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
aheinecke@5167: # GNU General Public License for more details.
aheinecke@5167: #
aheinecke@5167: # You should have received a copy of the GNU General Public License
aheinecke@5167: # along with this program; if not, write to the Free Software
aheinecke@5167: # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
aheinecke@5167: 
aheinecke@5167: set -e
aheinecke@5167: 
aheinecke@5167: # Default settings
aheinecke@5167: DEFAULT_HOST=localhost
aheinecke@5167: DEFAULT_PORT=1521
aheinecke@5167: DEFAULT_USER=flys_dami
aheinecke@5167: DEFAULT_PASS=flys_dami
aheinecke@5167: DEFAULT_LOG=$PWD/logs
aheinecke@5167: DEFAULT_BACKEND_NAME="XE"
aheinecke@5167: JAR="hydr_morph/importer.jar"
aheinecke@5167: IMPORTER_DRY_RUN=false
tom@6449: IMPORTER_MAINVALUE_TYPES=QWTD-
aheinecke@5167: IMPORTER_ANNOTATION_TYPES="conf/annotation-types.xml"
aheinecke@5167: 
aheinecke@5167: 
aheinecke@5167: MIN_MEMORY="8024m"
aheinecke@5167: 
tom@6651: # Default encoding. Change here if necessary
tom@6651: export LC_ALL=de_DE@euro
tom@6651: 
aheinecke@5169: if [ -z "$OPTIONAL_LIBS" ]; then
aheinecke@5212:     OPTIONAL_LIBS="$(dirname $0)/opt"
aheinecke@5169: fi
aheinecke@5169: 
aheinecke@5167: if [ -d "$OPTIONAL_LIBS" ]; then
aheinecke@5167:     export PATH="$OPTIONAL_LIBS/bin:$PATH"
aheinecke@5167:     export LD_LIBRARY_PATH="$OPTIONAL_LIBS/lib:$LD_LIBRARY_PATH"
aheinecke@5167:     export LD_LIBRARY_PATH="$OPTIONAL_LIBS/lib64:$LD_LIBRARY_PATH"
aheinecke@5167:     export PYTHONPATH="$OPTIONAL_LIBS/lib/python2.6/site-packages:$PYTHONPATH"
aheinecke@5167:     export PYTHONPATH="$OPTIONAL_LIBS/lib64/python2.6/site-packages:$PYTHONPATH"
aheinecke@5167:     export GDAL_DATA="$OPTIONAL_LIBS/share/gdal"
aheinecke@5167: fi
aheinecke@5167: 
aheinecke@5167: usage(){
aheinecke@5167:     cat << EOF
aheinecke@5167: 
aheinecke@5167: usage: $0 [options] gew_file
aheinecke@5167: 
aheinecke@5167: Import a river described by the gew_file
aheinecke@5167: 
aheinecke@5167: OPTIONS:
aheinecke@5167:    -?, --help                      Show this message
aheinecke@5167:    -u, --username=<username>       Database username. Default: $DEFAULT_USER
aheinecke@5167:    -w, --password=<password>       Database password. Default: $DEFAULT_PASS
aheinecke@5167:    -h, --host=<host>               Connect to database on host <host>.
aheinecke@5167:                                    Default: $DEFAULT_HOST
aheinecke@5167:    -p, --port=<number>             Use port number <number>. Default: $DEFAULT_PORT
aheinecke@5167:    -d, --db-name=<database_name>   Name of the database / backend. Default: $DEFAULT_BACKEND_NAME
aheinecke@5167:    -l, --log-dir=<directory>       Directory in which to create the log files.
tom@7728:                                    Default: $PWD/logs
aheinecke@5167:    --postgres                      Database is PostgreSQL
aheinecke@5167:    --skip-hydro                    Skip import of hydrological data
aheinecke@5167:    --skip-morpho                   Skip import of morphological data
aheinecke@5167:    --skip-geo                      Skip import of geographic data
tom@6825:    --skip-prf                      Skip import of cross section data
aheinecke@5167: EOF
aheinecke@5167: exit 0
aheinecke@5167: }
aheinecke@5167: 
tom@7728: OPTS=`getopt -o ?u:w:h:p:d:l: \
tom@7728:      -l help,username:,password:,host:,port:,db-name:,log-dir:,skip-hydro,skip-morpho,skip-geo,skip-prf,postgres \
aheinecke@5167:      -n $0 -- "$@"`
aheinecke@5167: if [ $? != 0 ] ; then usage; fi
aheinecke@5167: eval set -- "$OPTS"
aheinecke@5167: while true ; do
aheinecke@5167:   case "$1" in
aheinecke@5167:     "-?"|"--help")
aheinecke@5167:       usage;;
aheinecke@5167:     "--")
aheinecke@5167:       shift
aheinecke@5167:       break;;
aheinecke@5167:     "-u"|"--username")
aheinecke@5192:       DBUSER=$2
aheinecke@5167:       shift 2;;
aheinecke@5167:     "-w"|"--password")
aheinecke@5192:       DBPASS=$2
aheinecke@5167:       shift 2;;
aheinecke@5167:     "-h"|"--host")
aheinecke@5192:       DBHOST=$2
aheinecke@5167:       shift 2;;
aheinecke@5167:     "-p"|"--port")
aheinecke@5192:       DBPORT=$2
aheinecke@5167:       shift 2;;
aheinecke@5167:     "-l"|"--log-dir")
aheinecke@5167:       LOG=$2
aheinecke@5167:       shift 2;;
aheinecke@5167:     "-d"|"--db-name")
aheinecke@5167:       BACKEND_NAME=$2
aheinecke@5167:       shift 2;;
aheinecke@5167:     "--skip-hydro")
aheinecke@5167:       SKIP_HYDRO="TRUE"
aheinecke@5167:       shift;;
aheinecke@5167:     "--skip-morpho")
aheinecke@5167:       SKIP_MORPHO="TRUE"
aheinecke@5167:       shift;;
tom@6825:     "--skip-prf")
tom@6825:       SKIP_PRF="TRUE"
aheinecke@5167:       shift;;
aheinecke@5167:     "--skip-geo")
aheinecke@5167:       SKIP_GEO="TRUE"
aheinecke@5167:       shift;;
aheinecke@5167:     "--postgres")
aheinecke@5167:       POSTGRES="TRUE"
aheinecke@5167:       shift;;
aheinecke@5167:     *)
aheinecke@5167:       echo "Unknown Option $1"
aheinecke@5167:       usage;;
aheinecke@5167:   esac
aheinecke@5167: done
aheinecke@5167: 
aheinecke@5191: if [ -z $DBUSER ]; then
aheinecke@5192:   DBUSER=$DEFAULT_USER
aheinecke@5167: fi
aheinecke@5191: if [ -z $DBPASS ]; then
aheinecke@5192:   DBPASS=$DEFAULT_PASS
aheinecke@5167: fi
aheinecke@5191: if [ -z $DBPORT ]; then
aheinecke@5192:   DBPORT=$DEFAULT_PORT
aheinecke@5167: fi
aheinecke@5191: if [ -z $DBHOST ]; then
aheinecke@5192:   DBHOST=$DEFAULT_HOST
aheinecke@5167: fi
aheinecke@5167: if [ -z $BACKEND_NAME ]; then
aheinecke@5167:   BACKEND_NAME=$DEFAULT_BACKEND_NAME
aheinecke@5167: fi
tom@7728: if [ -z $LOG ]; then
aheinecke@5167:   LOG=$DEFAULT_LOG
aheinecke@5167: fi
aheinecke@5167: 
aheinecke@5167: if [ $# != 1 ]; then
aheinecke@5167:     usage
aheinecke@5167: fi
aheinecke@5167: 
aheinecke@5167: if [ ! -r $1 ]; then
aheinecke@5167:     echo "Could not open $1 please ensure it exists and is readable"
aheinecke@5167: fi
aheinecke@5167: 
aheinecke@5167: GEW_FILE="$1"
tom@5459: RIVER_NAME=$(grep "Gew.sser" "$1" | sed 's/Gew.sser: //')
aheinecke@5167: DATE=$(date +%Y.%m.%d_%H%M)
tom@5459: LOG_DIR=${LOG}/`basename $GEW_FILE .gew`-$DATE
aheinecke@5167: mkdir -p ${LOG_DIR}
aheinecke@5167: 
tom@7728: cat > "$LOG_DIR/log4j.properties" << "EOF"
tom@7728: log4j.rootLogger=DEBUG, IMPORTER
tom@7728: log4j.appender.IMPORTER.layout=org.apache.log4j.PatternLayout
tom@7728: log4j.appender.IMPORTER.layout.ConversionPattern=%d [%t] %-5p %c - %m%n
tom@7728: log4j.appender.IMPORTER=org.apache.log4j.RollingFileAppender
tom@7728: log4j.appender.IMPORTER.File=./import.log
tom@7728: log4j.appender.IMPORTER.MaxFileSize=100000KB
tom@7728: log4j.appender.IMPORTER.MaxBackupIndex=10
tom@7728: EOF
tom@7728: 
aheinecke@5355: if [ "$POSTGRES" = "TRUE" ]; then
aheinecke@5167:     JAR=$(echo "$JAR" | sed 's/importer/importer_psql/')
aheinecke@5167:     if [ ! -r "$JAR" ]; then
aheinecke@5167:       echo "Could not find Postgres importer $JAR"
aheinecke@5167:       exit 1
aheinecke@5167:     fi
aheinecke@5191:     OGR_CONNECTION="PG:dbname=$BACKEND_NAME host=$DBHOST port=$DBPORT \
aheinecke@5191:       user=$DBUSER password=$DBPASS"
aheinecke@5167:     BACKEND_DB_PREFIX="jdbc:postgresql:"
aheinecke@5167:     BACKEND_DB_DRIVER="org.postgresql.Driver"
aheinecke@5167:     BACKEND_DB_DIALECT="org.hibernate.dialect.PostgreSQLDialect"
aheinecke@5167: else
aheinecke@5167:     BACKEND_DB_PREFIX="jdbc:oracle:thin:@"
aheinecke@5167:     BACKEND_DB_DRIVER="oracle.jdbc.OracleDriver"
aheinecke@5167:     BACKEND_DB_DIALECT="org.hibernate.dialect.OracleDialect"
aheinecke@5167: fi
aheinecke@5167: 
aheinecke@5191: BACKEND_URL=$BACKEND_DB_PREFIX//$DBHOST:$DBPORT/$BACKEND_NAME
aheinecke@5167: 
aheinecke@5167: echo "Importing $RIVER_NAME into $BACKEND_URL."
aheinecke@5167: 
aheinecke@5167: import_hydro(){
aheinecke@5167:     LOG_FILE=${LOG_DIR}/hydro.log
aheinecke@5167:     echo Importing Hydrological data.
aheinecke@5167:     echo Logging into: $LOG_FILE
tom@7728:     sed -i 's!\(log4j.appender.IMPORTER.File=\).*!\1'"$LOG_FILE"'!' \
tom@7728:         $LOG_DIR/log4j.properties
aheinecke@5167:     java -jar \
aheinecke@5167:     -Xmx$MIN_MEMORY \
aheinecke@5167:     -server \
aheinecke@5167:     -Dlog4j.configuration=file://$LOG_DIR/log4j.properties \
aheinecke@5191:     -Dflys.backend.user=$DBUSER \
aheinecke@5191:     -Dflys.backend.password=$DBPASS \
aheinecke@5167:     -Dflys.backend.url=$BACKEND_URL \
aheinecke@5167:     -Dflys.backend.driver=$BACKEND_DB_DRIVER \
aheinecke@5167:     -Dflys.backend.dialect=$BACKEND_DB_DIALECT \
aheinecke@5167:     -Dflys.backend.importer.infogew.file="$GEW_FILE" \
aheinecke@5167:     -Dflys.backend.main.value.types=$IMPORTER_MAINVALUE_TYPES \
aheinecke@5167:     -Dflys.backend.importer.annotation.types=$IMPORTER_ANNOTATION_TYPES \
aheinecke@5167:     -Dflys.backend.importer.dry.run=$IMPORTER_DRY_RUN \
aheinecke@5167:     -Dflys.backend.importer.skip.annotations=false \
aheinecke@5167:     -Dflys.backend.importer.skip.bwastr=false \
aheinecke@5167:     -Dflys.backend.importer.skip.extra.wsts=false \
aheinecke@5167:     -Dflys.backend.importer.skip.fixations=false \
aheinecke@5167:     -Dflys.backend.importer.skip.flood.water=false \
aheinecke@5167:     -Dflys.backend.importer.skip.flood.protection=false \
aheinecke@5167:     -Dflys.backend.importer.skip.gauges=false \
tom@7326:     -Dflys.backend.importer.skip.historical.discharge.tables=true \
aheinecke@5167:     -Dflys.backend.importer.skip.hyks=false \
aheinecke@5167:     -Dflys.backend.importer.skip.official.lines=false \
tom@6825:     -Dflys.backend.importer.skip.prfs=true \
tom@6825:     -Dflys.backend.importer.skip.w80s=true \
tom@6825:     -Dflys.backend.importer.skip.w80.csvs=true \
tom@6825:     -Dflys.backend.importer.skip.da50s=true \
tom@6825:     -Dflys.backend.importer.skip.da66s=true \
tom@6825:     -Dflys.backend.importer.skip.wst=false \
tom@6417:     -Dflys.backend.importer.skip.measurement.stations=true \
aheinecke@5167:     -Dflys.backend.importer.skip.waterlevel.differences=true \
aheinecke@5167:     -Dflys.backend.importer.skip.waterlevels=true \
aheinecke@5167:     -Dflys.backend.importer.skip.sq.relation=true \
aheinecke@5167:     -Dflys.backend.importer.skip.sediment.density=true \
aheinecke@5167:     -Dflys.backend.importer.skip.sediment.yield=true \
aheinecke@5167:     -Dflys.backend.importer.skip.morphological.width=true \
aheinecke@5167:     -Dflys.backend.importer.skip.flow.velocity=true \
aheinecke@5167:     -Dflys.backend.importer.skip.bed.height.single=true \
aheinecke@5167:     $JAR
aheinecke@5167: }
aheinecke@5167: 
aheinecke@5167: import_morpho(){
aheinecke@5167:     LOG_FILE=${LOG_DIR}/morpho.log
aheinecke@5167:     echo Importing Morphological data.
aheinecke@5167:     echo Logging into: $LOG_FILE
tom@7728:     sed -i 's!\(log4j.appender.IMPORTER.File=\).*!\1'"$LOG_FILE"'!' \
tom@7728:         $LOG_DIR/log4j.properties
aheinecke@5167:     java -jar \
aheinecke@5167:     -Xmx$MIN_MEMORY \
aheinecke@5167:     -server \
aheinecke@5167:     -Dlog4j.configuration=file://$LOG_DIR/log4j.properties \
aheinecke@5191:     -Dflys.backend.user=$DBUSER \
aheinecke@5191:     -Dflys.backend.password=$DBPASS \
aheinecke@5167:     -Dflys.backend.url=$BACKEND_URL \
aheinecke@5167:     -Dflys.backend.driver=$BACKEND_DB_DRIVER \
aheinecke@5167:     -Dflys.backend.dialect=$BACKEND_DB_DIALECT \
aheinecke@5167:     -Dflys.backend.importer.infogew.file="$GEW_FILE" \
aheinecke@5167:     -Dflys.backend.main.value.types=$IMPORTER_MAINVALUE_TYPES \
aheinecke@5167:     -Dflys.backend.importer.annotation.types=$IMPORTER_ANNOTATION_TYPES \
aheinecke@5167:     -Dflys.backend.importer.dry.run=$IMPORTER_DRY_RUN \
aheinecke@5167:     -Dflys.backend.importer.skip.annotations=true \
aheinecke@5167:     -Dflys.backend.importer.skip.bwastr=true \
aheinecke@5167:     -Dflys.backend.importer.skip.extra.wsts=true \
aheinecke@5167:     -Dflys.backend.importer.skip.fixations=true \
aheinecke@5167:     -Dflys.backend.importer.skip.flood.water=true \
aheinecke@5167:     -Dflys.backend.importer.skip.flood.protection=true \
aheinecke@5167:     -Dflys.backend.importer.skip.gauges=true \
aheinecke@5167:     -Dflys.backend.importer.skip.historical.discharge.tables=true \
aheinecke@5167:     -Dflys.backend.importer.skip.hyks=true \
aheinecke@5167:     -Dflys.backend.importer.skip.official.lines=true \
aheinecke@5167:     -Dflys.backend.importer.skip.prfs=true \
aheinecke@5167:     -Dflys.backend.importer.skip.w80s=true \
tom@6706:     -Dflys.backend.importer.skip.w80.csvs=true \
tom@6825:     -Dflys.backend.importer.skip.da50s=true \
tom@6825:     -Dflys.backend.importer.skip.da66s=true \
aheinecke@5167:     -Dflys.backend.importer.skip.wst=true \
tom@6417:     -Dflys.backend.importer.skip.measurement.stations=false \
aheinecke@5167:     -Dflys.backend.importer.skip.waterlevel.differences=false \
aheinecke@5167:     -Dflys.backend.importer.skip.waterlevels=false \
aheinecke@5167:     -Dflys.backend.importer.skip.sq.relation=false \
aheinecke@5167:     -Dflys.backend.importer.skip.sediment.density=false \
aheinecke@5167:     -Dflys.backend.importer.skip.sediment.yield=false \
aheinecke@5167:     -Dflys.backend.importer.skip.morphological.width=false \
aheinecke@5167:     -Dflys.backend.importer.skip.flow.velocity=false \
aheinecke@5167:     -Dflys.backend.importer.skip.bed.height.single=false \
aheinecke@5167:     $JAR
aheinecke@5167: }
aheinecke@5167: 
tom@6825: import_prf(){
tom@6825:     LOG_FILE=${LOG_DIR}/prf.log
tom@6825:     echo Importing cross section data.
aheinecke@5167:     echo Logging into: $LOG_FILE
tom@7728:     sed -i 's!\(log4j.appender.IMPORTER.File=\).*!\1'"$LOG_FILE"'!' \
tom@7728:         $LOG_DIR/log4j.properties
aheinecke@5167:     java -jar \
aheinecke@5167:     -Xmx$MIN_MEMORY \
aheinecke@5167:     -server \
aheinecke@5167:     -Dlog4j.configuration=file://$LOG_DIR/log4j.properties \
aheinecke@5191:     -Dflys.backend.user=$DBUSER \
aheinecke@5191:     -Dflys.backend.password=$DBPASS \
aheinecke@5167:     -Dflys.backend.url=$BACKEND_URL \
aheinecke@5167:     -Dflys.backend.driver=$BACKEND_DB_DRIVER \
aheinecke@5167:     -Dflys.backend.dialect=$BACKEND_DB_DIALECT \
aheinecke@5167:     -Dflys.backend.importer.infogew.file="$GEW_FILE" \
aheinecke@5167:     -Dflys.backend.main.value.types=$IMPORTER_MAINVALUE_TYPES \
aheinecke@5167:     -Dflys.backend.importer.annotation.types=$IMPORTER_ANNOTATION_TYPES \
aheinecke@5167:     -Dflys.backend.importer.dry.run=$IMPORTER_DRY_RUN \
aheinecke@5167:     -Dflys.backend.importer.skip.annotations=true \
aheinecke@5167:     -Dflys.backend.importer.skip.bwastr=true \
aheinecke@5167:     -Dflys.backend.importer.skip.extra.wsts=true \
aheinecke@5167:     -Dflys.backend.importer.skip.fixations=true \
aheinecke@5167:     -Dflys.backend.importer.skip.flood.water=true \
aheinecke@5167:     -Dflys.backend.importer.skip.flood.protection=true \
aheinecke@5167:     -Dflys.backend.importer.skip.gauges=true \
aheinecke@5167:     -Dflys.backend.importer.skip.historical.discharge.tables=true \
aheinecke@5167:     -Dflys.backend.importer.skip.hyks=true \
aheinecke@5167:     -Dflys.backend.importer.skip.official.lines=true \
tom@6825:     -Dflys.backend.importer.skip.prfs=false \
tom@6825:     -Dflys.backend.importer.skip.w80s=false \
tom@6825:     -Dflys.backend.importer.skip.w80.csvs=false \
tom@6825:     -Dflys.backend.importer.skip.da50s=false \
tom@6825:     -Dflys.backend.importer.skip.da66s=false \
tom@6825:     -Dflys.backend.importer.skip.wst=true \
tom@6417:     -Dflys.backend.importer.skip.measurement.stations=true \
aheinecke@5167:     -Dflys.backend.importer.skip.waterlevel.differences=true \
aheinecke@5167:     -Dflys.backend.importer.skip.waterlevels=true \
aheinecke@5167:     -Dflys.backend.importer.skip.sq.relation=true \
aheinecke@5167:     -Dflys.backend.importer.skip.sediment.density=true \
aheinecke@5167:     -Dflys.backend.importer.skip.sediment.yield=true \
aheinecke@5167:     -Dflys.backend.importer.skip.morphological.width=true \
aheinecke@5167:     -Dflys.backend.importer.skip.flow.velocity=true \
aheinecke@5167:     -Dflys.backend.importer.skip.bed.height.single=true \
aheinecke@5167:     $JAR
aheinecke@5167: }
aheinecke@5167: 
aheinecke@5167: import_geo(){
aheinecke@5167:     LOG_FILE=${LOG_DIR}/geo.log
aheinecke@5167:     echo Importing Geographic data.
aheinecke@5167:     echo Logging into: $LOG_FILE
aheinecke@5167: 
aheinecke@5169:     RIVER_PATH=$(grep "WSTDatei:" "$GEW_FILE" | awk '{print $2}')
aheinecke@5169:     RIVER_PATH=$(dirname "$RIVER_PATH")/../..
aheinecke@5169:     RIVER_PATH=$(readlink -f "$RIVER_PATH")
aheinecke@5169: 
aheinecke@5169:     exec python $(dirname $0)/geodaesie/shpimporter.py \
aheinecke@5167:     --directory $RIVER_PATH \
tom@5563:     --river_name "$RIVER_NAME" \
aheinecke@5167:     --ogr_connection "$OGR_CONNECTION" \
tom@6658:     --host $DBHOST/$BACKEND_NAME \
aheinecke@5191:     --user $DBUSER \
aheinecke@5191:     --password $DBPASS \
aheinecke@5169:     --verbose 1 \
aheinecke@5169:     > "$LOG_FILE" 2>&1
aheinecke@5167: }
aheinecke@5167: 
aheinecke@5167: 
aheinecke@5167: if [ "$SKIP_HYDRO" != "TRUE" ]; then
aheinecke@5167: import_hydro
aheinecke@5167: fi
tom@6825: if [ "$SKIP_PRF" != "TRUE" ]; then
tom@6825: import_prf
aheinecke@5167: fi
aheinecke@5167: if [ "$SKIP_MORPHO" != "TRUE" ]; then
aheinecke@5167: import_morpho
aheinecke@5167: fi
aheinecke@5167: if [ "$SKIP_GEO" != "TRUE" ]; then
aheinecke@5167: import_geo
aheinecke@5167: fi