view flys-backend/contrib/run_hydr_morph.sh @ 5779:ebec12def170

Datacage: Add a pool of builders to make it multi threadable. XML DOM is not thread safe. Therefore the old implementation only allowed one thread to use the builder at a time. As the complexity of the configuration has increased over time this has become a bottleneck of the whole application because it took quiet some time to build a result. Furthermore the builder code path is visited very frequent. So many concurrent requests were piled up resulting in long waits for the users. To mitigate this problem a round robin pool of builders is used now. Each of the pooled builders has an independent copy of the XML template and can be run in parallel. The number of builders is determined by the system property 'flys.datacage.pool.size'. It defaults to 4.
author Sascha L. Teichmann <teichmann@intevation.de>
date Sun, 21 Apr 2013 12:48:09 +0200
parents 4f65d833680f
children b9ae676a9afe
line wrap: on
line source
#!/bin/bash

######################### CONFIG OPTIONS ############################
INFO_GEW="/vol1/projects/Geospatial/flys-3.0/testdaten/saar.gew"
BACKEND_USER="flys28"
BACKEND_PASS="flys28"
BACKEND_HOST="czech-republic.atlas.intevation.de"
BACKEND_PORT="1521"
BACKEND_NAME="XE"
LOG4J_CONFIG="conf/log4j.properties"
JAR="hydr_morph/importer.jar"
#####################################################################


########################## Oracle Settings ##########################
BACKEND_DB_PREFIX="jdbc:oracle:thin:@"
BACKEND_DB_DRIVER="oracle.jdbc.OracleDriver"
BACKEND_DB_DIALECT="org.hibernate.dialect.OracleDialect"
BACKEND_URL=$BACKEND_DB_PREFIX//$BACKEND_HOST:$BACKEND_PORT/$BACKEND_NAME
#####################################################################


######################## Custom Importer Settings ###################
IMPORTER_DRY_RUN=false
IMPORTER_MAINVALUE_TYPES=QWTD
IMPORTER_ANNOTATION_TYPES="conf/annotation-types.xml"

IMPORTER_SKIP_ANNOTATIONS=false
IMPORTER_SKIP_BWASTR=false
IMPORTER_SKIP_DA50S=false
IMPORTER_SKIP_DA66S=false
IMPORTER_SKIP_EXTRA_WST=false
IMPORTER_SKIP_FIXATIONS=false
IMPORTER_SKIP_FLOOD_WATER=false
IMPORTER_SKIP_FLOOD_PROTECTION=false
IMPORTER_SKIP_GAUGES=false
IMPORTER_SKIP_HISTORICAL_DISCHARGE_GAUGES=false
IMPORTER_SKIP_HYKS=false
IMPORTER_SKIP_OFFICIAL_LINES=false
IMPORTER_SKIP_PRFS=false
IMPORTER_SKIP_W80S=false
IMPORTER_SKIP_WST=false

IMPORTER_SKIP_BED_HEIGHT_SINGLE=false
IMPORTER_SKIP_BED_HEIGHT_EPOCH=false
IMPORTER_SKIP_FLOW_VELOCITY=false
IMPORTER_SKIP_MORPHOLOGICAL_WIDTH=false
IMPORTER_SKIP_SEDIMENT_DENSITY=false
IMPORTER_SKIP_SEDIMENT_YIELD=false
IMPORTER_SKIP_SQ_RELATION=false
IMPORTER_SKIP_WATERLEVELS=false
IMPORTER_SKIP_WATERLEVEL_DIFFERENCES=false
#####################################################################

#MIN_MEMORY="8192m"
MIN_MEMORY="1024m"


######################### Run Importer ##############################
OPTIONAL_LIBS="${DIR}"/../opt
if [ -d "$OPTIONAL_LIBS" ]; then
    export PATH="$OPTIONAL_LIBS/bin:$PATH"
    export LD_LIBRARY_PATH="$OPTIONAL_LIBS/lib:$LD_LIBRARY_PATH"
    export LD_LIBRARY_PATH="$OPTIONAL_LIBS/lib64:$LD_LIBRARY_PATH"
fi

export LC_ALL=de_DE@euro # Workaround encoding problem

exec java -jar \
    -Xmx$MIN_MEMORY \
    -server \
    -Dlog4j.configuration=file://`readlink -f $LOG4J_CONFIG` \
    -Dflys.backend.importer.infogew.file=$INFO_GEW \
    -Dflys.backend.main.value.types=$IMPORTER_MAINVALUE_TYPES \
    -Dflys.backend.importer.annotation.types=$IMPORTER_ANNOTATION_TYPES \
    -Dflys.backend.importer.dry.run=$IMPORTER_DRY_RUN \
    -Dflys.backend.importer.skip.annotations=$IMPORTER_SKIP_ANNOTATIONS \
    -Dflys.backend.importer.skip.bed.height.single=$IMPORTER_SKIP_BED_HEIGHT_SINGLE \
    -Dflys.backend.importer.skip.bed.height.epoch=$IMPORTER_SKIP_BED_HEIGHT_EPOCH \
    -Dflys.backend.importer.skip.bwastr=$IMPORTER_SKIP_BWASTR \
    -Dflys.backend.importer.skip.da50s=$IMPORTER_SKIP_DA50S \
    -Dflys.backend.importer.skip.da66s=$IMPORTER_SKIP_DA66S \
    -Dflys.backend.importer.skip.extra.wsts=$IMPORTER_SKIP_EXTRA_WST \
    -Dflys.backend.importer.skip.fixations=$IMPORTER_SKIP_FIXATIONS \
    -Dflys.backend.importer.skip.flood.water=$IMPORTER_SKIP_FLOOD_WATER \
    -Dflys.backend.importer.skip.flood.protection=$IMPORTER_SKIP_FLOOD_PROTECTION \
    -Dflys.backend.importer.skip.flow.velocity=$IMPORTER_SKIP_FLOW_VELOCITY \
    -Dflys.backend.importer.skip.gauges=$IMPORTER_SKIP_GAUGES \
    -Dflys.backend.importer.skip.historical.discharge.tables=$IMPORTER_SKIP_HISTORICAL_DISCHARGE_GAUGES \
    -Dflys.backend.importer.skip.hyks=$IMPORTER_SKIP_HYKS \
    -Dflys.backend.importer.skip.morphological.width=$IMPORTER_SKIP_MORPHOLOGICAL_WIDTH \
    -Dflys.backend.importer.skip.official.lines=$IMPORTER_SKIP_OFFICIAL_LINES \
    -Dflys.backend.importer.skip.prfs=$IMPORTER_SKIP_PRFS \
    -Dflys.backend.importer.skip.sediment.density=$IMPORTER_SKIP_SEDIMENT_DENSITY \
    -Dflys.backend.importer.skip.sediment.yield=$IMPORTER_SKIP_SEDIMENT_YIELD \
    -Dflys.backend.importer.skip.sq.relation=$IMPORTER_SKIP_SQ_RELATION \
    -Dflys.backend.importer.skip.w80s=$IMPORTER_SKIP_W80S \
    -Dflys.backend.importer.skip.waterlevels=$IMPORTER_SKIP_WATERLEVELS \
    -Dflys.backend.importer.skip.waterlevel.differences=$IMPORTER_SKIP_WATERLEVEL_DIFFERENCES \
    -Dflys.backend.importer.skip.wst=$IMPORTER_SKIP_WST \
    -Dflys.backend.user=$BACKEND_USER \
    -Dflys.backend.password=$BACKEND_PASS \
    -Dflys.backend.url=$BACKEND_URL \
    -Dflys.backend.driver=$BACKEND_DB_DRIVER \
    -Dflys.backend.dialect=$BACKEND_DB_DIALECT \
     $JAR

http://dive4elements.wald.intevation.org