Mercurial > dive4elements > river

comparison backend/contrib/import_river.sh @ 5838:5aa05a7a34b7

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Rename modules to more fitting names.
author Sascha L. Teichmann <teichmann@intevation.de>
date Thu, 25 Apr 2013 15:23:37 +0200
parents flys-backend/contrib/import_river.sh@673f68bb19a4
children 14b5674e57a4
comparison
equal deleted inserted replaced
5837:d9901a08d0a6 5838:5aa05a7a34b7
1 #!/bin/bash
2 # Import script for rivers
3 #
4 # Authors:
5 # Andre Heinecke <aheinecke@intevation.de>
6 #
7 # Copyright:
8 # Copyright (C) 2013 Intevation GmbH
9 #
10 # This program is free software; you can redistribute it and/or
11 # modify it under the terms of the GNU General Public License
12 # as published by the Free Software Foundation; either version 2
13 # of the License, or (at your option) any later version.
14 #
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU General Public License for more details.
19 #
20 # You should have received a copy of the GNU General Public License
21 # along with this program; if not, write to the Free Software
22 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
23
24 set -e
25
26 # Default settings
27 DEFAULT_HOST=localhost
28 DEFAULT_PORT=1521
29 DEFAULT_USER=flys_dami
30 DEFAULT_PASS=flys_dami
31 DEFAULT_LOG=$PWD/logs
32 DEFAULT_BACKEND_NAME="XE"
33 JAR="hydr_morph/importer.jar"
34 IMPORTER_DRY_RUN=false
35 IMPORTER_MAINVALUE_TYPES=QWTD
36 IMPORTER_ANNOTATION_TYPES="conf/annotation-types.xml"
37
38
39 MIN_MEMORY="8024m"
40
41 if [ -z "$OPTIONAL_LIBS" ]; then
42 OPTIONAL_LIBS="$(dirname $0)/opt"
43 fi
44
45 if [ -d "$OPTIONAL_LIBS" ]; then
46 export PATH="$OPTIONAL_LIBS/bin:$PATH"
47 export LD_LIBRARY_PATH="$OPTIONAL_LIBS/lib:$LD_LIBRARY_PATH"
48 export LD_LIBRARY_PATH="$OPTIONAL_LIBS/lib64:$LD_LIBRARY_PATH"
49 export PYTHONPATH="$OPTIONAL_LIBS/lib/python2.6/site-packages:$PYTHONPATH"
50 export PYTHONPATH="$OPTIONAL_LIBS/lib64/python2.6/site-packages:$PYTHONPATH"
51 export GDAL_DATA="$OPTIONAL_LIBS/share/gdal"
52 fi
53
54 usage(){
55 cat << EOF
56
57 usage: $0 [options] gew_file
58
59 Import a river described by the gew_file
60
61 OPTIONS:
62 -?, --help Show this message
63 -u, --username=<username> Database username. Default: $DEFAULT_USER
64 -w, --password=<password> Database password. Default: $DEFAULT_PASS
65 -h, --host=<host> Connect to database on host <host>.
66 Default: $DEFAULT_HOST
67 -p, --port=<number> Use port number <number>. Default: $DEFAULT_PORT
68 -d, --db-name=<database_name> Name of the database / backend. Default: $DEFAULT_BACKEND_NAME
69 -l, --log-dir=<directory> Directory in which to create the log files.
70 Default: $LOG_DIR
71 --postgres Database is PostgreSQL
72 --skip-hydro Skip import of hydrological data
73 --skip-morpho Skip import of morphological data
74 --skip-geo Skip import of geographic data
75 --skip-wst Skip import of wst data
76 EOF
77 exit 0
78 }
79
80 OPTS=`getopt -o ?u:w:h:p:d: \
81 -l help,username:,password:,host:,port:,db-name:,skip-hydro,skip-morpho,skip-geo,skip-wst,postgres \
82 -n $0 -- "$@"`
83 if [ $? != 0 ] ; then usage; fi
84 eval set -- "$OPTS"
85 while true ; do
86 case "$1" in
87 "-?"|"--help")
88 usage;;
89 "--")
90 shift
91 break;;
92 "-u"|"--username")
93 DBUSER=$2
94 shift 2;;
95 "-w"|"--password")
96 DBPASS=$2
97 shift 2;;
98 "-h"|"--host")
99 DBHOST=$2
100 shift 2;;
101 "-p"|"--port")
102 DBPORT=$2
103 shift 2;;
104 "-l"|"--log-dir")
105 LOG=$2
106 shift 2;;
107 "-d"|"--db-name")
108 BACKEND_NAME=$2
109 shift 2;;
110 "--skip-hydro")
111 SKIP_HYDRO="TRUE"
112 shift;;
113 "--skip-morpho")
114 SKIP_MORPHO="TRUE"
115 shift;;
116 "--skip-wst")
117 SKIP_WST="TRUE"
118 shift;;
119 "--skip-geo")
120 SKIP_GEO="TRUE"
121 shift;;
122 "--postgres")
123 POSTGRES="TRUE"
124 shift;;
125 *)
126 echo "Unknown Option $1"
127 usage;;
128 esac
129 done
130
131 if [ -z $DBUSER ]; then
132 DBUSER=$DEFAULT_USER
133 fi
134 if [ -z $DBPASS ]; then
135 DBPASS=$DEFAULT_PASS
136 fi
137 if [ -z $DBPORT ]; then
138 DBPORT=$DEFAULT_PORT
139 fi
140 if [ -z $DBHOST ]; then
141 DBHOST=$DEFAULT_HOST
142 fi
143 if [ -z $BACKEND_NAME ]; then
144 BACKEND_NAME=$DEFAULT_BACKEND_NAME
145 fi
146 if [ -z $LOGDIR ]; then
147 LOG=$DEFAULT_LOG
148 fi
149
150 if [ $# != 1 ]; then
151 usage
152 fi
153
154 if [ ! -r $1 ]; then
155 echo "Could not open $1 please ensure it exists and is readable"
156 fi
157
158 GEW_FILE="$1"
159 RIVER_NAME=$(grep "Gew.sser" "$1" | sed 's/Gew.sser: //')
160 DATE=$(date +%Y.%m.%d_%H%M)
161 LOG_DIR=${LOG}/`basename $GEW_FILE .gew`-$DATE
162 mkdir -p ${LOG_DIR}
163
164 if [ "$POSTGRES" = "TRUE" ]; then
165 JAR=$(echo "$JAR" | sed 's/importer/importer_psql/')
166 if [ ! -r "$JAR" ]; then
167 echo "Could not find Postgres importer $JAR"
168 exit 1
169 fi
170 OGR_CONNECTION="PG:dbname=$BACKEND_NAME host=$DBHOST port=$DBPORT \
171 user=$DBUSER password=$DBPASS"
172 BACKEND_DB_PREFIX="jdbc:postgresql:"
173 BACKEND_DB_DRIVER="org.postgresql.Driver"
174 BACKEND_DB_DIALECT="org.hibernate.dialect.PostgreSQLDialect"
175 else
176 BACKEND_DB_PREFIX="jdbc:oracle:thin:@"
177 BACKEND_DB_DRIVER="oracle.jdbc.OracleDriver"
178 BACKEND_DB_DIALECT="org.hibernate.dialect.OracleDialect"
179 fi
180
181 BACKEND_URL=$BACKEND_DB_PREFIX//$DBHOST:$DBPORT/$BACKEND_NAME
182
183 echo "Importing $RIVER_NAME into $BACKEND_URL."
184
185 import_hydro(){
186 LOG_FILE=${LOG_DIR}/hydro.log
187 echo Importing Hydrological data.
188 echo Logging into: $LOG_FILE
189 sed 's!./import.log!'"$LOG_FILE"'!' conf/log4j.properties > $LOG_DIR/log4j.properties
190 java -jar \
191 -Xmx$MIN_MEMORY \
192 -server \
193 -Dlog4j.configuration=file://$LOG_DIR/log4j.properties \
194 -Dflys.backend.user=$DBUSER \
195 -Dflys.backend.password=$DBPASS \
196 -Dflys.backend.url=$BACKEND_URL \
197 -Dflys.backend.driver=$BACKEND_DB_DRIVER \
198 -Dflys.backend.dialect=$BACKEND_DB_DIALECT \
199 -Dflys.backend.importer.infogew.file="$GEW_FILE" \
200 -Dflys.backend.main.value.types=$IMPORTER_MAINVALUE_TYPES \
201 -Dflys.backend.importer.annotation.types=$IMPORTER_ANNOTATION_TYPES \
202 -Dflys.backend.importer.dry.run=$IMPORTER_DRY_RUN \
203 -Dflys.backend.importer.skip.annotations=false \
204 -Dflys.backend.importer.skip.bwastr=false \
205 -Dflys.backend.importer.skip.da50s=false \
206 -Dflys.backend.importer.skip.da66s=false \
207 -Dflys.backend.importer.skip.extra.wsts=false \
208 -Dflys.backend.importer.skip.fixations=false \
209 -Dflys.backend.importer.skip.flood.water=false \
210 -Dflys.backend.importer.skip.flood.protection=false \
211 -Dflys.backend.importer.skip.gauges=false \
212 -Dflys.backend.importer.skip.historical.discharge.tables=false \
213 -Dflys.backend.importer.skip.hyks=false \
214 -Dflys.backend.importer.skip.official.lines=false \
215 -Dflys.backend.importer.skip.prfs=false \
216 -Dflys.backend.importer.skip.w80s=false \
217 -Dflys.backend.importer.skip.wst=true \
218 -Dflys.backend.importer.skip.waterlevel.differences=true \
219 -Dflys.backend.importer.skip.waterlevels=true \
220 -Dflys.backend.importer.skip.sq.relation=true \
221 -Dflys.backend.importer.skip.sediment.density=true \
222 -Dflys.backend.importer.skip.sediment.yield=true \
223 -Dflys.backend.importer.skip.morphological.width=true \
224 -Dflys.backend.importer.skip.flow.velocity=true \
225 -Dflys.backend.importer.skip.bed.height.single=true \
226 -Dflys.backend.importer.skip.bed.height.epoch=true \
227 $JAR
228 }
229
230 import_morpho(){
231 LOG_FILE=${LOG_DIR}/morpho.log
232 echo Importing Morphological data.
233 echo Logging into: $LOG_FILE
234 sed 's!./import.log!'"$LOG_FILE"'!' conf/log4j.properties > $LOG_DIR/log4j.properties
235 java -jar \
236 -Xmx$MIN_MEMORY \
237 -server \
238 -Dlog4j.configuration=file://$LOG_DIR/log4j.properties \
239 -Dflys.backend.user=$DBUSER \
240 -Dflys.backend.password=$DBPASS \
241 -Dflys.backend.url=$BACKEND_URL \
242 -Dflys.backend.driver=$BACKEND_DB_DRIVER \
243 -Dflys.backend.dialect=$BACKEND_DB_DIALECT \
244 -Dflys.backend.importer.infogew.file="$GEW_FILE" \
245 -Dflys.backend.main.value.types=$IMPORTER_MAINVALUE_TYPES \
246 -Dflys.backend.importer.annotation.types=$IMPORTER_ANNOTATION_TYPES \
247 -Dflys.backend.importer.dry.run=$IMPORTER_DRY_RUN \
248 -Dflys.backend.importer.skip.annotations=true \
249 -Dflys.backend.importer.skip.bwastr=true \
250 -Dflys.backend.importer.skip.da50s=true \
251 -Dflys.backend.importer.skip.da66s=true \
252 -Dflys.backend.importer.skip.extra.wsts=true \
253 -Dflys.backend.importer.skip.fixations=true \
254 -Dflys.backend.importer.skip.flood.water=true \
255 -Dflys.backend.importer.skip.flood.protection=true \
256 -Dflys.backend.importer.skip.gauges=true \
257 -Dflys.backend.importer.skip.historical.discharge.tables=true \
258 -Dflys.backend.importer.skip.hyks=true \
259 -Dflys.backend.importer.skip.official.lines=true \
260 -Dflys.backend.importer.skip.prfs=true \
261 -Dflys.backend.importer.skip.w80s=true \
262 -Dflys.backend.importer.skip.wst=true \
263 -Dflys.backend.importer.skip.waterlevel.differences=false \
264 -Dflys.backend.importer.skip.waterlevels=false \
265 -Dflys.backend.importer.skip.sq.relation=false \
266 -Dflys.backend.importer.skip.sediment.density=false \
267 -Dflys.backend.importer.skip.sediment.yield=false \
268 -Dflys.backend.importer.skip.morphological.width=false \
269 -Dflys.backend.importer.skip.flow.velocity=false \
270 -Dflys.backend.importer.skip.bed.height.single=false \
271 -Dflys.backend.importer.skip.bed.height.epoch=false \
272 $JAR
273 }
274
275 import_wst(){
276 LOG_FILE=${LOG_DIR}/wst.log
277 echo Importing WST data.
278 echo Logging into: $LOG_FILE
279 sed 's!./import.log!'"$LOG_FILE"'!' conf/log4j.properties > $LOG_DIR/log4j.properties
280 java -jar \
281 -Xmx$MIN_MEMORY \
282 -server \
283 -Dlog4j.configuration=file://$LOG_DIR/log4j.properties \
284 -Dflys.backend.user=$DBUSER \
285 -Dflys.backend.password=$DBPASS \
286 -Dflys.backend.url=$BACKEND_URL \
287 -Dflys.backend.driver=$BACKEND_DB_DRIVER \
288 -Dflys.backend.dialect=$BACKEND_DB_DIALECT \
289 -Dflys.backend.importer.infogew.file="$GEW_FILE" \
290 -Dflys.backend.main.value.types=$IMPORTER_MAINVALUE_TYPES \
291 -Dflys.backend.importer.annotation.types=$IMPORTER_ANNOTATION_TYPES \
292 -Dflys.backend.importer.dry.run=$IMPORTER_DRY_RUN \
293 -Dflys.backend.importer.skip.annotations=true \
294 -Dflys.backend.importer.skip.bwastr=true \
295 -Dflys.backend.importer.skip.da50s=true \
296 -Dflys.backend.importer.skip.da66s=true \
297 -Dflys.backend.importer.skip.extra.wsts=true \
298 -Dflys.backend.importer.skip.fixations=true \
299 -Dflys.backend.importer.skip.flood.water=true \
300 -Dflys.backend.importer.skip.flood.protection=true \
301 -Dflys.backend.importer.skip.gauges=true \
302 -Dflys.backend.importer.skip.historical.discharge.tables=true \
303 -Dflys.backend.importer.skip.hyks=true \
304 -Dflys.backend.importer.skip.official.lines=true \
305 -Dflys.backend.importer.skip.prfs=true \
306 -Dflys.backend.importer.skip.w80s=true \
307 -Dflys.backend.importer.skip.wst=false \
308 -Dflys.backend.importer.skip.waterlevel.differences=true \
309 -Dflys.backend.importer.skip.waterlevels=true \
310 -Dflys.backend.importer.skip.sq.relation=true \
311 -Dflys.backend.importer.skip.sediment.density=true \
312 -Dflys.backend.importer.skip.sediment.yield=true \
313 -Dflys.backend.importer.skip.morphological.width=true \
314 -Dflys.backend.importer.skip.flow.velocity=true \
315 -Dflys.backend.importer.skip.bed.height.single=true \
316 -Dflys.backend.importer.skip.bed.height.epoch=true \
317 $JAR
318 }
319
320 import_geo(){
321 LOG_FILE=${LOG_DIR}/geo.log
322 echo Importing Geographic data.
323 echo Logging into: $LOG_FILE
324
325 RIVER_PATH=$(grep "WSTDatei:" "$GEW_FILE" | awk '{print $2}')
326 RIVER_PATH=$(dirname "$RIVER_PATH")/../..
327 RIVER_PATH=$(readlink -f "$RIVER_PATH")
328
329 exec python $(dirname $0)/geodaesie/shpimporter.py \
330 --directory $RIVER_PATH \
331 --river_name "$RIVER_NAME" \
332 --ogr_connection "$OGR_CONNECTION" \
333 --host $DBHOST \
334 --user $DBUSER \
335 --password $DBPASS \
336 --verbose 1 \
337 > "$LOG_FILE" 2>&1
338 }
339
340
341 if [ "$SKIP_HYDRO" != "TRUE" ]; then
342 import_hydro
343 fi
344 if [ "$SKIP_WST" != "TRUE" ]; then
345 import_wst
346 fi
347 if [ "$SKIP_MORPHO" != "TRUE" ]; then
348 import_morpho
349 fi
350 if [ "$SKIP_GEO" != "TRUE" ]; then
351 import_geo
352 fi
http://dive4elements.wald.intevation.org