Mercurial > dive4elements > river
annotate flys-backend/contrib/shpimporter/utils.py @ 4974:a92241d40228 dami
Handle transformation errors
| author | Andre Heinecke <aheinecke@intevation.de> |
|---|---|
| date | Wed, 13 Feb 2013 17:45:35 +0100 |
| parents | 174fbaa3d344 |
| children | a6ee62a070b0 |
| rev | line source |
|---|---|
|
2798
5a654f2e35bc
Added a python tool to import shapefiles into database.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff
changeset
|
1 import os |
|
4874
b1d7e600b43b
(importer) Add utility function to convert paths to utf-8
Andre Heinecke <aheinecke@intevation.de>
parents:
3654
diff
changeset
|
2 import sys |
|
3654
59ca5dab2782
Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
2798
diff
changeset
|
3 from shpimporter import DEBUG, INFO, ERROR |
|
4970
174fbaa3d344
Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents:
4935
diff
changeset
|
4 try: |
|
174fbaa3d344
Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents:
4935
diff
changeset
|
5 from osgeo import ogr |
|
174fbaa3d344
Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents:
4935
diff
changeset
|
6 except ImportErrror: |
|
174fbaa3d344
Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents:
4935
diff
changeset
|
7 import ogr |
|
2798
5a654f2e35bc
Added a python tool to import shapefiles into database.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff
changeset
|
8 |
|
5a654f2e35bc
Added a python tool to import shapefiles into database.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff
changeset
|
9 SHP='.shp' |
|
4970
174fbaa3d344
Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents:
4935
diff
changeset
|
10 SQL_SELECT_ID="SELECT id FROM %s WHERE %s = '%s'" |
|
2798
5a654f2e35bc
Added a python tool to import shapefiles into database.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff
changeset
|
11 |
|
5a654f2e35bc
Added a python tool to import shapefiles into database.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff
changeset
|
12 def findShapefiles(path): |
|
5a654f2e35bc
Added a python tool to import shapefiles into database.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff
changeset
|
13 shapes = [] |
|
5a654f2e35bc
Added a python tool to import shapefiles into database.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff
changeset
|
14 |
|
5a654f2e35bc
Added a python tool to import shapefiles into database.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff
changeset
|
15 for root, dirs, files in os.walk(path): |
|
5a654f2e35bc
Added a python tool to import shapefiles into database.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff
changeset
|
16 if len(files) == 0: |
|
5a654f2e35bc
Added a python tool to import shapefiles into database.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff
changeset
|
17 continue |
|
5a654f2e35bc
Added a python tool to import shapefiles into database.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff
changeset
|
18 |
|
3654
59ca5dab2782
Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
2798
diff
changeset
|
19 DEBUG("Processing directory '%s' with %i files " % (root, len(files))) |
|
2798
5a654f2e35bc
Added a python tool to import shapefiles into database.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff
changeset
|
20 |
|
5a654f2e35bc
Added a python tool to import shapefiles into database.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff
changeset
|
21 for f in files: |
|
5a654f2e35bc
Added a python tool to import shapefiles into database.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff
changeset
|
22 idx = f.find(SHP) |
|
5a654f2e35bc
Added a python tool to import shapefiles into database.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff
changeset
|
23 if (idx+len(SHP)) == len(f): |
|
5a654f2e35bc
Added a python tool to import shapefiles into database.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff
changeset
|
24 shapes.append((f.replace(SHP, ''), root + "/" + f)) |
|
5a654f2e35bc
Added a python tool to import shapefiles into database.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff
changeset
|
25 |
|
5a654f2e35bc
Added a python tool to import shapefiles into database.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff
changeset
|
26 return shapes |
|
5a654f2e35bc
Added a python tool to import shapefiles into database.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff
changeset
|
27 |
|
4970
174fbaa3d344
Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents:
4935
diff
changeset
|
28 def getRiverId(dbconn, name): |
|
174fbaa3d344
Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents:
4935
diff
changeset
|
29 """ |
|
174fbaa3d344
Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents:
4935
diff
changeset
|
30 Returns the id of the river "name" |
|
174fbaa3d344
Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents:
4935
diff
changeset
|
31 Dbconn must be a python database connection api compliant object |
|
174fbaa3d344
Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents:
4935
diff
changeset
|
32 """ |
|
174fbaa3d344
Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents:
4935
diff
changeset
|
33 cur = dbconn.cursor() |
|
174fbaa3d344
Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents:
4935
diff
changeset
|
34 select_stmt = SQL_SELECT_ID % ("rivers", "name", name) |
|
174fbaa3d344
Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents:
4935
diff
changeset
|
35 cur.execute(select_stmt) |
|
174fbaa3d344
Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents:
4935
diff
changeset
|
36 row = cur.fetchone() |
|
174fbaa3d344
Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents:
4935
diff
changeset
|
37 if row: |
|
174fbaa3d344
Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents:
4935
diff
changeset
|
38 return row[0] |
|
174fbaa3d344
Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents:
4935
diff
changeset
|
39 else: |
|
174fbaa3d344
Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents:
4935
diff
changeset
|
40 return 0 |
|
174fbaa3d344
Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents:
4935
diff
changeset
|
41 |
|
4935
c0a58558b817
Importer: - Handle regular expressions for attribute names
Andre Heinecke <aheinecke@intevation.de>
parents:
4887
diff
changeset
|
42 def getUTF8(string): |
|
c0a58558b817
Importer: - Handle regular expressions for attribute names
Andre Heinecke <aheinecke@intevation.de>
parents:
4887
diff
changeset
|
43 """ |
|
c0a58558b817
Importer: - Handle regular expressions for attribute names
Andre Heinecke <aheinecke@intevation.de>
parents:
4887
diff
changeset
|
44 Tries to convert the string to a UTF-8 encoding by first checking if it |
|
c0a58558b817
Importer: - Handle regular expressions for attribute names
Andre Heinecke <aheinecke@intevation.de>
parents:
4887
diff
changeset
|
45 is UTF-8 and then trying cp1252 |
|
c0a58558b817
Importer: - Handle regular expressions for attribute names
Andre Heinecke <aheinecke@intevation.de>
parents:
4887
diff
changeset
|
46 """ |
|
c0a58558b817
Importer: - Handle regular expressions for attribute names
Andre Heinecke <aheinecke@intevation.de>
parents:
4887
diff
changeset
|
47 try: |
|
c0a58558b817
Importer: - Handle regular expressions for attribute names
Andre Heinecke <aheinecke@intevation.de>
parents:
4887
diff
changeset
|
48 return unicode.encode(unicode(string, "UTF-8"), "UTF-8") |
|
c0a58558b817
Importer: - Handle regular expressions for attribute names
Andre Heinecke <aheinecke@intevation.de>
parents:
4887
diff
changeset
|
49 except UnicodeDecodeError: |
|
c0a58558b817
Importer: - Handle regular expressions for attribute names
Andre Heinecke <aheinecke@intevation.de>
parents:
4887
diff
changeset
|
50 # Probably European Windows names so lets try again |
|
c0a58558b817
Importer: - Handle regular expressions for attribute names
Andre Heinecke <aheinecke@intevation.de>
parents:
4887
diff
changeset
|
51 return unicode.encode(unicode(string, "cp1252"), "UTF-8") |
|
c0a58558b817
Importer: - Handle regular expressions for attribute names
Andre Heinecke <aheinecke@intevation.de>
parents:
4887
diff
changeset
|
52 |
|
4874
b1d7e600b43b
(importer) Add utility function to convert paths to utf-8
Andre Heinecke <aheinecke@intevation.de>
parents:
3654
diff
changeset
|
53 def getUTF8Path(path): |
|
b1d7e600b43b
(importer) Add utility function to convert paths to utf-8
Andre Heinecke <aheinecke@intevation.de>
parents:
3654
diff
changeset
|
54 """ |
|
b1d7e600b43b
(importer) Add utility function to convert paths to utf-8
Andre Heinecke <aheinecke@intevation.de>
parents:
3654
diff
changeset
|
55 Tries to convert path to utf-8 by first checking the filesystemencoding |
|
b1d7e600b43b
(importer) Add utility function to convert paths to utf-8
Andre Heinecke <aheinecke@intevation.de>
parents:
3654
diff
changeset
|
56 and trying the default windows encoding afterwards. |
|
b1d7e600b43b
(importer) Add utility function to convert paths to utf-8
Andre Heinecke <aheinecke@intevation.de>
parents:
3654
diff
changeset
|
57 Returns a valid UTF-8 encoded unicode object or throws a UnicodeDecodeError |
|
b1d7e600b43b
(importer) Add utility function to convert paths to utf-8
Andre Heinecke <aheinecke@intevation.de>
parents:
3654
diff
changeset
|
58 """ |
|
b1d7e600b43b
(importer) Add utility function to convert paths to utf-8
Andre Heinecke <aheinecke@intevation.de>
parents:
3654
diff
changeset
|
59 try: |
|
b1d7e600b43b
(importer) Add utility function to convert paths to utf-8
Andre Heinecke <aheinecke@intevation.de>
parents:
3654
diff
changeset
|
60 return unicode.encode(unicode(path, sys.getfilesystemencoding()), "UTF-8") |
|
b1d7e600b43b
(importer) Add utility function to convert paths to utf-8
Andre Heinecke <aheinecke@intevation.de>
parents:
3654
diff
changeset
|
61 except UnicodeDecodeError: |
|
4887
1f6e544f7a7f
Importer: Use cp1252 instead of latin-9 to guess filename encodings
Andre Heinecke <aheinecke@intevation.de>
parents:
4884
diff
changeset
|
62 # Probably European Windows names so lets try again |
|
1f6e544f7a7f
Importer: Use cp1252 instead of latin-9 to guess filename encodings
Andre Heinecke <aheinecke@intevation.de>
parents:
4884
diff
changeset
|
63 return unicode.encode(unicode(path, "cp1252"), "UTF-8") |