|
0
|
1 #!/usr/bin/env python3 |
|
|
2 |
|
|
3 """ Fetch issues from a roundup-tracker and save them in a databse. |
|
|
4 |
|
|
5 author: Sascha L. Teichmann <sascha.teichmann@intevation.de> |
|
|
6 author: Bernhard Reiter <bernhard@intevation.de> |
|
|
7 author: Sean Engelhardt <sean.engelhardt@intevation.de> |
|
|
8 |
|
|
9 (c) 2010,2015 by Intevation GmbH |
|
|
10 |
|
|
11 This is Free Software unter the terms of the |
|
|
12 GNU GENERAL PUBLIC LICENSE Version 3 or later. |
|
|
13 See http://www.gnu.org/licenses/gpl-3.0.txt for details |
|
|
14 """ |
|
|
15 |
|
|
16 import http.cookiejar |
|
|
17 import urllib.parse |
|
|
18 import urllib.request |
|
|
19 import csv |
|
|
20 import io |
|
|
21 import sqlite3 as db |
|
|
22 import os |
|
|
23 import roundup_content_data as rcd |
|
|
24 |
|
|
25 |
|
|
26 # Types of URLS |
|
|
27 BASE_URL = "http://10.42.7.199:8917/demo/" |
|
|
28 SEARCH_URL = "issue?@action=export_csv&@columns=title,priority&@filter=status&@pagesize=50&@startwith=0&status=-1,1,2,3,4,5,6,7" |
|
|
29 REPORT_URL = BASE_URL + SEARCH_URL |
|
|
30 |
|
|
31 |
|
|
32 LOGIN_PARAMETERS = ( |
|
|
33 ("__login_name", "demo"), |
|
|
34 ("__login_password", "demo"), |
|
|
35 ("@action", "Login"), |
|
|
36 ) |
|
|
37 |
|
|
38 |
|
|
39 # SQL-COMMANDS |
|
|
40 |
|
|
41 |
|
|
42 def connect_to_server(): |
|
|
43 enc_data = urllib.parse.urlencode(LOGIN_PARAMETERS).encode() |
|
|
44 cj = http.cookiejar.CookieJar() |
|
|
45 opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj)) |
|
|
46 req = urllib.request.Request(url=BASE_URL, data=enc_data) |
|
|
47 opener.open(req) |
|
|
48 return opener |
|
|
49 |
|
|
50 |
|
|
51 def get_issues_as_csv(opener): |
|
|
52 csv_req = urllib.request.Request(url=REPORT_URL) |
|
|
53 f = opener.open(csv_req) |
|
|
54 |
|
|
55 reader = csv.DictReader(io.TextIOWrapper(f)) |
|
|
56 |
|
|
57 return reader |
|
|
58 |
|
|
59 |
|
|
60 def check_create_database(database_file): |
|
|
61 if not os.path.isfile(database_file): |
|
|
62 con = None |
|
|
63 cur = None |
|
|
64 try: |
|
|
65 con = db.connect(database_file) |
|
|
66 cur = con.cursor() |
|
|
67 try: |
|
|
68 cur.execute(rcd.CREATE_DB) |
|
|
69 con.commit() |
|
|
70 os.chmod(rcd.DATABASE_FILE, 0o744) |
|
|
71 except: |
|
|
72 con.rollback() |
|
|
73 raise |
|
|
74 finally: |
|
|
75 if cur: |
|
|
76 cur.close() |
|
|
77 if con: |
|
|
78 con.close() |
|
|
79 |
|
|
80 |
|
|
81 def issues_to_quantities(rows): |
|
|
82 quantities = [0] * len(rcd.COLUMNS) |
|
|
83 for row in rows: |
|
|
84 quantities[int(row["priority"]) - 1] += 1 |
|
|
85 return quantities |
|
|
86 |
|
|
87 |
|
|
88 def save_issues_to_db(quantities, database_file): |
|
|
89 check_create_database(database_file) |
|
|
90 |
|
|
91 cur = None |
|
|
92 con = None |
|
|
93 |
|
|
94 try: |
|
|
95 con = db.connect(database_file) |
|
|
96 cur = con.cursor() |
|
|
97 try: |
|
|
98 cur.execute(rcd.INSERT_NEW, quantities) |
|
|
99 con.commit() |
|
|
100 except: |
|
|
101 con.rollback() |
|
|
102 raise |
|
|
103 finally: |
|
|
104 if cur: |
|
|
105 cur.close() |
|
|
106 if con: |
|
|
107 con.close() |
|
|
108 |
|
|
109 |
|
|
110 def save_stats_in_db(): |
|
|
111 try: |
|
|
112 opener = connect_to_server() |
|
|
113 current_issues_csv = get_issues_as_csv(opener) |
|
|
114 opener.close() |
|
|
115 |
|
|
116 quantities = issues_to_quantities(current_issues_csv) |
|
|
117 |
|
|
118 save_issues_to_db(quantities, rcd.DATABASE_FILE) |
|
|
119 except urllib.error.URLError: |
|
|
120 print("No Valid Connection to server : " + BASE_URL) |
|
|
121 |
|
|
122 |
|
|
123 save_stats_in_db() |
