annotate collect_issues @ 3:99c68ebfb3b9

Streamlined to example use. * removed superfluous file * added example file for usage as python module
author Bernhard Reiter <bernhard@intevation.de>
date Mon, 30 Nov 2015 17:46:22 +0100
parents 3f139db894f1
children
rev   line source
0
3f139db894f1 initial commit
sean
parents:
diff changeset
1 #!/usr/bin/env python3
3f139db894f1 initial commit
sean
parents:
diff changeset
2
3f139db894f1 initial commit
sean
parents:
diff changeset
3 """ Fetch issues from a roundup-tracker and save them in a databse.
3f139db894f1 initial commit
sean
parents:
diff changeset
4
3f139db894f1 initial commit
sean
parents:
diff changeset
5 author: Sascha L. Teichmann <sascha.teichmann@intevation.de>
3f139db894f1 initial commit
sean
parents:
diff changeset
6 author: Bernhard Reiter <bernhard@intevation.de>
3f139db894f1 initial commit
sean
parents:
diff changeset
7 author: Sean Engelhardt <sean.engelhardt@intevation.de>
3f139db894f1 initial commit
sean
parents:
diff changeset
8
3f139db894f1 initial commit
sean
parents:
diff changeset
9 (c) 2010,2015 by Intevation GmbH
3f139db894f1 initial commit
sean
parents:
diff changeset
10
3f139db894f1 initial commit
sean
parents:
diff changeset
11 This is Free Software unter the terms of the
3f139db894f1 initial commit
sean
parents:
diff changeset
12 GNU GENERAL PUBLIC LICENSE Version 3 or later.
3f139db894f1 initial commit
sean
parents:
diff changeset
13 See http://www.gnu.org/licenses/gpl-3.0.txt for details
3f139db894f1 initial commit
sean
parents:
diff changeset
14 """
3f139db894f1 initial commit
sean
parents:
diff changeset
15
3f139db894f1 initial commit
sean
parents:
diff changeset
16 import http.cookiejar
3f139db894f1 initial commit
sean
parents:
diff changeset
17 import urllib.parse
3f139db894f1 initial commit
sean
parents:
diff changeset
18 import urllib.request
3f139db894f1 initial commit
sean
parents:
diff changeset
19 import csv
3f139db894f1 initial commit
sean
parents:
diff changeset
20 import io
3f139db894f1 initial commit
sean
parents:
diff changeset
21 import sqlite3 as db
3f139db894f1 initial commit
sean
parents:
diff changeset
22 import os
3f139db894f1 initial commit
sean
parents:
diff changeset
23 import roundup_content_data as rcd
3f139db894f1 initial commit
sean
parents:
diff changeset
24
3f139db894f1 initial commit
sean
parents:
diff changeset
25
3f139db894f1 initial commit
sean
parents:
diff changeset
26 # Types of URLS
3f139db894f1 initial commit
sean
parents:
diff changeset
27 BASE_URL = "http://10.42.7.199:8917/demo/"
3f139db894f1 initial commit
sean
parents:
diff changeset
28 SEARCH_URL = "issue?@action=export_csv&@columns=title,priority&@filter=status&@pagesize=50&@startwith=0&status=-1,1,2,3,4,5,6,7"
3f139db894f1 initial commit
sean
parents:
diff changeset
29 REPORT_URL = BASE_URL + SEARCH_URL
3f139db894f1 initial commit
sean
parents:
diff changeset
30
3f139db894f1 initial commit
sean
parents:
diff changeset
31
3f139db894f1 initial commit
sean
parents:
diff changeset
32 LOGIN_PARAMETERS = (
3f139db894f1 initial commit
sean
parents:
diff changeset
33 ("__login_name", "demo"),
3f139db894f1 initial commit
sean
parents:
diff changeset
34 ("__login_password", "demo"),
3f139db894f1 initial commit
sean
parents:
diff changeset
35 ("@action", "Login"),
3f139db894f1 initial commit
sean
parents:
diff changeset
36 )
3f139db894f1 initial commit
sean
parents:
diff changeset
37
3f139db894f1 initial commit
sean
parents:
diff changeset
38
3f139db894f1 initial commit
sean
parents:
diff changeset
39 # SQL-COMMANDS
3f139db894f1 initial commit
sean
parents:
diff changeset
40
3f139db894f1 initial commit
sean
parents:
diff changeset
41
3f139db894f1 initial commit
sean
parents:
diff changeset
42 def connect_to_server():
3f139db894f1 initial commit
sean
parents:
diff changeset
43 enc_data = urllib.parse.urlencode(LOGIN_PARAMETERS).encode()
3f139db894f1 initial commit
sean
parents:
diff changeset
44 cj = http.cookiejar.CookieJar()
3f139db894f1 initial commit
sean
parents:
diff changeset
45 opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj))
3f139db894f1 initial commit
sean
parents:
diff changeset
46 req = urllib.request.Request(url=BASE_URL, data=enc_data)
3f139db894f1 initial commit
sean
parents:
diff changeset
47 opener.open(req)
3f139db894f1 initial commit
sean
parents:
diff changeset
48 return opener
3f139db894f1 initial commit
sean
parents:
diff changeset
49
3f139db894f1 initial commit
sean
parents:
diff changeset
50
3f139db894f1 initial commit
sean
parents:
diff changeset
51 def get_issues_as_csv(opener):
3f139db894f1 initial commit
sean
parents:
diff changeset
52 csv_req = urllib.request.Request(url=REPORT_URL)
3f139db894f1 initial commit
sean
parents:
diff changeset
53 f = opener.open(csv_req)
3f139db894f1 initial commit
sean
parents:
diff changeset
54
3f139db894f1 initial commit
sean
parents:
diff changeset
55 reader = csv.DictReader(io.TextIOWrapper(f))
3f139db894f1 initial commit
sean
parents:
diff changeset
56
3f139db894f1 initial commit
sean
parents:
diff changeset
57 return reader
3f139db894f1 initial commit
sean
parents:
diff changeset
58
3f139db894f1 initial commit
sean
parents:
diff changeset
59
3f139db894f1 initial commit
sean
parents:
diff changeset
60 def check_create_database(database_file):
3f139db894f1 initial commit
sean
parents:
diff changeset
61 if not os.path.isfile(database_file):
3f139db894f1 initial commit
sean
parents:
diff changeset
62 con = None
3f139db894f1 initial commit
sean
parents:
diff changeset
63 cur = None
3f139db894f1 initial commit
sean
parents:
diff changeset
64 try:
3f139db894f1 initial commit
sean
parents:
diff changeset
65 con = db.connect(database_file)
3f139db894f1 initial commit
sean
parents:
diff changeset
66 cur = con.cursor()
3f139db894f1 initial commit
sean
parents:
diff changeset
67 try:
3f139db894f1 initial commit
sean
parents:
diff changeset
68 cur.execute(rcd.CREATE_DB)
3f139db894f1 initial commit
sean
parents:
diff changeset
69 con.commit()
3f139db894f1 initial commit
sean
parents:
diff changeset
70 os.chmod(rcd.DATABASE_FILE, 0o744)
3f139db894f1 initial commit
sean
parents:
diff changeset
71 except:
3f139db894f1 initial commit
sean
parents:
diff changeset
72 con.rollback()
3f139db894f1 initial commit
sean
parents:
diff changeset
73 raise
3f139db894f1 initial commit
sean
parents:
diff changeset
74 finally:
3f139db894f1 initial commit
sean
parents:
diff changeset
75 if cur:
3f139db894f1 initial commit
sean
parents:
diff changeset
76 cur.close()
3f139db894f1 initial commit
sean
parents:
diff changeset
77 if con:
3f139db894f1 initial commit
sean
parents:
diff changeset
78 con.close()
3f139db894f1 initial commit
sean
parents:
diff changeset
79
3f139db894f1 initial commit
sean
parents:
diff changeset
80
3f139db894f1 initial commit
sean
parents:
diff changeset
81 def issues_to_quantities(rows):
3f139db894f1 initial commit
sean
parents:
diff changeset
82 quantities = [0] * len(rcd.COLUMNS)
3f139db894f1 initial commit
sean
parents:
diff changeset
83 for row in rows:
3f139db894f1 initial commit
sean
parents:
diff changeset
84 quantities[int(row["priority"]) - 1] += 1
3f139db894f1 initial commit
sean
parents:
diff changeset
85 return quantities
3f139db894f1 initial commit
sean
parents:
diff changeset
86
3f139db894f1 initial commit
sean
parents:
diff changeset
87
3f139db894f1 initial commit
sean
parents:
diff changeset
88 def save_issues_to_db(quantities, database_file):
3f139db894f1 initial commit
sean
parents:
diff changeset
89 check_create_database(database_file)
3f139db894f1 initial commit
sean
parents:
diff changeset
90
3f139db894f1 initial commit
sean
parents:
diff changeset
91 cur = None
3f139db894f1 initial commit
sean
parents:
diff changeset
92 con = None
3f139db894f1 initial commit
sean
parents:
diff changeset
93
3f139db894f1 initial commit
sean
parents:
diff changeset
94 try:
3f139db894f1 initial commit
sean
parents:
diff changeset
95 con = db.connect(database_file)
3f139db894f1 initial commit
sean
parents:
diff changeset
96 cur = con.cursor()
3f139db894f1 initial commit
sean
parents:
diff changeset
97 try:
3f139db894f1 initial commit
sean
parents:
diff changeset
98 cur.execute(rcd.INSERT_NEW, quantities)
3f139db894f1 initial commit
sean
parents:
diff changeset
99 con.commit()
3f139db894f1 initial commit
sean
parents:
diff changeset
100 except:
3f139db894f1 initial commit
sean
parents:
diff changeset
101 con.rollback()
3f139db894f1 initial commit
sean
parents:
diff changeset
102 raise
3f139db894f1 initial commit
sean
parents:
diff changeset
103 finally:
3f139db894f1 initial commit
sean
parents:
diff changeset
104 if cur:
3f139db894f1 initial commit
sean
parents:
diff changeset
105 cur.close()
3f139db894f1 initial commit
sean
parents:
diff changeset
106 if con:
3f139db894f1 initial commit
sean
parents:
diff changeset
107 con.close()
3f139db894f1 initial commit
sean
parents:
diff changeset
108
3f139db894f1 initial commit
sean
parents:
diff changeset
109
3f139db894f1 initial commit
sean
parents:
diff changeset
110 def save_stats_in_db():
3f139db894f1 initial commit
sean
parents:
diff changeset
111 try:
3f139db894f1 initial commit
sean
parents:
diff changeset
112 opener = connect_to_server()
3f139db894f1 initial commit
sean
parents:
diff changeset
113 current_issues_csv = get_issues_as_csv(opener)
3f139db894f1 initial commit
sean
parents:
diff changeset
114 opener.close()
3f139db894f1 initial commit
sean
parents:
diff changeset
115
3f139db894f1 initial commit
sean
parents:
diff changeset
116 quantities = issues_to_quantities(current_issues_csv)
3f139db894f1 initial commit
sean
parents:
diff changeset
117
3f139db894f1 initial commit
sean
parents:
diff changeset
118 save_issues_to_db(quantities, rcd.DATABASE_FILE)
3f139db894f1 initial commit
sean
parents:
diff changeset
119 except urllib.error.URLError:
3f139db894f1 initial commit
sean
parents:
diff changeset
120 print("No Valid Connection to server : " + BASE_URL)
3f139db894f1 initial commit
sean
parents:
diff changeset
121
3f139db894f1 initial commit
sean
parents:
diff changeset
122
3f139db894f1 initial commit
sean
parents:
diff changeset
123 save_stats_in_db()
This site is hosted by Intevation GmbH (Datenschutzerklärung und Impressum | Privacy Policy and Imprint)