Mercurial > roundup-cc > roundup-cc
comparison collect_issues @ 0:3f139db894f1
initial commit
| author | sean |
|---|---|
| date | Thu, 02 Apr 2015 09:51:19 +0200 |
| parents | |
| children |
comparison
equal
deleted
inserted
replaced
| -1:000000000000 | 0:3f139db894f1 |
|---|---|
| 1 #!/usr/bin/env python3 | |
| 2 | |
| 3 """ Fetch issues from a roundup-tracker and save them in a databse. | |
| 4 | |
| 5 author: Sascha L. Teichmann <sascha.teichmann@intevation.de> | |
| 6 author: Bernhard Reiter <bernhard@intevation.de> | |
| 7 author: Sean Engelhardt <sean.engelhardt@intevation.de> | |
| 8 | |
| 9 (c) 2010,2015 by Intevation GmbH | |
| 10 | |
| 11 This is Free Software unter the terms of the | |
| 12 GNU GENERAL PUBLIC LICENSE Version 3 or later. | |
| 13 See http://www.gnu.org/licenses/gpl-3.0.txt for details | |
| 14 """ | |
| 15 | |
| 16 import http.cookiejar | |
| 17 import urllib.parse | |
| 18 import urllib.request | |
| 19 import csv | |
| 20 import io | |
| 21 import sqlite3 as db | |
| 22 import os | |
| 23 import roundup_content_data as rcd | |
| 24 | |
| 25 | |
| 26 # Types of URLS | |
| 27 BASE_URL = "http://10.42.7.199:8917/demo/" | |
| 28 SEARCH_URL = "issue?@action=export_csv&@columns=title,priority&@filter=status&@pagesize=50&@startwith=0&status=-1,1,2,3,4,5,6,7" | |
| 29 REPORT_URL = BASE_URL + SEARCH_URL | |
| 30 | |
| 31 | |
| 32 LOGIN_PARAMETERS = ( | |
| 33 ("__login_name", "demo"), | |
| 34 ("__login_password", "demo"), | |
| 35 ("@action", "Login"), | |
| 36 ) | |
| 37 | |
| 38 | |
| 39 # SQL-COMMANDS | |
| 40 | |
| 41 | |
| 42 def connect_to_server(): | |
| 43 enc_data = urllib.parse.urlencode(LOGIN_PARAMETERS).encode() | |
| 44 cj = http.cookiejar.CookieJar() | |
| 45 opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj)) | |
| 46 req = urllib.request.Request(url=BASE_URL, data=enc_data) | |
| 47 opener.open(req) | |
| 48 return opener | |
| 49 | |
| 50 | |
| 51 def get_issues_as_csv(opener): | |
| 52 csv_req = urllib.request.Request(url=REPORT_URL) | |
| 53 f = opener.open(csv_req) | |
| 54 | |
| 55 reader = csv.DictReader(io.TextIOWrapper(f)) | |
| 56 | |
| 57 return reader | |
| 58 | |
| 59 | |
| 60 def check_create_database(database_file): | |
| 61 if not os.path.isfile(database_file): | |
| 62 con = None | |
| 63 cur = None | |
| 64 try: | |
| 65 con = db.connect(database_file) | |
| 66 cur = con.cursor() | |
| 67 try: | |
| 68 cur.execute(rcd.CREATE_DB) | |
| 69 con.commit() | |
| 70 os.chmod(rcd.DATABASE_FILE, 0o744) | |
| 71 except: | |
| 72 con.rollback() | |
| 73 raise | |
| 74 finally: | |
| 75 if cur: | |
| 76 cur.close() | |
| 77 if con: | |
| 78 con.close() | |
| 79 | |
| 80 | |
| 81 def issues_to_quantities(rows): | |
| 82 quantities = [0] * len(rcd.COLUMNS) | |
| 83 for row in rows: | |
| 84 quantities[int(row["priority"]) - 1] += 1 | |
| 85 return quantities | |
| 86 | |
| 87 | |
| 88 def save_issues_to_db(quantities, database_file): | |
| 89 check_create_database(database_file) | |
| 90 | |
| 91 cur = None | |
| 92 con = None | |
| 93 | |
| 94 try: | |
| 95 con = db.connect(database_file) | |
| 96 cur = con.cursor() | |
| 97 try: | |
| 98 cur.execute(rcd.INSERT_NEW, quantities) | |
| 99 con.commit() | |
| 100 except: | |
| 101 con.rollback() | |
| 102 raise | |
| 103 finally: | |
| 104 if cur: | |
| 105 cur.close() | |
| 106 if con: | |
| 107 con.close() | |
| 108 | |
| 109 | |
| 110 def save_stats_in_db(): | |
| 111 try: | |
| 112 opener = connect_to_server() | |
| 113 current_issues_csv = get_issues_as_csv(opener) | |
| 114 opener.close() | |
| 115 | |
| 116 quantities = issues_to_quantities(current_issues_csv) | |
| 117 | |
| 118 save_issues_to_db(quantities, rcd.DATABASE_FILE) | |
| 119 except urllib.error.URLError: | |
| 120 print("No Valid Connection to server : " + BASE_URL) | |
| 121 | |
| 122 | |
| 123 save_stats_in_db() |