sean@1: #!/usr/bin/env python3 sean@1: """ Fetch issues from a roundup-tracker and save them in a databse. sean@1: sean@1: author: Sascha L. Teichmann sean@1: author: Bernhard Reiter sean@1: author: Sean Engelhardt sean@1: bernhard@15: (c) 2010, 2015, 2018 by Intevation GmbH sean@1: sean@1: This is Free Software unter the terms of the sean@1: GNU GENERAL PUBLIC LICENSE Version 3 or later. sean@1: See http://www.gnu.org/licenses/gpl-3.0.txt for details sean@1: bernhard@17: For usage see examples/. sean@1: """ sean@1: sean@1: import http.cookiejar sean@1: import urllib.parse sean@1: import urllib.request sean@1: import csv sean@1: import io sean@1: import sqlite3 as db sean@1: import os sean@1: sean@1: mschieder@25: # Getting all priority in their order. mschieder@25: CHECK_ROUNDUP_ORDER_PRIO = "priority?@action=export_csv&@columns=id,order" mschieder@25: # Getting all statuses in their order. mschieder@25: CHECK_ROUNDUP_ORDER_STATUS = "status?@action=export_csv&@columns=id,order" mschieder@25: # Getting keywords and their ids. mschieder@20: CHECK_KEYWORD_ORDER = "keyword?@action=export_csv&@columns=id,name" mschieder@25: # Getting states and their ids. mschieder@20: CHECK_ROUNDUP_SEARCH_VALUES = "status?@action=export_csv&@columns=id,name&@filter=open&open=1" mschieder@25: # Getting the priority of each issue with the filter status and keywords mschieder@25: SEARCH_ROUNDUP_PRIO = "issue?@action=export_csv&@columns=priority&@filter=status,keyword&@pagesize=500&@startwith=0&status={search_values}&keyword={keyword_values}" mschieder@25: # Getting the status of each issue with the filter keywords mschieder@25: SEARCH_ROUNDUP_STATUS = "issue?@action=export_csv&@columns=status&@filter=keyword&@pagesize=500&@startwith=0&keyword={keyword_values}" sean@1: sean@1: sean@1: def connect_to_server(params, baseurl): sean@1: enc_data = urllib.parse.urlencode(params).encode() sean@1: cj = http.cookiejar.CookieJar() sean@1: opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj)) sean@1: req = urllib.request.Request(url=baseurl, data=enc_data) sean@1: opener.open(req) sean@1: return opener sean@1: sean@1: sean@1: def get_csv_from_server(opener, roundup_url, sub_url): sean@1: csv_req = urllib.request.Request(url=roundup_url+sub_url) sean@1: f = opener.open(csv_req) sean@1: csv_reader = csv.DictReader(io.TextIOWrapper(f)) sean@1: return csv_reader sean@1: sean@1: sean@1: def check_create_database(database_file, sql_create_db): sean@1: if not os.path.isfile(database_file): sean@1: con = None sean@1: cur = None sean@1: try: sean@1: con = db.connect(database_file) sean@1: cur = con.cursor() sean@1: try: sean@1: cur.execute(sql_create_db) sean@1: con.commit() sean@1: os.chmod(database_file, 0o644) sean@1: except: sean@1: con.rollback() sean@1: raise sean@1: finally: sean@1: if cur: sean@1: cur.close() sean@1: if con: sean@1: con.close() sean@1: mschieder@20: def get_keyword_ids(opener, baseurl, keywords): mschieder@20: if keywords == [""]: mschieder@20: return "" mschieder@20: mschieder@20: keywords_csv = get_csv_from_server(opener, baseurl, CHECK_KEYWORD_ORDER) mschieder@20: keywords_dict = {} mschieder@20: for x in keywords_csv: mschieder@20: keywords_dict[x["name"]] = x["id"] mschieder@20: mschieder@20: keywords_ids = [] mschieder@20: for x in keywords: mschieder@20: keywords_ids.append(keywords_dict[x]) mschieder@20: mschieder@20: return ",".join(keywords_ids) mschieder@20: mschieder@20: def get_status_ids(opener, baseurl, status): mschieder@20: mschieder@20: if status == [""]: mschieder@20: return "" mschieder@20: mschieder@25: status_csv = get_csv_from_server(opener, baseurl, CHECK_ROUNDUP_SEARCH_VALUES) mschieder@25: mschieder@20: status_dict = {} mschieder@20: for x in status_csv: mschieder@20: status_dict[x["name"]] = x["id"] mschieder@20: mschieder@20: staus_ids = ["-1"] mschieder@20: for x in status: mschieder@20: staus_ids.append(status_dict[x]) mschieder@20: mschieder@20: return ",".join(staus_ids) mschieder@20: sean@1: sean@1: def issues_to_quantities(issue_csv, columns, orders_csv): bernhard@15: """Count issues per priority. sean@1: bernhard@15: Returns: a list of ints, containing how often a prio occurred [:-1] bernhard@15: in order of the priorities, with the last being the "None" prio bernhard@15: """ bernhard@15: bernhard@15: quantities = [0] * (len(columns) +1) sean@1: order_dict = {} sean@1: sean@1: #convert the csv-dict reader to real dict sean@1: for row in orders_csv: sean@1: order_dict[row["id"]] = int(float(row["order"])) # int(float()) because the order-value is indeed "1.0, 2.0" etc sean@1: sean@1: for issue in issue_csv: mschieder@25: priority = issue[issue_csv.fieldnames[0]] sean@1: sean@2: if priority.isdigit() == True : sean@1: quantities[order_dict[priority] -1 ] += 1 bernhard@15: else: # no priority set bernhard@15: quantities[-1] += 1 sean@1: mschieder@25: # print("quantities : " + str(quantities)) sean@1: sean@1: return quantities sean@1: sean@1: sean@1: def save_issues_to_db(quantities, database_file, sql_create_db, sql_insert_in_db): sean@1: check_create_database(database_file, sql_create_db) sean@1: sean@1: cur = None sean@1: con = None sean@1: sean@1: try: sean@1: con = db.connect(database_file) sean@1: cur = con.cursor() sean@1: try: sean@1: cur.execute(sql_insert_in_db, quantities) sean@1: con.commit() sean@1: except: sean@1: con.rollback() sean@1: raise sean@1: finally: sean@1: if cur: sean@1: cur.close() sean@1: if con: sean@1: con.close() sean@1: sean@1: mschieder@25: def save_stats_in_db(search, login_parmeters, baseurl, db_file, columns, sql_create_db, sql_insert_in_db, keywords, status, include_no_prio=False): sean@1: try: sean@1: sean@1: opener = connect_to_server(login_parmeters, baseurl) sean@1: mschieder@20: keywords_ids_url = get_keyword_ids(opener, baseurl, keywords) mschieder@20: mschieder@25: if search == "prio": mschieder@25: order_csv = get_csv_from_server(opener, baseurl, CHECK_ROUNDUP_ORDER_PRIO) mschieder@25: status_ids_url = get_status_ids(opener, baseurl, status) mschieder@25: formated_search_url = SEARCH_ROUNDUP_PRIO.format(search_values=status_ids_url, keyword_values=keywords_ids_url) mschieder@25: elif search == "status": mschieder@25: order_csv = get_csv_from_server(opener, baseurl, CHECK_ROUNDUP_ORDER_STATUS) mschieder@25: formated_search_url = SEARCH_ROUNDUP_STATUS.format(keyword_values=keywords_ids_url) sean@1: sean@1: current_issues_csv = get_csv_from_server(opener, baseurl, formated_search_url) sean@1: sean@1: opener.close() sean@1: sean@1: quantities = issues_to_quantities(current_issues_csv, columns, order_csv) bernhard@15: if not include_no_prio: bernhard@15: quantities = quantities[:-1] sean@1: bernhard@9: save_issues_to_db(quantities, db_file, sql_create_db, sql_insert_in_db) sean@1: sean@1: except urllib.error.URLError as e: sean@1: print("No Valid Connection to server : " + baseurl + "\nerror: " + str(e))