Fetch jira issues and link to bug fixes

Result output is dict with timestamps and hashes

code/fetch_jira_bugs/fetch.py

@@ -0,0 +1,51 @@
+""" Fetch issues that match given jql query """
+__author__ = "Kristian Berg"
+__copyright__ = "Copyright (c) 2018 Axis Communications AB"
+__license__ = "MIT"
+
+import urllib.request as url
+import json
+import os
+import argparse
+
+def fetch():
+    """ Fetch issues that match given jql query """
+    # Jira Query Language string which filters for resolved issues of type bug
+    jql = 'project = JENKINS '\
+        + 'AND issuetype = Bug '\
+        + 'AND status in (Resolved, Closed) '\
+        + 'AND resolution = Fixed '\
+        + 'AND component = core '\
+        + 'AND created <= "2018-02-20 10:34" '\
+        + 'ORDER BY created DESC'
+    jql = jql.replace(' ', '%20')
+
+    start_at = 0
+
+    # max_results parameter is capped at 1000, specifying a higher value will
+    # still return only the first 1000 results
+    max_results = 1000
+
+    os.makedirs('issues/', exist_ok=True)
+    request = 'https://issues.jenkins-ci.org/rest/api/2/search?'\
+        + 'jql={}&start_at={}&max_results={}'
+
+    # Do small request to establish value of 'total'
+    with url.urlopen(request.format(jql, start_at, '1')) as conn:
+        contents = json.loads(conn.read().decode('utf-8'))
+        total = contents['total']
+
+    # Fetch all matching issues and write to file(s)
+    print('Total issue matches: ' + str(total))
+    print('Progress: | = ' + str(max_results) + ' issues')
+    while start_at < total:
+        with url.urlopen(request.format(jql, start_at, max_results)) as conn:
+            with open('issues/res' + str(start_at) + '.json', 'w') as f:
+                f.write(conn.read().decode('utf-8'))
+        print('|', end='', flush='True')
+        start_at += max_results
+
+    print('\nDone!')
+
+if __name__ == '__main__':
+    fetch()

code/fetch_jira_bugs/find_bug_fixes.py

@@ -0,0 +1,108 @@
+""" Identify bugfixes in Jenkins repository given a list of issues """
+__author__ = "Kristian Berg"
+__copyright__ = "Copyright (c) 2018 Axis Communications AB"
+__license__ = "MIT"
+
+import os
+import json
+import re
+import argparse
+
+def find_bug_fixes(issue_path, gitlog_path):
+    """ Identify bugfixes in Jenkins repository given a list of issues """
+
+    i = 0 # Used to display progress
+    no_matches = []
+    matches_per_issue = {}
+    total_matches = 0
+
+    issue_list = build_issue_list(issue_path)
+    with open(gitlog_path) as f:
+        gitlog = json.loads(f.read())
+
+    for key in issue_list:
+        nbr = key.split('-')[1]
+        matches = []
+
+        for commit in gitlog:
+            pattern = r'JENKINS-{nbr}\D|#{nbr}\D|HUDSON-{nbr}\D'.format(nbr=nbr)
+            if re.search(pattern, commit):
+                if re.search(r'#{nbr}\D'.format(nbr=nbr), commit) \
+                    and not re.search('[Ff]ix', commit):
+                    pass
+                else:
+                    matches.append(commit)
+        total_matches += len(matches)
+        matches_per_issue[key] = len(matches)
+
+        if matches:
+            selected_commit = commit_selector_heuristic(matches)
+            if not selected_commit:
+                no_matches.append(key)
+            else:
+                issue_list[key]['hash'] = \
+                    re.search('(?<=^commit )[a-z0-9]+(?=\n)', \
+                    selected_commit).group(0)
+                issue_list[key]['commitdate'] = \
+                    re.search('(?<=\nDate:   )[0-9 -:+]+(?=\n)',\
+                    selected_commit).group(0)
+        else:
+            no_matches.append(key)
+
+        # Progress counter
+        i += 1
+        if i % 10 == 0:
+            print(i, end='\r')
+
+    print('Total issues: ' + str(len(issue_list)))
+    print('Issues matched to a bugfix: ' + str(len(issue_list) - len(no_matches)))
+    print('Percent of issues matched to a bugfix: ' + \
+          str((len(issue_list) - len(no_matches)) / len(issue_list)))
+    for key in no_matches:
+        issue_list.pop(key)
+
+    return issue_list
+
+
+def build_issue_list(path):
+    """ Helper method for find_bug_fixes """
+    issue_list = {}
+    for filename in os.listdir(path):
+        with open(path + '/' + filename) as f:
+            for issue in json.loads(f.read())['issues']:
+                issue_list[issue['key']] = {}
+
+                created_date = issue['fields']['created'].replace('T', ' ')
+                created_date = created_date.replace('.000', ' ')
+                issue_list[issue['key']]['creationdate'] = created_date
+
+                res_date = issue['fields']['resolutiondate'].replace('T', ' ')
+                res_date = res_date.replace('.000', ' ')
+                issue_list[issue['key']]['resolutiondate'] = res_date
+    return issue_list
+
+def commit_selector_heuristic(commits):
+    """ Helper method for find_bug_fixes.
+    Commits are assumed to be ordered in reverse chronological order.
+    Given said order, pick first commit that does not match the pattern.
+    If all commits match, return newest one. """
+    for commit in commits:
+        if not re.search('[Mm]erge|[Cc]herry|[Nn]oting', commit):
+            return commit
+    return commits[0]
+
+def main():
+    """ Main method """
+    parser = argparse.ArgumentParser(description='Identify bugfixes')
+    parser.add_argument('gitlog', metavar='G', type=str,
+                        help='Path to json file containing gitlog')
+    parser.add_argument('issue_list', metavar='I', type=str,
+                        help='Path to directory containing issue json files')
+    args = parser.parse_args()
+
+    issue_list = find_bug_fixes(args.issue_list, args.gitlog)
+    with open('issue_list.json', 'w') as f:
+        f.write(json.dumps(issue_list))
+
+if __name__ == '__main__':
+    main()

code/fetch_jira_bugs/git_log_to_array.py

@@ -0,0 +1,29 @@
+""" Dump the git log to a file """
+__author__ = "Kristian Berg"
+__copyright__ = "Copyright (c) 2018 Axis Communications AB"
+__license__ = "MIT"
+
+import subprocess
+import sys
+import json
+
+# Commits are saved in reverse chronological order from newest to oldest
+if __name__ == '__main__':
+    path_to_repo = sys.argv[1]
+
+    hashes = subprocess.run(['git', 'rev-list', '02d6908ada70fcf8012833ddef628bc09c6f8389'], cwd=path_to_repo,
+        stdout=subprocess.PIPE).stdout.decode('ascii').split()
+
+    logs = []
+    i = 0
+    for hash in hashes:
+        entry = subprocess.run(['git', 'show', '--quiet', '--date=iso', hash],
+            cwd=path_to_repo, stdout=subprocess.PIPE)\
+            .stdout.decode(errors='replace')
+        logs.append(entry)
+        i += 1
+        if i % 10 == 0:
+            print(i, end='\r')
+
+    with open('gitlog.json', 'w') as f:
+        f.write(json.dumps(logs))