| #!/usr/bin/env python3 |
| |
| # This file is part of the LibreOffice project. |
| # |
| # This Source Code Form is subject to the terms of the Mozilla Public |
| # License, v. 2.0. If a copy of the MPL was not distributed with this |
| # file, You can obtain one at http://mozilla.org/MPL/2.0/. |
| |
| # Use this script to update |
| # https://wiki.documentfoundation.org/MissingUnitTests |
| |
| import os |
| import datetime |
| import subprocess |
| import sys |
| import re |
| import json |
| import requests |
| |
| def splitList(lst, n): |
| for i in range(0, len(lst), n): |
| yield lst[i:i + n] |
| |
| def whiteboardNotes(whiteboard): |
| if not whiteboard: |
| return '' |
| if ' ' in whiteboard: |
| whiteboardList = reversed(whiteboard.split(' ')) |
| for w in whiteboardList: |
| if w.startswith("unitTestNotes"): |
| return w.split(':')[1] |
| elif whiteboard.startswith("unitTestNotes"): |
| return whiteboard.split(':')[1] |
| |
| return '' |
| |
| def linesModified(commit_hash): |
| repoPath = os.path.dirname(os.path.abspath(__file__)) + '/..' |
| commits = subprocess.check_output( |
| ['git', '-C', repoPath, 'show', commit_hash, '--shortstat'], |
| stderr=subprocess.DEVNULL) |
| linesmodified = commits.decode('utf-8', 'ignore').split('\n\n') |
| stats = linesmodified[-1].strip("\n").split(",") |
| insertions = 0 |
| deletions = 0 |
| # Make sure we have the file stats, a file stat must have |
| # file(s) changed in it |
| if "files changed" in linesmodified[-1] or "file changed" in linesmodified[-1]: |
| fileschanged = re.findall(r'\b\d+\b', stats[0])[0] |
| for element in stats: |
| if "+" in element: |
| insertions = re.findall(r'\b\d+\b', element)[0] |
| elif "-" in element: |
| deletions = re.findall(r'\b\d+\b', element)[0] |
| return [fileschanged, insertions, deletions] |
| else: |
| return ["not found", "not found", "not found"] |
| |
| def main(ignoredBugs): |
| results = { |
| 'export': { |
| 'docx': {}, |
| 'doc': {}, |
| 'pptx': {}, |
| 'xlsx': {}, |
| 'xhtml': {}, |
| 'html': {}, |
| }, |
| 'writer': { |
| 'undo': {}, |
| 'autoformat': {}, |
| 'autocorrect': {}, |
| 'others': {}, |
| }, |
| 'calc': { |
| 'import': {}, |
| 'format': {}, |
| 'others': {}, |
| }, |
| 'impress': { |
| 'drawingml': {}, |
| 'slidesorter': {}, |
| 'others': {}, |
| }, |
| |
| } |
| hasTestSet = set() |
| |
| repoPath = os.path.dirname(os.path.abspath(__file__)) + '/..' |
| branch = subprocess.check_output( |
| ['git', '-C', repoPath, 'rev-parse', '--abbrev-ref', 'HEAD'], |
| stderr=subprocess.DEVNULL) |
| last_hash = subprocess.check_output( |
| ['git', '-C', repoPath, 'rev-parse', 'HEAD'], |
| stderr=subprocess.DEVNULL) |
| output = subprocess.check_output( |
| ['git', '-C', repoPath, 'log', '--since="2012-01-01', '--name-only' ,'--pretty=format:"%s%n%ad%n%H"', '--date=format:"%Y/%m/%d"'], |
| stderr=subprocess.DEVNULL) |
| commits = output.decode('utf-8', 'ignore').split('\n\n') |
| |
| for commit in reversed(commits): |
| |
| commitInfo = commit.split('\n') |
| |
| summary = commitInfo[0].strip('"').lower() |
| |
| # Check for bugIds in the summary. Ignore those with a '-' after the digits. |
| # Those are used for file names ( e.g. tdf129410-1.ods ) |
| bugIds = re.findall("\\b(?:bug|fdo|tdf|lo)[#:]?(\\d+)(?!-)\\b", summary) |
| if bugIds is None or len(bugIds) == 0: |
| continue |
| |
| for bugId in bugIds: |
| |
| isIgnored = False |
| for i in ignoredBugs: |
| if i in summary: |
| isIgnored = True |
| if isIgnored: |
| continue |
| |
| if bugId in hasTestSet: |
| continue |
| |
| commitHash = commitInfo[2].strip('"') |
| date = commitInfo[1].strip('"') |
| infoList = [date, summary, commitHash] |
| |
| changedFiles = "".join(commitInfo[3:]) |
| if 'qa' in changedFiles: |
| hasTestSet.add(bugId) |
| continue |
| |
| elif 'sw/source/filter/ww8/docx' in changedFiles or \ |
| 'writerfilter/source/dmapper' in changedFiles or \ |
| 'starmath/source/ooxmlimport' in changedFiles: |
| results['export']['docx'][bugId] = infoList |
| |
| elif 'sw/source/filter/ww8/ww8' in changedFiles: |
| results['export']['doc'][bugId] = infoList |
| |
| elif 'sc/source/filter/excel/xe' in changedFiles: |
| results['export']['xlsx'][bugId] = infoList |
| |
| elif 'oox/source/export/' in changedFiles: |
| results['export']['pptx'][bugId] = infoList |
| |
| elif 'filter/source/xslt/odf2xhtml/export' in changedFiles: |
| results['export']['xhtml'][bugId] = infoList |
| |
| elif 'sw/source/filter/html/' in changedFiles: |
| results['export']['html'][bugId] = infoList |
| |
| elif 'sw/source/core/undo/' in changedFiles: |
| results['writer']['undo'][bugId] = infoList |
| |
| elif 'sw/source/core/edit/autofmt' in changedFiles: |
| results['writer']['autoformat'][bugId] = infoList |
| |
| elif 'sw/source/core/edit/acorrect' in changedFiles: |
| results['writer']['autocorrect'][bugId] = infoList |
| |
| elif 'drawingml' in changedFiles: |
| results['impress']['drawingml'][bugId] = infoList |
| |
| elif 'sd/source/ui/slidesorter/' in changedFiles: |
| results['impress']['slidesorter'][bugId] = infoList |
| |
| elif 'sc/source/core/tool/interpr' in changedFiles: |
| results['calc']['import'][bugId] = infoList |
| |
| elif 'svl/source/numbers/' in changedFiles: |
| results['calc']['format'][bugId] = infoList |
| |
| # Keep the following if statements at the end |
| |
| elif 'sc/source/core/' in changedFiles: |
| results['calc']['others'][bugId] = infoList |
| |
| elif 'sw/source/core/' in changedFiles: |
| results['writer']['others'][bugId] = infoList |
| |
| elif 'sd/source/core/' in changedFiles: |
| results['impress']['others'][bugId] = infoList |
| |
| listOfBugIdsWithoutTest = [] |
| for k,v in results.items(): |
| for k1, v1 in v.items(): |
| for bugId, info in v1.items(): |
| if bugId not in hasTestSet: |
| listOfBugIdsWithoutTest.append(bugId) |
| |
| |
| bugzillaJson = [] |
| resultList = [] |
| fixList = [] |
| #Split the list into different chunks for the requests, otherwise it fails |
| for chunk in splitList(listOfBugIdsWithoutTest, 50): |
| urlGet = 'https://bugs.documentfoundation.org/rest/bug?id=' + ','.join(chunk) |
| rGet = requests.get(urlGet) |
| rawData = json.loads(rGet.text) |
| rGet.close() |
| bugzillaJson.extend(rawData['bugs']) |
| |
| for k,v in results.items(): |
| for k1, v1 in v.items(): |
| for bugId, info in v1.items(): |
| |
| resolution = '' |
| keywords = [] |
| priority = '' |
| notes = '' |
| for bug in bugzillaJson: |
| if str(bug['id']) == str(bugId): |
| resolution = bug['resolution'] |
| keywords = bug['keywords'] |
| priority = bug['priority'] |
| notes = whiteboardNotes(bug['whiteboard']) |
| break |
| |
| # Only care about FIXED bugs |
| # Ignore performance bugs and accessibility bugs |
| if resolution and resolution == 'FIXED' and 'perf' not in keywords \ |
| and 'accessibility' not in keywords: |
| stats = linesModified(info[2]) |
| fixList.append({ |
| "id": bugId, |
| "date": info[0], |
| "priority": priority.upper(), |
| "summary": info[1], |
| "maintopic":k, |
| "subtopic":k1, |
| "fileschanged": stats[0], |
| "insertions": stats[1], |
| "deletions": stats[2], |
| "notes": notes |
| }) |
| |
| resultList.append([{ |
| "Generator": os.path.basename(sys.argv[0]), |
| "Date": str(datetime.datetime.now()), |
| "Commits": str(len(commits)), |
| "Branch": branch.decode().strip(), |
| "Hash": str(last_hash.decode().strip()), |
| }]) |
| resultList.append(fixList) |
| print(json.dumps(resultList)) |
| |
| def usage(): |
| message = """usage: {program} [bugs to ignore (each one is one argument)] |
| |
| Sample: {program} 10000 10001 10002""" |
| print(message.format(program = os.path.basename(sys.argv[0]))) |
| |
| if __name__ == '__main__': |
| |
| args = set() |
| if len(sys.argv) > 1: |
| arg1 = sys.argv[1] |
| if arg1 == '-h' or arg1 == "--help": |
| usage() |
| sys.exit(1) |
| for i in sys.argv: |
| if i.isdigit(): |
| args.add(i) |
| |
| main(sorted(args)) |