Skip to content
GitLab
Commit 899ed7ce authored by Dimitri Podborski's avatar Dimitri Podborski
Browse files

CSV and CLI improvements, +bugfix

parent 5ce89157
Hide whitespace changes
Inline Side-by-side
automation/helpers.py
View file @ 899ed7ce
......@@ -163,7 +163,7 @@ def find_project(projects, url_or_path, path_root = 'MPEG/'):
'''
if url_or_path is None:
return None
striped_url_or_path = url_or_path.replace(' ', '').strip()
striped_url_or_path = url_or_path.replace(' ', '').strip().strip('/')
for project in projects:
if striped_url_or_path == project['url']:
return project
......@@ -213,7 +213,7 @@ def create_issue_metadata(document, details):
if len(details['documents']) > 0:
last_doc = max(details['documents'], key=lambda x:x['version'])
version = str(last_doc['version'])
title = document['title'].replace('(', '').replace(')', '')
title = document['title'].replace('(', '').replace(')', '').replace(',', '')
meta = '[meta]: # ({},{},{},{})'.format(document['mdms_id'], document['document'], title, version)
return meta
......
systems.py
View file @ 899ed7ce
'''
Script for MPEG Systems Subgroups
This script allows you to do the following things:
Some of the things you can do with this script are listed below:
1. Print information about input documents on MDMS and GitLab (optional):
e.g.: python systems.py -m m55958,m55959,m56121 -p FileFormat/CENC
2. Open issues based on the information provided in a CSV file:
e.g.: python systems.py --csv Contribs.csv -o
1. Open issues based on the information provided in a CSV file:
e.g.: python systems.py -o --csv Contribs.csv
2. Open issues based on CLI options:
e.g.: python systems.py -o -m 55958,55959,56121 -p FileFormat/CENC
3. Generate an output document based on the information provided in a CSV file:
e.g.: python systems.py --csv Contribs.csv -d --template template.docx
e.g.: python systems.py -d --csv Contribs.csv --template template.docx
4. Close issues based on the information provided in a CSV file:
e.g.: python systems.py -c --csv Contribs.csv
5. Close issues based on CLI options:
e.g.: python systems.py -c -m m55958,m55959,m56121 -p FileFormat/CENC
6. Print information about input documents on MDMS and GitLab:
e.g.: python systems.py -l -m m55958,m55959,m56121 -p FileFormat/CENC
NOTE 1: The script stores data from mdms and gitlab to ./data folder to minimize the number of
requests to both systems and to impove the performance. You can use the -u option to update
......@@ -22,6 +28,7 @@ import csv
import os
import sys
from automation import gitlab, mdms, helpers
from datetime import datetime
__version__ = '1.3'
......@@ -31,16 +38,26 @@ GITLAB_USERS_PATH = os.path.join(DATA_PATH, 'gitlab_users.json')
MEETINGS_PATH = os.path.join(DATA_PATH, 'meetings.json')
SYSTEMS_GROUP_ID = 727 # GitLab Group ID for Systems Subgroup
def print_infos(docs, project_url_or_path, gitlab_projects, input_docs):
print(' * Dump document information')
docs = docs.replace('m','').replace('M','').strip().split(',')
for doc in docs:
print('-'*51+ '\n' + 'm' + doc + '\n' + '. '*25 + '.')
print('MDMS metadata')
document = helpers.find_document(input_docs, 'm'+doc)
class Entry:
number: str
project_or_url: str
# doc = Entry()
# doc.title = 'asdf'
# print(doc)
def print_infos(table_entries):
print('\nDump information')
for entry in table_entries:
document = entry['document']
project = entry['project']
if not document:
print(' Document not found. Try updating the database (-u) or select another meeting (--meeting).')
continue
print('-'*51+ '\n' + document['document'] + '\n' + '. '*25 + '.')
print('MDMS metadata')
details = mdms.get_document_details(document['mdms_id'])
if details is None:
print(' Skip', document['document'])
......@@ -58,31 +75,29 @@ def print_infos(docs, project_url_or_path, gitlab_projects, input_docs):
if len(details['documents']) > 0:
last_doc = max(details['documents'], key=lambda x:x['version'])
print(' Last version: version', last_doc['version'], 'from', last_doc['timestamp'])
if project_url_or_path:
print(' URL:', last_doc['path'])
if project is not None:
print('GitLab metadata')
project = helpers.find_project(gitlab_projects, project_url_or_path, 'MPEG/Systems')
if not project:
print(' Could not find a GitLab project with', project_url_or_path)
else:
issues = gitlab.get_issues(project['id'])
issue_with_title, issue_with_meta, meta_last_doc_version = helpers.find_issue(issues, document)
if issue_with_title is not None:
print(' User opened an issue:', issue_with_title.web_url)
print(' Labels:', issue_with_title.labels)
comments = issue_with_title.notes.list(lazy=True)
print(' Comments:', len(comments))
if len(comments) > 0:
last_comment = max(comments, key=lambda x:x.id)
print(' Last comment from', last_comment.author['username'], 'at', last_comment.updated_at)
if issue_with_meta is not None:
print(' Automatically generated issue:', issue_with_meta.web_url)
print(' Last document version found in the meta tag:', meta_last_doc_version)
print(' Labels:', issue_with_meta.labels)
comments = issue_with_meta.notes.list(lazy=True)
print(' Comments:', len(comments))
if len(comments) > 0:
last_comment = max(comments, key=lambda x:x.id)
print(' Last comment from', last_comment.author['username'], 'at', last_comment.updated_at)
issues = gitlab.get_issues(project['id'])
issue_with_title, issue_with_meta, meta_last_doc_version = helpers.find_issue(issues, document)
if issue_with_title is not None:
print(' User opened an issue:', issue_with_title.web_url)
print(' Labels:', issue_with_title.labels)
comments = issue_with_title.notes.list(lazy=True)
print(' Comments:', len(comments))
if len(comments) > 0:
last_comment = max(comments, key=lambda x:x.id)
print(' Last comment from', last_comment.author['username'], 'at', last_comment.updated_at)
if issue_with_meta is not None:
print(' Automatically generated issue:', issue_with_meta.web_url)
print(' Last document version found in the meta tag:', meta_last_doc_version)
print(' Labels:', issue_with_meta.labels)
comments = issue_with_meta.notes.list(lazy=True)
print(' Comments:', len(comments))
if len(comments) > 0:
last_comment = max(comments, key=lambda x:x.id)
print(' Last comment from', last_comment.author['username'], 'at', last_comment.updated_at)
def open_new_issue(project_id, document, test, meeting_start):
issue_title = helpers.create_issue_title(document)
......@@ -92,19 +107,19 @@ def open_new_issue(project_id, document, test, meeting_start):
return
issue_description = helpers.create_issue_description(document, document_details)
issue_lables = []
timestamp = datetime.now()
if len(document_details['documents']) > 0:
issue_lables.append(gitlab.Label.DocAvailable.value)
first_doc = min(document_details['documents'], key=lambda x:x['version'])
v1_timestamp = first_doc['timestamp']
if helpers.is_document_late(meeting_start, v1_timestamp):
issue_lables.append(gitlab.Label.Late.value)
else:
timestamp = first_doc['timestamp']
if helpers.is_document_late(meeting_start, timestamp):
issue_lables.append(gitlab.Label.Late.value)
if not test:
print(' * Open issue', issue_title, '| Labels', issue_lables)
print(' * {}: Open issue with title "{}" | Lables={}'.format(document['document'], issue_title, issue_lables))
gitlab.open_issue(project_id, issue_title, issue_description, issue_lables)
else:
print(' * Test open issue', issue_title, '| Labels', issue_lables)
print(' * {}: Test open issue with title "{}" | Lables={}'.format(document['document'], issue_title, issue_lables))
def close_issue(issue, test):
if not test:
......@@ -114,7 +129,7 @@ def close_issue(issue, test):
print(' * Test close issue:', issue.web_url)
def open_issues(table_entries, test, gitlab_members, meeting_start):
print(' * Open issues. TestMode =', test)
print('\nOpen {} issues. TestMode={}'.format(len(table_entries), test))
counter = 0
for entry in table_entries:
document = entry['document']
......@@ -128,47 +143,50 @@ def open_issues(table_entries, test, gitlab_members, meeting_start):
open_new_issue(project_id, document, test, meeting_start)
counter += 1
elif issue_with_title is not None and issue_with_meta is None:
print(' * *** ATTENTION *** Issue with the same document number in the title aleady found on GitLab.')
print(' - Issue:', issue_with_title.web_url)
print(' * {}: ATTENTION Another issue with the same document number in the title was found on GitLab.'.format(document['document']))
print(' - Issue URL:', issue_with_title.web_url)
print(' - Issue Title:', issue_with_title.title)
print(' - Should we still open a new one?')
user_input = input('Type y or n: ')
user_input = input(' Type y or n: ')
if 'y' in user_input:
open_new_issue(project_id, document, test, meeting_start)
counter += 1
else:
print(' * Skip', document['document'], document['title'])
print(' * {} Skip "{}"'.format(document['document'], document['title']))
else:
if issue_with_title is not None:
print(' * *** ATTENTION *** We found multiple issues with "{}" in the title. One with and one without metadata tag.'.format(document['document']))
print(' * {0}: ATTENTION We found multiple issues with "{0}" in the title. One with, and one without metadata tag.'.format(document['document']))
print(' - Issue without metadata tag:', issue_with_title.web_url)
document_details = mdms.get_document_details(document['mdms_id'])
if document_details is None:
print(' * Skip', document['document'])
print(' * {}: Skip. Could not get document details from MDMS.'.format(document['document']))
continue
last_doc_version = 0
if len(document_details['documents']) > 0:
last_doc = max(document_details['documents'], key=lambda x:x['version'])
last_doc_version = last_doc['version']
if last_doc_version > meta_last_doc_version:
print(' * NOTE: A new version of the "{}" was added to MDMS after the issue was opened on GitLab.'.format(document['document']))
print(' - Issue:', issue_with_meta.web_url)
print(' * {}: A new version of the document was added to MDMS after the issue was opened on GitLab.'.format(document['document']))
print(' - Issue URL:', issue_with_meta.web_url)
print(' - Issue Title:', issue_with_meta.title)
print(' - Should we update the metadata table and add the new version to it?')
user_input = input('Type y or n: ')
user_input = input(' Type y or n: ')
if 'y' in user_input:
new_description = helpers.get_updated_issue_description(issue_with_meta.description, document, document_details)
if not test:
print(' - Update issue description of {}.'.format(document['document']))
issue_with_meta.description = new_description
issue_with_meta.save()
else:
print(' * Test update issue description of', document['document'])
print(' - Test update issue description of {}.'.format(document['document']))
else:
print(' * Skip', document['document'], document['title'])
print(' - Skip "{}"'.format(document['title']))
else:
print(' * No update required for', document['document'], document['title'])
print(' * Opened issues:', counter)
print(' * {}: No update required for "{}"'.format(document['document'], document['title']))
print('Opened issues:', counter)
def close_issues(table_entries, test):
print(' * Close issues. TestMode =', test)
print('\nClose issues. TestMode =', test)
counter = 0
for entry in table_entries:
document = entry['document']
......@@ -197,8 +215,8 @@ def close_issues(table_entries, test):
print(' * Skip', document['document'], document['title'])
print(' * Closed issues:', counter)
def create_output_doc(table_entries, template_path):
print(' * Create Output Document')
def create_output_doc(table_entries, output_path, template_path):
print('\nCreate Output Document')
# iterate over the CSV table and gather all the data
projects = {}
projects_data = {}
......@@ -226,7 +244,6 @@ def create_output_doc(table_entries, template_path):
for contribution in projects_data[project_id]:
formatter.add_contribution(contribution)
# save the file
output_path = csv_path.replace('.csv', '') + '.docx'
print(' * Save output document:', output_path)
formatter.save(output_path)
......@@ -301,7 +318,7 @@ def parse_cli(docs, project_url_or_path, close_flag, gitlab_projects, input_docs
docs = docs.replace('m','').replace('M','').strip().split(',')
project = helpers.find_project(gitlab_projects, project_url_or_path, 'MPEG/Systems')
if not project:
print(' Could not find a GitLab project with', project_url_or_path)
print(' - Could not find a GitLab project with project URL or path "{}"'.format(project_url_or_path))
return table_entries
for doc in docs:
document = helpers.find_document(input_docs, 'm'+doc)
......@@ -320,48 +337,45 @@ print('* MPEG Systems script version', __version__, '*')
print('*'*35 + '\n')
# program options
usage_examples= '''Examples:
python3 systems.py -m m55958,m55959,m56121 -p FileFormat/CENC --meeting 133
python3 systems.py --csv Contribs.csv -o
python3 systems.py --csv Contribs.csv -d
python systems.py -l -m m55958,m55959,m56121 -p FileFormat/CENC --meeting 133
python systems.py -o -d --csv Contribs.csv
python systems.py -c --csv Contribs.csv
python systems.py -d -m m55958,m55959,m56121 -p FileFormat/CENC --meeting 133
'''
parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter,
description='A tool for managing the GitLab issues for MPEG Systems Subgroups.',
epilog=usage_examples)
parser.add_argument('-u', help='Update MDMS documents database.', action='store_true')
parser.add_argument('-U', help='Update all databases.', action='store_true')
parser.add_argument('-o', '--open', help='Open GitLab issues.', action='store_true')
parser.add_argument('-c', '--close', help='Close GitLab issues.', action='store_true')
parser.add_argument('-d', '--docx', help='Generate output word document.', action='store_true')
parser.add_argument('-l', '--list', help='List information about the contribution(s).', action='store_true')
parser.add_argument('-u', '--update', help='Update MDMS documents database.', action='store_true')
parser.add_argument('-U', '--UPDATE', help='Update all databases.', action='store_true')
parser.add_argument('-i', '--csv', help='Input CSV file. Header row shall include "Number" and ("Project URL" or "Sub Group" and "Project Name").')
parser.add_argument('-m', '--documents', help='Comma separated MDMS document number(s). e.g.: m12345,...', type=str)
parser.add_argument('-p', '--project', help='GitLab project URL or "SubGroup/ProjectName".', type=str)
parser.add_argument('--meeting', help='MPEG meeting number. If not set, the latest meeting is used.', default=-1, type=int)
parser.add_argument('--csv', help='Input CSV file. Header row shall include "Number" and ("Project URL" or "Sub Group" and "Project Name").')
parser.add_argument('-o', help='Open issues mode. Open issues based on CSV file.', action='store_true')
parser.add_argument('-d', help='Generate output document mode. Generate output document based on CSV file.', action='store_true')
parser.add_argument('-c', help='Close issues mode. Close issues based on CSV file.', action='store_true')
parser.add_argument('--template', help='Document template path.', type=str)
parser.add_argument('--test', help='Test mode. If set, no issues will be opened.', action='store_true')
parser.add_argument('-m', help='Documents (comma separated values). List information about the contribution(s).', type=str)
parser.add_argument('-p', help='GitLab project URL or "SubGroup/ProjectName". This option is used with -m option.', type=str)
parser.add_argument('-t', '--template', help='Document template path if you want to use your .docx template for output document.', type=str)
parser.add_argument('--test', help='Test mode. If set, no issues will be opened or closed.', action='store_true')
args = parser.parse_args()
print("args.o",args.o)
print("args.d",args.d)
print("args.c",args.c)
print("")
print("args.csv",args.csv)
print("args.m",args.m)
print("args.p",args.p)
if not args.open and not args.docx and not args.close and not args.list:
print('There is no work to be done. Check the CLI options. Use -h for help.')
sys.exit(1)
# get gitlab projects (and update if needed)
if not os.path.isfile(GITLAB_PROJECTS_PATH) or args.U:
if not os.path.isfile(GITLAB_PROJECTS_PATH) or args.UPDATE:
print(' * Update GitLab projects data')
projects = gitlab.get_projects()
helpers.store_json_data(GITLAB_PROJECTS_PATH, projects)
gitlab_projects = helpers.load_json_data(GITLAB_PROJECTS_PATH)
# get gitlab members (and update if needed)
if not os.path.isfile(GITLAB_USERS_PATH) or args.U:
if not os.path.isfile(GITLAB_USERS_PATH) or args.UPDATE:
print(' * Update GitLab users data')
members = gitlab.get_members(SYSTEMS_GROUP_ID)
helpers.store_json_data(GITLAB_USERS_PATH, members)
gitlab_members = helpers.load_json_data(GITLAB_USERS_PATH)
# get MPEG meetings (and update if needed)
if not os.path.isfile(MEETINGS_PATH) or args.U:
if not os.path.isfile(MEETINGS_PATH) or args.UPDATE:
print(' * Update MPEG meetings data')
meetings = mdms.get_meetings()
if len(meetings) == 0:
......@@ -386,7 +400,7 @@ if not meeting:
print(' * Operating on MPEG#{} ({}) from {} to {}'.format(meeting['number'], meeting['name'], meeting['start_date'], meeting['end_date']))
# get input documents (and update if needed)
input_docs_path = os.path.join(DATA_PATH, 'input_docs_{}.json'.format(meeting['number']))
if not os.path.isfile(input_docs_path) or args.U or args.u:
if not os.path.isfile(input_docs_path) or args.UPDATE or args.update:
print(' * Update MPEG input documents data for MPEG#', meeting['number'])
input_docs = mdms.get_input_documents(meeting['id'])
if len(input_docs) == 0:
......@@ -395,20 +409,21 @@ if not os.path.isfile(input_docs_path) or args.U or args.u:
helpers.store_json_data(input_docs_path, input_docs)
input_docs = helpers.load_json_data(input_docs_path)
output_path = 'output.docx'
table_entries = []
if args.csv is not None:
table_entries = parse_csv(args.csv, gitlab_projects, input_docs)
elif args.m is not None and args.p is not None:
table_entries = parse_cli(args.m, args.p, args.c, gitlab_projects, input_docs)
output_path = args.csv.replace('.csv', '') + '.docx'
elif args.documents is not None:
table_entries = parse_cli(args.documents, args.project, args.close, gitlab_projects, input_docs)
# do some action
if args.m is not None and not (args.o or args.d or args.c):
print_infos(args.m, args.p, gitlab_projects, input_docs)
if args.o:
if args.list:
print_infos(table_entries)
if args.open:
meeting_start = helpers.try_parsing_date(meeting['start_date'])
open_issues(table_entries, args.test, gitlab_members, meeting_start)
if args.d:
create_output_doc(table_entries, args.template)
if args.c:
if args.close:
close_issues(table_entries, args.test)
if args.docx:
create_output_doc(table_entries, output_path, args.template)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment