Commit 689d85ef authored by Dimitri Podborski's avatar Dimitri Podborski
Browse files

add --fileformat option

This option can be used to work with already opened issues on GitLab. No
need to provide a csv or cli document numbers.
parent a7c8a24f
......@@ -25,9 +25,11 @@ import argparse
import csv
import os
import sys
import requests
from datetime import datetime
from automation import gitlab, mdms, helpers
import requests
from automation import gitlab, helpers, mdms
__version__ = '1.4'
......@@ -37,6 +39,9 @@ GITLAB_USERS_PATH = os.path.join(DATA_PATH, 'gitlab_users.json')
MEETINGS_PATH = os.path.join(DATA_PATH, 'meetings.json')
SYSTEMS_GROUP_ID = 727 # GitLab Group ID for Systems Subgroup
PROJECTS_FF = ['isobmff', 'HEIF', 'NALuFF', 'FFConformanceRefSoft', 'rawvideo', 'Text',
'eventmessage', 'General', 'DerivedVis', 'CENC', 'Metrics', 'PartialFF', 'MP4FF', 'Audio']
def download_url(url, save_path, chunk_size=128):
r = requests.get(url, auth=(mdms.MPEG_LOGIN, mdms.MPEG_PWD), stream=True)
......@@ -68,7 +73,7 @@ def fetch_contributions(table_entries):
download_url(url, filename)
def print_infos(table_entries):
def print_infos(table_entries, project_url, gitlab_projects):
print('\nDump information')
for entry in table_entries:
document = entry['document']
......@@ -94,13 +99,15 @@ def print_infos(table_entries):
print(' Authors:', authors)
if len(details['documents']) > 0:
last_doc = max(details['documents'], key=lambda x: x['version'])
print(' Last version: version', last_doc['version'], 'from', last_doc['timestamp'])
print(' Last version: version',
last_doc['version'], 'from', last_doc['timestamp'])
print(' URL:', last_doc['path'])
if project is not None:
print('GitLab metadata')
issues = gitlab.get_issues(project['id'])
issue_with_title, issue_with_meta, meta_last_doc_version = helpers.find_issue(issues, document)
issue_with_title, issue_with_meta, meta_last_doc_version = helpers.find_issue(
issues, document)
if issue_with_title is not None:
print(' User opened an issue:', issue_with_title.web_url)
print(' Labels:', issue_with_title.labels)
......@@ -108,16 +115,30 @@ def print_infos(table_entries):
print(' Comments:', len(comments))
if len(comments) > 0:
last_comment = max(comments, key=lambda x: x.id)
print(' Last comment from', last_comment.author['username'], 'at', last_comment.updated_at)
print(' Last comment from',
last_comment.author['username'], 'at', last_comment.updated_at)
if issue_with_meta is not None:
print(' Automatically generated issue:', issue_with_meta.web_url)
print(' Last document version found in the meta tag:', meta_last_doc_version)
print(' Automatically generated issue:',
issue_with_meta.web_url)
print(' Last document version found in the meta tag:',
meta_last_doc_version)
print(' Labels:', issue_with_meta.labels)
comments = issue_with_meta.notes.list(lazy=True)
print(' Comments:', len(comments))
if len(comments) > 0:
last_comment = max(comments, key=lambda x: x.id)
print(' Last comment from', last_comment.author['username'], 'at', last_comment.updated_at)
print(' Last comment from',
last_comment.author['username'], 'at', last_comment.updated_at)
project = helpers.find_project(gitlab_projects, project_url)
if project is not None:
print('Auto-generated contribution list')
issues = gitlab.get_issues(project['id'])
contribution_list = []
for issue in issues:
meta = helpers.get_issue_metadata(issue.description)
if meta is not None:
contribution_list.append(meta['document'])
print(','.join(contribution_list))
def open_new_issue(project_id, document, test, meeting_start, gitlab_members):
......@@ -127,21 +148,26 @@ def open_new_issue(project_id, document, test, meeting_start, gitlab_members):
if document_details is None:
print(' No document details found. Skip', document['document'])
return False
issue_description = helpers.create_issue_description(document, document_details)
issue_description = helpers.create_issue_description(
document, document_details)
if len(usernames) > 0:
issue_description += '\n_for:_ ' + ''.join('@' + str(u) + ', ' for u in usernames)
issue_description += '\n_for:_ ' + \
''.join('@' + str(u) + ', ' for u in usernames)
issue_lables = []
timestamp = datetime.now()
if len(document_details['documents']) > 0:
issue_lables.append(gitlab.Label.DocAvailable.value)
first_doc = min(document_details['documents'], key=lambda x: x['version'])
first_doc = min(
document_details['documents'], key=lambda x: x['version'])
timestamp = first_doc['timestamp']
if helpers.is_document_late(meeting_start, timestamp):
issue_lables.append(gitlab.Label.Late.value)
if not test:
print(' * {}: Open issue with title "{}" | Lables={}'.format(document['document'], issue_title, issue_lables))
gitlab.open_issue(project_id, issue_title, issue_description, issue_lables)
print(' * {}: Open issue with title "{}" | Lables={}'.format(
document['document'], issue_title, issue_lables))
gitlab.open_issue(project_id, issue_title,
issue_description, issue_lables)
return True
else:
print(' * {}: Test open issue with title "{}" | Lables={}'.format(document['document'], issue_title,
......@@ -151,11 +177,13 @@ def open_new_issue(project_id, document, test, meeting_start, gitlab_members):
def close_issue(issue, test, document):
if not test:
print(' * {}: Close issue: {}'.format(document['document'], issue.web_url))
print(
' * {}: Close issue: {}'.format(document['document'], issue.web_url))
gitlab.close_issue(issue)
return True
else:
print(' * {}: Test close issue: {}'.format(document['document'], issue.web_url))
print(
' * {}: Test close issue: {}'.format(document['document'], issue.web_url))
return False
......@@ -167,10 +195,12 @@ def open_issues(table_entries, test, gitlab_members, meeting_start):
project_id = entry['project']['id']
issues = gitlab.get_issues(project_id)
issue_with_title, issue_with_meta, meta_last_doc_version = helpers.find_issue(issues, document)
issue_with_title, issue_with_meta, meta_last_doc_version = helpers.find_issue(
issues, document)
if issue_with_title is None and issue_with_meta is None:
was_opened = open_new_issue(project_id, document, test, meeting_start, gitlab_members)
was_opened = open_new_issue(
project_id, document, test, meeting_start, gitlab_members)
if was_opened:
counter += 1
elif issue_with_title is not None and issue_with_meta is None:
......@@ -182,30 +212,36 @@ def open_issues(table_entries, test, gitlab_members, meeting_start):
print(' - Should we still open a new one?')
user_input = input(' Type y or n: ')
if 'y' in user_input:
was_opened = open_new_issue(project_id, document, test, meeting_start, gitlab_members)
was_opened = open_new_issue(
project_id, document, test, meeting_start, gitlab_members)
if was_opened:
counter += 1
else:
print(' * {} Skip "{}"'.format(document['document'], document['title']))
print(
' * {} Skip "{}"'.format(document['document'], document['title']))
else:
if issue_with_title is not None:
print('* {0}: ATTENTION We found multiple issues with "{0}" in the title. One with, and one without '
'metadata tag.'.format(document['document']))
print(' - Issue without metadata tag:', issue_with_title.web_url)
print(' - Issue without metadata tag:',
issue_with_title.web_url)
document_details = mdms.get_document_details(document['mdms_id'])
if document_details is None:
print(' * {}: Skip. Could not get document details from MDMS.'.format(document['document']))
print(
' * {}: Skip. Could not get document details from MDMS.'.format(document['document']))
continue
last_doc_version = 0
if len(document_details['documents']) > 0:
last_doc = max(document_details['documents'], key=lambda x: x['version'])
last_doc = max(
document_details['documents'], key=lambda x: x['version'])
last_doc_version = last_doc['version']
if last_doc_version > meta_last_doc_version:
print(' * {}: A new version of the document was added to MDMS '
'after the issue was opened on GitLab.'.format(document['document']))
print(' - Issue URL:', issue_with_meta.web_url)
print(' - Issue Title:', issue_with_meta.title)
print(' - Should we update the metadata table and add the new version to it?')
print(
' - Should we update the metadata table and add the new version to it?')
user_input = input(' Type y or n: ')
if 'y' in user_input:
new_description = helpers.get_updated_issue_description(issue_with_meta.description, document,
......@@ -213,15 +249,18 @@ def open_issues(table_entries, test, gitlab_members, meeting_start):
if 'DocAvailable' not in issue_with_meta.labels:
issue_with_meta.labels.append('DocAvailable')
if not test:
print(' - Update issue description of {}.'.format(document['document']))
print(
' - Update issue description of {}.'.format(document['document']))
issue_with_meta.description = new_description
issue_with_meta.save()
else:
print(' - Test update issue description of {}.'.format(document['document']))
print(
' - Test update issue description of {}.'.format(document['document']))
else:
print(' - Skip "{}"'.format(document['title']))
else:
print(' * {}: No update required for "{}"'.format(document['document'], document['title']))
print(
' * {}: No update required for "{}"'.format(document['document'], document['title']))
print('Opened issues:', counter)
......@@ -239,9 +278,11 @@ def close_issues(table_entries, test, force):
continue
issues = gitlab.get_issues(project_id)
issue_with_title, issue_with_meta, _meta_last_doc_version = helpers.find_issue(issues, document)
issue_with_title, issue_with_meta, _meta_last_doc_version = helpers.find_issue(
issues, document)
if issue_with_meta is None and issue_with_title is None:
print(' * {}: No issue to close in: {}'.format(document['document'], entry['project']['url']))
print(
' * {}: No issue to close in: {}'.format(document['document'], entry['project']['url']))
if issue_with_meta is not None:
was_closed = close_issue(issue_with_meta, test, document)
if was_closed:
......@@ -276,7 +317,8 @@ def create_output_doc(table_entries, output_path, template_path):
continue
project = entry['project']
issues = gitlab.get_issues(project['id'])
issue_with_title, issue_with_meta, _meta_last_doc_version = helpers.find_issue(issues, document)
issue_with_title, issue_with_meta, _meta_last_doc_version = helpers.find_issue(
issues, document)
if not project['id'] in projects:
projects[project['id']] = project
if project['id'] in projects_data:
......@@ -367,7 +409,8 @@ def parse_cli(docs, project_url, close_flag, gitlab_projects, input_docs):
docs = docs.replace('m', '').replace('M', '').strip().split(',')
project = helpers.find_project(gitlab_projects, project_url)
if not project:
print(' - Could not find a GitLab project with project URL "{}"'.format(project_url))
print(
' - Could not find a GitLab project with project URL "{}"'.format(project_url))
return table_entries
for doc in docs:
document = helpers.find_document(input_docs, 'm' + doc)
......@@ -383,6 +426,36 @@ def parse_cli(docs, project_url, close_flag, gitlab_projects, input_docs):
return table_entries
def derive_fileformat(gitlab_projects, input_docs):
"""return table_entries based on already opened issues in the FIleFormat group"""
proj_urls = []
table_entries = []
for proj in PROJECTS_FF:
proj_urls.append(os.path.join(
'http://mpegx.int-evry.fr/software/MPEG/Systems/FileFormat', proj))
for project_url in proj_urls:
print(f'gather contributions from {project_url}')
project = helpers.find_project(gitlab_projects, project_url)
if project is not None:
issues = gitlab.get_issues(project['id'])
for issue in issues:
meta = helpers.get_issue_metadata(issue.description)
if meta is not None:
document = helpers.find_document(
input_docs, meta['document'])
if not document:
print(
f'WARNING: Document "{meta["document"]}" not found. Try updating the database (-U) or select another meeting (--meeting).')
continue
table_entries.append({
'project': project,
'document': document,
'close': False
})
return table_entries
def main():
print('*' * 35)
print('* MPEG Systems script version', __version__, '*')
......@@ -397,22 +470,34 @@ def main():
parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter,
description='A tool for managing the GitLab issues for MPEG Systems Subgroups.',
epilog=usage_examples)
parser.add_argument('-o', '--open', help='Mode: Open GitLab issues.', action='store_true')
parser.add_argument('-c', '--close', help='Mode: Close GitLab issues.', action='store_true')
parser.add_argument('-d', '--docx', help='Mode: Generate output word document.', action='store_true')
parser.add_argument('-l', '--list', help='Mode: List information about the contribution(s).', action='store_true')
parser.add_argument('-f', '--fetch', help='Mode: Download contributions.', action='store_true')
parser.add_argument('-C', '--CLOSE', help='Force closing GitLab issues.', action='store_true')
parser.add_argument('-U', '--UPDATE', help='Update all databases.', action='store_true')
parser.add_argument('-i', '--csv', help='Input CSV file. Header row shall include "Number" and "Project URL"')
parser.add_argument('-m', '--documents', help='Comma separated MDMS document number(s). e.g.: m12345,...', type=str)
parser.add_argument('-p', '--project', help='GitLab project URL or "SubGroup/ProjectName".', type=str)
parser.add_argument(
'-o', '--open', help='Mode: Open GitLab issues.', action='store_true')
parser.add_argument(
'-c', '--close', help='Mode: Close GitLab issues.', action='store_true')
parser.add_argument(
'-d', '--docx', help='Mode: Generate output word document.', action='store_true')
parser.add_argument(
'-l', '--list', help='Mode: List information about the contribution(s).', action='store_true')
parser.add_argument(
'-f', '--fetch', help='Mode: Download contributions.', action='store_true')
parser.add_argument(
'-C', '--CLOSE', help='Force closing GitLab issues.', action='store_true')
parser.add_argument(
'-U', '--UPDATE', help='Update all databases.', action='store_true')
parser.add_argument(
'-i', '--csv', help='Input CSV file. Header row shall include "Number" and "Project URL"')
parser.add_argument(
'-m', '--documents', help='Comma separated MDMS document number(s). e.g.: m12345,...', type=str)
parser.add_argument(
'--fileformat', help='Derive input from already opened issues in FileFormat group', action='store_true')
parser.add_argument(
'-p', '--project', help='GitLab project URL or "SubGroup/ProjectName".', type=str)
parser.add_argument('--meeting', help='MPEG meeting number. If not set, the latest meeting is used.', default=-1,
type=int)
parser.add_argument('-t', '--template', help='Document template path if you want to use your .docx template for '
'output document.', type=str)
parser.add_argument('--test', help='Test mode. If set, no issues will be opened or closed.', action='store_true')
parser.add_argument('-n', '--notify', help='Notify (@mention) authors in opened issues.', action='store_true')
parser.add_argument(
'--test', help='Test mode. If set, no issues will be opened or closed.', action='store_true')
args = parser.parse_args()
if not args.open and not args.docx and not args.close and not args.list and not args.fetch:
......@@ -442,7 +527,8 @@ def main():
# get MPEG meeting we want to work on
meeting = helpers.find_meeting(meetings, args.meeting)
if not meeting:
print('Warning: Could not find meeting #{}. Update MDMS database and see if it was added'.format(args.meeting))
print('Warning: Could not find meeting #{}. Update MDMS database and see if it was added'.format(
args.meeting))
meetings = mdms.get_meetings()
if len(meetings) == 0:
print(' * ERROR, could not get meetings from MDMS.')
......@@ -456,7 +542,8 @@ def main():
print(' * Operating on MPEG#{} ({}) from {} to {}'.format(meeting['number'], meeting['name'], meeting['start_date'],
meeting['end_date']))
# get input documents (and update if needed)
input_docs_path = os.path.join(DATA_PATH, 'input_docs_{}.json'.format(meeting['number']))
input_docs_path = os.path.join(
DATA_PATH, 'input_docs_{}.json'.format(meeting['number']))
print(' * Update MPEG input documents data for MPEG#', meeting['number'])
input_docs = mdms.get_input_documents(meeting['id'])
if len(input_docs) == 0:
......@@ -470,15 +557,16 @@ def main():
table_entries = parse_csv(args.csv, gitlab_projects, input_docs)
output_path = args.csv.replace('.csv', '') + '.docx'
elif args.documents is not None:
table_entries = parse_cli(args.documents, args.project, args.close, gitlab_projects, input_docs)
table_entries = parse_cli(
args.documents, args.project, args.close, gitlab_projects, input_docs)
elif args.fileformat:
table_entries = derive_fileformat(gitlab_projects, input_docs)
# do some action
# do some work
if args.list:
print_infos(table_entries)
print_infos(table_entries, args.project, gitlab_projects)
if args.open:
meeting_start = helpers.try_parsing_date(meeting['start_date'])
if not args.notify:
gitlab_members = None
open_issues(table_entries, args.test, gitlab_members, meeting_start)
if args.close:
close_issues(table_entries, args.test, args.CLOSE)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment