""" Script for MPEG Systems Subgroups Some of the things you can do with this script are listed below: 1. Open issues based on the information provided in a CSV file: e.g.: python systems.py -o --csv Contribs.csv 2. Open issues based on CLI options: e.g.: python systems.py -o -m 55958,55959,56121 -p http://... 3. Generate an output document based on the information provided in a CSV file: e.g.: python systems.py -d --csv Contribs.csv --template template.docx 4. Close issues based on the information provided in a CSV file: e.g.: python systems.py -c --csv Contribs.csv 5. Close issues based on CLI options: e.g.: python systems.py -c -m m55958,m55959,m56121 -p http://... 6. Print information about input documents on MDMS and GitLab: e.g.: python systems.py -l -m m55958,m55959,m56121 -p http://... NOTE 1: The script stores data from mdms and gitlab to ./data folder to minimize the number of requests to both systems and to impove the performance. You can use the -U option to update all data. NOTE 2: You can use --test option to 'test run' the script. It will not create or close any issues on GitLab. """ import argparse import csv import os import sys from datetime import datetime import requests from automation import gitlab, helpers, mdms __version__ = '1.4' DATA_PATH = './data' GITLAB_PROJECTS_PATH = os.path.join(DATA_PATH, 'gitlab_projects.json') GITLAB_USERS_PATH = os.path.join(DATA_PATH, 'gitlab_users.json') MEETINGS_PATH = os.path.join(DATA_PATH, 'meetings.json') SYSTEMS_GROUP_ID = 727 # GitLab Group ID for Systems Subgroup PROJECTS_FF = ['isobmff', 'HEIF', 'NALuFF', 'FFConformanceRefSoft', 'rawvideo', 'Text', 'eventmessage', 'General', 'DerivedVis', 'CENC', 'Metrics', 'PartialFF', 'MP4FF', 'Audio'] def download_url(url, save_path, chunk_size=128): r = requests.get(url, auth=(mdms.MPEG_LOGIN, mdms.MPEG_PWD), stream=True) with open(save_path, 'wb') as fd: for chunk in r.iter_content(chunk_size=chunk_size): fd.write(chunk) def fetch_contributions(table_entries): print('\nDownload contributions') for entry in table_entries: path = os.path.join(DATA_PATH, 'contributions') document = entry['document'] project = entry['project'] if not document: print('WARNING: Document not found. Try updating the database (-U) or select another meeting (--meeting).') continue url = document['latest_version_url'] if url is None: print(document['document'], 'Skip [no document]') continue if project is not None: path = os.path.join(path, project['path_with_namespace']) if not os.path.exists(path) and len(path) > 0: os.makedirs(path) folder, filename = os.path.split(url) filename = os.path.join(path, filename) print(document['document'], ' -> ', filename) download_url(url, filename) def print_infos(table_entries, project_url, gitlab_projects): print('\nDump information') for entry in table_entries: document = entry['document'] project = entry['project'] if not document: print('WARNING: Document not found. Try updating the database (-U) or select another meeting (--meeting).') continue print('-' * 51 + '\n' + document['document'] + '\n' + '. ' * 25 + '.') print('MDMS metadata') details = mdms.get_document_details(document['mdms_id']) if details is None: print(' Skip', document['document']) continue print(' Title:', document['title']) if details['organizations']: print(' Organizations:', details['organizations']) authors = '' for author in document['authors']: authors += author['name'] if author['email']: authors += ' (' + author['email'] + ')' authors += ', ' print(' Authors:', authors) if len(details['documents']) > 0: last_doc = max(details['documents'], key=lambda x: x['version']) print(' Last version: version', last_doc['version'], 'from', last_doc['timestamp']) print(' URL:', last_doc['path']) if project is not None: print('GitLab metadata') issues = gitlab.get_issues(project['id']) issue_with_title, issue_with_meta, meta_last_doc_version = helpers.find_issue( issues, document) if issue_with_title is not None: print(' User opened an issue:', issue_with_title.web_url) print(' Labels:', issue_with_title.labels) comments = issue_with_title.notes.list(lazy=True) print(' Comments:', len(comments)) if len(comments) > 0: last_comment = max(comments, key=lambda x: x.id) print(' Last comment from', last_comment.author['username'], 'at', last_comment.updated_at) if issue_with_meta is not None: print(' Automatically generated issue:', issue_with_meta.web_url) print(' Last document version found in the meta tag:', meta_last_doc_version) print(' Labels:', issue_with_meta.labels) comments = issue_with_meta.notes.list(lazy=True) print(' Comments:', len(comments)) if len(comments) > 0: last_comment = max(comments, key=lambda x: x.id) print(' Last comment from', last_comment.author['username'], 'at', last_comment.updated_at) project = helpers.find_project(gitlab_projects, project_url) if project is not None: print('Auto-generated contribution list') issues = gitlab.get_issues(project['id']) contribution_list = [] for issue in issues: meta = helpers.get_issue_metadata(issue.description) if meta is not None: contribution_list.append(meta['document']) print(','.join(contribution_list)) def open_new_issue(project_id, document, test, meeting_start, gitlab_members): usernames = helpers.find_gitlab_users(gitlab_members, document) issue_title = helpers.create_issue_title(document) document_details = mdms.get_document_details(document['mdms_id']) if document_details is None: print(' No document details found. Skip', document['document']) return False issue_description = helpers.create_issue_description( document, document_details) if len(usernames) > 0: issue_description += '\n_for:_ ' + \ ''.join('@' + str(u) + ', ' for u in usernames) issue_lables = [] timestamp = datetime.now() if len(document_details['documents']) > 0: issue_lables.append(gitlab.Label.DocAvailable.value) first_doc = min( document_details['documents'], key=lambda x: x['version']) timestamp = first_doc['timestamp'] if helpers.is_document_late(meeting_start, timestamp): issue_lables.append(gitlab.Label.Late.value) if not test: print(' * {}: Open issue with title "{}" | Lables={}'.format( document['document'], issue_title, issue_lables)) gitlab.open_issue(project_id, issue_title, issue_description, issue_lables) return True else: print(' * {}: Test open issue with title "{}" | Lables={}'.format(document['document'], issue_title, issue_lables)) return False def close_issue(issue, test, document): if not test: print( ' * {}: Close issue: {}'.format(document['document'], issue.web_url)) gitlab.close_issue(issue) return True else: print( ' * {}: Test close issue: {}'.format(document['document'], issue.web_url)) return False def open_issues(table_entries, test, gitlab_members, meeting_start): print('\nOpen {} issues. TestMode={}'.format(len(table_entries), test)) counter = 0 for entry in table_entries: document = entry['document'] project_id = entry['project']['id'] issues = gitlab.get_issues(project_id) issue_with_title, issue_with_meta, meta_last_doc_version = helpers.find_issue( issues, document) if issue_with_title is None and issue_with_meta is None: was_opened = open_new_issue( project_id, document, test, meeting_start, gitlab_members) if was_opened: counter += 1 elif issue_with_title is not None and issue_with_meta is None: print( ' * {}: ATTENTION Another issue with the same document number in the title was found on GitLab.'.format( document['document'])) print(' - Issue URL:', issue_with_title.web_url) print(' - Issue Title:', issue_with_title.title) print(' - Should we still open a new one?') user_input = input(' Type y or n: ') if 'y' in user_input: was_opened = open_new_issue( project_id, document, test, meeting_start, gitlab_members) if was_opened: counter += 1 else: print( ' * {} Skip "{}"'.format(document['document'], document['title'])) else: if issue_with_title is not None: print('* {0}: ATTENTION We found multiple issues with "{0}" in the title. One with, and one without ' 'metadata tag.'.format(document['document'])) print(' - Issue without metadata tag:', issue_with_title.web_url) document_details = mdms.get_document_details(document['mdms_id']) if document_details is None: print( ' * {}: Skip. Could not get document details from MDMS.'.format(document['document'])) continue last_doc_version = 0 if len(document_details['documents']) > 0: last_doc = max( document_details['documents'], key=lambda x: x['version']) last_doc_version = last_doc['version'] if last_doc_version > meta_last_doc_version: print(' * {}: A new version of the document was added to MDMS ' 'after the issue was opened on GitLab.'.format(document['document'])) print(' - Issue URL:', issue_with_meta.web_url) print(' - Issue Title:', issue_with_meta.title) print( ' - Should we update the metadata table and add the new version to it?') user_input = input(' Type y or n: ') if 'y' in user_input: new_description = helpers.get_updated_issue_description(issue_with_meta.description, document, document_details) if 'DocAvailable' not in issue_with_meta.labels: issue_with_meta.labels.append('DocAvailable') if not test: print( ' - Update issue description of {}.'.format(document['document'])) issue_with_meta.description = new_description issue_with_meta.save() else: print( ' - Test update issue description of {}.'.format(document['document'])) else: print(' - Skip "{}"'.format(document['title'])) else: print( ' * {}: No update required for "{}"'.format(document['document'], document['title'])) print('Opened issues:', counter) def close_issues(table_entries, test, force): print('\nClose {} issues. TestMode={}'.format(len(table_entries), test)) counter = 0 for entry in table_entries: document = entry['document'] project_id = entry['project']['id'] close_flag = entry['close'] if force: close_flag = True if not close_flag: continue issues = gitlab.get_issues(project_id) issue_with_title, issue_with_meta, _meta_last_doc_version = helpers.find_issue( issues, document) if issue_with_meta is None and issue_with_title is None: print( ' * {}: No issue to close in: {}'.format(document['document'], entry['project']['url'])) if issue_with_meta is not None: was_closed = close_issue(issue_with_meta, test, document) if was_closed: counter += 1 if issue_with_title is not None: print(' * {}: ATTENTION User created issue with the same document number in the title ' 'was found on GitLab.'.format(document['document'])) print(' - Issue URL:', issue_with_title.web_url) print(' - Issue Title:', issue_with_title.title) print(' - Should we also close it?') user_input = input(' Type y or n: ') if 'y' in user_input: was_closed = close_issue(issue_with_title, test, document) if was_closed: counter += 1 else: print(' * Skip "{}"'.format(issue_with_title.title)) print('Closed issues:', counter) def create_output_doc(table_entries, output_path, template_path): print('\nCreate Output Document') # iterate over the CSV table and gather all the data projects = {} projects_data = {} for entry in table_entries: document = entry['document'] print(' * Process document', document['document'], document['title']) details = mdms.get_document_details(document['mdms_id']) if details is None: print(' * Skip', document['document']) continue project = entry['project'] issues = gitlab.get_issues(project['id']) issue_with_title, issue_with_meta, _meta_last_doc_version = helpers.find_issue( issues, document) if not project['id'] in projects: projects[project['id']] = project if project['id'] in projects_data: projects_data[project['id']].append( {'document': document, 'details': details, 'issue_meta': issue_with_meta, 'issue_title': issue_with_title}) else: projects_data[project['id']] = [{'document': document, 'details': details, 'issue_meta': issue_with_meta, 'issue_title': issue_with_title}] # now iterate over all projects and write the docment formatter = helpers.DocumentFormatter(template_path) for project_id in projects_data: formatter.add_project(projects[project_id]) # formatter.add_paragraph() for contribution in projects_data[project_id]: formatter.add_contribution(contribution) # save the file print(' * Save output document:', output_path) formatter.save(output_path) def parse_csv(csv_file, projects, docs): table_entries = [] with open(csv_file, 'r', encoding='utf-8-sig') as f: sample = f.readline() f.seek(0) dialect = csv.Sniffer().sniff(sample) # find out the delimeter type has_header = csv.Sniffer().has_header(sample) reader = csv.reader(f, dialect) if not has_header: print('Error: Input CSV file has no header.') sys.exit(-1) header = next(reader) doc_number_idx = None project_url_idx = None subgroup_idx = None project_name_idx = None close_idx = None for n in range(len(header)): if 'number' == header[n].replace(' ', '').strip().lower(): doc_number_idx = n elif 'projecturl' == header[n].replace(' ', '').strip().lower(): project_url_idx = n elif 'subgroup' == header[n].replace(' ', '').strip().lower(): subgroup_idx = n elif 'projectname' == header[n].replace(' ', '').strip().lower(): project_name_idx = n elif 'closeissue' == header[n].replace(' ', '').strip().lower(): close_idx = n if doc_number_idx is None or not ( (subgroup_idx is not None and project_name_idx is not None) or project_url_idx is not None): print('Error: CSV header should have, "number" and ("project url" or ("sub group" and "project name")) ' 'fields') sys.exit(-1) for row in reader: # skip empty lines if len(' '.join(row).strip()) == 0: continue project_url = None if project_url_idx is not None: if len(row[project_url_idx]) > 0: project_url = row[project_url_idx] if project_url is None: print('NOTE: No project URL found:', row) continue close_flag = False if close_idx is not None: if 'true' in row[close_idx].lower() or '1' == row[close_idx]: close_flag = True project = helpers.find_project(projects, project_url) doc = helpers.find_document(docs, row[doc_number_idx]) if not project: print('WARNING: No project on GitLab for:', row) elif not doc: print('WARNING: Document not found:', row) else: table_entries.append({ 'project': project, 'document': doc, 'close': close_flag }) return table_entries def parse_cli(docs, project_url, close_flag, gitlab_projects, input_docs): table_entries = [] docs = docs.replace('m', '').replace('M', '').strip().split(',') project = helpers.find_project(gitlab_projects, project_url) if not project: print( ' - Could not find a GitLab project with project URL "{}"'.format(project_url)) return table_entries for doc in docs: document = helpers.find_document(input_docs, 'm' + doc) if not document: print('WARNING: Document "m{}" not found. Try updating the database (-U) or select another meeting (' '--meeting).'.format(doc)) continue table_entries.append({ 'project': project, 'document': document, 'close': close_flag }) return table_entries def derive_fileformat(gitlab_projects, input_docs): """return table_entries based on already opened issues in the FIleFormat group""" proj_urls = [] table_entries = [] for proj in PROJECTS_FF: proj_urls.append(os.path.join( 'http://mpegx.int-evry.fr/software/MPEG/Systems/FileFormat', proj)) for project_url in proj_urls: print(f'gather contributions from {project_url}') project = helpers.find_project(gitlab_projects, project_url) if project is not None: issues = gitlab.get_issues(project['id']) for issue in issues: meta = helpers.get_issue_metadata(issue.description) if meta is not None: document = helpers.find_document( input_docs, meta['document']) if not document: print( f'WARNING: Document "{meta["document"]}" not found. Try updating the database (-U) or select another meeting (--meeting).') continue table_entries.append({ 'project': project, 'document': document, 'close': False }) return table_entries def main(): print('*' * 35) print('* MPEG Systems script version', __version__, '*') print('*' * 35 + '\n') # program options usage_examples = '''Examples: python systems.py -l -m m55958,m55959,m56121 -p FileFormat/CENC --meeting 133 python systems.py -o -d --csv Contribs.csv python systems.py -c --csv Contribs.csv python systems.py -d -m m55958,m55959,m56121 -p FileFormat/CENC --meeting 133 ''' parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, description='A tool for managing the GitLab issues for MPEG Systems Subgroups.', epilog=usage_examples) parser.add_argument( '-o', '--open', help='Mode: Open GitLab issues.', action='store_true') parser.add_argument( '-c', '--close', help='Mode: Close GitLab issues.', action='store_true') parser.add_argument( '-d', '--docx', help='Mode: Generate output word document.', action='store_true') parser.add_argument( '-l', '--list', help='Mode: List information about the contribution(s).', action='store_true') parser.add_argument( '-f', '--fetch', help='Mode: Download contributions.', action='store_true') parser.add_argument( '-C', '--CLOSE', help='Force closing GitLab issues.', action='store_true') parser.add_argument( '-U', '--UPDATE', help='Update all databases.', action='store_true') parser.add_argument( '-i', '--csv', help='Input CSV file. Header row shall include "Number" and "Project URL"') parser.add_argument( '-m', '--documents', help='Comma separated MDMS document number(s). e.g.: m12345,...', type=str) parser.add_argument( '--fileformat', help='Derive input from already opened issues in FileFormat group', action='store_true') parser.add_argument( '-p', '--project', help='GitLab project URL or "SubGroup/ProjectName".', type=str) parser.add_argument('--meeting', help='MPEG meeting number. If not set, the latest meeting is used.', default=-1, type=int) parser.add_argument('-t', '--template', help='Document template path if you want to use your .docx template for ' 'output document.', type=str) parser.add_argument( '--test', help='Test mode. If set, no issues will be opened or closed.', action='store_true') args = parser.parse_args() if not args.open and not args.docx and not args.close and not args.list and not args.fetch: print('There is no work to be done. Check the CLI options. Use -h for help.') sys.exit(1) # get gitlab projects (and update if needed) if not os.path.isfile(GITLAB_PROJECTS_PATH) or args.UPDATE: print(' * Update GitLab projects data') projects = gitlab.get_projects() helpers.store_json_data(GITLAB_PROJECTS_PATH, projects) gitlab_projects = helpers.load_json_data(GITLAB_PROJECTS_PATH) # get gitlab members (and update if needed) if not os.path.isfile(GITLAB_USERS_PATH) or args.UPDATE: print(' * Update GitLab users data') members = gitlab.get_members(SYSTEMS_GROUP_ID) helpers.store_json_data(GITLAB_USERS_PATH, members) gitlab_members = helpers.load_json_data(GITLAB_USERS_PATH) # get MPEG meetings (and update if needed) if not os.path.isfile(MEETINGS_PATH) or args.UPDATE: print(' * Update MPEG meetings data') meetings = mdms.get_meetings() if len(meetings) == 0: print(' * ERROR, could not get meetings from MDMS. Check your password.') sys.exit(1) helpers.store_json_data(MEETINGS_PATH, meetings) meetings = helpers.load_json_data(MEETINGS_PATH) # get MPEG meeting we want to work on meeting = helpers.find_meeting(meetings, args.meeting) if not meeting: print('Warning: Could not find meeting #{}. Update MDMS database and see if it was added'.format( args.meeting)) meetings = mdms.get_meetings() if len(meetings) == 0: print(' * ERROR, could not get meetings from MDMS.') sys.exit(1) helpers.store_json_data(MEETINGS_PATH, meetings) meetings = helpers.load_json_data(MEETINGS_PATH) meeting = helpers.find_meeting(meetings, args.meeting) if not meeting: print('Error: No such meeting found on MDMS. Check your input!') sys.exit(-1) print(' * Operating on MPEG#{} ({}) from {} to {}'.format(meeting['number'], meeting['name'], meeting['start_date'], meeting['end_date'])) # get input documents (and update if needed) input_docs_path = os.path.join( DATA_PATH, 'input_docs_{}.json'.format(meeting['number'])) print(' * Update MPEG input documents data for MPEG#', meeting['number']) input_docs = mdms.get_input_documents(meeting['id']) if len(input_docs) == 0: print(' * ERROR, could not get input documents from MDMS. Check your password.') sys.exit(1) helpers.store_json_data(input_docs_path, input_docs) output_path = 'output.docx' table_entries = [] if args.csv is not None: table_entries = parse_csv(args.csv, gitlab_projects, input_docs) output_path = args.csv.replace('.csv', '') + '.docx' elif args.documents is not None: table_entries = parse_cli( args.documents, args.project, args.close, gitlab_projects, input_docs) elif args.fileformat: table_entries = derive_fileformat(gitlab_projects, input_docs) # do some work if args.list: print_infos(table_entries, args.project, gitlab_projects) if args.open: meeting_start = helpers.try_parsing_date(meeting['start_date']) open_issues(table_entries, args.test, gitlab_members, meeting_start) if args.close: close_issues(table_entries, args.test, args.CLOSE) if args.docx: create_output_doc(table_entries, output_path, args.template) if args.fetch: fetch_contributions(table_entries) if __name__ == "__main__": main()