add --fileformat option

This option can be used to work with already opened issues on GitLab. No
need to provide a csv or cli document numbers.

systems.py

@@ -25,9 +25,11 @@ import argparse
 import csv
 import os
 import sys
-import requests
 from datetime import datetime
-from automation import gitlab, mdms, helpers
+
+import requests
+
+from automation import gitlab, helpers, mdms
 
 __version__ = '1.4'
 
@@ -37,6 +39,9 @@ GITLAB_USERS_PATH = os.path.join(DATA_PATH, 'gitlab_users.json')
 MEETINGS_PATH = os.path.join(DATA_PATH, 'meetings.json')
 SYSTEMS_GROUP_ID = 727  # GitLab Group ID for Systems Subgroup
 
+PROJECTS_FF = ['isobmff', 'HEIF', 'NALuFF', 'FFConformanceRefSoft', 'rawvideo', 'Text',
+               'eventmessage', 'General', 'DerivedVis', 'CENC', 'Metrics', 'PartialFF', 'MP4FF', 'Audio']
+
 
 def download_url(url, save_path, chunk_size=128):
     r = requests.get(url, auth=(mdms.MPEG_LOGIN, mdms.MPEG_PWD), stream=True)
@@ -68,7 +73,7 @@ def fetch_contributions(table_entries):
         download_url(url, filename)
 
 
-def print_infos(table_entries):
+def print_infos(table_entries, project_url, gitlab_projects):
     print('\nDump information')
     for entry in table_entries:
         document = entry['document']
@@ -94,13 +99,15 @@ def print_infos(table_entries):
         print('  Authors:', authors)
         if len(details['documents']) > 0:
             last_doc = max(details['documents'], key=lambda x: x['version'])
-            print('  Last version: version', last_doc['version'], 'from', last_doc['timestamp'])
+            print('  Last version: version',
+                  last_doc['version'], 'from', last_doc['timestamp'])
             print('  URL:', last_doc['path'])
 
         if project is not None:
             print('GitLab metadata')
             issues = gitlab.get_issues(project['id'])
-            issue_with_title, issue_with_meta, meta_last_doc_version = helpers.find_issue(issues, document)
+            issue_with_title, issue_with_meta, meta_last_doc_version = helpers.find_issue(
+                issues, document)
             if issue_with_title is not None:
                 print('  User opened an issue:', issue_with_title.web_url)
                 print('  Labels:', issue_with_title.labels)
@@ -108,16 +115,30 @@ def print_infos(table_entries):
                 print('  Comments:', len(comments))
                 if len(comments) > 0:
                     last_comment = max(comments, key=lambda x: x.id)
-                    print('  Last comment from', last_comment.author['username'], 'at', last_comment.updated_at)
+                    print('  Last comment from',
+                          last_comment.author['username'], 'at', last_comment.updated_at)
             if issue_with_meta is not None:
-                print('  Automatically generated issue:', issue_with_meta.web_url)
-                print('  Last document version found in the meta tag:', meta_last_doc_version)
+                print('  Automatically generated issue:',
+                      issue_with_meta.web_url)
+                print('  Last document version found in the meta tag:',
+                      meta_last_doc_version)
                 print('  Labels:', issue_with_meta.labels)
                 comments = issue_with_meta.notes.list(lazy=True)
                 print('  Comments:', len(comments))
                 if len(comments) > 0:
                     last_comment = max(comments, key=lambda x: x.id)
-                    print('  Last comment from', last_comment.author['username'], 'at', last_comment.updated_at)
+                    print('  Last comment from',
+                          last_comment.author['username'], 'at', last_comment.updated_at)
+    project = helpers.find_project(gitlab_projects, project_url)
+    if project is not None:
+        print('Auto-generated contribution list')
+        issues = gitlab.get_issues(project['id'])
+        contribution_list = []
+        for issue in issues:
+            meta = helpers.get_issue_metadata(issue.description)
+            if meta is not None:
+                contribution_list.append(meta['document'])
+        print(','.join(contribution_list))
 
 
 def open_new_issue(project_id, document, test, meeting_start, gitlab_members):
@@ -127,21 +148,26 @@ def open_new_issue(project_id, document, test, meeting_start, gitlab_members):
     if document_details is None:
         print(' No document details found. Skip', document['document'])
         return False
-    issue_description = helpers.create_issue_description(document, document_details)
+    issue_description = helpers.create_issue_description(
+        document, document_details)
     if len(usernames) > 0:
-        issue_description += '\n_for:_ ' + ''.join('@' + str(u) + ', ' for u in usernames)
+        issue_description += '\n_for:_ ' + \
+            ''.join('@' + str(u) + ', ' for u in usernames)
     issue_lables = []
     timestamp = datetime.now()
     if len(document_details['documents']) > 0:
         issue_lables.append(gitlab.Label.DocAvailable.value)
-        first_doc = min(document_details['documents'], key=lambda x: x['version'])
+        first_doc = min(
+            document_details['documents'], key=lambda x: x['version'])
         timestamp = first_doc['timestamp']
     if helpers.is_document_late(meeting_start, timestamp):
         issue_lables.append(gitlab.Label.Late.value)
 
     if not test:
-        print(' * {}: Open issue with title "{}" | Lables={}'.format(document['document'], issue_title, issue_lables))
-        gitlab.open_issue(project_id, issue_title, issue_description, issue_lables)
+        print(' * {}: Open issue with title "{}" | Lables={}'.format(
+            document['document'], issue_title, issue_lables))
+        gitlab.open_issue(project_id, issue_title,
+                          issue_description, issue_lables)
         return True
     else:
         print(' * {}: Test open issue with title "{}" | Lables={}'.format(document['document'], issue_title,
@@ -151,11 +177,13 @@ def open_new_issue(project_id, document, test, meeting_start, gitlab_members):
 
 def close_issue(issue, test, document):
     if not test:
-        print(' * {}: Close issue: {}'.format(document['document'], issue.web_url))
+        print(
+            ' * {}: Close issue: {}'.format(document['document'], issue.web_url))
         gitlab.close_issue(issue)
         return True
     else:
-        print(' * {}: Test close issue: {}'.format(document['document'], issue.web_url))
+        print(
+            ' * {}: Test close issue: {}'.format(document['document'], issue.web_url))
     return False
 
 
@@ -167,10 +195,12 @@ def open_issues(table_entries, test, gitlab_members, meeting_start):
         project_id = entry['project']['id']
 
         issues = gitlab.get_issues(project_id)
-        issue_with_title, issue_with_meta, meta_last_doc_version = helpers.find_issue(issues, document)
+        issue_with_title, issue_with_meta, meta_last_doc_version = helpers.find_issue(
+            issues, document)
 
         if issue_with_title is None and issue_with_meta is None:
-            was_opened = open_new_issue(project_id, document, test, meeting_start, gitlab_members)
+            was_opened = open_new_issue(
+                project_id, document, test, meeting_start, gitlab_members)
             if was_opened:
                 counter += 1
         elif issue_with_title is not None and issue_with_meta is None:
@@ -182,30 +212,36 @@ def open_issues(table_entries, test, gitlab_members, meeting_start):
             print('   - Should we still open a new one?')
             user_input = input('   Type y or n: ')
             if 'y' in user_input:
-                was_opened = open_new_issue(project_id, document, test, meeting_start, gitlab_members)
+                was_opened = open_new_issue(
+                    project_id, document, test, meeting_start, gitlab_members)
                 if was_opened:
                     counter += 1
             else:
-                print(' * {} Skip "{}"'.format(document['document'], document['title']))
+                print(
+                    ' * {} Skip "{}"'.format(document['document'], document['title']))
         else:
             if issue_with_title is not None:
                 print('* {0}: ATTENTION We found multiple issues with "{0}" in the title. One with, and one without '
                       'metadata tag.'.format(document['document']))
-                print('   - Issue without metadata tag:', issue_with_title.web_url)
+                print('   - Issue without metadata tag:',
+                      issue_with_title.web_url)
             document_details = mdms.get_document_details(document['mdms_id'])
             if document_details is None:
-                print(' * {}: Skip. Could not get document details from MDMS.'.format(document['document']))
+                print(
+                    ' * {}: Skip. Could not get document details from MDMS.'.format(document['document']))
                 continue
             last_doc_version = 0
             if len(document_details['documents']) > 0:
-                last_doc = max(document_details['documents'], key=lambda x: x['version'])
+                last_doc = max(
+                    document_details['documents'], key=lambda x: x['version'])
                 last_doc_version = last_doc['version']
             if last_doc_version > meta_last_doc_version:
                 print(' * {}: A new version of the document was added to MDMS '
                       'after the issue was opened on GitLab.'.format(document['document']))
                 print('   - Issue URL:', issue_with_meta.web_url)
                 print('   - Issue Title:', issue_with_meta.title)
-                print('   - Should we update the metadata table and add the new version to it?')
+                print(
+                    '   - Should we update the metadata table and add the new version to it?')
                 user_input = input('   Type y or n: ')
                 if 'y' in user_input:
                     new_description = helpers.get_updated_issue_description(issue_with_meta.description, document,
@@ -213,15 +249,18 @@ def open_issues(table_entries, test, gitlab_members, meeting_start):
                     if 'DocAvailable' not in issue_with_meta.labels:
                         issue_with_meta.labels.append('DocAvailable')
                     if not test:
-                        print('   - Update issue description of {}.'.format(document['document']))
+                        print(
+                            '   - Update issue description of {}.'.format(document['document']))
                         issue_with_meta.description = new_description
                         issue_with_meta.save()
                     else:
-                        print('   - Test update issue description of {}.'.format(document['document']))
+                        print(
+                            '   - Test update issue description of {}.'.format(document['document']))
                 else:
                     print('   - Skip "{}"'.format(document['title']))
             else:
-                print(' * {}: No update required for "{}"'.format(document['document'], document['title']))
+                print(
+                    ' * {}: No update required for "{}"'.format(document['document'], document['title']))
     print('Opened issues:', counter)
 
 
@@ -239,9 +278,11 @@ def close_issues(table_entries, test, force):
             continue
 
         issues = gitlab.get_issues(project_id)
-        issue_with_title, issue_with_meta, _meta_last_doc_version = helpers.find_issue(issues, document)
+        issue_with_title, issue_with_meta, _meta_last_doc_version = helpers.find_issue(
+            issues, document)
         if issue_with_meta is None and issue_with_title is None:
-            print(' * {}: No issue to close in: {}'.format(document['document'], entry['project']['url']))
+            print(
+                ' * {}: No issue to close in: {}'.format(document['document'], entry['project']['url']))
         if issue_with_meta is not None:
             was_closed = close_issue(issue_with_meta, test, document)
             if was_closed:
@@ -276,7 +317,8 @@ def create_output_doc(table_entries, output_path, template_path):
             continue
         project = entry['project']
         issues = gitlab.get_issues(project['id'])
-        issue_with_title, issue_with_meta, _meta_last_doc_version = helpers.find_issue(issues, document)
+        issue_with_title, issue_with_meta, _meta_last_doc_version = helpers.find_issue(
+            issues, document)
         if not project['id'] in projects:
             projects[project['id']] = project
         if project['id'] in projects_data:
@@ -367,7 +409,8 @@ def parse_cli(docs, project_url, close_flag, gitlab_projects, input_docs):
     docs = docs.replace('m', '').replace('M', '').strip().split(',')
     project = helpers.find_project(gitlab_projects, project_url)
     if not project:
-        print('   - Could not find a GitLab project with project URL "{}"'.format(project_url))
+        print(
+            '   - Could not find a GitLab project with project URL "{}"'.format(project_url))
         return table_entries
     for doc in docs:
         document = helpers.find_document(input_docs, 'm' + doc)
@@ -383,6 +426,36 @@ def parse_cli(docs, project_url, close_flag, gitlab_projects, input_docs):
     return table_entries
 
 
+def derive_fileformat(gitlab_projects, input_docs):
+    """return table_entries based on already opened issues in the FIleFormat group"""
+    proj_urls = []
+    table_entries = []
+    for proj in PROJECTS_FF:
+        proj_urls.append(os.path.join(
+            'http://mpegx.int-evry.fr/software/MPEG/Systems/FileFormat', proj))
+
+    for project_url in proj_urls:
+        print(f'gather contributions from {project_url}')
+        project = helpers.find_project(gitlab_projects, project_url)
+        if project is not None:
+            issues = gitlab.get_issues(project['id'])
+            for issue in issues:
+                meta = helpers.get_issue_metadata(issue.description)
+                if meta is not None:
+                    document = helpers.find_document(
+                        input_docs, meta['document'])
+                    if not document:
+                        print(
+                            f'WARNING: Document "{meta["document"]}" not found. Try updating the database (-U) or select another meeting (--meeting).')
+                        continue
+                    table_entries.append({
+                        'project': project,
+                        'document': document,
+                        'close': False
+                    })
+    return table_entries
+
+
 def main():
     print('*' * 35)
     print('* MPEG Systems script version', __version__, '*')
@@ -397,22 +470,34 @@ def main():
     parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter,
                                      description='A tool for managing the GitLab issues for MPEG Systems Subgroups.',
                                      epilog=usage_examples)
-    parser.add_argument('-o', '--open', help='Mode: Open GitLab issues.', action='store_true')
-    parser.add_argument('-c', '--close', help='Mode: Close GitLab issues.', action='store_true')
-    parser.add_argument('-d', '--docx', help='Mode: Generate output word document.', action='store_true')
-    parser.add_argument('-l', '--list', help='Mode: List information about the contribution(s).', action='store_true')
-    parser.add_argument('-f', '--fetch', help='Mode: Download contributions.', action='store_true')
-    parser.add_argument('-C', '--CLOSE', help='Force closing GitLab issues.', action='store_true')
-    parser.add_argument('-U', '--UPDATE', help='Update all databases.', action='store_true')
-    parser.add_argument('-i', '--csv', help='Input CSV file. Header row shall include "Number" and "Project URL"')
-    parser.add_argument('-m', '--documents', help='Comma separated MDMS document number(s). e.g.: m12345,...', type=str)
-    parser.add_argument('-p', '--project', help='GitLab project URL or "SubGroup/ProjectName".', type=str)
+    parser.add_argument(
+        '-o', '--open', help='Mode: Open GitLab issues.', action='store_true')
+    parser.add_argument(
+        '-c', '--close', help='Mode: Close GitLab issues.', action='store_true')
+    parser.add_argument(
+        '-d', '--docx', help='Mode: Generate output word document.', action='store_true')
+    parser.add_argument(
+        '-l', '--list', help='Mode: List information about the contribution(s).', action='store_true')
+    parser.add_argument(
+        '-f', '--fetch', help='Mode: Download contributions.', action='store_true')
+    parser.add_argument(
+        '-C', '--CLOSE', help='Force closing GitLab issues.', action='store_true')
+    parser.add_argument(
+        '-U', '--UPDATE', help='Update all databases.', action='store_true')
+    parser.add_argument(
+        '-i', '--csv', help='Input CSV file. Header row shall include "Number" and "Project URL"')
+    parser.add_argument(
+        '-m', '--documents', help='Comma separated MDMS document number(s). e.g.: m12345,...', type=str)
+    parser.add_argument(
+        '--fileformat', help='Derive input from already opened issues in FileFormat group', action='store_true')
+    parser.add_argument(
+        '-p', '--project', help='GitLab project URL or "SubGroup/ProjectName".', type=str)
     parser.add_argument('--meeting', help='MPEG meeting number. If not set, the latest meeting is used.', default=-1,
                         type=int)
     parser.add_argument('-t', '--template', help='Document template path if you want to use your .docx template for '
                                                  'output document.', type=str)
-    parser.add_argument('--test', help='Test mode. If set, no issues will be opened or closed.', action='store_true')
-    parser.add_argument('-n', '--notify', help='Notify (@mention) authors in opened issues.', action='store_true')
+    parser.add_argument(
+        '--test', help='Test mode. If set, no issues will be opened or closed.', action='store_true')
     args = parser.parse_args()
 
     if not args.open and not args.docx and not args.close and not args.list and not args.fetch:
@@ -442,7 +527,8 @@ def main():
     # get MPEG meeting we want to work on
     meeting = helpers.find_meeting(meetings, args.meeting)
     if not meeting:
-        print('Warning: Could not find meeting #{}. Update MDMS database and see if it was added'.format(args.meeting))
+        print('Warning: Could not find meeting #{}. Update MDMS database and see if it was added'.format(
+            args.meeting))
         meetings = mdms.get_meetings()
         if len(meetings) == 0:
             print(' * ERROR, could not get meetings from MDMS.')
@@ -456,7 +542,8 @@ def main():
     print(' * Operating on MPEG#{} ({}) from {} to {}'.format(meeting['number'], meeting['name'], meeting['start_date'],
                                                               meeting['end_date']))
     # get input documents (and update if needed)
-    input_docs_path = os.path.join(DATA_PATH, 'input_docs_{}.json'.format(meeting['number']))
+    input_docs_path = os.path.join(
+        DATA_PATH, 'input_docs_{}.json'.format(meeting['number']))
     print(' * Update MPEG input documents data for MPEG#', meeting['number'])
     input_docs = mdms.get_input_documents(meeting['id'])
     if len(input_docs) == 0:
@@ -470,15 +557,16 @@ def main():
         table_entries = parse_csv(args.csv, gitlab_projects, input_docs)
         output_path = args.csv.replace('.csv', '') + '.docx'
     elif args.documents is not None:
-        table_entries = parse_cli(args.documents, args.project, args.close, gitlab_projects, input_docs)
+        table_entries = parse_cli(
+            args.documents, args.project, args.close, gitlab_projects, input_docs)
+    elif args.fileformat:
+        table_entries = derive_fileformat(gitlab_projects, input_docs)
 
-    # do some action
+    # do some work
     if args.list:
-        print_infos(table_entries)
+        print_infos(table_entries, args.project, gitlab_projects)
     if args.open:
         meeting_start = helpers.try_parsing_date(meeting['start_date'])
-        if not args.notify:
-            gitlab_members = None
         open_issues(table_entries, args.test, gitlab_members, meeting_start)
     if args.close:
         close_issues(table_entries, args.test, args.CLOSE)