improve update issues for FF

systems.py

@@ -31,7 +31,7 @@ import requests
 
 from automation import gitlab, helpers, mdms
 
-__version__ = '1.4'
+__version__ = '1.5'
 
 DATA_PATH = './data'
 GITLAB_PROJECTS_PATH = os.path.join(DATA_PATH, 'gitlab_projects.json')
@@ -39,18 +39,17 @@ GITLAB_USERS_PATH = os.path.join(DATA_PATH, 'gitlab_users.json')
 MEETINGS_PATH = os.path.join(DATA_PATH, 'meetings.json')
 SYSTEMS_GROUP_ID = 727  # GitLab Group ID for Systems Subgroup
 
-PROJECTS_FF = ['isobmff', 'HEIF', 'NALuFF', 'FFConformanceRefSoft', 'rawvideo', 'Text',
-               'eventmessage', 'General', 'DerivedVis', 'CENC', 'Metrics', 'PartialFF', 'MP4FF', 'Audio']
-
 
 def download_url(url, save_path, chunk_size=128):
-    r = requests.get(url, auth=(mdms.MPEG_LOGIN, mdms.MPEG_PWD), stream=True)
-    with open(save_path, 'wb') as fd:
-        for chunk in r.iter_content(chunk_size=chunk_size):
-            fd.write(chunk)
+    '''Download a file from the given URL'''
+    res = requests.get(url, auth=(mdms.MPEG_LOGIN, mdms.MPEG_PWD), stream=True)
+    with open(save_path, 'wb') as file_descriptor:
+        for chunk in res.iter_content(chunk_size=chunk_size):
+            file_descriptor.write(chunk)
 
 
 def fetch_contributions(table_entries):
+    '''Download all files from the entries'''
     print('\nDownload contributions')
     for entry in table_entries:
         path = os.path.join(DATA_PATH, 'contributions')
@@ -67,13 +66,14 @@ def fetch_contributions(table_entries):
             path = os.path.join(path, project['path_with_namespace'])
         if not os.path.exists(path) and len(path) > 0:
             os.makedirs(path)
-        folder, filename = os.path.split(url)
+        _folder, filename = os.path.split(url)
         filename = os.path.join(path, filename)
         print(document['document'], ' -> ', filename)
         download_url(url, filename)
 
 
 def print_infos(table_entries, project_url, gitlab_projects):
+    '''Print information about contributions and issues'''
     print('\nDump information')
     for entry in table_entries:
         document = entry['document']
@@ -142,6 +142,7 @@ def print_infos(table_entries, project_url, gitlab_projects):
 
 
 def open_new_issue(project_id, document, test, meeting_start, gitlab_members):
+    '''Open a new GitLab issue'''
     usernames = helpers.find_gitlab_users(gitlab_members, document)
     issue_title = helpers.create_issue_title(document)
     document_details = mdms.get_document_details(document['mdms_id'])
@@ -169,25 +170,25 @@ def open_new_issue(project_id, document, test, meeting_start, gitlab_members):
         gitlab.open_issue(project_id, issue_title,
                           issue_description, issue_lables)
         return True
-    else:
-        print(' * {}: Test open issue with title "{}" | Lables={}'.format(document['document'], issue_title,
-                                                                          issue_lables))
+    print(' * {}: Test open issue with title "{}" | Lables={}'.format(
+        document['document'], issue_title, issue_lables))
     return False
 
 
 def close_issue(issue, test, document):
+    '''Close issue'''
     if not test:
         print(
             ' * {}: Close issue: {}'.format(document['document'], issue.web_url))
         gitlab.close_issue(issue)
         return True
-    else:
-        print(
-            ' * {}: Test close issue: {}'.format(document['document'], issue.web_url))
+    print(
+        ' * {}: Test close issue: {}'.format(document['document'], issue.web_url))
     return False
 
 
 def open_issues(table_entries, test, gitlab_members, meeting_start):
+    '''open issues using table entries'''
     print('\nOpen {} issues. TestMode={}'.format(len(table_entries), test))
     counter = 0
     for entry in table_entries:
@@ -428,31 +429,34 @@ def parse_cli(docs, project_url, close_flag, gitlab_projects, input_docs):
 
 def derive_fileformat(gitlab_projects, input_docs):
     """return table_entries based on already opened issues in the FIleFormat group"""
-    proj_urls = []
+
     table_entries = []
-    for proj in PROJECTS_FF:
-        proj_urls.append(os.path.join(
-            'http://mpegx.int-evry.fr/software/MPEG/Systems/FileFormat', proj))
+    ff_projects = [
+        p for p in gitlab_projects if "MPEG/Systems/FileFormat" in p["path_with_namespace"]]
 
-    for project_url in proj_urls:
-        print(f'gather contributions from {project_url}')
-        project = helpers.find_project(gitlab_projects, project_url)
-        if project is not None:
-            issues = gitlab.get_issues(project['id'])
-            for issue in issues:
-                meta = helpers.get_issue_metadata(issue.description)
-                if meta is not None:
-                    document = helpers.find_document(
-                        input_docs, meta['document'])
-                    if not document:
-                        print(
-                            f'WARNING: Document "{meta["document"]}" not found. Try updating the database (-U) or select another meeting (--meeting).')
+    for project in ff_projects:
+        print(f'gather contributions from {project["path_with_namespace"]}')
+        issues = gitlab.get_issues(project['id'])
+        for issue in issues:
+            meta = helpers.get_issue_metadata(issue.description)
+            if meta is not None:
+                document = helpers.find_document(input_docs, meta['document'])
+                if not document:
+                    # document is probably from the previous meeting, search for it
+                    doc_nr = meta['document'].replace('m', '')
+                    print(f'Search for m{doc_nr}')
+                    search_result = mdms.find_documents(number=doc_nr, category=mdms.SearchCategory.INPUT)
+                    if len(search_result) == 0:
+                        print(f'WARNING: Document m{doc_nr} not found.')
                         continue
-                    table_entries.append({
-                        'project': project,
-                        'document': document,
-                        'close': False
-                    })
+                    document = search_result[0]
+
+                table_entries.append({
+                    'project': project,
+                    'document': document,
+                    'close': False
+                })
+    print(f'Got {len(table_entries)} issues.')
     return table_entries