Commit a3b07438 authored by Dimitri Podborski's avatar Dimitri Podborski
Browse files

improve update issues for FF

parent 689d85ef
...@@ -31,7 +31,7 @@ import requests ...@@ -31,7 +31,7 @@ import requests
from automation import gitlab, helpers, mdms from automation import gitlab, helpers, mdms
__version__ = '1.4' __version__ = '1.5'
DATA_PATH = './data' DATA_PATH = './data'
GITLAB_PROJECTS_PATH = os.path.join(DATA_PATH, 'gitlab_projects.json') GITLAB_PROJECTS_PATH = os.path.join(DATA_PATH, 'gitlab_projects.json')
...@@ -39,18 +39,17 @@ GITLAB_USERS_PATH = os.path.join(DATA_PATH, 'gitlab_users.json') ...@@ -39,18 +39,17 @@ GITLAB_USERS_PATH = os.path.join(DATA_PATH, 'gitlab_users.json')
MEETINGS_PATH = os.path.join(DATA_PATH, 'meetings.json') MEETINGS_PATH = os.path.join(DATA_PATH, 'meetings.json')
SYSTEMS_GROUP_ID = 727 # GitLab Group ID for Systems Subgroup SYSTEMS_GROUP_ID = 727 # GitLab Group ID for Systems Subgroup
PROJECTS_FF = ['isobmff', 'HEIF', 'NALuFF', 'FFConformanceRefSoft', 'rawvideo', 'Text',
'eventmessage', 'General', 'DerivedVis', 'CENC', 'Metrics', 'PartialFF', 'MP4FF', 'Audio']
def download_url(url, save_path, chunk_size=128): def download_url(url, save_path, chunk_size=128):
r = requests.get(url, auth=(mdms.MPEG_LOGIN, mdms.MPEG_PWD), stream=True) '''Download a file from the given URL'''
with open(save_path, 'wb') as fd: res = requests.get(url, auth=(mdms.MPEG_LOGIN, mdms.MPEG_PWD), stream=True)
for chunk in r.iter_content(chunk_size=chunk_size): with open(save_path, 'wb') as file_descriptor:
fd.write(chunk) for chunk in res.iter_content(chunk_size=chunk_size):
file_descriptor.write(chunk)
def fetch_contributions(table_entries): def fetch_contributions(table_entries):
'''Download all files from the entries'''
print('\nDownload contributions') print('\nDownload contributions')
for entry in table_entries: for entry in table_entries:
path = os.path.join(DATA_PATH, 'contributions') path = os.path.join(DATA_PATH, 'contributions')
...@@ -67,13 +66,14 @@ def fetch_contributions(table_entries): ...@@ -67,13 +66,14 @@ def fetch_contributions(table_entries):
path = os.path.join(path, project['path_with_namespace']) path = os.path.join(path, project['path_with_namespace'])
if not os.path.exists(path) and len(path) > 0: if not os.path.exists(path) and len(path) > 0:
os.makedirs(path) os.makedirs(path)
folder, filename = os.path.split(url) _folder, filename = os.path.split(url)
filename = os.path.join(path, filename) filename = os.path.join(path, filename)
print(document['document'], ' -> ', filename) print(document['document'], ' -> ', filename)
download_url(url, filename) download_url(url, filename)
def print_infos(table_entries, project_url, gitlab_projects): def print_infos(table_entries, project_url, gitlab_projects):
'''Print information about contributions and issues'''
print('\nDump information') print('\nDump information')
for entry in table_entries: for entry in table_entries:
document = entry['document'] document = entry['document']
...@@ -142,6 +142,7 @@ def print_infos(table_entries, project_url, gitlab_projects): ...@@ -142,6 +142,7 @@ def print_infos(table_entries, project_url, gitlab_projects):
def open_new_issue(project_id, document, test, meeting_start, gitlab_members): def open_new_issue(project_id, document, test, meeting_start, gitlab_members):
'''Open a new GitLab issue'''
usernames = helpers.find_gitlab_users(gitlab_members, document) usernames = helpers.find_gitlab_users(gitlab_members, document)
issue_title = helpers.create_issue_title(document) issue_title = helpers.create_issue_title(document)
document_details = mdms.get_document_details(document['mdms_id']) document_details = mdms.get_document_details(document['mdms_id'])
...@@ -169,25 +170,25 @@ def open_new_issue(project_id, document, test, meeting_start, gitlab_members): ...@@ -169,25 +170,25 @@ def open_new_issue(project_id, document, test, meeting_start, gitlab_members):
gitlab.open_issue(project_id, issue_title, gitlab.open_issue(project_id, issue_title,
issue_description, issue_lables) issue_description, issue_lables)
return True return True
else: print(' * {}: Test open issue with title "{}" | Lables={}'.format(
print(' * {}: Test open issue with title "{}" | Lables={}'.format(document['document'], issue_title, document['document'], issue_title, issue_lables))
issue_lables))
return False return False
def close_issue(issue, test, document): def close_issue(issue, test, document):
'''Close issue'''
if not test: if not test:
print( print(
' * {}: Close issue: {}'.format(document['document'], issue.web_url)) ' * {}: Close issue: {}'.format(document['document'], issue.web_url))
gitlab.close_issue(issue) gitlab.close_issue(issue)
return True return True
else: print(
print( ' * {}: Test close issue: {}'.format(document['document'], issue.web_url))
' * {}: Test close issue: {}'.format(document['document'], issue.web_url))
return False return False
def open_issues(table_entries, test, gitlab_members, meeting_start): def open_issues(table_entries, test, gitlab_members, meeting_start):
'''open issues using table entries'''
print('\nOpen {} issues. TestMode={}'.format(len(table_entries), test)) print('\nOpen {} issues. TestMode={}'.format(len(table_entries), test))
counter = 0 counter = 0
for entry in table_entries: for entry in table_entries:
...@@ -428,31 +429,34 @@ def parse_cli(docs, project_url, close_flag, gitlab_projects, input_docs): ...@@ -428,31 +429,34 @@ def parse_cli(docs, project_url, close_flag, gitlab_projects, input_docs):
def derive_fileformat(gitlab_projects, input_docs): def derive_fileformat(gitlab_projects, input_docs):
"""return table_entries based on already opened issues in the FIleFormat group""" """return table_entries based on already opened issues in the FIleFormat group"""
proj_urls = []
table_entries = [] table_entries = []
for proj in PROJECTS_FF: ff_projects = [
proj_urls.append(os.path.join( p for p in gitlab_projects if "MPEG/Systems/FileFormat" in p["path_with_namespace"]]
'http://mpegx.int-evry.fr/software/MPEG/Systems/FileFormat', proj))
for project_url in proj_urls: for project in ff_projects:
print(f'gather contributions from {project_url}') print(f'gather contributions from {project["path_with_namespace"]}')
project = helpers.find_project(gitlab_projects, project_url) issues = gitlab.get_issues(project['id'])
if project is not None: for issue in issues:
issues = gitlab.get_issues(project['id']) meta = helpers.get_issue_metadata(issue.description)
for issue in issues: if meta is not None:
meta = helpers.get_issue_metadata(issue.description) document = helpers.find_document(input_docs, meta['document'])
if meta is not None: if not document:
document = helpers.find_document( # document is probably from the previous meeting, search for it
input_docs, meta['document']) doc_nr = meta['document'].replace('m', '')
if not document: print(f'Search for m{doc_nr}')
print( search_result = mdms.find_documents(number=doc_nr, category=mdms.SearchCategory.INPUT)
f'WARNING: Document "{meta["document"]}" not found. Try updating the database (-U) or select another meeting (--meeting).') if len(search_result) == 0:
print(f'WARNING: Document m{doc_nr} not found.')
continue continue
table_entries.append({ document = search_result[0]
'project': project,
'document': document, table_entries.append({
'close': False 'project': project,
}) 'document': document,
'close': False
})
print(f'Got {len(table_entries)} issues.')
return table_entries return table_entries
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment