Commit a3b07438 authored by Dimitri Podborski's avatar Dimitri Podborski
Browse files

improve update issues for FF

parent 689d85ef
......@@ -31,7 +31,7 @@ import requests
from automation import gitlab, helpers, mdms
__version__ = '1.4'
__version__ = '1.5'
DATA_PATH = './data'
GITLAB_PROJECTS_PATH = os.path.join(DATA_PATH, 'gitlab_projects.json')
......@@ -39,18 +39,17 @@ GITLAB_USERS_PATH = os.path.join(DATA_PATH, 'gitlab_users.json')
MEETINGS_PATH = os.path.join(DATA_PATH, 'meetings.json')
SYSTEMS_GROUP_ID = 727 # GitLab Group ID for Systems Subgroup
PROJECTS_FF = ['isobmff', 'HEIF', 'NALuFF', 'FFConformanceRefSoft', 'rawvideo', 'Text',
'eventmessage', 'General', 'DerivedVis', 'CENC', 'Metrics', 'PartialFF', 'MP4FF', 'Audio']
def download_url(url, save_path, chunk_size=128):
r = requests.get(url, auth=(mdms.MPEG_LOGIN, mdms.MPEG_PWD), stream=True)
with open(save_path, 'wb') as fd:
for chunk in r.iter_content(chunk_size=chunk_size):
fd.write(chunk)
'''Download a file from the given URL'''
res = requests.get(url, auth=(mdms.MPEG_LOGIN, mdms.MPEG_PWD), stream=True)
with open(save_path, 'wb') as file_descriptor:
for chunk in res.iter_content(chunk_size=chunk_size):
file_descriptor.write(chunk)
def fetch_contributions(table_entries):
'''Download all files from the entries'''
print('\nDownload contributions')
for entry in table_entries:
path = os.path.join(DATA_PATH, 'contributions')
......@@ -67,13 +66,14 @@ def fetch_contributions(table_entries):
path = os.path.join(path, project['path_with_namespace'])
if not os.path.exists(path) and len(path) > 0:
os.makedirs(path)
folder, filename = os.path.split(url)
_folder, filename = os.path.split(url)
filename = os.path.join(path, filename)
print(document['document'], ' -> ', filename)
download_url(url, filename)
def print_infos(table_entries, project_url, gitlab_projects):
'''Print information about contributions and issues'''
print('\nDump information')
for entry in table_entries:
document = entry['document']
......@@ -142,6 +142,7 @@ def print_infos(table_entries, project_url, gitlab_projects):
def open_new_issue(project_id, document, test, meeting_start, gitlab_members):
'''Open a new GitLab issue'''
usernames = helpers.find_gitlab_users(gitlab_members, document)
issue_title = helpers.create_issue_title(document)
document_details = mdms.get_document_details(document['mdms_id'])
......@@ -169,25 +170,25 @@ def open_new_issue(project_id, document, test, meeting_start, gitlab_members):
gitlab.open_issue(project_id, issue_title,
issue_description, issue_lables)
return True
else:
print(' * {}: Test open issue with title "{}" | Lables={}'.format(document['document'], issue_title,
issue_lables))
print(' * {}: Test open issue with title "{}" | Lables={}'.format(
document['document'], issue_title, issue_lables))
return False
def close_issue(issue, test, document):
'''Close issue'''
if not test:
print(
' * {}: Close issue: {}'.format(document['document'], issue.web_url))
gitlab.close_issue(issue)
return True
else:
print(
' * {}: Test close issue: {}'.format(document['document'], issue.web_url))
print(
' * {}: Test close issue: {}'.format(document['document'], issue.web_url))
return False
def open_issues(table_entries, test, gitlab_members, meeting_start):
'''open issues using table entries'''
print('\nOpen {} issues. TestMode={}'.format(len(table_entries), test))
counter = 0
for entry in table_entries:
......@@ -428,31 +429,34 @@ def parse_cli(docs, project_url, close_flag, gitlab_projects, input_docs):
def derive_fileformat(gitlab_projects, input_docs):
"""return table_entries based on already opened issues in the FIleFormat group"""
proj_urls = []
table_entries = []
for proj in PROJECTS_FF:
proj_urls.append(os.path.join(
'http://mpegx.int-evry.fr/software/MPEG/Systems/FileFormat', proj))
ff_projects = [
p for p in gitlab_projects if "MPEG/Systems/FileFormat" in p["path_with_namespace"]]
for project_url in proj_urls:
print(f'gather contributions from {project_url}')
project = helpers.find_project(gitlab_projects, project_url)
if project is not None:
issues = gitlab.get_issues(project['id'])
for issue in issues:
meta = helpers.get_issue_metadata(issue.description)
if meta is not None:
document = helpers.find_document(
input_docs, meta['document'])
if not document:
print(
f'WARNING: Document "{meta["document"]}" not found. Try updating the database (-U) or select another meeting (--meeting).')
for project in ff_projects:
print(f'gather contributions from {project["path_with_namespace"]}')
issues = gitlab.get_issues(project['id'])
for issue in issues:
meta = helpers.get_issue_metadata(issue.description)
if meta is not None:
document = helpers.find_document(input_docs, meta['document'])
if not document:
# document is probably from the previous meeting, search for it
doc_nr = meta['document'].replace('m', '')
print(f'Search for m{doc_nr}')
search_result = mdms.find_documents(number=doc_nr, category=mdms.SearchCategory.INPUT)
if len(search_result) == 0:
print(f'WARNING: Document m{doc_nr} not found.')
continue
table_entries.append({
'project': project,
'document': document,
'close': False
})
document = search_result[0]
table_entries.append({
'project': project,
'document': document,
'close': False
})
print(f'Got {len(table_entries)} issues.')
return table_entries
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment