Commit b96339fb authored by Dimitri Podborski's avatar Dimitri Podborski
Browse files

bugfix in parse meetings

parent 0b82940e
......@@ -140,6 +140,8 @@ def find_meeting(meetings, meeting_number):
'''
Find and return a meeting using the meeting_number. If meeting_number < 0 return the latest meeting.
'''
if len(meetings) == 0:
return None
if meeting_number < 0:
return max(meetings, key=lambda x:x['number'])
for meeting in meetings:
......
......@@ -41,9 +41,12 @@ class MDMSParser:
cols = rows[n].find_all('td', recursive=False)
try:
last_input = None
last_output = None
if len(cols[5].string) > 0:
last_output = int(cols[5].string)
if len(cols[4].text) > 0:
last_input = int(cols[4].text)
if len(cols[5].text) > 0:
last_output = int(cols[5].text)
parsed_href = urlparse(cols[1].a['href'])
meeting_id = int(parse_qs(parsed_href.query)['id_meeting'][0])
......@@ -54,7 +57,7 @@ class MDMSParser:
'name': cols[1].text.strip(),
'start_date': datetime.strptime(cols[2].text, '%Y-%m-%d'),
'end_date': datetime.strptime(cols[3].text, '%Y-%m-%d'),
'last_input': int(cols[4].text),
'last_input': last_input,
'last_output': last_output
})
except:
......@@ -392,6 +395,8 @@ def get_current_meeting():
{ 'number', 'id', 'name', 'start_date', 'end_date', 'last_input', 'last_output' }
'''
meetings = get_meetings()
if len(meetings) == 0:
return None
return max(meetings, key=lambda x:x['number'])
def get_input_documents(meeting_id, standard=Standard.ALL, subgroup=Subgroup.ALL):
......@@ -421,7 +426,7 @@ def get_document_details(document_id):
if not response.status_code == 200:
print('HTTP response {} != 200'.format(response.status_code))
print('\t{}'.format(response.text.replace('\n', '\n\t')))
return []
return None
parser = MDMSParser()
details = parser.parse_document_details(response.text)
for n in range(len(details['documents'])):
......
......@@ -23,7 +23,7 @@ import os
import sys
from automation import gitlab, mdms, helpers
__version__ = '1.0'
__version__ = '1.1'
DATA_PATH = './data'
GITLAB_PROJECTS_PATH = os.path.join(DATA_PATH, 'gitlab_projects.json')
......@@ -42,6 +42,9 @@ def print_infos(docs, project_url_or_path, gitlab_projects, input_docs):
print(' Document not found in the provided meeting. Try updating the database (-u option).')
else:
details = mdms.get_document_details(document['mdms_id'])
if details is None:
print(' Skip', document['document'])
continue
print(' Title:', document['title'])
if details['organizations']:
print(' Organizations:', details['organizations'])
......@@ -84,6 +87,9 @@ def print_infos(docs, project_url_or_path, gitlab_projects, input_docs):
def open_new_issue(project_id, document, test):
issue_title = helpers.create_issue_title(document)
document_details = mdms.get_document_details(document['mdms_id'])
if document_details is None:
print(' Skip', document['document'])
return
issue_description = helpers.create_issue_description(document, document_details)
issue_lables = []
if len(document_details['documents']) > 0:
......@@ -127,6 +133,9 @@ def open_issues(csv_path, test, gitlab_projects, input_docs, gitlab_members, mee
print(' * *** ATTENTION *** We found multiple issues with "{}" in the title. One with and one without metadata tag.'.format(document['document']))
print(' - Issue without metadata tag:', issue_with_title.web_url)
document_details = mdms.get_document_details(document['mdms_id'])
if document_details is None:
print(' * Skip', document['document'])
continue
last_doc_version = 0
if len(document_details['documents']) > 0:
last_doc = max(document_details['documents'], key=lambda x:x['version'])
......@@ -158,6 +167,9 @@ def create_output_doc(csv_path, gitlab_projects, input_docs, template_path):
document = entry['document']
print(' * Process document', document['document'], document['title'])
details = mdms.get_document_details(document['mdms_id'])
if details is None:
print(' * Skip', document['document'])
continue
project = entry['project']
issues = gitlab.get_issues(project['id'])
issue_with_title, issue_with_meta, meta_last_doc_version = helpers.find_issue(issues, document)
......@@ -224,9 +236,9 @@ def parse_csv(csv_file, projects, docs):
project = helpers.find_project(projects, project_url_or_path)
doc = helpers.find_document(docs, row[doc_number_idx])
if not project:
print('WARNING: No such project on GitLab! Could not parse row:', row)
print('WARNING: No project on GitLab for:', row)
elif not doc:
print('WARNING: No such document on MDMS! Could not parse row:', row)
print('WARNING: Document not found:', row)
else:
issues.append({
'project': project,
......@@ -281,6 +293,9 @@ gitlab_members = helpers.load_json_data(GITLAB_USERS_PATH)
if not os.path.isfile(MEETINGS_PATH) or args.U:
print(' * Update MPEG meetings data')
meetings = mdms.get_meetings()
if len(meetings) == 0:
print(' * ERROR, could not get meetings from MDMS. Check your password.')
sys.exit(1)
helpers.store_json_data(MEETINGS_PATH, meetings)
meetings = helpers.load_json_data(MEETINGS_PATH)
# get MPEG meeting we want to work on
......@@ -288,6 +303,9 @@ meeting = helpers.find_meeting(meetings, args.meeting)
if not meeting:
print('Warning: Could not find meeting #{}. Update MDMS database and see if it was added'.format(args.meeting))
meetings = mdms.get_meetings()
if len(meetings) == 0:
print(' * ERROR, could not get meetings from MDMS.')
sys.exit(1)
helpers.store_json_data(MEETINGS_PATH, meetings)
meetings = helpers.load_json_data(MEETINGS_PATH)
meeting = helpers.find_meeting(meetings, args.meeting)
......@@ -300,6 +318,9 @@ input_docs_path = os.path.join(DATA_PATH, 'input_docs_{}.json'.format(meeting['n
if not os.path.isfile(input_docs_path) or args.U or args.u:
print(' * Update MPEG input documents data for MPEG#', meeting['number'])
input_docs = mdms.get_input_documents(meeting['id'])
if len(input_docs) == 0:
print(' * ERROR, could not get input documents from MDMS. Check your password.')
sys.exit(1)
helpers.store_json_data(input_docs_path, input_docs)
input_docs = helpers.load_json_data(input_docs_path)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment