bugfix in parse meetings

automation/helpers.py

@@ -140,6 +140,8 @@ def find_meeting(meetings, meeting_number):
   '''
   Find and return a meeting using the meeting_number. If meeting_number < 0 return the latest meeting.
   '''
+  if len(meetings) == 0:
+    return None
   if meeting_number < 0:
     return max(meetings, key=lambda x:x['number'])
   for meeting in meetings:

automation/mdms.py

@@ -41,9 +41,12 @@ class MDMSParser:
       cols = rows[n].find_all('td', recursive=False)
       
       try:
+        last_input = None
         last_output = None
-        if len(cols[5].string) > 0:
-          last_output = int(cols[5].string)
+        if len(cols[4].text) > 0:
+          last_input = int(cols[4].text)
+        if len(cols[5].text) > 0:
+          last_output = int(cols[5].text)
 
         parsed_href = urlparse(cols[1].a['href'])
         meeting_id = int(parse_qs(parsed_href.query)['id_meeting'][0])
@@ -54,7 +57,7 @@ class MDMSParser:
           'name': cols[1].text.strip(),
           'start_date': datetime.strptime(cols[2].text, '%Y-%m-%d'),
           'end_date': datetime.strptime(cols[3].text, '%Y-%m-%d'),
-          'last_input': int(cols[4].text),
+          'last_input': last_input,
           'last_output': last_output
         })
       except:
@@ -392,6 +395,8 @@ def get_current_meeting():
   { 'number', 'id', 'name', 'start_date', 'end_date', 'last_input', 'last_output' }
   '''
   meetings = get_meetings()
+  if len(meetings) == 0:
+    return None
   return max(meetings, key=lambda x:x['number'])
 
 def get_input_documents(meeting_id, standard=Standard.ALL, subgroup=Subgroup.ALL):
@@ -421,7 +426,7 @@ def get_document_details(document_id):
   if not response.status_code == 200:
       print('HTTP response {} != 200'.format(response.status_code))
       print('\t{}'.format(response.text.replace('\n', '\n\t')))
-      return []
+      return None
   parser = MDMSParser()
   details = parser.parse_document_details(response.text)
   for n in range(len(details['documents'])):

systems.py

@@ -23,7 +23,7 @@ import os
 import sys
 from automation import gitlab, mdms, helpers
 
-__version__ = '1.0'
+__version__ = '1.1'
 
 DATA_PATH = './data'
 GITLAB_PROJECTS_PATH = os.path.join(DATA_PATH, 'gitlab_projects.json')
@@ -42,6 +42,9 @@ def print_infos(docs, project_url_or_path, gitlab_projects, input_docs):
       print('  Document not found in the provided meeting. Try updating the database (-u option).')
     else:
       details = mdms.get_document_details(document['mdms_id'])
+      if details is None:
+        print('  Skip', document['document'])
+        continue
       print('  Title:', document['title'])
       if details['organizations']:
         print('  Organizations:', details['organizations'])
@@ -84,6 +87,9 @@ def print_infos(docs, project_url_or_path, gitlab_projects, input_docs):
 def open_new_issue(project_id, document, test):
   issue_title = helpers.create_issue_title(document)
   document_details = mdms.get_document_details(document['mdms_id'])
+  if document_details is None:
+    print('  Skip', document['document'])
+    return
   issue_description = helpers.create_issue_description(document, document_details)
   issue_lables = []
   if len(document_details['documents']) > 0:
@@ -127,6 +133,9 @@ def open_issues(csv_path, test, gitlab_projects, input_docs, gitlab_members, mee
         print(' * *** ATTENTION *** We found multiple issues with "{}" in the title. One with and one without metadata tag.'.format(document['document']))
         print('   - Issue without metadata tag:', issue_with_title.web_url)
       document_details = mdms.get_document_details(document['mdms_id'])
+      if document_details is None:
+        print(' * Skip', document['document'])
+        continue
       last_doc_version = 0
       if len(document_details['documents']) > 0:
         last_doc = max(document_details['documents'], key=lambda x:x['version'])
@@ -158,6 +167,9 @@ def create_output_doc(csv_path, gitlab_projects, input_docs, template_path):
     document = entry['document']
     print(' * Process document', document['document'], document['title'])
     details = mdms.get_document_details(document['mdms_id'])
+    if details is None:
+      print(' * Skip', document['document'])
+      continue
     project = entry['project']
     issues = gitlab.get_issues(project['id'])
     issue_with_title, issue_with_meta, meta_last_doc_version = helpers.find_issue(issues, document)
@@ -224,9 +236,9 @@ def parse_csv(csv_file, projects, docs):
       project = helpers.find_project(projects, project_url_or_path)
       doc = helpers.find_document(docs, row[doc_number_idx])
       if not project:
-        print('WARNING: No such project on GitLab! Could not parse row:', row)
+        print('WARNING: No project on GitLab for:', row)
       elif not doc:
-        print('WARNING: No such document on MDMS! Could not parse row:', row)
+        print('WARNING: Document not found:', row)
       else:
         issues.append({
           'project': project,
@@ -281,6 +293,9 @@ gitlab_members = helpers.load_json_data(GITLAB_USERS_PATH)
 if not os.path.isfile(MEETINGS_PATH) or args.U:
   print(' * Update MPEG meetings data')
   meetings = mdms.get_meetings()
+  if len(meetings) == 0:
+    print(' * ERROR, could not get meetings from MDMS. Check your password.')
+    sys.exit(1)
   helpers.store_json_data(MEETINGS_PATH, meetings)
 meetings = helpers.load_json_data(MEETINGS_PATH)
 # get MPEG meeting we want to work on
@@ -288,6 +303,9 @@ meeting = helpers.find_meeting(meetings, args.meeting)
 if not meeting:
   print('Warning: Could not find meeting #{}. Update MDMS database and see if it was added'.format(args.meeting))
   meetings = mdms.get_meetings()
+  if len(meetings) == 0:
+    print(' * ERROR, could not get meetings from MDMS.')
+    sys.exit(1)
   helpers.store_json_data(MEETINGS_PATH, meetings)
   meetings = helpers.load_json_data(MEETINGS_PATH)
   meeting = helpers.find_meeting(meetings, args.meeting)
@@ -300,6 +318,9 @@ input_docs_path = os.path.join(DATA_PATH, 'input_docs_{}.json'.format(meeting['n
 if not os.path.isfile(input_docs_path) or args.U or args.u:
   print(' * Update MPEG input documents data for MPEG#', meeting['number'])
   input_docs = mdms.get_input_documents(meeting['id'])
+  if len(input_docs) == 0:
+    print(' * ERROR, could not get input documents from MDMS. Check your password.')
+    sys.exit(1)
   helpers.store_json_data(input_docs_path, input_docs)
 input_docs = helpers.load_json_data(input_docs_path)