implement find_documents API

automation/mdms.py

@@ -99,6 +99,104 @@ class MDMSParser:
                 pass
         return None
 
+    def parse_search_docs(self, html):
+        docs = []
+        soup = bs4.BeautifulSoup(html, features='lxml')
+        for i in soup.select('br'):  # replace <br/> with a space, it makes checking headers easier
+            i.replace_with(' ')
+
+        form = soup.find('body').find('form', id='documents')
+        if not form:
+            print('Error: No form with id="documents" found. Did MDMS layout change?')
+            return []
+        table_main = form.find('table')
+        if not table_main:
+            print('Error: No main table element found. Did MDMS layout change?')
+            return None
+        rows = table_main.find_all('tr', recursive=False)
+
+        for n in range(len(rows)):
+            if n == 0:  # check header first
+                header = ['number', 'meeting', 'created', 'Uploaded', 'Group SubGroup', 'title', 'Author(s)', '']
+                if not self.check_table_header(header, rows[n]):
+                    print('Error: Wrong table header. Did layout of MDMS change?')
+                    return []
+                continue
+            cols = rows[n].find_all('td', recursive=False)
+
+            try:
+                if len(cols[0].text) == 0:
+                    continue
+                # get document ID on MDMS
+                parsed_href = urlparse(cols[0].a['href'])
+                mdms_id = int(parse_qs(parsed_href.query)['id'][0])
+                container_url = urljoin(DOCUMENT_URL, '?id={}'.format(mdms_id))
+
+                # get timestamp of the last uploaded version
+                created_timestamp = self.try_parsing_date(cols[2].text)
+                uploaded_timestamp = self.try_parsing_date(cols[3].text)
+
+                authors = self._get_authors(cols[6])
+
+                # get latest document link (if available)
+                latest_url = None
+                if len(cols) == 8:
+                    if not cols[7].find('a') is None:
+                        latest_url = urljoin(CURRENT_MEETING_URL, cols[7].find('a')['href'])
+
+                docs.append({
+                    'mdms_id': mdms_id,
+                    'document': cols[0].text,
+                    'meeting': cols[1].text,
+                    'created': created_timestamp,
+                    'last_version_uploaded': uploaded_timestamp,
+                    'sub_group_text': cols[4].text,
+                    'title': cols[5].text.strip(),
+                    'authors': authors,
+                    'latest_version_url': latest_url,
+                    'container': container_url
+                })
+            except:  # TODO: catch properly
+                print('Error: Could not parse search documents data. Did MDMS layout change?')
+                return []
+        return docs
+
+    def _get_authors(self, col):
+        authors = []
+        for entry in col.contents:
+            if isinstance(entry, bs4.Tag):
+                email = None
+                try:
+                    parsed_href = urlparse(entry['href'])
+                    email = parsed_href.path
+                    author_data = self.parse_author_entry(entry.text)
+                    name = entry.text
+                    if author_data:
+                        name = author_data[0]  # clean version of the name
+                        # sometimes people type name and email wrong in MDMS and they are flipped
+                        if '@' not in email and author_data[1]:
+                            name = email
+                            email = author_data[1]
+                except KeyError:
+                    # sometimes Author's field is formatted with fake html tags.
+                    print('Bad HTML format in Authors field: ', entry)
+                    name = entry.text
+                    pass
+
+                authors.append({
+                    'name': name,
+                    'email': email
+                })
+            else:
+                for author_entry in entry.string.replace(' and ', ',').split(','):
+                    author_data = self.parse_author_entry(author_entry)
+                    if author_data:
+                        authors.append({
+                            'name': author_data[0],
+                            'email': author_data[1]
+                        })
+        return authors
+
     def parse_input_docs(self, html):
         docs = []
         soup = bs4.BeautifulSoup(html, features='lxml')
@@ -138,39 +236,7 @@ class MDMSParser:
                 created_timestamp = self.try_parsing_date(cols[1].text)
 
                 # get authors
-                authors = []
-                for entry in cols[5].contents:
-                    if isinstance(entry, bs4.Tag):
-                        email = None
-                        try:
-                            parsed_href = urlparse(entry['href'])
-                            email = parsed_href.path
-                            author_data = self.parse_author_entry(entry.text)
-                            name = entry.text
-                            if author_data:
-                                name = author_data[0]  # clean version of the name
-                                # sometimes people type name and email wrong in MDMS and they are flipped
-                                if '@' not in email and author_data[1]:
-                                    name = email
-                                    email = author_data[1]
-                        except KeyError:
-                            # sometimes Author's field is formatted with fake html tags.
-                            print('Bad HTML format in Authors field: ', entry)
-                            name = entry.text
-                            pass
-
-                        authors.append({
-                            'name': name,
-                            'email': email
-                        })
-                    else:
-                        for author_entry in entry.string.replace(' and ', ',').split(','):
-                            author_data = self.parse_author_entry(author_entry)
-                            if author_data:
-                                authors.append({
-                                    'name': author_data[0],
-                                    'email': author_data[1]
-                                })
+                authors = self._get_authors(cols[5])
 
                 # get latest document link (if available)
                 latest_url = None
@@ -455,6 +521,22 @@ def find_documents(title='',
                    subgroup=Subgroup.ALL):
     """
     Find documents using the search URL.
-    TODO: Fire a POST request to SEARCH_URL and parse the result
     """
-    raise NotImplemented
+    query = '?search_title={}' \
+            '&search_number={}'\
+            '&search_category={}' \
+            '&search_author={}'\
+            '&search_id_group={}' \
+            '&search_sub_group={}' \
+            '&id_meeting=0' \
+            '&submit=Search' \
+            '&meeting=0'.format(title, number, category.value, author, group.value, subgroup.value)
+
+    url = urljoin(SEARCH_URL, query)
+    response = requests.post(url, auth=(MPEG_LOGIN, MPEG_PWD))
+    if not response.status_code == 200:
+        print('HTTP response {} != 200'.format(response.status_code))
+        print('\t{}'.format(response.text.replace('\n', '\n\t')))
+        return None
+    parser = MDMSParser()
+    return parser.parse_search_docs(response.text)

systems.py

@@ -210,7 +210,6 @@ def open_issues(table_entries, test, gitlab_members, meeting_start):
                 if 'y' in user_input:
                     new_description = helpers.get_updated_issue_description(issue_with_meta.description, document,
                                                                             document_details)
-                    current_labels = issue_with_meta.labels
                     if 'DocAvailable' not in issue_with_meta.labels:
                         issue_with_meta.labels.append('DocAvailable')
                     if not test: