Merge branch 'refactoring' into 'master'

release 1.4 Closes #197 and #199 See merge request !5

Merge branch 'refactoring' into 'master'
release 1.4 Closes #197 and #199 See merge request !5
952b1d18 · Dimitri Podborski · 183bbcbd · 4d1a3662 · 952b1d18 · 952b1d18
Commit 952b1d18 authored Jul 06, 2021 by Dimitri Podborski
--- a/.gitignore
+++ b/.gitignore
@@ -46,6 +46,11 @@ Temporary Items
 # Local History for Visual Studio Code
 .history/
+####################################
+# IntelliJ
+####################################
+.idea/*
 ####################################
 # Python
 ####################################

--- a/README.md
+++ b/README.md
@@ -51,32 +51,29 @@ Below are a few examples:
 1. Open issues based on the information provided in a CSV file:  
 e.g.: `python systems.py -o --csv Contribs.csv`
 2. Open issues based on CLI options:  
-e.g.: `python systems.py -o -m 55958,55959,56121 -p FileFormat/CENC --meeting 133`
+e.g.: `python systems.py -o -m 55958,55959,56121 -p http://... --meeting 133`
 3. Generate an output document based on the information provided in a CSV file. Use a template as a basis (`--template` is optional):  
 e.g.: `python systems.py -d --csv Contribs.csv --template template.docx`
 4. Close issues based on the information provided in a CSV file:  
 e.g.: `python systems.py -c --csv Contribs.csv`
 5. Close issues based on CLI options:  
-e.g.: `python systems.py -c -m m55958,m55959,m56121 -p FileFormat/CENC --meeting 133`
+e.g.: `python systems.py -c -m m55958,m55959,m56121 -p http://... --meeting 133`
 6. Print information about input documents on MDMS and GitLab:  
-e.g.: `python systems.py -l -m m55958,m55959,m56121 -p FileFormat/CENC --meeting 133`
+e.g.: `python systems.py -l -m m55958,m55959,m56121 -p http://... --meeting 133`
 The CSV file must have a header row with the folowing entries:
- **Number** - MPEG document number with entries like `m12345`
+- **Number** - MPEG document numbers
- To determine which GitLab project needs to be used make sure that your CSV file has either:
+- **Project URL** - a full URL to your GitLab project
-  - **Project URL** - a full URL to your GitLab project (*recommended*)
+- **Close issue** - (optional) if you want to close multiple issues at once. Supported values are `0`, `1`, `TRUE`, `FALSE`, `true` and `false`.
-  - or **Sub Group** and **Project Name** - two last elements of the Project URL. (non-case-sensitive)  
-  e.g.: http://mpegx.int-evry.fr/software/MPEG/Systems/PCC-SYS/V-PCC &rarr;  `Sub Group=PCC-SYS`, `Project Name=V-PCC`.  
-  e.g.: http://mpegx.int-evry.fr/software/MPEG/Systems/FileFormat/isobmff &rarr;  `Sub Group=FileFormat`, `Project Name=ISOBMFF`
- **Close issue** - is required if you want to close multiple issues at once. Supported values are `0`, `1`, `TRUE`, `FALSE`, `true` and `false`.
-The example CSV below has both `Project URL` and (`Sub Group` with `Project Name`) but you can also have one of these in your CSV. The CSV delimiter is determined automatically.
+The CSV delimiter is determined automatically. The order of columns does not matter. CSV file example:
 ```csv
-Number;Title;Project URL;Sub Group;Project Name
+Number;Whatever column;Project URL
-m55958;On item encryption;http://mpegx.int-evry.fr/software/MPEG/Systems/FileFormat/CENC;FileFormat;CENC
+m55958;On item encryption;http://mpegx.int-evry.fr/software/MPEG/Systems/FileFormat/CENC
-m55959;On multi-key encryption;http://mpegx.int-evry.fr/software/MPEG/Systems/FileFormat/CENC;FileFormat;CENC
+m55959;On multi-key encryption;http://mpegx.int-evry.fr/software/MPEG/Systems/FileFormat/CENC
+...
 ```
 ## 3. generate_ballot_issues.py

--- a/automation/__init__.py
+++ b/automation/__init__.py
-# init automation package 
+# init automation package
\ No newline at end of file
--- a/automation/gitlab.py
+++ b/automation/gitlab.py
 # -*- coding: utf-8 -*-
-'''
+"""
 This is the interface to MPEG GitLab API.
-'''
+"""
 import os
 import gitlab
@@ -10,98 +10,107 @@ from enum import Enum, unique
 BASE_URL = 'http://mpegx.int-evry.fr/software'
 TOKEN = os.environ.get('GITLAB_TOKEN')
 @unique
 class Label(Enum):
-  Accepted = 'Accepted'
+    Accepted = 'Accepted'
-  BallotComment = 'BallotComment'
+    BallotComment = 'BallotComment'
-  Combined = 'Combined'
+    Combined = 'Combined'
-  DocAvailable = 'DocAvailable'
+    DocAvailable = 'DocAvailable'
-  Editorial = 'Editorial'
+    Editorial = 'Editorial'
-  Late = 'Late'
+    Late = 'Late'
-  NeedsRevision = 'NeedsRevision'
+    NeedsRevision = 'NeedsRevision'
-  Noted = 'Noted'
+    Noted = 'Noted'
-  Postponed = 'Postponed'
+    Postponed = 'Postponed'
-  ProbableAgreement = 'ProbableAgreement'
+    ProbableAgreement = 'ProbableAgreement'
-  Rejected = 'Rejected'
+    Rejected = 'Rejected'
-  Revised = 'Revised'
+    Revised = 'Revised'
-  SeeDoCR = 'SeeDoCR'
+    SeeDoCR = 'SeeDoCR'
-  Withdrawn = 'Withdrawn'
+    Withdrawn = 'Withdrawn'
 # private token authentication
 GL = gitlab.Gitlab(BASE_URL, private_token=TOKEN)
 try:
-  GL.auth()
+    GL.auth()
-  print('GitLab API: Authenticated as "{}"'.format(GL.user.username))
+    print('GitLab API: Authenticated as "{}"'.format(GL.user.username))
 except gitlab.exceptions.GitlabAuthenticationError:
-  print('Error: Could not authenticate. Please set the valid private GITLAB_TOKEN env. variable.')
+    print('Error: Could not authenticate. Please set the valid private GITLAB_TOKEN env. variable.')
-  GL = None
+    GL = None
 def _get_project(project_id):
-  if not GL:
+    if not GL:
-    print('Error: GitLab API authentication failed.')
+        print('Error: GitLab API authentication failed.')
-    return
+        return
-  try:
+    try:
-    project = GL.projects.get(project_id)
+        project = GL.projects.get(project_id)
-  except gitlab.exceptions.GitlabGetError as err:
+    except gitlab.exceptions.GitlabGetError as err:
-    print('project_id', project_id, err)
+        print('project_id', project_id, err)
-    return None
+        return None
-  return project
+    return project
 # --------------------------------------------------------------------------------------------------
 # Interfaces
 # --------------------------------------------------------------------------------------------------
 def get_projects():
-  if not GL:
+    if not GL:
-    print('Error: GitLab API authentication failed.')
+        print('Error: GitLab API authentication failed.')
-    return []
+        return []
-  projects = GL.projects.list(all=True)
+    projects = GL.projects.list(all=True)
-  projects_stripped = []
+    projects_stripped = []
-  for project in projects:
+    for project in projects:
-    projects_stripped.append({
+        projects_stripped.append({
-      'id': project.id,
+            'id': project.id,
-      'name': project.name,
+            'name': project.name,
-      'url': project.web_url,
+            'url': project.web_url,
-      'path_with_namespace': project.path_with_namespace,
+            'path_with_namespace': project.path_with_namespace,
-      'description': project.description
+            'description': project.description
-    })
+        })
-  return projects_stripped
+    return projects_stripped
 def get_members(group_id):
-  if not GL:
+    if not GL:
-    print('Error: GitLab API authentication failed.')
+        print('Error: GitLab API authentication failed.')
-    return []
+        return []
-  group = GL.groups.get(group_id)
+    group = GL.groups.get(group_id)
-  subgroups = group.subgroups.list()
+    subgroups = group.subgroups.list()
-  members_stripped = {}
+    members_stripped = {}
-  for subgroup in subgroups:
+    for subgroup in subgroups:
-    real_group = GL.groups.get(subgroup.id, lazy=True)
+        real_group = GL.groups.get(subgroup.id, lazy=True)
-    members = real_group.members.all(all=True)
+        members = real_group.members.all(all=True)
-    for member in members:
+        for member in members:
-      if not member.username in members_stripped:
+            if member.username not in members_stripped:
-        members_stripped[member.username] = {
+                members_stripped[member.username] = {
-          'id': member.id,
+                    'id': member.id,
-          'name': member.name,
+                    'name': member.name,
-          'url': member.web_url
+                    'url': member.web_url
-        }
+                }
-  return members_stripped
+    return members_stripped
 def get_issues(project_id):
-  project = _get_project(project_id)
+    project = _get_project(project_id)
-  if not project:
+    if not project:
-    return []
+        return []
-  issues = project.issues.list(state='opened', all=True)
+    issues = project.issues.list(state='opened', all=True)
-  return issues
+    return issues
-def open_issue(project_id, title, description, labels=[]):
-  project = _get_project(project_id)
-  if not project:
-    return
-  issue = project.issues.create({'title': title, 'description': description, 'labels': labels})
-  issue.save()
-def close_issue(issue):
-  if isinstance(issue, gitlab.v4.objects.ProjectIssue):
+def open_issue(project_id, title, description, labels=None):
-    issue.state_event = 'close'
+    project = _get_project(project_id)
+    if not project:
+        return
+    if labels is None:
+        labels = []
+    issue = project.issues.create({'title': title, 'description': description, 'labels': labels})
    issue.save()
+def close_issue(issue):
+    if isinstance(issue, gitlab.v4.objects.ProjectIssue):
+        issue.state_event = 'close'
+        issue.save()
--- a/automation/helpers.py
+++ b/automation/helpers.py
 # -*- coding: utf-8 -*-
-'''
+"""
 Some helper functions
-'''
+"""
 import json
 import os
 import re
 from datetime import datetime, timedelta
 from docx import Document, opc, oxml, shared
-from docx.enum.text import WD_ALIGN_PARAGRAPH # pylint: disable=E0611
+from docx.enum.text import WD_ALIGN_PARAGRAPH  # pylint: disable=E0611
 OPENING_TAG = '[//]: # ( !!! ATTENTION !!! DO NOT MODIFY BEFORE AND AFTER THIS LINE)'
 CLOSING_TAG = '[//]: # ( !!! ATTENTION !!! YOU CAN MODIFY AFTER THIS LINE)'
+DEADLINE_DAYS = 6  # Number of days to be subtracted from Monday
 class DocumentFormatter:
-  def __init__(self, template_path):
+    def __init__(self, template_path):
-    self.__doc = Document(docx = template_path)
+        self.__doc = Document(docx=template_path)
-  def save(self, output_path):
+    def save(self, output_path):
-    self.__doc.save(output_path)
+        self.__doc.save(output_path)
-  # https://github.com/python-openxml/python-docx/issues/74#issuecomment-261169410
+    # https://github.com/python-openxml/python-docx/issues/74#issuecomment-261169410
-  def add_hyperlink(self, paragraph, url, text):
+    @staticmethod
-    part = paragraph.part
+    def add_hyperlink(paragraph, url, text):
-    r_id = part.relate_to(url, opc.constants.RELATIONSHIP_TYPE.HYPERLINK, is_external = True)
+        part = paragraph.part
-    hyperlink = oxml.shared.OxmlElement('w:hyperlink')
+        r_id = part.relate_to(url, opc.constants.RELATIONSHIP_TYPE.HYPERLINK, is_external=True)
-    hyperlink.set(oxml.shared.qn('r:id'), r_id, )
+        hyperlink = oxml.shared.OxmlElement('w:hyperlink')
-    run = oxml.shared.OxmlElement('w:r')
+        hyperlink.set(oxml.shared.qn('r:id'), r_id, )
-    rPr = oxml.shared.OxmlElement('w:rPr')
+        run = oxml.shared.OxmlElement('w:r')
-    c = oxml.shared.OxmlElement('w:color')
+        r_pr = oxml.shared.OxmlElement('w:rPr')
-    c.set(oxml.shared.qn('w:val'), '0000EE')
+        c = oxml.shared.OxmlElement('w:color')
-    rPr.append(c)
+        c.set(oxml.shared.qn('w:val'), '0000EE')
-    u = oxml.shared.OxmlElement('w:u')
+        r_pr.append(c)
-    u.set(oxml.shared.qn('w:val'), 'single')
+        u = oxml.shared.OxmlElement('w:u')
-    rPr.append(u)
+        u.set(oxml.shared.qn('w:val'), 'single')
-    run.append(rPr)
+        r_pr.append(u)
-    run.text = text
+        run.append(r_pr)
-    hyperlink.append(run)
+        run.text = text
-    paragraph._p.append(hyperlink)
+        hyperlink.append(run)
+        paragraph._p.append(hyperlink)
-  def add_project(self, project):
-    project_description = project['description'].strip()
+    def add_project(self, project):
-    project_url = project['url']
+        project_description = project['description'].strip()
-    project_name = project['name']
+        project_url = project['url']
-    h = self.__doc.add_heading('', 2)
+        project_name = project['name']
-    self.add_hyperlink(h, project_url, project_name)
+        h = self.__doc.add_heading('', 2)
-    p = None
+        self.add_hyperlink(h, project_url, project_name)
-    if len(project_description) > 0:
+        if len(project_description) > 0:
-      p = self.__doc.add_paragraph(project_description)
+            p = self.__doc.add_paragraph(project_description)
-    else:
+        else:
-      p = self.__doc.add_paragraph('[no project description supplied]')
+            p = self.__doc.add_paragraph('[no project description supplied]')
-    p.paragraph_format.keep_with_next = True
+        p.paragraph_format.keep_with_next = True
-  def add_contribution(self, contribution):
+    def add_contribution(self, contribution):
-    document = contribution['document']
+        document = contribution['document']
-    details = contribution['details']
+        details = contribution['details']
-    issue_meta = contribution['issue_meta']
+        issue_meta = contribution['issue_meta']
-    issue_title = contribution['issue_title']
+        issue_title = contribution['issue_title']
-    # Create a heading 3 with the document number (linked to a container) and title
+        # Create a heading 3 with the document number (linked to a container) and title
-    h = self.__doc.add_heading('', 3)
+        h = self.__doc.add_heading('', 3)
-    self.add_hyperlink(h, document['container'], document['document'])
+        self.add_hyperlink(h, document['container'], document['document'])
-    h.add_run(' ' + document['title'])
+        h.add_run(' ' + document['title'])
-    # Create a 4x2 table with all borders
+        # Create a 4x2 table with all borders
-    table = self.__doc.add_table(rows = 4, cols = 2)
+        table = self.__doc.add_table(rows=4, cols=2)
-    table.style = 'Table Grid'
+        table.style = 'Table Grid'
-    # Set the text of all the cells
+        # Set the text of all the cells
-    table.rows[0].cells[0].text = 'Authors'
+        table.rows[0].cells[0].text = 'Authors'
-    if details['authors_string'] is not None:
+        if details['authors_string'] is not None:
-      table.rows[0].cells[1].text = details['authors_string']
+            table.rows[0].cells[1].text = details['authors_string']
-    table.rows[0].cells[1].paragraphs[0].alignment = WD_ALIGN_PARAGRAPH.LEFT
+        table.rows[0].cells[1].paragraphs[0].alignment = WD_ALIGN_PARAGRAPH.LEFT
-    table.rows[1].cells[0].text = 'Abstract'
+        table.rows[1].cells[0].text = 'Abstract'
-    if details['abstract'] is not None:
+        if details['abstract'] is not None:
-      table.rows[1].cells[1].text = details['abstract']
+            table.rows[1].cells[1].text = details['abstract']
-    table.rows[2].cells[0].text = 'Gitlab'
+        table.rows[2].cells[0].text = 'Gitlab'
-    issues_added = 0
+        issues_added = 0
-    if issue_meta is not None:
+        if issue_meta is not None:
-      self.add_hyperlink(table.rows[2].cells[1].paragraphs[0], issue_meta.web_url, issue_meta.references['full'])
+            self.add_hyperlink(table.rows[2].cells[1].paragraphs[0], issue_meta.web_url, issue_meta.references['full'])
-      issues_added += 1
+            issues_added += 1
-    if issue_title is not None:
+        if issue_title is not None:
-      if issues_added > 0:
+            if issues_added > 0:
-        p = table.rows[2].cells[1].add_paragraph()
+                p = table.rows[2].cells[1].add_paragraph()
-        self.add_hyperlink(p, issue_title.web_url, issue_title.references['full'])
+                self.add_hyperlink(p, issue_title.web_url, issue_title.references['full'])
-      else:
+            else:
-        self.add_hyperlink(table.rows[2].cells[1].paragraphs[0], issue_title.web_url, issue_title.references['full'])
+                self.add_hyperlink(table.rows[2].cells[1].paragraphs[0], issue_title.web_url,
-      issues_added += 1
+                                   issue_title.references['full'])
-    table.rows[3].cells[0].text = 'Disposition'
+            issues_added += 1
+        table.rows[3].cells[0].text = 'Disposition'
-    # Set column widths
-    for cell in table.columns[0].cells:
+        # Set column widths
-      cell.width = shared.Cm(2)
+        for cell in table.columns[0].cells:
-    for cell in table.columns[1].cells:
+            cell.width = shared.Cm(2)
-      cell.width = shared.Cm(14)
+        for cell in table.columns[1].cells:
+            cell.width = shared.Cm(14)
-    p = self.__doc.add_paragraph('<minutes>')
-    p.paragraph_format.space_before = shared.Pt(8)
+        p = self.__doc.add_paragraph('<minutes>')
+        p.paragraph_format.space_before = shared.Pt(8)
 def is_document_late(meeting_start, v1_upload_timestamp):
-  '''
+    """
-  meeting_start and v1_upload_timestamp shall be datetime objects
+    meeting_start and v1_upload_timestamp shall be datetime objects
-  '''
+    """
-  meeting_start = meeting_start.replace(hour=0, minute=0, second=0) # paranoia
+    meeting_start = meeting_start.replace(hour=0, minute=0, second=0)  # paranoia
-  deadline = meeting_start - timedelta(days=4) # End of Wed.
+    deadline = meeting_start - timedelta(days=DEADLINE_DAYS)
-  diff = deadline - v1_upload_timestamp
+    diff = deadline - v1_upload_timestamp
-  if diff.total_seconds() <= 0:
+    if diff.total_seconds() <= 0:
-    return True
+        return True
-  return False
+    return False
 def try_parsing_date(text):
-  '''
+    """
-  Try parsing the timestamp, if not possible return None
+    Try parsing the timestamp, if not possible return None
-  '''
+    """
-  for fmt in ('%Y-%m-%d %H:%M:%S', '%Y-%m-%d'):
+    for fmt in ('%Y-%m-%d %H:%M:%S', '%Y-%m-%d'):
-    try:
+        try:
-      return datetime.strptime(text.strip(), fmt)
+            return datetime.strptime(text.strip(), fmt)
-    except ValueError:
+        except ValueError:
-      pass
+            pass
-  return None
+    return None
 def load_json_data(json_path):
-  '''
+    """
-  Load json file from json_path and return the data.
+    Load json file from json_path and return the data.
-  '''
+    """
-  with open(json_path, 'r', encoding='utf-8') as f:
+    with open(json_path, 'r', encoding='utf-8') as f:
-    data = json.load(f)
+        data = json.load(f)
-    return data
+        return data
 def store_json_data(json_path, data):
-  '''
+    """
-  Store data as a json file to json_path. datetime objects are stored as strings.
+    Store data as a json file to json_path. datetime objects are stored as strings.
-  '''
+    """
-  dir_name = os.path.dirname(json_path)
+    dir_name = os.path.dirname(json_path)
-  if not os.path.exists(dir_name) and len(dir_name) > 0:
+    if not os.path.exists(dir_name) and len(dir_name) > 0:
-    os.makedirs(dir_name)
+        os.makedirs(dir_name)
-  with open(json_path, 'w', encoding='utf-8') as f:
+    with open(json_path, 'w', encoding='utf-8') as f:
-    json.dump(data, f, ensure_ascii=False, indent=2, default=str)
+        json.dump(data, f, ensure_ascii=False, indent=2, default=str)
 def find_meeting(meetings, meeting_number):
-  '''
+    """
-  Find and return a meeting using the meeting_number. If meeting_number < 0 return the latest meeting.
+    Find and return a meeting using the meeting_number. If meeting_number < 0 return the latest meeting.
-  '''
+    """
-  if len(meetings) == 0:
+    if len(meetings) == 0:
+        return None
+    if meeting_number < 0:
+        return max(meetings, key=lambda x: x['number'])
+    for meeting in meetings:
+        if meeting['number'] == meeting_number:
+            return meeting
    return None
-  if meeting_number < 0:
-    return max(meetings, key=lambda x:x['number'])
-  for meeting in meetings:
-    if meeting['number'] == meeting_number:
-      return meeting
-  return None
 def find_document(documents, document_number):
-  striped_doc_nr = document_number.replace(' ', '').strip().lower()
+    striped_doc_nr = document_number.replace(' ', '').strip().lower()
-  for doc in documents:
+    for doc in documents:
-    if striped_doc_nr in doc['document']:
+        if striped_doc_nr in doc['document']:
-      return doc
+            return doc
-  return None
-def find_project(projects, url_or_path, path_root = 'MPEG/'):
-  '''
-  Search for gitlab project based on URL or path_with_namespace. 
-  '''
-  if url_or_path is None:
    return None
-  striped_url_or_path = url_or_path.replace(' ', '').strip().strip('/')
-  for project in projects:
-    if striped_url_or_path == project['url']:
+def find_project(projects, url_or_path):
-      return project
+    """
-    path_ns = project['path_with_namespace'].lower()
+    Search for gitlab project based on URL.
-    if striped_url_or_path.lower() in path_ns and path_ns.startswith(path_root.lower()):
+    """
-      return project
+    if url_or_path is None:
-  return None
+        return None
+    striped_url_or_path = url_or_path.replace(' ', '').strip().strip('/')
+    for project in projects:
+        if project['url'] in striped_url_or_path:
+            return project