Skip to content
GitLab
Commit 952b1d18 authored by Dimitri Podborski's avatar Dimitri Podborski 😂
Browse files

Merge branch 'refactoring' into 'master' 

release 1.4

Closes #197 and #199

See merge request !5
parents 183bbcbd 4d1a3662
Hide whitespace changes
Inline Side-by-side
.gitignore
View file @ 952b1d18
......@@ -46,6 +46,11 @@ Temporary Items
# Local History for Visual Studio Code
.history/
####################################
# IntelliJ
####################################
.idea/*
####################################
# Python
####################################
......
README.md
View file @ 952b1d18
......@@ -51,32 +51,29 @@ Below are a few examples:
1. Open issues based on the information provided in a CSV file:
e.g.: `python systems.py -o --csv Contribs.csv`
2. Open issues based on CLI options:
e.g.: `python systems.py -o -m 55958,55959,56121 -p FileFormat/CENC --meeting 133`
e.g.: `python systems.py -o -m 55958,55959,56121 -p http://... --meeting 133`
3. Generate an output document based on the information provided in a CSV file. Use a template as a basis (`--template` is optional):
e.g.: `python systems.py -d --csv Contribs.csv --template template.docx`
4. Close issues based on the information provided in a CSV file:
e.g.: `python systems.py -c --csv Contribs.csv`
5. Close issues based on CLI options:
e.g.: `python systems.py -c -m m55958,m55959,m56121 -p FileFormat/CENC --meeting 133`
e.g.: `python systems.py -c -m m55958,m55959,m56121 -p http://... --meeting 133`
6. Print information about input documents on MDMS and GitLab:
e.g.: `python systems.py -l -m m55958,m55959,m56121 -p FileFormat/CENC --meeting 133`
e.g.: `python systems.py -l -m m55958,m55959,m56121 -p http://... --meeting 133`
The CSV file must have a header row with the folowing entries:
- **Number** - MPEG document number with entries like `m12345`
- To determine which GitLab project needs to be used make sure that your CSV file has either:
- **Project URL** - a full URL to your GitLab project (*recommended*)
- or **Sub Group** and **Project Name** - two last elements of the Project URL. (non-case-sensitive)
e.g.: http://mpegx.int-evry.fr/software/MPEG/Systems/PCC-SYS/V-PCC → `Sub Group=PCC-SYS`, `Project Name=V-PCC`.
e.g.: http://mpegx.int-evry.fr/software/MPEG/Systems/FileFormat/isobmff → `Sub Group=FileFormat`, `Project Name=ISOBMFF`
- **Close issue** - is required if you want to close multiple issues at once. Supported values are `0`, `1`, `TRUE`, `FALSE`, `true` and `false`.
- **Number** - MPEG document numbers
- **Project URL** - a full URL to your GitLab project
- **Close issue** - (optional) if you want to close multiple issues at once. Supported values are `0`, `1`, `TRUE`, `FALSE`, `true` and `false`.
The example CSV below has both `Project URL` and (`Sub Group` with `Project Name`) but you can also have one of these in your CSV. The CSV delimiter is determined automatically.
The CSV delimiter is determined automatically. The order of columns does not matter. CSV file example:
```csv
Number;Title;Project URL;Sub Group;Project Name
m55958;On item encryption;http://mpegx.int-evry.fr/software/MPEG/Systems/FileFormat/CENC;FileFormat;CENC
m55959;On multi-key encryption;http://mpegx.int-evry.fr/software/MPEG/Systems/FileFormat/CENC;FileFormat;CENC
Number;Whatever column;Project URL
m55958;On item encryption;http://mpegx.int-evry.fr/software/MPEG/Systems/FileFormat/CENC
m55959;On multi-key encryption;http://mpegx.int-evry.fr/software/MPEG/Systems/FileFormat/CENC
...
```
## 3. generate_ballot_issues.py
......
automation/__init__.py
View file @ 952b1d18
# init automation package
\ No newline at end of file
# init automation package
automation/gitlab.py
View file @ 952b1d18
# -*- coding: utf-8 -*-
'''
"""
This is the interface to MPEG GitLab API.
'''
"""
import os
import gitlab
......@@ -10,98 +10,107 @@ from enum import Enum, unique
BASE_URL = 'http://mpegx.int-evry.fr/software'
TOKEN = os.environ.get('GITLAB_TOKEN')
@unique
class Label(Enum):
Accepted = 'Accepted'
BallotComment = 'BallotComment'
Combined = 'Combined'
DocAvailable = 'DocAvailable'
Editorial = 'Editorial'
Late = 'Late'
NeedsRevision = 'NeedsRevision'
Noted = 'Noted'
Postponed = 'Postponed'
ProbableAgreement = 'ProbableAgreement'
Rejected = 'Rejected'
Revised = 'Revised'
SeeDoCR = 'SeeDoCR'
Withdrawn = 'Withdrawn'
Accepted = 'Accepted'
BallotComment = 'BallotComment'
Combined = 'Combined'
DocAvailable = 'DocAvailable'
Editorial = 'Editorial'
Late = 'Late'
NeedsRevision = 'NeedsRevision'
Noted = 'Noted'
Postponed = 'Postponed'
ProbableAgreement = 'ProbableAgreement'
Rejected = 'Rejected'
Revised = 'Revised'
SeeDoCR = 'SeeDoCR'
Withdrawn = 'Withdrawn'
# private token authentication
GL = gitlab.Gitlab(BASE_URL, private_token=TOKEN)
try:
GL.auth()
print('GitLab API: Authenticated as "{}"'.format(GL.user.username))
GL.auth()
print('GitLab API: Authenticated as "{}"'.format(GL.user.username))
except gitlab.exceptions.GitlabAuthenticationError:
print('Error: Could not authenticate. Please set the valid private GITLAB_TOKEN env. variable.')
GL = None
print('Error: Could not authenticate. Please set the valid private GITLAB_TOKEN env. variable.')
GL = None
def _get_project(project_id):
if not GL:
print('Error: GitLab API authentication failed.')
return
try:
project = GL.projects.get(project_id)
except gitlab.exceptions.GitlabGetError as err:
print('project_id', project_id, err)
return None
return project
if not GL:
print('Error: GitLab API authentication failed.')
return
try:
project = GL.projects.get(project_id)
except gitlab.exceptions.GitlabGetError as err:
print('project_id', project_id, err)
return None
return project
# --------------------------------------------------------------------------------------------------
# Interfaces
# --------------------------------------------------------------------------------------------------
def get_projects():
if not GL:
print('Error: GitLab API authentication failed.')
return []
projects = GL.projects.list(all=True)
projects_stripped = []
for project in projects:
projects_stripped.append({
'id': project.id,
'name': project.name,
'url': project.web_url,
'path_with_namespace': project.path_with_namespace,
'description': project.description
})
return projects_stripped
if not GL:
print('Error: GitLab API authentication failed.')
return []
projects = GL.projects.list(all=True)
projects_stripped = []
for project in projects:
projects_stripped.append({
'id': project.id,
'name': project.name,
'url': project.web_url,
'path_with_namespace': project.path_with_namespace,
'description': project.description
})
return projects_stripped
def get_members(group_id):
if not GL:
print('Error: GitLab API authentication failed.')
return []
group = GL.groups.get(group_id)
subgroups = group.subgroups.list()
members_stripped = {}
for subgroup in subgroups:
real_group = GL.groups.get(subgroup.id, lazy=True)
members = real_group.members.all(all=True)
for member in members:
if not member.username in members_stripped:
members_stripped[member.username] = {
'id': member.id,
'name': member.name,
'url': member.web_url
}
return members_stripped
if not GL:
print('Error: GitLab API authentication failed.')
return []
group = GL.groups.get(group_id)
subgroups = group.subgroups.list()
members_stripped = {}
for subgroup in subgroups:
real_group = GL.groups.get(subgroup.id, lazy=True)
members = real_group.members.all(all=True)
for member in members:
if member.username not in members_stripped:
members_stripped[member.username] = {
'id': member.id,
'name': member.name,
'url': member.web_url
}
return members_stripped
def get_issues(project_id):
project = _get_project(project_id)
if not project:
return []
issues = project.issues.list(state='opened', all=True)
return issues
def open_issue(project_id, title, description, labels=[]):
project = _get_project(project_id)
if not project:
return
issue = project.issues.create({'title': title, 'description': description, 'labels': labels})
issue.save()
project = _get_project(project_id)
if not project:
return []
issues = project.issues.list(state='opened', all=True)
return issues
def close_issue(issue):
if isinstance(issue, gitlab.v4.objects.ProjectIssue):
issue.state_event = 'close'
def open_issue(project_id, title, description, labels=None):
project = _get_project(project_id)
if not project:
return
if labels is None:
labels = []
issue = project.issues.create({'title': title, 'description': description, 'labels': labels})
issue.save()
def close_issue(issue):
if isinstance(issue, gitlab.v4.objects.ProjectIssue):
issue.state_event = 'close'
issue.save()
automation/helpers.py
View file @ 952b1d18
# -*- coding: utf-8 -*-
'''
"""
Some helper functions
'''
"""
import json
import os
import re
from datetime import datetime, timedelta
from docx import Document, opc, oxml, shared
from docx.enum.text import WD_ALIGN_PARAGRAPH # pylint: disable=E0611
from docx.enum.text import WD_ALIGN_PARAGRAPH # pylint: disable=E0611
OPENING_TAG = '[//]: # ( !!! ATTENTION !!! DO NOT MODIFY BEFORE AND AFTER THIS LINE)'
CLOSING_TAG = '[//]: # ( !!! ATTENTION !!! YOU CAN MODIFY AFTER THIS LINE)'
DEADLINE_DAYS = 6 # Number of days to be subtracted from Monday
class DocumentFormatter:
def __init__(self, template_path):
self.__doc = Document(docx = template_path)
def save(self, output_path):
self.__doc.save(output_path)
# https://github.com/python-openxml/python-docx/issues/74#issuecomment-261169410
def add_hyperlink(self, paragraph, url, text):
part = paragraph.part
r_id = part.relate_to(url, opc.constants.RELATIONSHIP_TYPE.HYPERLINK, is_external = True)
hyperlink = oxml.shared.OxmlElement('w:hyperlink')
hyperlink.set(oxml.shared.qn('r:id'), r_id, )
run = oxml.shared.OxmlElement('w:r')
rPr = oxml.shared.OxmlElement('w:rPr')
c = oxml.shared.OxmlElement('w:color')
c.set(oxml.shared.qn('w:val'), '0000EE')
rPr.append(c)
u = oxml.shared.OxmlElement('w:u')
u.set(oxml.shared.qn('w:val'), 'single')
rPr.append(u)
run.append(rPr)
run.text = text
hyperlink.append(run)
paragraph._p.append(hyperlink)
def add_project(self, project):
project_description = project['description'].strip()
project_url = project['url']
project_name = project['name']
h = self.__doc.add_heading('', 2)
self.add_hyperlink(h, project_url, project_name)
p = None
if len(project_description) > 0:
p = self.__doc.add_paragraph(project_description)
else:
p = self.__doc.add_paragraph('[no project description supplied]')
p.paragraph_format.keep_with_next = True
def add_contribution(self, contribution):
document = contribution['document']
details = contribution['details']
issue_meta = contribution['issue_meta']
issue_title = contribution['issue_title']
# Create a heading 3 with the document number (linked to a container) and title
h = self.__doc.add_heading('', 3)
self.add_hyperlink(h, document['container'], document['document'])
h.add_run(' ' + document['title'])
# Create a 4x2 table with all borders
table = self.__doc.add_table(rows = 4, cols = 2)
table.style = 'Table Grid'
# Set the text of all the cells
table.rows[0].cells[0].text = 'Authors'
if details['authors_string'] is not None:
table.rows[0].cells[1].text = details['authors_string']
table.rows[0].cells[1].paragraphs[0].alignment = WD_ALIGN_PARAGRAPH.LEFT
table.rows[1].cells[0].text = 'Abstract'
if details['abstract'] is not None:
table.rows[1].cells[1].text = details['abstract']
table.rows[2].cells[0].text = 'Gitlab'
issues_added = 0
if issue_meta is not None:
self.add_hyperlink(table.rows[2].cells[1].paragraphs[0], issue_meta.web_url, issue_meta.references['full'])
issues_added += 1
if issue_title is not None:
if issues_added > 0:
p = table.rows[2].cells[1].add_paragraph()
self.add_hyperlink(p, issue_title.web_url, issue_title.references['full'])
else:
self.add_hyperlink(table.rows[2].cells[1].paragraphs[0], issue_title.web_url, issue_title.references['full'])
issues_added += 1
table.rows[3].cells[0].text = 'Disposition'
# Set column widths
for cell in table.columns[0].cells:
cell.width = shared.Cm(2)
for cell in table.columns[1].cells:
cell.width = shared.Cm(14)
p = self.__doc.add_paragraph('<minutes>')
p.paragraph_format.space_before = shared.Pt(8)
def __init__(self, template_path):
self.__doc = Document(docx=template_path)
def save(self, output_path):
self.__doc.save(output_path)
# https://github.com/python-openxml/python-docx/issues/74#issuecomment-261169410
@staticmethod
def add_hyperlink(paragraph, url, text):
part = paragraph.part
r_id = part.relate_to(url, opc.constants.RELATIONSHIP_TYPE.HYPERLINK, is_external=True)
hyperlink = oxml.shared.OxmlElement('w:hyperlink')
hyperlink.set(oxml.shared.qn('r:id'), r_id, )
run = oxml.shared.OxmlElement('w:r')
r_pr = oxml.shared.OxmlElement('w:rPr')
c = oxml.shared.OxmlElement('w:color')
c.set(oxml.shared.qn('w:val'), '0000EE')
r_pr.append(c)
u = oxml.shared.OxmlElement('w:u')
u.set(oxml.shared.qn('w:val'), 'single')
r_pr.append(u)
run.append(r_pr)
run.text = text
hyperlink.append(run)
paragraph._p.append(hyperlink)
def add_project(self, project):
project_description = project['description'].strip()
project_url = project['url']
project_name = project['name']
h = self.__doc.add_heading('', 2)
self.add_hyperlink(h, project_url, project_name)
if len(project_description) > 0:
p = self.__doc.add_paragraph(project_description)
else:
p = self.__doc.add_paragraph('[no project description supplied]')
p.paragraph_format.keep_with_next = True
def add_contribution(self, contribution):
document = contribution['document']
details = contribution['details']
issue_meta = contribution['issue_meta']
issue_title = contribution['issue_title']
# Create a heading 3 with the document number (linked to a container) and title
h = self.__doc.add_heading('', 3)
self.add_hyperlink(h, document['container'], document['document'])
h.add_run(' ' + document['title'])
# Create a 4x2 table with all borders
table = self.__doc.add_table(rows=4, cols=2)
table.style = 'Table Grid'
# Set the text of all the cells
table.rows[0].cells[0].text = 'Authors'
if details['authors_string'] is not None:
table.rows[0].cells[1].text = details['authors_string']
table.rows[0].cells[1].paragraphs[0].alignment = WD_ALIGN_PARAGRAPH.LEFT
table.rows[1].cells[0].text = 'Abstract'
if details['abstract'] is not None:
table.rows[1].cells[1].text = details['abstract']
table.rows[2].cells[0].text = 'Gitlab'
issues_added = 0
if issue_meta is not None:
self.add_hyperlink(table.rows[2].cells[1].paragraphs[0], issue_meta.web_url, issue_meta.references['full'])
issues_added += 1
if issue_title is not None:
if issues_added > 0:
p = table.rows[2].cells[1].add_paragraph()
self.add_hyperlink(p, issue_title.web_url, issue_title.references['full'])
else:
self.add_hyperlink(table.rows[2].cells[1].paragraphs[0], issue_title.web_url,
issue_title.references['full'])
issues_added += 1
table.rows[3].cells[0].text = 'Disposition'
# Set column widths
for cell in table.columns[0].cells:
cell.width = shared.Cm(2)
for cell in table.columns[1].cells:
cell.width = shared.Cm(14)
p = self.__doc.add_paragraph('<minutes>')
p.paragraph_format.space_before = shared.Pt(8)
def is_document_late(meeting_start, v1_upload_timestamp):
'''
meeting_start and v1_upload_timestamp shall be datetime objects
'''
meeting_start = meeting_start.replace(hour=0, minute=0, second=0) # paranoia
deadline = meeting_start - timedelta(days=4) # End of Wed.
diff = deadline - v1_upload_timestamp
if diff.total_seconds() <= 0:
return True
return False
"""
meeting_start and v1_upload_timestamp shall be datetime objects
"""
meeting_start = meeting_start.replace(hour=0, minute=0, second=0) # paranoia
deadline = meeting_start - timedelta(days=DEADLINE_DAYS)
diff = deadline - v1_upload_timestamp
if diff.total_seconds() <= 0:
return True
return False
def try_parsing_date(text):
'''
Try parsing the timestamp, if not possible return None
'''
for fmt in ('%Y-%m-%d %H:%M:%S', '%Y-%m-%d'):
try:
return datetime.strptime(text.strip(), fmt)
except ValueError:
pass
return None
"""
Try parsing the timestamp, if not possible return None
"""
for fmt in ('%Y-%m-%d %H:%M:%S', '%Y-%m-%d'):
try:
return datetime.strptime(text.strip(), fmt)
except ValueError:
pass
return None
def load_json_data(json_path):
'''
Load json file from json_path and return the data.
'''
with open(json_path, 'r', encoding='utf-8') as f:
data = json.load(f)
return data
"""
Load json file from json_path and return the data.
"""
with open(json_path, 'r', encoding='utf-8') as f:
data = json.load(f)
return data
def store_json_data(json_path, data):
'''
Store data as a json file to json_path. datetime objects are stored as strings.
'''
dir_name = os.path.dirname(json_path)
if not os.path.exists(dir_name) and len(dir_name) > 0:
os.makedirs(dir_name)
with open(json_path, 'w', encoding='utf-8') as f:
json.dump(data, f, ensure_ascii=False, indent=2, default=str)
"""
Store data as a json file to json_path. datetime objects are stored as strings.
"""
dir_name = os.path.dirname(json_path)
if not os.path.exists(dir_name) and len(dir_name) > 0:
os.makedirs(dir_name)
with open(json_path, 'w', encoding='utf-8') as f:
json.dump(data, f, ensure_ascii=False, indent=2, default=str)
def find_meeting(meetings, meeting_number):
'''
Find and return a meeting using the meeting_number. If meeting_number < 0 return the latest meeting.
'''
if len(meetings) == 0:
"""
Find and return a meeting using the meeting_number. If meeting_number < 0 return the latest meeting.
"""
if len(meetings) == 0:
return None
if meeting_number < 0:
return max(meetings, key=lambda x: x['number'])
for meeting in meetings:
if meeting['number'] == meeting_number:
return meeting
return None
if meeting_number < 0:
return max(meetings, key=lambda x:x['number'])
for meeting in meetings:
if meeting['number'] == meeting_number:
return meeting
return None
def find_document(documents, document_number):
striped_doc_nr = document_number.replace(' ', '').strip().lower()
for doc in documents:
if striped_doc_nr in doc['document']:
return doc
return None
def find_project(projects, url_or_path, path_root = 'MPEG/'):
'''
Search for gitlab project based on URL or path_with_namespace.
'''
if url_or_path is None:
striped_doc_nr = document_number.replace(' ', '').strip().lower()
for doc in documents:
if striped_doc_nr in doc['document']:
return doc
return None
striped_url_or_path = url_or_path.replace(' ', '').strip().strip('/')
for project in projects:
if striped_url_or_path == project['url']:
return project
path_ns = project['path_with_namespace'].lower()
if striped_url_or_path.lower() in path_ns and path_ns.startswith(path_root.lower()):
return project
return None
def find_project(projects, url_or_path):
"""
Search for gitlab project based on URL.
"""
if url_or_path is None:
return None
striped_url_or_path = url_or_path.replace(' ', '').strip().strip('/')
for project in projects:
if project['url'] in striped_url_or_path:
return project
return None
def find_issue(issues, document):
title_only_hit = None
metadata_hit = None
last_version = 0
for issue in issues:
if document['document'] in issue.title:
meta = get_issue_metadata(issue.description)
if meta == None:
title_only_hit = issue
else:
if int(meta['mdms_id']) == document['mdms_id']:
metadata_hit = issue
if len(meta['version']) > 0:
last_version = int(meta['version'])
else:
print('WARNING. We found a GitLab issue with the document number in the title and with metadata tag in description. But the metadata tag has wrong document id in it.')
return title_only_hit, metadata_hit, last_version
title_only_hit = None