Skip to content

GitLab

Commit 72944f6f authored by Dimitri Podborski's avatar Dimitri Podborski
Browse files

pip 8

parent 7aaf00e2
Hide whitespace changes
Inline Side-by-side
automation/__init__.py
View file @ 72944f6f
# init automation package
\ No newline at end of file
# init automation package
automation/gitlab.py
View file @ 72944f6f
# -*- coding: utf-8 -*-
'''
"""
This is the interface to MPEG GitLab API.
'''
"""
import os
import gitlab
......@@ -10,98 +10,107 @@ from enum import Enum, unique
BASE_URL = 'http://mpegx.int-evry.fr/software'
TOKEN = os.environ.get('GITLAB_TOKEN')
@unique
class Label(Enum):
Accepted = 'Accepted'
BallotComment = 'BallotComment'
Combined = 'Combined'
DocAvailable = 'DocAvailable'
Editorial = 'Editorial'
Late = 'Late'
NeedsRevision = 'NeedsRevision'
Noted = 'Noted'
Postponed = 'Postponed'
ProbableAgreement = 'ProbableAgreement'
Rejected = 'Rejected'
Revised = 'Revised'
SeeDoCR = 'SeeDoCR'
Withdrawn = 'Withdrawn'
Accepted = 'Accepted'
BallotComment = 'BallotComment'
Combined = 'Combined'
DocAvailable = 'DocAvailable'
Editorial = 'Editorial'
Late = 'Late'
NeedsRevision = 'NeedsRevision'
Noted = 'Noted'
Postponed = 'Postponed'
ProbableAgreement = 'ProbableAgreement'
Rejected = 'Rejected'
Revised = 'Revised'
SeeDoCR = 'SeeDoCR'
Withdrawn = 'Withdrawn'
# private token authentication
GL = gitlab.Gitlab(BASE_URL, private_token=TOKEN)
try:
GL.auth()
print('GitLab API: Authenticated as "{}"'.format(GL.user.username))
GL.auth()
print('GitLab API: Authenticated as "{}"'.format(GL.user.username))
except gitlab.exceptions.GitlabAuthenticationError:
print('Error: Could not authenticate. Please set the valid private GITLAB_TOKEN env. variable.')
GL = None
print('Error: Could not authenticate. Please set the valid private GITLAB_TOKEN env. variable.')
GL = None
def _get_project(project_id):
if not GL:
print('Error: GitLab API authentication failed.')
return
try:
project = GL.projects.get(project_id)
except gitlab.exceptions.GitlabGetError as err:
print('project_id', project_id, err)
return None
return project
if not GL:
print('Error: GitLab API authentication failed.')
return
try:
project = GL.projects.get(project_id)
except gitlab.exceptions.GitlabGetError as err:
print('project_id', project_id, err)
return None
return project
# --------------------------------------------------------------------------------------------------
# Interfaces
# --------------------------------------------------------------------------------------------------
def get_projects():
if not GL:
print('Error: GitLab API authentication failed.')
return []
projects = GL.projects.list(all=True)
projects_stripped = []
for project in projects:
projects_stripped.append({
'id': project.id,
'name': project.name,
'url': project.web_url,
'path_with_namespace': project.path_with_namespace,
'description': project.description
})
return projects_stripped
if not GL:
print('Error: GitLab API authentication failed.')
return []
projects = GL.projects.list(all=True)
projects_stripped = []
for project in projects:
projects_stripped.append({
'id': project.id,
'name': project.name,
'url': project.web_url,
'path_with_namespace': project.path_with_namespace,
'description': project.description
})
return projects_stripped
def get_members(group_id):
if not GL:
print('Error: GitLab API authentication failed.')
return []
group = GL.groups.get(group_id)
subgroups = group.subgroups.list()
members_stripped = {}
for subgroup in subgroups:
real_group = GL.groups.get(subgroup.id, lazy=True)
members = real_group.members.all(all=True)
for member in members:
if not member.username in members_stripped:
members_stripped[member.username] = {
'id': member.id,
'name': member.name,
'url': member.web_url
}
return members_stripped
if not GL:
print('Error: GitLab API authentication failed.')
return []
group = GL.groups.get(group_id)
subgroups = group.subgroups.list()
members_stripped = {}
for subgroup in subgroups:
real_group = GL.groups.get(subgroup.id, lazy=True)
members = real_group.members.all(all=True)
for member in members:
if member.username not in members_stripped:
members_stripped[member.username] = {
'id': member.id,
'name': member.name,
'url': member.web_url
}
return members_stripped
def get_issues(project_id):
project = _get_project(project_id)
if not project:
return []
issues = project.issues.list(state='opened', all=True)
return issues
def open_issue(project_id, title, description, labels=[]):
project = _get_project(project_id)
if not project:
return
issue = project.issues.create({'title': title, 'description': description, 'labels': labels})
issue.save()
project = _get_project(project_id)
if not project:
return []
issues = project.issues.list(state='opened', all=True)
return issues
def close_issue(issue):
if isinstance(issue, gitlab.v4.objects.ProjectIssue):
issue.state_event = 'close'
def open_issue(project_id, title, description, labels=None):
project = _get_project(project_id)
if not project:
return
if labels is None:
labels = []
issue = project.issues.create({'title': title, 'description': description, 'labels': labels})
issue.save()
def close_issue(issue):
if isinstance(issue, gitlab.v4.objects.ProjectIssue):
issue.state_event = 'close'
issue.save()
automation/helpers.py
View file @ 72944f6f
# -*- coding: utf-8 -*-
'''
"""
Some helper functions
'''
"""
import json
import os
import re
from datetime import datetime, timedelta
from docx import Document, opc, oxml, shared
from docx.enum.text import WD_ALIGN_PARAGRAPH # pylint: disable=E0611
from docx.enum.text import WD_ALIGN_PARAGRAPH # pylint: disable=E0611
OPENING_TAG = '[//]: # ( !!! ATTENTION !!! DO NOT MODIFY BEFORE AND AFTER THIS LINE)'
CLOSING_TAG = '[//]: # ( !!! ATTENTION !!! YOU CAN MODIFY AFTER THIS LINE)'
class DocumentFormatter:
def __init__(self, template_path):
self.__doc = Document(docx = template_path)
def save(self, output_path):
self.__doc.save(output_path)
# https://github.com/python-openxml/python-docx/issues/74#issuecomment-261169410
def add_hyperlink(self, paragraph, url, text):
part = paragraph.part
r_id = part.relate_to(url, opc.constants.RELATIONSHIP_TYPE.HYPERLINK, is_external = True)
hyperlink = oxml.shared.OxmlElement('w:hyperlink')
hyperlink.set(oxml.shared.qn('r:id'), r_id, )
run = oxml.shared.OxmlElement('w:r')
rPr = oxml.shared.OxmlElement('w:rPr')
c = oxml.shared.OxmlElement('w:color')
c.set(oxml.shared.qn('w:val'), '0000EE')
rPr.append(c)
u = oxml.shared.OxmlElement('w:u')
u.set(oxml.shared.qn('w:val'), 'single')
rPr.append(u)
run.append(rPr)
run.text = text
hyperlink.append(run)
paragraph._p.append(hyperlink)
def add_project(self, project):
project_description = project['description'].strip()
project_url = project['url']
project_name = project['name']
h = self.__doc.add_heading('', 2)
self.add_hyperlink(h, project_url, project_name)
p = None
if len(project_description) > 0:
p = self.__doc.add_paragraph(project_description)
else:
p = self.__doc.add_paragraph('[no project description supplied]')
p.paragraph_format.keep_with_next = True
def add_contribution(self, contribution):
document = contribution['document']
details = contribution['details']
issue_meta = contribution['issue_meta']
issue_title = contribution['issue_title']
# Create a heading 3 with the document number (linked to a container) and title
h = self.__doc.add_heading('', 3)
self.add_hyperlink(h, document['container'], document['document'])
h.add_run(' ' + document['title'])
# Create a 4x2 table with all borders
table = self.__doc.add_table(rows = 4, cols = 2)
table.style = 'Table Grid'
# Set the text of all the cells
table.rows[0].cells[0].text = 'Authors'
if details['authors_string'] is not None:
table.rows[0].cells[1].text = details['authors_string']
table.rows[0].cells[1].paragraphs[0].alignment = WD_ALIGN_PARAGRAPH.LEFT
table.rows[1].cells[0].text = 'Abstract'
if details['abstract'] is not None:
table.rows[1].cells[1].text = details['abstract']
table.rows[2].cells[0].text = 'Gitlab'
issues_added = 0
if issue_meta is not None:
self.add_hyperlink(table.rows[2].cells[1].paragraphs[0], issue_meta.web_url, issue_meta.references['full'])
issues_added += 1
if issue_title is not None:
if issues_added > 0:
p = table.rows[2].cells[1].add_paragraph()
self.add_hyperlink(p, issue_title.web_url, issue_title.references['full'])
else:
self.add_hyperlink(table.rows[2].cells[1].paragraphs[0], issue_title.web_url, issue_title.references['full'])
issues_added += 1
table.rows[3].cells[0].text = 'Disposition'
# Set column widths
for cell in table.columns[0].cells:
cell.width = shared.Cm(2)
for cell in table.columns[1].cells:
cell.width = shared.Cm(14)
p = self.__doc.add_paragraph('<minutes>')
p.paragraph_format.space_before = shared.Pt(8)
def __init__(self, template_path):
self.__doc = Document(docx=template_path)
def save(self, output_path):
self.__doc.save(output_path)
# https://github.com/python-openxml/python-docx/issues/74#issuecomment-261169410
@staticmethod
def add_hyperlink(paragraph, url, text):
part = paragraph.part
r_id = part.relate_to(url, opc.constants.RELATIONSHIP_TYPE.HYPERLINK, is_external=True)
hyperlink = oxml.shared.OxmlElement('w:hyperlink')
hyperlink.set(oxml.shared.qn('r:id'), r_id, )
run = oxml.shared.OxmlElement('w:r')
r_pr = oxml.shared.OxmlElement('w:rPr')
c = oxml.shared.OxmlElement('w:color')
c.set(oxml.shared.qn('w:val'), '0000EE')
r_pr.append(c)
u = oxml.shared.OxmlElement('w:u')
u.set(oxml.shared.qn('w:val'), 'single')
r_pr.append(u)
run.append(r_pr)
run.text = text
hyperlink.append(run)
paragraph._p.append(hyperlink)
def add_project(self, project):
project_description = project['description'].strip()
project_url = project['url']
project_name = project['name']
h = self.__doc.add_heading('', 2)
self.add_hyperlink(h, project_url, project_name)
if len(project_description) > 0:
p = self.__doc.add_paragraph(project_description)
else:
p = self.__doc.add_paragraph('[no project description supplied]')
p.paragraph_format.keep_with_next = True
def add_contribution(self, contribution):
document = contribution['document']
details = contribution['details']
issue_meta = contribution['issue_meta']
issue_title = contribution['issue_title']
# Create a heading 3 with the document number (linked to a container) and title
h = self.__doc.add_heading('', 3)
self.add_hyperlink(h, document['container'], document['document'])
h.add_run(' ' + document['title'])
# Create a 4x2 table with all borders
table = self.__doc.add_table(rows=4, cols=2)
table.style = 'Table Grid'
# Set the text of all the cells
table.rows[0].cells[0].text = 'Authors'
if details['authors_string'] is not None:
table.rows[0].cells[1].text = details['authors_string']
table.rows[0].cells[1].paragraphs[0].alignment = WD_ALIGN_PARAGRAPH.LEFT
table.rows[1].cells[0].text = 'Abstract'
if details['abstract'] is not None:
table.rows[1].cells[1].text = details['abstract']
table.rows[2].cells[0].text = 'Gitlab'
issues_added = 0
if issue_meta is not None:
self.add_hyperlink(table.rows[2].cells[1].paragraphs[0], issue_meta.web_url, issue_meta.references['full'])
issues_added += 1
if issue_title is not None:
if issues_added > 0:
p = table.rows[2].cells[1].add_paragraph()
self.add_hyperlink(p, issue_title.web_url, issue_title.references['full'])
else:
self.add_hyperlink(table.rows[2].cells[1].paragraphs[0], issue_title.web_url,
issue_title.references['full'])
issues_added += 1
table.rows[3].cells[0].text = 'Disposition'
# Set column widths
for cell in table.columns[0].cells:
cell.width = shared.Cm(2)
for cell in table.columns[1].cells:
cell.width = shared.Cm(14)
p = self.__doc.add_paragraph('<minutes>')
p.paragraph_format.space_before = shared.Pt(8)
def is_document_late(meeting_start, v1_upload_timestamp):
'''
meeting_start and v1_upload_timestamp shall be datetime objects
'''
meeting_start = meeting_start.replace(hour=0, minute=0, second=0) # paranoia
deadline = meeting_start - timedelta(days=7) # End of Sunday
diff = deadline - v1_upload_timestamp
if diff.total_seconds() <= 0:
return True
return False
"""
meeting_start and v1_upload_timestamp shall be datetime objects
"""
meeting_start = meeting_start.replace(hour=0, minute=0, second=0) # paranoia
deadline = meeting_start - timedelta(days=7) # End of Sunday
diff = deadline - v1_upload_timestamp
if diff.total_seconds() <= 0:
return True
return False
def try_parsing_date(text):
'''
Try parsing the timestamp, if not possible return None
'''
for fmt in ('%Y-%m-%d %H:%M:%S', 'Y-%m-%d'):
try:
return datetime.strptime(text.strip(), fmt)
except ValueError:
pass
return None
"""
Try parsing the timestamp, if not possible return None
"""
for fmt in ('%Y-%m-%d %H:%M:%S', 'Y-%m-%d'):
try:
return datetime.strptime(text.strip(), fmt)
except ValueError:
pass
return None
def load_json_data(json_path):
'''
Load json file from json_path and return the data.
'''
with open(json_path, 'r', encoding='utf-8') as f:
data = json.load(f)
return data
"""
Load json file from json_path and return the data.
"""
with open(json_path, 'r', encoding='utf-8') as f:
data = json.load(f)
return data
def store_json_data(json_path, data):
'''
Store data as a json file to json_path. datetime objects are stored as strings.
'''
dir_name = os.path.dirname(json_path)
if not os.path.exists(dir_name) and len(dir_name) > 0:
os.makedirs(dir_name)
with open(json_path, 'w', encoding='utf-8') as f:
json.dump(data, f, ensure_ascii=False, indent=2, default=str)
"""
Store data as a json file to json_path. datetime objects are stored as strings.
"""
dir_name = os.path.dirname(json_path)
if not os.path.exists(dir_name) and len(dir_name) > 0:
os.makedirs(dir_name)
with open(json_path, 'w', encoding='utf-8') as f:
json.dump(data, f, ensure_ascii=False, indent=2, default=str)
def find_meeting(meetings, meeting_number):
'''
Find and return a meeting using the meeting_number. If meeting_number < 0 return the latest meeting.
'''
if len(meetings) == 0:
"""
Find and return a meeting using the meeting_number. If meeting_number < 0 return the latest meeting.
"""
if len(meetings) == 0:
return None
if meeting_number < 0:
return max(meetings, key=lambda x: x['number'])
for meeting in meetings:
if meeting['number'] == meeting_number:
return meeting
return None
if meeting_number < 0:
return max(meetings, key=lambda x:x['number'])
for meeting in meetings:
if meeting['number'] == meeting_number:
return meeting
return None
def find_document(documents, document_number):
striped_doc_nr = document_number.replace(' ', '').strip().lower()
for doc in documents:
if striped_doc_nr in doc['document']:
return doc
return None
def find_project(projects, url_or_path, path_root = 'MPEG/'):
'''
Search for gitlab project based on URL or path_with_namespace.
'''
if url_or_path is None:
striped_doc_nr = document_number.replace(' ', '').strip().lower()
for doc in documents:
if striped_doc_nr in doc['document']:
return doc
return None
striped_url_or_path = url_or_path.replace(' ', '').strip().strip('/')
for project in projects:
if striped_url_or_path == project['url']:
return project
path_ns = project['path_with_namespace'].lower()
if striped_url_or_path.lower() in path_ns and path_ns.startswith(path_root.lower()):
return project
return None
def find_project(projects, url_or_path, path_root='MPEG/'):
"""
Search for gitlab project based on URL or path_with_namespace.
"""
if url_or_path is None:
return None
striped_url_or_path = url_or_path.replace(' ', '').strip().strip('/')
for project in projects:
if striped_url_or_path == project['url']:
return project
path_ns = project['path_with_namespace'].lower()
if striped_url_or_path.lower() in path_ns and path_ns.startswith(path_root.lower()):
return project
return None
def find_issue(issues, document):
title_only_hit = None
metadata_hit = None
last_version = 0
for issue in issues:
if document['document'] in issue.title:
meta = get_issue_metadata(issue.description)
if meta == None:
title_only_hit = issue
else:
if int(meta['mdms_id']) == document['mdms_id']:
metadata_hit = issue
if len(meta['version']) > 0:
last_version = int(meta['version'])
else:
print('WARNING. We found a GitLab issue with the document number in the title and with metadata tag in description. But the metadata tag has wrong document id in it.')
return title_only_hit, metadata_hit, last_version
title_only_hit = None
metadata_hit = None
last_version = 0
for issue in issues:
if document['document'] in issue.title:
meta = get_issue_metadata(issue.description)
if meta is None:
title_only_hit = issue
else:
if int(meta['mdms_id']) == document['mdms_id']:
metadata_hit = issue
if len(meta['version']) > 0:
last_version = int(meta['version'])
else:
print('WARNING. We found a GitLab issue with the document number in the title and with metadata '
'tag in description. But the metadata tag has wrong document id in it.')
return title_only_hit, metadata_hit, last_version
def get_issue_metadata(description):
'''
Find and parse the metada from the description of the issue
'''
pattern = '[meta]: # ('
pos1 = description.find(pattern)
if pos1 < 0:
return None
pos2 = description.find(')', pos1 + len(pattern))
meta_str = description[pos1+len(pattern):pos2]
meta = meta_str.split(',')
if len(meta) != 4:
return None
return {'mdms_id': meta[0], 'document': meta[1], 'title': meta[2], 'version': meta[3]}
"""
Find and parse the metada from the description of the issue
"""
pattern = '[meta]: # ('
pos1 = description.find(pattern)
if pos1 < 0:
return None
pos2 = description.find(')', pos1 + len(pattern))
meta_str = description[pos1 + len(pattern):pos2]
meta = meta_str.split(',')
if len(meta) != 4:
return None
return {'mdms_id': meta[0], 'document': meta[1], 'title': meta[2], 'version': meta[3]}
def create_issue_metadata(document, details):
'''
Create a metadata tag
'''
version = ''
if len(details['documents']) > 0:
last_doc = max(details['documents'], key=lambda x:x['version'])
version = str(last_doc['version'])
title = document['title'].replace('(', '').replace(')', '').replace(',', '')