Commit ee3a80d7 authored by Dimitri Podborski's avatar Dimitri Podborski
Browse files

initial commit for the automation script

parent e682b053
#! /usr/local/bin/python3
# coding: utf-8
"""
Script for managing the GitLab issues for MPEG
To work properly this script requires following:
1. GITLAB_TOKEN environment variable to be set to your private token.
Create your token here: http://mpegx.int-evry.fr/software/profile/personal_access_tokens
2. GitLabI API module: pip install --upgrade python-gitlab
3. CSV module: pip install csv
4. curl installed on your system
5. Set the following environment variables: MPEG_LOGIN, MPEG_PWD
"""
__author__ = "Dimitri Podborski"
__version__ = "0.0.1"
import os
import sys
import argparse
import subprocess
import gitlab
from bs4 import BeautifulSoup
# constants
TOKEN = os.environ.get('GITLAB_TOKEN')
BASEURL = "https://dms.mpeg.expert/"
FIND_CONTRIBUTION_URL = "https://dms.mpeg.expert/doc_end_user/searchAcross.php"
GET_CONTRIBUTION_URL = "https://dms.mpeg.expert/doc_end_user/current_document.php"
# TODO: let the user input it, or search it
PROJECT_ID = 272 # this ID is my private repo for testing. You can figure out your ID by running list_all_projects
# TODO: this (and other metadata) should be passed from the CSV file as input
contributions = ['55686', '56083']
def command_to_string(command):
cmd = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True)
cmd_out, cmd_err = cmd.communicate()
if not cmd_err == None:
print("ERROR while executing {}\n\nstderr:\n{}".format(command, cmd_err))
sys.exit(-1)
return cmd_out.decode("utf-8")
# TODO: improve this, to be more resilient to other layout (in case it changes one day)
def get_table_entry(alltext, docNr):
# title, autors, org, abstract, related, ahg, subGr, gr, standard, activity, doc, docNr
entries = []
soup = BeautifulSoup(alltext)
table = soup.select_one("table")
headers = [th.text.encode("utf-8") for th in table.select("tr th")]
n = 0
for row in table.select("tr + tr"):
n += 1
tds = row.find_all("td")
if len(tds) != 3:
break
entries.append(tds[1].text.encode("utf-8").decode("utf-8").strip())
entries.append('m' + docNr)
return entries
def get_projects():
projects = gl.projects.list(all=True)
print("{} projects found".format(len(projects)))
return projects
def list_all_projects():
projects = get_projects()
for project in projects:
print("ID {}:\t\"{}\"\tDescr.: {}".format(project.id, project.name, project.description))
def get_issues(project_id):
project = gl.projects.get(project_id)
issues = project.issues.list()
for issue in issues:
print('')
print(issue)
print(issue.labels)
return issues
def open_issue(project_id, title, description, labels=[]):
project = gl.projects.get(project_id)
issue = project.issues.create({'title': title, 'description': description, 'labels': labels})
issue.save()
parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter,
description='A tool for managing the GitLab issues for MPEG.\n\n'
'Description TBD:\n'
' ...\n'
' ...')
parser.add_argument('-u', '--mpeguser', default='sc29wg11', help='MPEG user name')
parser.add_argument('-p', '--mpegpwd', help='MPEG password') # TODO: let the use also type it in
parser.add_argument('-l', '--listProjects', help='Print a list of all GitLab projects')
args = parser.parse_args()
if not TOKEN:
print("No private token found, please set the GITLAB_TOKEN environment variable")
sys.exit(1)
# private token authentication
gl = gitlab.Gitlab('http://mpegx.int-evry.fr/software', private_token=TOKEN)
# get entries for each contribution
entries = []
for contrNr in contributions:
searchCmd = "curl -s -X POST -d 'search_title=&search_number=" + contrNr + "&search_category=&search_author=&search_id_group=1&search_sub_group=1&id_meeting=&submit=Search&meeting=' -u $MPEG_LOGIN:$MPEG_PWD https://dms.mpeg.expert/doc_end_user/searchAcross.php"
alltext = command_to_string(searchCmd)
# get document ID: NOTE: there is probably a better way to do it, than making a two requests each time
pattern = "current_document.php?id="
pos1 = alltext.find(pattern)
pos2 = alltext.find('&', pos1+len(pattern)+1)
docID = alltext[pos1+len(pattern):pos2]
getCmd = "curl -s -X POST -d 'id=" + docID + "&id_meeting=' -u $MPEG_LOGIN:$MPEG_PWD https://dms.mpeg.expert/doc_end_user/current_document.php"
alltext = command_to_string(getCmd)
entry = get_table_entry(alltext, contrNr)
# TODO: rewrite this dirty hack
findPattern = 'href="../'
startUrl = alltext.rfind(findPattern)
endUrl = alltext.find('">', startUrl + len(findPattern))
contribUrl = BASEURL + alltext[startUrl+len(findPattern): endUrl]
if 'doc_end_user/documents' not in contribUrl and '.zip' not in contribUrl:
contribUrl = None
entry.append(contribUrl)
print("get metadata for {}".format(contrNr))
entries.append(entry)
print("\n\n\n")
# TODO: move it to its own method
n = 0
for line in entries:
title = line[12] + ' ' + line[0]
authors = line[1]
org = line[2]
abstract = line[3]
doc = line[10]
docNr = line[12]
docURL = line[13]
n += 1
description = "{}\n\n{}\n\nAuthors:\n\n{}".format(abstract, org, authors)
labels=[]
if docURL:
description += "\n\n[{}]({})".format(docNr, docURL)
labels.append('DocAvailable')
description += "\n\n_automatically generated issue_"
print("open issue {}: {}".format(n, title))
open_issue(PROJECT_ID, title, description, labels)
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment