initial commit for the automation script

mpegAutomation.py

+#! /usr/local/bin/python3
+# coding: utf-8
+""" 
+  Script for managing the GitLab issues for MPEG
+
+  To work properly this script requires following:
+  1.  GITLAB_TOKEN environment variable to be set to your private token.
+      Create your token here: http://mpegx.int-evry.fr/software/profile/personal_access_tokens
+  2.  GitLabI API module: pip install --upgrade python-gitlab
+  3.  CSV module: pip install csv
+  4.  curl installed on your system
+  5.  Set the following environment variables: MPEG_LOGIN, MPEG_PWD
+"""
+
+__author__ = "Dimitri Podborski"
+__version__ = "0.0.1"
+
+import os
+import sys
+import argparse
+import subprocess
+import gitlab
+from bs4 import BeautifulSoup
+
+# constants
+TOKEN = os.environ.get('GITLAB_TOKEN')
+BASEURL = "https://dms.mpeg.expert/"
+FIND_CONTRIBUTION_URL = "https://dms.mpeg.expert/doc_end_user/searchAcross.php"
+GET_CONTRIBUTION_URL = "https://dms.mpeg.expert/doc_end_user/current_document.php"
+# TODO: let the user input it, or search it
+PROJECT_ID = 272 # this ID is my private repo for testing. You can figure out your ID by running list_all_projects
+
+# TODO: this (and other metadata) should be passed from the CSV file as input
+contributions = ['55686', '56083']
+
+def command_to_string(command):
+  cmd = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True)
+  cmd_out, cmd_err = cmd.communicate()
+  if not cmd_err == None:
+    print("ERROR while executing {}\n\nstderr:\n{}".format(command, cmd_err))
+    sys.exit(-1)
+  return cmd_out.decode("utf-8") 
+
+# TODO: improve this, to be more resilient to other layout (in case it changes one day)
+def get_table_entry(alltext, docNr):
+  # title, autors, org, abstract, related, ahg, subGr, gr, standard, activity, doc, docNr
+  entries = []
+  soup = BeautifulSoup(alltext)
+  table = soup.select_one("table")
+  headers = [th.text.encode("utf-8") for th in table.select("tr th")]
+  n = 0
+  for row in table.select("tr + tr"):
+    n += 1
+    tds = row.find_all("td")
+    if len(tds) != 3:
+      break
+    entries.append(tds[1].text.encode("utf-8").decode("utf-8").strip())
+  entries.append('m' + docNr)
+  return entries
+
+def get_projects():
+    projects = gl.projects.list(all=True)
+    print("{} projects found".format(len(projects)))
+    return projects
+
+def list_all_projects():
+    projects = get_projects()
+    for project in projects:
+        print("ID {}:\t\"{}\"\tDescr.: {}".format(project.id, project.name, project.description))
+
+def get_issues(project_id):
+    project = gl.projects.get(project_id)
+    issues = project.issues.list()
+    for issue in issues:
+        print('')
+        print(issue)
+        print(issue.labels)
+    return issues
+
+def open_issue(project_id, title, description, labels=[]):
+    project = gl.projects.get(project_id)
+    issue = project.issues.create({'title': title, 'description': description, 'labels': labels})
+    issue.save()
+
+
+parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, 
+  description='A tool for managing the GitLab issues for MPEG.\n\n'
+              'Description TBD:\n'
+              '  ...\n'
+              '  ...')
+parser.add_argument('-u', '--mpeguser', default='sc29wg11', help='MPEG user name')
+parser.add_argument('-p', '--mpegpwd', help='MPEG password') # TODO: let the use also type it in
+parser.add_argument('-l', '--listProjects', help='Print a list of all GitLab projects')
+args = parser.parse_args()
+
+if not TOKEN:
+  print("No private token found, please set the GITLAB_TOKEN environment variable")
+  sys.exit(1)
+
+# private token authentication
+gl = gitlab.Gitlab('http://mpegx.int-evry.fr/software', private_token=TOKEN)
+
+# get entries for each contribution
+entries = []
+for contrNr in contributions:
+  searchCmd = "curl -s -X POST -d 'search_title=&search_number=" + contrNr + "&search_category=&search_author=&search_id_group=1&search_sub_group=1&id_meeting=&submit=Search&meeting=' -u $MPEG_LOGIN:$MPEG_PWD https://dms.mpeg.expert/doc_end_user/searchAcross.php"
+  
+  alltext = command_to_string(searchCmd)
+  
+  # get document ID: NOTE: there is probably a better way to do it, than making a two requests each time
+  pattern = "current_document.php?id="
+  pos1 = alltext.find(pattern)
+  pos2 = alltext.find('&', pos1+len(pattern)+1)
+  docID = alltext[pos1+len(pattern):pos2]
+
+  getCmd = "curl -s -X POST -d 'id=" + docID + "&id_meeting=' -u $MPEG_LOGIN:$MPEG_PWD https://dms.mpeg.expert/doc_end_user/current_document.php"
+
+  alltext = command_to_string(getCmd)
+  entry = get_table_entry(alltext, contrNr)
+
+  # TODO: rewrite this dirty hack
+  findPattern = 'href="../'
+  startUrl = alltext.rfind(findPattern)
+  endUrl = alltext.find('">', startUrl + len(findPattern))
+  contribUrl = BASEURL + alltext[startUrl+len(findPattern): endUrl]
+  if 'doc_end_user/documents' not in contribUrl and '.zip' not in contribUrl:
+    contribUrl = None
+  entry.append(contribUrl)
+
+  print("get metadata for {}".format(contrNr))
+  entries.append(entry)
+
+print("\n\n\n")
+
+
+
+
+
+# TODO: move it to its own method 
+n = 0
+for line in entries:
+  title = line[12] + ' ' + line[0]
+  authors = line[1]
+  org = line[2]
+  abstract = line[3]
+  doc = line[10]
+  docNr = line[12]
+  docURL = line[13]
+  n += 1
+
+  description = "{}\n\n{}\n\nAuthors:\n\n{}".format(abstract, org, authors)
+
+  labels=[]
+  if docURL:
+    description += "\n\n[{}]({})".format(docNr, docURL)
+    labels.append('DocAvailable')
+      
+  description += "\n\n_automatically generated issue_"
+
+  print("open issue {}: {}".format(n, title))
+  open_issue(PROJECT_ID, title, description, labels)
\ No newline at end of file