Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in / Register
Toggle navigation
Menu
Open sidebar
Dimitri Podborski
AutomationTools
Commits
ee3a80d7
Commit
ee3a80d7
authored
Jan 08, 2021
by
Dimitri Podborski
Browse files
initial commit for the automation script
parent
e682b053
Changes
1
Hide whitespace changes
Inline
Side-by-side
mpegAutomation.py
0 → 100644
View file @
ee3a80d7
#! /usr/local/bin/python3
# coding: utf-8
"""
Script for managing the GitLab issues for MPEG
To work properly this script requires following:
1. GITLAB_TOKEN environment variable to be set to your private token.
Create your token here: http://mpegx.int-evry.fr/software/profile/personal_access_tokens
2. GitLabI API module: pip install --upgrade python-gitlab
3. CSV module: pip install csv
4. curl installed on your system
5. Set the following environment variables: MPEG_LOGIN, MPEG_PWD
"""
__author__
=
"Dimitri Podborski"
__version__
=
"0.0.1"
import
os
import
sys
import
argparse
import
subprocess
import
gitlab
from
bs4
import
BeautifulSoup
# constants
TOKEN
=
os
.
environ
.
get
(
'GITLAB_TOKEN'
)
BASEURL
=
"https://dms.mpeg.expert/"
FIND_CONTRIBUTION_URL
=
"https://dms.mpeg.expert/doc_end_user/searchAcross.php"
GET_CONTRIBUTION_URL
=
"https://dms.mpeg.expert/doc_end_user/current_document.php"
# TODO: let the user input it, or search it
PROJECT_ID
=
272
# this ID is my private repo for testing. You can figure out your ID by running list_all_projects
# TODO: this (and other metadata) should be passed from the CSV file as input
contributions
=
[
'55686'
,
'56083'
]
def
command_to_string
(
command
):
cmd
=
subprocess
.
Popen
(
command
,
stdout
=
subprocess
.
PIPE
,
shell
=
True
)
cmd_out
,
cmd_err
=
cmd
.
communicate
()
if
not
cmd_err
==
None
:
print
(
"ERROR while executing {}
\n\n
stderr:
\n
{}"
.
format
(
command
,
cmd_err
))
sys
.
exit
(
-
1
)
return
cmd_out
.
decode
(
"utf-8"
)
# TODO: improve this, to be more resilient to other layout (in case it changes one day)
def
get_table_entry
(
alltext
,
docNr
):
# title, autors, org, abstract, related, ahg, subGr, gr, standard, activity, doc, docNr
entries
=
[]
soup
=
BeautifulSoup
(
alltext
)
table
=
soup
.
select_one
(
"table"
)
headers
=
[
th
.
text
.
encode
(
"utf-8"
)
for
th
in
table
.
select
(
"tr th"
)]
n
=
0
for
row
in
table
.
select
(
"tr + tr"
):
n
+=
1
tds
=
row
.
find_all
(
"td"
)
if
len
(
tds
)
!=
3
:
break
entries
.
append
(
tds
[
1
].
text
.
encode
(
"utf-8"
).
decode
(
"utf-8"
).
strip
())
entries
.
append
(
'm'
+
docNr
)
return
entries
def
get_projects
():
projects
=
gl
.
projects
.
list
(
all
=
True
)
print
(
"{} projects found"
.
format
(
len
(
projects
)))
return
projects
def
list_all_projects
():
projects
=
get_projects
()
for
project
in
projects
:
print
(
"ID {}:
\t\"
{}
\"\t
Descr.: {}"
.
format
(
project
.
id
,
project
.
name
,
project
.
description
))
def
get_issues
(
project_id
):
project
=
gl
.
projects
.
get
(
project_id
)
issues
=
project
.
issues
.
list
()
for
issue
in
issues
:
print
(
''
)
print
(
issue
)
print
(
issue
.
labels
)
return
issues
def
open_issue
(
project_id
,
title
,
description
,
labels
=
[]):
project
=
gl
.
projects
.
get
(
project_id
)
issue
=
project
.
issues
.
create
({
'title'
:
title
,
'description'
:
description
,
'labels'
:
labels
})
issue
.
save
()
parser
=
argparse
.
ArgumentParser
(
formatter_class
=
argparse
.
RawTextHelpFormatter
,
description
=
'A tool for managing the GitLab issues for MPEG.
\n\n
'
'Description TBD:
\n
'
' ...
\n
'
' ...'
)
parser
.
add_argument
(
'-u'
,
'--mpeguser'
,
default
=
'sc29wg11'
,
help
=
'MPEG user name'
)
parser
.
add_argument
(
'-p'
,
'--mpegpwd'
,
help
=
'MPEG password'
)
# TODO: let the use also type it in
parser
.
add_argument
(
'-l'
,
'--listProjects'
,
help
=
'Print a list of all GitLab projects'
)
args
=
parser
.
parse_args
()
if
not
TOKEN
:
print
(
"No private token found, please set the GITLAB_TOKEN environment variable"
)
sys
.
exit
(
1
)
# private token authentication
gl
=
gitlab
.
Gitlab
(
'http://mpegx.int-evry.fr/software'
,
private_token
=
TOKEN
)
# get entries for each contribution
entries
=
[]
for
contrNr
in
contributions
:
searchCmd
=
"curl -s -X POST -d 'search_title=&search_number="
+
contrNr
+
"&search_category=&search_author=&search_id_group=1&search_sub_group=1&id_meeting=&submit=Search&meeting=' -u $MPEG_LOGIN:$MPEG_PWD https://dms.mpeg.expert/doc_end_user/searchAcross.php"
alltext
=
command_to_string
(
searchCmd
)
# get document ID: NOTE: there is probably a better way to do it, than making a two requests each time
pattern
=
"current_document.php?id="
pos1
=
alltext
.
find
(
pattern
)
pos2
=
alltext
.
find
(
'&'
,
pos1
+
len
(
pattern
)
+
1
)
docID
=
alltext
[
pos1
+
len
(
pattern
):
pos2
]
getCmd
=
"curl -s -X POST -d 'id="
+
docID
+
"&id_meeting=' -u $MPEG_LOGIN:$MPEG_PWD https://dms.mpeg.expert/doc_end_user/current_document.php"
alltext
=
command_to_string
(
getCmd
)
entry
=
get_table_entry
(
alltext
,
contrNr
)
# TODO: rewrite this dirty hack
findPattern
=
'href="../'
startUrl
=
alltext
.
rfind
(
findPattern
)
endUrl
=
alltext
.
find
(
'">'
,
startUrl
+
len
(
findPattern
))
contribUrl
=
BASEURL
+
alltext
[
startUrl
+
len
(
findPattern
):
endUrl
]
if
'doc_end_user/documents'
not
in
contribUrl
and
'.zip'
not
in
contribUrl
:
contribUrl
=
None
entry
.
append
(
contribUrl
)
print
(
"get metadata for {}"
.
format
(
contrNr
))
entries
.
append
(
entry
)
print
(
"
\n\n\n
"
)
# TODO: move it to its own method
n
=
0
for
line
in
entries
:
title
=
line
[
12
]
+
' '
+
line
[
0
]
authors
=
line
[
1
]
org
=
line
[
2
]
abstract
=
line
[
3
]
doc
=
line
[
10
]
docNr
=
line
[
12
]
docURL
=
line
[
13
]
n
+=
1
description
=
"{}
\n\n
{}
\n\n
Authors:
\n\n
{}"
.
format
(
abstract
,
org
,
authors
)
labels
=
[]
if
docURL
:
description
+=
"
\n\n
[{}]({})"
.
format
(
docNr
,
docURL
)
labels
.
append
(
'DocAvailable'
)
description
+=
"
\n\n
_automatically generated issue_"
print
(
"open issue {}: {}"
.
format
(
n
,
title
))
open_issue
(
PROJECT_ID
,
title
,
description
,
labels
)
\ No newline at end of file
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment