Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in / Register
Toggle navigation
Menu
Open sidebar
Dimitri Podborski
AutomationTools
Commits
b96339fb
Commit
b96339fb
authored
Feb 12, 2021
by
Dimitri Podborski
Browse files
bugfix in parse meetings
parent
0b82940e
Changes
3
Hide whitespace changes
Inline
Side-by-side
automation/helpers.py
View file @
b96339fb
...
...
@@ -140,6 +140,8 @@ def find_meeting(meetings, meeting_number):
'''
Find and return a meeting using the meeting_number. If meeting_number < 0 return the latest meeting.
'''
if
len
(
meetings
)
==
0
:
return
None
if
meeting_number
<
0
:
return
max
(
meetings
,
key
=
lambda
x
:
x
[
'number'
])
for
meeting
in
meetings
:
...
...
automation/mdms.py
View file @
b96339fb
...
...
@@ -41,9 +41,12 @@ class MDMSParser:
cols
=
rows
[
n
].
find_all
(
'td'
,
recursive
=
False
)
try
:
last_input
=
None
last_output
=
None
if
len
(
cols
[
5
].
string
)
>
0
:
last_output
=
int
(
cols
[
5
].
string
)
if
len
(
cols
[
4
].
text
)
>
0
:
last_input
=
int
(
cols
[
4
].
text
)
if
len
(
cols
[
5
].
text
)
>
0
:
last_output
=
int
(
cols
[
5
].
text
)
parsed_href
=
urlparse
(
cols
[
1
].
a
[
'href'
])
meeting_id
=
int
(
parse_qs
(
parsed_href
.
query
)[
'id_meeting'
][
0
])
...
...
@@ -54,7 +57,7 @@ class MDMSParser:
'name'
:
cols
[
1
].
text
.
strip
(),
'start_date'
:
datetime
.
strptime
(
cols
[
2
].
text
,
'%Y-%m-%d'
),
'end_date'
:
datetime
.
strptime
(
cols
[
3
].
text
,
'%Y-%m-%d'
),
'last_input'
:
int
(
cols
[
4
].
text
)
,
'last_input'
:
last_input
,
'last_output'
:
last_output
})
except
:
...
...
@@ -392,6 +395,8 @@ def get_current_meeting():
{ 'number', 'id', 'name', 'start_date', 'end_date', 'last_input', 'last_output' }
'''
meetings
=
get_meetings
()
if
len
(
meetings
)
==
0
:
return
None
return
max
(
meetings
,
key
=
lambda
x
:
x
[
'number'
])
def
get_input_documents
(
meeting_id
,
standard
=
Standard
.
ALL
,
subgroup
=
Subgroup
.
ALL
):
...
...
@@ -421,7 +426,7 @@ def get_document_details(document_id):
if
not
response
.
status_code
==
200
:
print
(
'HTTP response {} != 200'
.
format
(
response
.
status_code
))
print
(
'
\t
{}'
.
format
(
response
.
text
.
replace
(
'
\n
'
,
'
\n\t
'
)))
return
[]
return
None
parser
=
MDMSParser
()
details
=
parser
.
parse_document_details
(
response
.
text
)
for
n
in
range
(
len
(
details
[
'documents'
])):
...
...
systems.py
View file @
b96339fb
...
...
@@ -23,7 +23,7 @@ import os
import
sys
from
automation
import
gitlab
,
mdms
,
helpers
__version__
=
'1.
0
'
__version__
=
'1.
1
'
DATA_PATH
=
'./data'
GITLAB_PROJECTS_PATH
=
os
.
path
.
join
(
DATA_PATH
,
'gitlab_projects.json'
)
...
...
@@ -42,6 +42,9 @@ def print_infos(docs, project_url_or_path, gitlab_projects, input_docs):
print
(
' Document not found in the provided meeting. Try updating the database (-u option).'
)
else
:
details
=
mdms
.
get_document_details
(
document
[
'mdms_id'
])
if
details
is
None
:
print
(
' Skip'
,
document
[
'document'
])
continue
print
(
' Title:'
,
document
[
'title'
])
if
details
[
'organizations'
]:
print
(
' Organizations:'
,
details
[
'organizations'
])
...
...
@@ -84,6 +87,9 @@ def print_infos(docs, project_url_or_path, gitlab_projects, input_docs):
def
open_new_issue
(
project_id
,
document
,
test
):
issue_title
=
helpers
.
create_issue_title
(
document
)
document_details
=
mdms
.
get_document_details
(
document
[
'mdms_id'
])
if
document_details
is
None
:
print
(
' Skip'
,
document
[
'document'
])
return
issue_description
=
helpers
.
create_issue_description
(
document
,
document_details
)
issue_lables
=
[]
if
len
(
document_details
[
'documents'
])
>
0
:
...
...
@@ -127,6 +133,9 @@ def open_issues(csv_path, test, gitlab_projects, input_docs, gitlab_members, mee
print
(
' * *** ATTENTION *** We found multiple issues with "{}" in the title. One with and one without metadata tag.'
.
format
(
document
[
'document'
]))
print
(
' - Issue without metadata tag:'
,
issue_with_title
.
web_url
)
document_details
=
mdms
.
get_document_details
(
document
[
'mdms_id'
])
if
document_details
is
None
:
print
(
' * Skip'
,
document
[
'document'
])
continue
last_doc_version
=
0
if
len
(
document_details
[
'documents'
])
>
0
:
last_doc
=
max
(
document_details
[
'documents'
],
key
=
lambda
x
:
x
[
'version'
])
...
...
@@ -158,6 +167,9 @@ def create_output_doc(csv_path, gitlab_projects, input_docs, template_path):
document
=
entry
[
'document'
]
print
(
' * Process document'
,
document
[
'document'
],
document
[
'title'
])
details
=
mdms
.
get_document_details
(
document
[
'mdms_id'
])
if
details
is
None
:
print
(
' * Skip'
,
document
[
'document'
])
continue
project
=
entry
[
'project'
]
issues
=
gitlab
.
get_issues
(
project
[
'id'
])
issue_with_title
,
issue_with_meta
,
meta_last_doc_version
=
helpers
.
find_issue
(
issues
,
document
)
...
...
@@ -224,9 +236,9 @@ def parse_csv(csv_file, projects, docs):
project
=
helpers
.
find_project
(
projects
,
project_url_or_path
)
doc
=
helpers
.
find_document
(
docs
,
row
[
doc_number_idx
])
if
not
project
:
print
(
'WARNING: No
such
project on GitLab
! Could not parse row
:'
,
row
)
print
(
'WARNING: No project on GitLab
for
:'
,
row
)
elif
not
doc
:
print
(
'WARNING:
No such document on MDMS! Could not parse row
:'
,
row
)
print
(
'WARNING:
Document not found
:'
,
row
)
else
:
issues
.
append
({
'project'
:
project
,
...
...
@@ -281,6 +293,9 @@ gitlab_members = helpers.load_json_data(GITLAB_USERS_PATH)
if
not
os
.
path
.
isfile
(
MEETINGS_PATH
)
or
args
.
U
:
print
(
' * Update MPEG meetings data'
)
meetings
=
mdms
.
get_meetings
()
if
len
(
meetings
)
==
0
:
print
(
' * ERROR, could not get meetings from MDMS. Check your password.'
)
sys
.
exit
(
1
)
helpers
.
store_json_data
(
MEETINGS_PATH
,
meetings
)
meetings
=
helpers
.
load_json_data
(
MEETINGS_PATH
)
# get MPEG meeting we want to work on
...
...
@@ -288,6 +303,9 @@ meeting = helpers.find_meeting(meetings, args.meeting)
if
not
meeting
:
print
(
'Warning: Could not find meeting #{}. Update MDMS database and see if it was added'
.
format
(
args
.
meeting
))
meetings
=
mdms
.
get_meetings
()
if
len
(
meetings
)
==
0
:
print
(
' * ERROR, could not get meetings from MDMS.'
)
sys
.
exit
(
1
)
helpers
.
store_json_data
(
MEETINGS_PATH
,
meetings
)
meetings
=
helpers
.
load_json_data
(
MEETINGS_PATH
)
meeting
=
helpers
.
find_meeting
(
meetings
,
args
.
meeting
)
...
...
@@ -300,6 +318,9 @@ input_docs_path = os.path.join(DATA_PATH, 'input_docs_{}.json'.format(meeting['n
if
not
os
.
path
.
isfile
(
input_docs_path
)
or
args
.
U
or
args
.
u
:
print
(
' * Update MPEG input documents data for MPEG#'
,
meeting
[
'number'
])
input_docs
=
mdms
.
get_input_documents
(
meeting
[
'id'
])
if
len
(
input_docs
)
==
0
:
print
(
' * ERROR, could not get input documents from MDMS. Check your password.'
)
sys
.
exit
(
1
)
helpers
.
store_json_data
(
input_docs_path
,
input_docs
)
input_docs
=
helpers
.
load_json_data
(
input_docs_path
)
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment