Commit 5ce89157 authored by Dimitri Podborski's avatar Dimitri Podborski
Browse files

Open,Close,Create output doc without CSV file

add the possibility to open/close/create output document based on CLI
options only (no CSV)
parent 36e3030b
...@@ -45,6 +45,8 @@ deactivate ...@@ -45,6 +45,8 @@ deactivate
e.g.: `python systems.py -m m55958,m55959,m56121 -p FileFormat/CENC --meeting 133` e.g.: `python systems.py -m m55958,m55959,m56121 -p FileFormat/CENC --meeting 133`
2. Open issues based on the information provided in a CSV file: 2. Open issues based on the information provided in a CSV file:
e.g.: `python systems.py --csv Contribs.csv -o` e.g.: `python systems.py --csv Contribs.csv -o`
3. Open issues based on the CLI options:
e.g.: `python systems.py -m m56345,m56346 -p http://mpegx.int-evry.fr/software/podborski/test`
3. Generate an output document based on the information provided in a CSV file. Use a template as a basis (`--template` is optional): 3. Generate an output document based on the information provided in a CSV file. Use a template as a basis (`--template` is optional):
e.g.: `python systems.py --csv Contribs.csv -d --template templates/WG03_input_template_dimitri.docx` e.g.: `python systems.py --csv Contribs.csv -d --template templates/WG03_input_template_dimitri.docx`
4. Close issues basd on the information provided in a CSV file: 4. Close issues basd on the information provided in a CSV file:
......
...@@ -245,7 +245,7 @@ def create_issue_description(document, details): ...@@ -245,7 +245,7 @@ def create_issue_description(document, details):
description += details['abstract'] description += details['abstract']
else: else:
description += '* [ ] please **add your abstract here**.\n' description += '* [ ] please **add your abstract here**.\n'
description += '* [ ] please also **add your abstract to MDMS** (this will be used when we create the output document).\n' description += '* [ ] please also **add your abstract to MDMS** (this can be used when we create the output document).\n'
description += '\n\n_automatically generated issue_' description += '\n\n_automatically generated issue_'
return description return description
......
...@@ -23,7 +23,7 @@ import os ...@@ -23,7 +23,7 @@ import os
import sys import sys
from automation import gitlab, mdms, helpers from automation import gitlab, mdms, helpers
__version__ = '1.2' __version__ = '1.3'
DATA_PATH = './data' DATA_PATH = './data'
GITLAB_PROJECTS_PATH = os.path.join(DATA_PATH, 'gitlab_projects.json') GITLAB_PROJECTS_PATH = os.path.join(DATA_PATH, 'gitlab_projects.json')
...@@ -41,24 +41,23 @@ def print_infos(docs, project_url_or_path, gitlab_projects, input_docs): ...@@ -41,24 +41,23 @@ def print_infos(docs, project_url_or_path, gitlab_projects, input_docs):
if not document: if not document:
print(' Document not found. Try updating the database (-u) or select another meeting (--meeting).') print(' Document not found. Try updating the database (-u) or select another meeting (--meeting).')
continue continue
else: details = mdms.get_document_details(document['mdms_id'])
details = mdms.get_document_details(document['mdms_id']) if details is None:
if details is None: print(' Skip', document['document'])
print(' Skip', document['document']) continue
continue print(' Title:', document['title'])
print(' Title:', document['title']) if details['organizations']:
if details['organizations']: print(' Organizations:', details['organizations'])
print(' Organizations:', details['organizations']) authors = ''
authors = '' for author in document['authors']:
for author in document['authors']: authors += author['name']
authors += author['name'] if author['email']:
if author['email']: authors += ' (' + author['email'] + ')'
authors += ' (' + author['email'] + ')' authors += ', '
authors += ', ' print(' Authors:', authors)
print(' Authors:', authors) if len(details['documents']) > 0:
if len(details['documents']) > 0: last_doc = max(details['documents'], key=lambda x:x['version'])
last_doc = max(details['documents'], key=lambda x:x['version']) print(' Last version: version', last_doc['version'], 'from', last_doc['timestamp'])
print(' Last version: version', last_doc['version'], 'from', last_doc['timestamp'])
if project_url_or_path: if project_url_or_path:
print('GitLab metadata') print('GitLab metadata')
project = helpers.find_project(gitlab_projects, project_url_or_path, 'MPEG/Systems') project = helpers.find_project(gitlab_projects, project_url_or_path, 'MPEG/Systems')
...@@ -85,7 +84,7 @@ def print_infos(docs, project_url_or_path, gitlab_projects, input_docs): ...@@ -85,7 +84,7 @@ def print_infos(docs, project_url_or_path, gitlab_projects, input_docs):
last_comment = max(comments, key=lambda x:x.id) last_comment = max(comments, key=lambda x:x.id)
print(' Last comment from', last_comment.author['username'], 'at', last_comment.updated_at) print(' Last comment from', last_comment.author['username'], 'at', last_comment.updated_at)
def open_new_issue(project_id, document, test): def open_new_issue(project_id, document, test, meeting_start):
issue_title = helpers.create_issue_title(document) issue_title = helpers.create_issue_title(document)
document_details = mdms.get_document_details(document['mdms_id']) document_details = mdms.get_document_details(document['mdms_id'])
if document_details is None: if document_details is None:
...@@ -114,9 +113,8 @@ def close_issue(issue, test): ...@@ -114,9 +113,8 @@ def close_issue(issue, test):
else: else:
print(' * Test close issue:', issue.web_url) print(' * Test close issue:', issue.web_url)
def open_issues(csv_path, test, gitlab_projects, input_docs, gitlab_members, meeting_start): def open_issues(table_entries, test, gitlab_members, meeting_start):
print(' * Open issues. TestMode =', test) print(' * Open issues. TestMode =', test)
table_entries = parse_csv(args.csv, gitlab_projects, input_docs)
counter = 0 counter = 0
for entry in table_entries: for entry in table_entries:
document = entry['document'] document = entry['document']
...@@ -127,7 +125,7 @@ def open_issues(csv_path, test, gitlab_projects, input_docs, gitlab_members, mee ...@@ -127,7 +125,7 @@ def open_issues(csv_path, test, gitlab_projects, input_docs, gitlab_members, mee
# TODO find gitlab_members and add them to the issue description, this will automatically notify the contributors # TODO find gitlab_members and add them to the issue description, this will automatically notify the contributors
if issue_with_title is None and issue_with_meta is None: if issue_with_title is None and issue_with_meta is None:
open_new_issue(project_id, document, test) open_new_issue(project_id, document, test, meeting_start)
counter += 1 counter += 1
elif issue_with_title is not None and issue_with_meta is None: elif issue_with_title is not None and issue_with_meta is None:
print(' * *** ATTENTION *** Issue with the same document number in the title aleady found on GitLab.') print(' * *** ATTENTION *** Issue with the same document number in the title aleady found on GitLab.')
...@@ -135,7 +133,7 @@ def open_issues(csv_path, test, gitlab_projects, input_docs, gitlab_members, mee ...@@ -135,7 +133,7 @@ def open_issues(csv_path, test, gitlab_projects, input_docs, gitlab_members, mee
print(' - Should we still open a new one?') print(' - Should we still open a new one?')
user_input = input('Type y or n: ') user_input = input('Type y or n: ')
if 'y' in user_input: if 'y' in user_input:
open_new_issue(project_id, document, test) open_new_issue(project_id, document, test, meeting_start)
counter += 1 counter += 1
else: else:
print(' * Skip', document['document'], document['title']) print(' * Skip', document['document'], document['title'])
...@@ -169,9 +167,8 @@ def open_issues(csv_path, test, gitlab_projects, input_docs, gitlab_members, mee ...@@ -169,9 +167,8 @@ def open_issues(csv_path, test, gitlab_projects, input_docs, gitlab_members, mee
print(' * No update required for', document['document'], document['title']) print(' * No update required for', document['document'], document['title'])
print(' * Opened issues:', counter) print(' * Opened issues:', counter)
def close_issues(csv_path, test, gitlab_projects, input_docs): def close_issues(table_entries, test):
print(' * Close issues. TestMode =', test) print(' * Close issues. TestMode =', test)
table_entries = parse_csv(args.csv, gitlab_projects, input_docs)
counter = 0 counter = 0
for entry in table_entries: for entry in table_entries:
document = entry['document'] document = entry['document']
...@@ -200,10 +197,9 @@ def close_issues(csv_path, test, gitlab_projects, input_docs): ...@@ -200,10 +197,9 @@ def close_issues(csv_path, test, gitlab_projects, input_docs):
print(' * Skip', document['document'], document['title']) print(' * Skip', document['document'], document['title'])
print(' * Closed issues:', counter) print(' * Closed issues:', counter)
def create_output_doc(csv_path, gitlab_projects, input_docs, template_path): def create_output_doc(table_entries, template_path):
print(' * Create Output Document') print(' * Create Output Document')
# iterate over the CSV table and gather all the data # iterate over the CSV table and gather all the data
table_entries = parse_csv(args.csv, gitlab_projects, input_docs)
projects = {} projects = {}
projects_data = {} projects_data = {}
for entry in table_entries: for entry in table_entries:
...@@ -235,7 +231,7 @@ def create_output_doc(csv_path, gitlab_projects, input_docs, template_path): ...@@ -235,7 +231,7 @@ def create_output_doc(csv_path, gitlab_projects, input_docs, template_path):
formatter.save(output_path) formatter.save(output_path)
def parse_csv(csv_file, projects, docs): def parse_csv(csv_file, projects, docs):
issues = [] table_entries = []
with open(csv_file, 'r', encoding='utf-8-sig') as f: with open(csv_file, 'r', encoding='utf-8-sig') as f:
sample = f.readline() sample = f.readline()
f.seek(0) f.seek(0)
...@@ -293,12 +289,31 @@ def parse_csv(csv_file, projects, docs): ...@@ -293,12 +289,31 @@ def parse_csv(csv_file, projects, docs):
elif not doc: elif not doc:
print('WARNING: Document not found:', row) print('WARNING: Document not found:', row)
else: else:
issues.append({ table_entries.append({
'project': project, 'project': project,
'document': doc, 'document': doc,
'close': close_flag 'close': close_flag
}) })
return issues return table_entries
def parse_cli(docs, project_url_or_path, close_flag, gitlab_projects, input_docs):
table_entries = []
docs = docs.replace('m','').replace('M','').strip().split(',')
project = helpers.find_project(gitlab_projects, project_url_or_path, 'MPEG/Systems')
if not project:
print(' Could not find a GitLab project with', project_url_or_path)
return table_entries
for doc in docs:
document = helpers.find_document(input_docs, 'm'+doc)
if not document:
print(' Document not found. Try updating the database (-u) or select another meeting (--meeting).')
continue
table_entries.append({
'project': project,
'document': document,
'close': close_flag
})
return table_entries
print('*'*35) print('*'*35)
print('* MPEG Systems script version', __version__, '*') print('* MPEG Systems script version', __version__, '*')
...@@ -324,14 +339,15 @@ parser.add_argument('--test', help='Test mode. If set, no issues will be opened. ...@@ -324,14 +339,15 @@ parser.add_argument('--test', help='Test mode. If set, no issues will be opened.
parser.add_argument('-m', help='Documents (comma separated values). List information about the contribution(s).', type=str) parser.add_argument('-m', help='Documents (comma separated values). List information about the contribution(s).', type=str)
parser.add_argument('-p', help='GitLab project URL or "SubGroup/ProjectName". This option is used with -m option.', type=str) parser.add_argument('-p', help='GitLab project URL or "SubGroup/ProjectName". This option is used with -m option.', type=str)
args = parser.parse_args() args = parser.parse_args()
# check command line options
if not (args.csv is not None and (args.o or args.d or args.c) or args.m): print("args.o",args.o)
print('Wrong arguments. Use option -h for help.') print("args.d",args.d)
parser.print_usage() print("args.c",args.c)
sys.exit(-1) print("")
if (args.o or args.d or args.c) and args.csv is None: print("args.csv",args.csv)
print('Wrong arguments. Options -o, -d and -c require --csv option. Use option -h for help.') print("args.m",args.m)
sys.exit(-1) print("args.p",args.p)
# get gitlab projects (and update if needed) # get gitlab projects (and update if needed)
if not os.path.isfile(GITLAB_PROJECTS_PATH) or args.U: if not os.path.isfile(GITLAB_PROJECTS_PATH) or args.U:
print(' * Update GitLab projects data') print(' * Update GitLab projects data')
...@@ -379,13 +395,20 @@ if not os.path.isfile(input_docs_path) or args.U or args.u: ...@@ -379,13 +395,20 @@ if not os.path.isfile(input_docs_path) or args.U or args.u:
helpers.store_json_data(input_docs_path, input_docs) helpers.store_json_data(input_docs_path, input_docs)
input_docs = helpers.load_json_data(input_docs_path) input_docs = helpers.load_json_data(input_docs_path)
table_entries = []
if args.csv is not None:
table_entries = parse_csv(args.csv, gitlab_projects, input_docs)
elif args.m is not None and args.p is not None:
table_entries = parse_cli(args.m, args.p, args.c, gitlab_projects, input_docs)
# do some action # do some action
if args.m: if args.m is not None and not (args.o or args.d or args.c):
print_infos(args.m, args.p, gitlab_projects, input_docs) print_infos(args.m, args.p, gitlab_projects, input_docs)
if args.o: if args.o:
meeting_start = helpers.try_parsing_date(meeting['start_date']) meeting_start = helpers.try_parsing_date(meeting['start_date'])
open_issues(args.csv, args.test, gitlab_projects, input_docs, gitlab_members, meeting_start) open_issues(table_entries, args.test, gitlab_members, meeting_start)
if args.d: if args.d:
create_output_doc(args.csv, gitlab_projects, input_docs, args.template) create_output_doc(table_entries, args.template)
if args.c: if args.c:
close_issues(args.csv, args.test, gitlab_projects, input_docs) close_issues(table_entries, args.test)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment