systems.py 25.9 KB
Newer Older
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
1
"""
2
3
Script for MPEG Systems Subgroups

4
Some of the things you can do with this script are listed below:
5

6
7
8
  1.  Open issues based on the information provided in a CSV file:
        e.g.: python systems.py -o --csv Contribs.csv
  2.  Open issues based on CLI options:
9
        e.g.: python systems.py -o -m 55958,55959,56121 -p http://...
10
  3.  Generate an output document based on the information provided in a CSV file:
11
12
13
14
        e.g.: python systems.py -d --csv Contribs.csv --template template.docx
  4.  Close issues based on the information provided in a CSV file:
        e.g.: python systems.py -c --csv Contribs.csv
  5.  Close issues based on CLI options:
15
        e.g.: python systems.py -c -m m55958,m55959,m56121 -p http://...
16
  6.  Print information about input documents on MDMS and GitLab:
17
        e.g.: python systems.py -l -m m55958,m55959,m56121 -p http://...
18

Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
19
NOTE 1: The script stores data from mdms and gitlab to ./data folder to minimize the number of
20
21
        requests to both systems and to impove the performance. You can use the -U option to update all data.
NOTE 2: You can use --test option to 'test run' the script. It will not create or close any issues on GitLab.
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
22
"""
23
24
25
26
27

import argparse
import csv
import os
import sys
28
from datetime import datetime
Dimitri Podborski's avatar
Dimitri Podborski committed
29
30
31
32

import requests

from automation import gitlab, helpers, mdms
33

Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
34
__version__ = '1.4'
35
36
37
38
39

DATA_PATH = './data'
GITLAB_PROJECTS_PATH = os.path.join(DATA_PATH, 'gitlab_projects.json')
GITLAB_USERS_PATH = os.path.join(DATA_PATH, 'gitlab_users.json')
MEETINGS_PATH = os.path.join(DATA_PATH, 'meetings.json')
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
40
41
SYSTEMS_GROUP_ID = 727  # GitLab Group ID for Systems Subgroup

Dimitri Podborski's avatar
Dimitri Podborski committed
42
43
44
PROJECTS_FF = ['isobmff', 'HEIF', 'NALuFF', 'FFConformanceRefSoft', 'rawvideo', 'Text',
               'eventmessage', 'General', 'DerivedVis', 'CENC', 'Metrics', 'PartialFF', 'MP4FF', 'Audio']

45

46
def download_url(url, save_path, chunk_size=128):
Dimitri Podborski's avatar
Dimitri Podborski committed
47
    r = requests.get(url, auth=(mdms.MPEG_LOGIN, mdms.MPEG_PWD), stream=True)
48
49
50
51
52
53
54
55
56
57
58
59
    with open(save_path, 'wb') as fd:
        for chunk in r.iter_content(chunk_size=chunk_size):
            fd.write(chunk)


def fetch_contributions(table_entries):
    print('\nDownload contributions')
    for entry in table_entries:
        path = os.path.join(DATA_PATH, 'contributions')
        document = entry['document']
        project = entry['project']
        if not document:
60
            print('WARNING: Document not found. Try updating the database (-U) or select another meeting (--meeting).')
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
            continue
        url = document['latest_version_url']
        if url is None:
            print(document['document'], 'Skip [no document]')
            continue
        if project is not None:
            path = os.path.join(path, project['path_with_namespace'])
        if not os.path.exists(path) and len(path) > 0:
            os.makedirs(path)
        folder, filename = os.path.split(url)
        filename = os.path.join(path, filename)
        print(document['document'], ' -> ', filename)
        download_url(url, filename)


Dimitri Podborski's avatar
Dimitri Podborski committed
76
def print_infos(table_entries, project_url, gitlab_projects):
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
77
78
79
80
81
    print('\nDump information')
    for entry in table_entries:
        document = entry['document']
        project = entry['project']
        if not document:
82
            print('WARNING: Document not found. Try updating the database (-U) or select another meeting (--meeting).')
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
            continue
        print('-' * 51 + '\n' + document['document'] + '\n' + '. ' * 25 + '.')
        print('MDMS metadata')
        details = mdms.get_document_details(document['mdms_id'])
        if details is None:
            print('  Skip', document['document'])
            continue
        print('  Title:', document['title'])
        if details['organizations']:
            print('  Organizations:', details['organizations'])
        authors = ''
        for author in document['authors']:
            authors += author['name']
            if author['email']:
                authors += ' (' + author['email'] + ')'
            authors += ', '
        print('  Authors:', authors)
        if len(details['documents']) > 0:
            last_doc = max(details['documents'], key=lambda x: x['version'])
Dimitri Podborski's avatar
Dimitri Podborski committed
102
103
            print('  Last version: version',
                  last_doc['version'], 'from', last_doc['timestamp'])
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
104
105
106
107
108
            print('  URL:', last_doc['path'])

        if project is not None:
            print('GitLab metadata')
            issues = gitlab.get_issues(project['id'])
Dimitri Podborski's avatar
Dimitri Podborski committed
109
110
            issue_with_title, issue_with_meta, meta_last_doc_version = helpers.find_issue(
                issues, document)
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
111
112
113
114
115
116
117
            if issue_with_title is not None:
                print('  User opened an issue:', issue_with_title.web_url)
                print('  Labels:', issue_with_title.labels)
                comments = issue_with_title.notes.list(lazy=True)
                print('  Comments:', len(comments))
                if len(comments) > 0:
                    last_comment = max(comments, key=lambda x: x.id)
Dimitri Podborski's avatar
Dimitri Podborski committed
118
119
                    print('  Last comment from',
                          last_comment.author['username'], 'at', last_comment.updated_at)
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
120
            if issue_with_meta is not None:
Dimitri Podborski's avatar
Dimitri Podborski committed
121
122
123
124
                print('  Automatically generated issue:',
                      issue_with_meta.web_url)
                print('  Last document version found in the meta tag:',
                      meta_last_doc_version)
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
125
126
127
128
129
                print('  Labels:', issue_with_meta.labels)
                comments = issue_with_meta.notes.list(lazy=True)
                print('  Comments:', len(comments))
                if len(comments) > 0:
                    last_comment = max(comments, key=lambda x: x.id)
Dimitri Podborski's avatar
Dimitri Podborski committed
130
131
132
133
134
135
136
137
138
139
140
141
                    print('  Last comment from',
                          last_comment.author['username'], 'at', last_comment.updated_at)
    project = helpers.find_project(gitlab_projects, project_url)
    if project is not None:
        print('Auto-generated contribution list')
        issues = gitlab.get_issues(project['id'])
        contribution_list = []
        for issue in issues:
            meta = helpers.get_issue_metadata(issue.description)
            if meta is not None:
                contribution_list.append(meta['document'])
        print(','.join(contribution_list))
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
142

143

144
def open_new_issue(project_id, document, test, meeting_start, gitlab_members):
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
145
146
147
148
149
150
    usernames = helpers.find_gitlab_users(gitlab_members, document)
    issue_title = helpers.create_issue_title(document)
    document_details = mdms.get_document_details(document['mdms_id'])
    if document_details is None:
        print(' No document details found. Skip', document['document'])
        return False
Dimitri Podborski's avatar
Dimitri Podborski committed
151
152
    issue_description = helpers.create_issue_description(
        document, document_details)
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
153
    if len(usernames) > 0:
Dimitri Podborski's avatar
Dimitri Podborski committed
154
155
        issue_description += '\n_for:_ ' + \
            ''.join('@' + str(u) + ', ' for u in usernames)
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
156
157
158
159
    issue_lables = []
    timestamp = datetime.now()
    if len(document_details['documents']) > 0:
        issue_lables.append(gitlab.Label.DocAvailable.value)
Dimitri Podborski's avatar
Dimitri Podborski committed
160
161
        first_doc = min(
            document_details['documents'], key=lambda x: x['version'])
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
162
163
164
165
166
        timestamp = first_doc['timestamp']
    if helpers.is_document_late(meeting_start, timestamp):
        issue_lables.append(gitlab.Label.Late.value)

    if not test:
Dimitri Podborski's avatar
Dimitri Podborski committed
167
168
169
170
        print(' * {}: Open issue with title "{}" | Lables={}'.format(
            document['document'], issue_title, issue_lables))
        gitlab.open_issue(project_id, issue_title,
                          issue_description, issue_lables)
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
171
172
173
174
        return True
    else:
        print(' * {}: Test open issue with title "{}" | Lables={}'.format(document['document'], issue_title,
                                                                          issue_lables))
Dimitri Podborski's avatar
Dimitri Podborski committed
175
    return False
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
176

177

Dimitri Podborski's avatar
Dimitri Podborski committed
178
def close_issue(issue, test, document):
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
179
    if not test:
Dimitri Podborski's avatar
Dimitri Podborski committed
180
181
        print(
            ' * {}: Close issue: {}'.format(document['document'], issue.web_url))
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
182
183
184
        gitlab.close_issue(issue)
        return True
    else:
Dimitri Podborski's avatar
Dimitri Podborski committed
185
186
        print(
            ' * {}: Test close issue: {}'.format(document['document'], issue.web_url))
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
187
188
    return False

Dimitri Podborski's avatar
Dimitri Podborski committed
189

190
def open_issues(table_entries, test, gitlab_members, meeting_start):
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
191
192
193
194
195
196
197
    print('\nOpen {} issues. TestMode={}'.format(len(table_entries), test))
    counter = 0
    for entry in table_entries:
        document = entry['document']
        project_id = entry['project']['id']

        issues = gitlab.get_issues(project_id)
Dimitri Podborski's avatar
Dimitri Podborski committed
198
199
        issue_with_title, issue_with_meta, meta_last_doc_version = helpers.find_issue(
            issues, document)
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
200
201

        if issue_with_title is None and issue_with_meta is None:
Dimitri Podborski's avatar
Dimitri Podborski committed
202
203
            was_opened = open_new_issue(
                project_id, document, test, meeting_start, gitlab_members)
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
204
205
206
207
208
209
210
211
212
213
214
            if was_opened:
                counter += 1
        elif issue_with_title is not None and issue_with_meta is None:
            print(
                ' * {}: ATTENTION Another issue with the same document number in the title was found on GitLab.'.format(
                    document['document']))
            print('   - Issue URL:', issue_with_title.web_url)
            print('   - Issue Title:', issue_with_title.title)
            print('   - Should we still open a new one?')
            user_input = input('   Type y or n: ')
            if 'y' in user_input:
Dimitri Podborski's avatar
Dimitri Podborski committed
215
216
                was_opened = open_new_issue(
                    project_id, document, test, meeting_start, gitlab_members)
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
217
218
219
                if was_opened:
                    counter += 1
            else:
Dimitri Podborski's avatar
Dimitri Podborski committed
220
221
                print(
                    ' * {} Skip "{}"'.format(document['document'], document['title']))
222
        else:
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
223
224
225
            if issue_with_title is not None:
                print('* {0}: ATTENTION We found multiple issues with "{0}" in the title. One with, and one without '
                      'metadata tag.'.format(document['document']))
Dimitri Podborski's avatar
Dimitri Podborski committed
226
227
                print('   - Issue without metadata tag:',
                      issue_with_title.web_url)
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
228
229
            document_details = mdms.get_document_details(document['mdms_id'])
            if document_details is None:
Dimitri Podborski's avatar
Dimitri Podborski committed
230
231
                print(
                    ' * {}: Skip. Could not get document details from MDMS.'.format(document['document']))
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
232
233
234
                continue
            last_doc_version = 0
            if len(document_details['documents']) > 0:
Dimitri Podborski's avatar
Dimitri Podborski committed
235
236
                last_doc = max(
                    document_details['documents'], key=lambda x: x['version'])
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
237
238
239
240
241
242
                last_doc_version = last_doc['version']
            if last_doc_version > meta_last_doc_version:
                print(' * {}: A new version of the document was added to MDMS '
                      'after the issue was opened on GitLab.'.format(document['document']))
                print('   - Issue URL:', issue_with_meta.web_url)
                print('   - Issue Title:', issue_with_meta.title)
Dimitri Podborski's avatar
Dimitri Podborski committed
243
244
                print(
                    '   - Should we update the metadata table and add the new version to it?')
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
245
246
247
248
                user_input = input('   Type y or n: ')
                if 'y' in user_input:
                    new_description = helpers.get_updated_issue_description(issue_with_meta.description, document,
                                                                            document_details)
Dimitri Podborski's avatar
Dimitri Podborski committed
249
250
                    if 'DocAvailable' not in issue_with_meta.labels:
                        issue_with_meta.labels.append('DocAvailable')
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
251
                    if not test:
Dimitri Podborski's avatar
Dimitri Podborski committed
252
253
                        print(
                            '   - Update issue description of {}.'.format(document['document']))
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
254
255
256
                        issue_with_meta.description = new_description
                        issue_with_meta.save()
                    else:
Dimitri Podborski's avatar
Dimitri Podborski committed
257
258
                        print(
                            '   - Test update issue description of {}.'.format(document['document']))
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
259
260
261
                else:
                    print('   - Skip "{}"'.format(document['title']))
            else:
Dimitri Podborski's avatar
Dimitri Podborski committed
262
263
                print(
                    ' * {}: No update required for "{}"'.format(document['document'], document['title']))
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
264
265
    print('Opened issues:', counter)

Dimitri Podborski's avatar
Dimitri Podborski committed
266

267
def close_issues(table_entries, test, force):
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
268
269
270
271
272
273
274
275
276
277
278
279
280
    print('\nClose {} issues. TestMode={}'.format(len(table_entries), test))
    counter = 0
    for entry in table_entries:
        document = entry['document']
        project_id = entry['project']['id']
        close_flag = entry['close']

        if force:
            close_flag = True
        if not close_flag:
            continue

        issues = gitlab.get_issues(project_id)
Dimitri Podborski's avatar
Dimitri Podborski committed
281
282
        issue_with_title, issue_with_meta, _meta_last_doc_version = helpers.find_issue(
            issues, document)
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
283
        if issue_with_meta is None and issue_with_title is None:
Dimitri Podborski's avatar
Dimitri Podborski committed
284
285
            print(
                ' * {}: No issue to close in: {}'.format(document['document'], entry['project']['url']))
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
        if issue_with_meta is not None:
            was_closed = close_issue(issue_with_meta, test, document)
            if was_closed:
                counter += 1
        if issue_with_title is not None:
            print(' * {}: ATTENTION User created issue with the same document number in the title '
                  'was found on GitLab.'.format(document['document']))
            print('   - Issue URL:', issue_with_title.web_url)
            print('   - Issue Title:', issue_with_title.title)
            print('   - Should we also close it?')
            user_input = input('   Type y or n: ')
            if 'y' in user_input:
                was_closed = close_issue(issue_with_title, test, document)
                if was_closed:
                    counter += 1
            else:
                print(' * Skip "{}"'.format(issue_with_title.title))
    print('Closed issues:', counter)

305

306
def create_output_doc(table_entries, output_path, template_path):
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
307
308
309
310
311
312
313
314
315
316
317
318
319
    print('\nCreate Output Document')
    # iterate over the CSV table and gather all the data
    projects = {}
    projects_data = {}
    for entry in table_entries:
        document = entry['document']
        print(' * Process document', document['document'], document['title'])
        details = mdms.get_document_details(document['mdms_id'])
        if details is None:
            print(' * Skip', document['document'])
            continue
        project = entry['project']
        issues = gitlab.get_issues(project['id'])
Dimitri Podborski's avatar
Dimitri Podborski committed
320
321
        issue_with_title, issue_with_meta, _meta_last_doc_version = helpers.find_issue(
            issues, document)
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
        if not project['id'] in projects:
            projects[project['id']] = project
        if project['id'] in projects_data:
            projects_data[project['id']].append(
                {'document': document, 'details': details, 'issue_meta': issue_with_meta,
                 'issue_title': issue_with_title})
        else:
            projects_data[project['id']] = [{'document': document, 'details': details, 'issue_meta': issue_with_meta,
                                             'issue_title': issue_with_title}]
    # now iterate over all projects and write the docment
    formatter = helpers.DocumentFormatter(template_path)
    for project_id in projects_data:
        formatter.add_project(projects[project_id])
        # formatter.add_paragraph()
        for contribution in projects_data[project_id]:
            formatter.add_contribution(contribution)
    # save the file
    print(' * Save output document:', output_path)
    formatter.save(output_path)

342
343

def parse_csv(csv_file, projects, docs):
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
    table_entries = []
    with open(csv_file, 'r', encoding='utf-8-sig') as f:
        sample = f.readline()
        f.seek(0)
        dialect = csv.Sniffer().sniff(sample)  # find out the delimeter type
        has_header = csv.Sniffer().has_header(sample)
        reader = csv.reader(f, dialect)
        if not has_header:
            print('Error: Input CSV file has no header.')
            sys.exit(-1)
        header = next(reader)
        doc_number_idx = None
        project_url_idx = None
        subgroup_idx = None
        project_name_idx = None
        close_idx = None
        for n in range(len(header)):
            if 'number' == header[n].replace(' ', '').strip().lower():
                doc_number_idx = n
            elif 'projecturl' == header[n].replace(' ', '').strip().lower():
                project_url_idx = n
            elif 'subgroup' == header[n].replace(' ', '').strip().lower():
                subgroup_idx = n
            elif 'projectname' == header[n].replace(' ', '').strip().lower():
                project_name_idx = n
            elif 'closeissue' == header[n].replace(' ', '').strip().lower():
                close_idx = n
        if doc_number_idx is None or not (
                (subgroup_idx is not None and project_name_idx is not None) or project_url_idx is not None):
            print('Error: CSV header should have, "number" and ("project url" or ("sub group" and "project name")) '
                  'fields')
            sys.exit(-1)

        for row in reader:
            # skip empty lines
            if len(' '.join(row).strip()) == 0:
                continue
381
            project_url = None
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
382
383
            if project_url_idx is not None:
                if len(row[project_url_idx]) > 0:
384
385
386
                    project_url = row[project_url_idx]
            if project_url is None:
                print('NOTE: No project URL found:', row)
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
387
388
389
390
391
                continue
            close_flag = False
            if close_idx is not None:
                if 'true' in row[close_idx].lower() or '1' == row[close_idx]:
                    close_flag = True
392
            project = helpers.find_project(projects, project_url)
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
393
394
395
396
397
398
399
400
401
402
403
404
405
            doc = helpers.find_document(docs, row[doc_number_idx])
            if not project:
                print('WARNING: No project on GitLab for:', row)
            elif not doc:
                print('WARNING: Document not found:', row)
            else:
                table_entries.append({
                    'project': project,
                    'document': doc,
                    'close': close_flag
                })
    return table_entries

406

407
def parse_cli(docs, project_url, close_flag, gitlab_projects, input_docs):
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
408
409
    table_entries = []
    docs = docs.replace('m', '').replace('M', '').strip().split(',')
410
    project = helpers.find_project(gitlab_projects, project_url)
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
411
    if not project:
Dimitri Podborski's avatar
Dimitri Podborski committed
412
413
        print(
            '   - Could not find a GitLab project with project URL "{}"'.format(project_url))
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
414
415
416
417
        return table_entries
    for doc in docs:
        document = helpers.find_document(input_docs, 'm' + doc)
        if not document:
418
            print('WARNING: Document "m{}" not found. Try updating the database (-U) or select another meeting ('
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
419
420
421
422
423
424
425
                  '--meeting).'.format(doc))
            continue
        table_entries.append({
            'project': project,
            'document': document,
            'close': close_flag
        })
426
    return table_entries
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
427
428


Dimitri Podborski's avatar
Dimitri Podborski committed
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
def derive_fileformat(gitlab_projects, input_docs):
    """return table_entries based on already opened issues in the FIleFormat group"""
    proj_urls = []
    table_entries = []
    for proj in PROJECTS_FF:
        proj_urls.append(os.path.join(
            'http://mpegx.int-evry.fr/software/MPEG/Systems/FileFormat', proj))

    for project_url in proj_urls:
        print(f'gather contributions from {project_url}')
        project = helpers.find_project(gitlab_projects, project_url)
        if project is not None:
            issues = gitlab.get_issues(project['id'])
            for issue in issues:
                meta = helpers.get_issue_metadata(issue.description)
                if meta is not None:
                    document = helpers.find_document(
                        input_docs, meta['document'])
                    if not document:
                        print(
                            f'WARNING: Document "{meta["document"]}" not found. Try updating the database (-U) or select another meeting (--meeting).')
                        continue
                    table_entries.append({
                        'project': project,
                        'document': document,
                        'close': False
                    })
    return table_entries


Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
459
460
461
462
463
464
465
466
467
468
469
470
471
472
def main():
    print('*' * 35)
    print('* MPEG Systems script version', __version__, '*')
    print('*' * 35 + '\n')
    # program options
    usage_examples = '''Examples:
      python systems.py -l -m m55958,m55959,m56121 -p FileFormat/CENC --meeting 133
      python systems.py -o -d --csv Contribs.csv
      python systems.py -c --csv Contribs.csv
      python systems.py -d -m m55958,m55959,m56121 -p FileFormat/CENC --meeting 133
    '''
    parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter,
                                     description='A tool for managing the GitLab issues for MPEG Systems Subgroups.',
                                     epilog=usage_examples)
Dimitri Podborski's avatar
Dimitri Podborski committed
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
    parser.add_argument(
        '-o', '--open', help='Mode: Open GitLab issues.', action='store_true')
    parser.add_argument(
        '-c', '--close', help='Mode: Close GitLab issues.', action='store_true')
    parser.add_argument(
        '-d', '--docx', help='Mode: Generate output word document.', action='store_true')
    parser.add_argument(
        '-l', '--list', help='Mode: List information about the contribution(s).', action='store_true')
    parser.add_argument(
        '-f', '--fetch', help='Mode: Download contributions.', action='store_true')
    parser.add_argument(
        '-C', '--CLOSE', help='Force closing GitLab issues.', action='store_true')
    parser.add_argument(
        '-U', '--UPDATE', help='Update all databases.', action='store_true')
    parser.add_argument(
        '-i', '--csv', help='Input CSV file. Header row shall include "Number" and "Project URL"')
    parser.add_argument(
        '-m', '--documents', help='Comma separated MDMS document number(s). e.g.: m12345,...', type=str)
    parser.add_argument(
        '--fileformat', help='Derive input from already opened issues in FileFormat group', action='store_true')
    parser.add_argument(
        '-p', '--project', help='GitLab project URL or "SubGroup/ProjectName".', type=str)
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
495
496
497
498
    parser.add_argument('--meeting', help='MPEG meeting number. If not set, the latest meeting is used.', default=-1,
                        type=int)
    parser.add_argument('-t', '--template', help='Document template path if you want to use your .docx template for '
                                                 'output document.', type=str)
Dimitri Podborski's avatar
Dimitri Podborski committed
499
500
    parser.add_argument(
        '--test', help='Test mode. If set, no issues will be opened or closed.', action='store_true')
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
501
502
    args = parser.parse_args()

503
    if not args.open and not args.docx and not args.close and not args.list and not args.fetch:
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
        print('There is no work to be done. Check the CLI options. Use -h for help.')
        sys.exit(1)
    # get gitlab projects (and update if needed)
    if not os.path.isfile(GITLAB_PROJECTS_PATH) or args.UPDATE:
        print(' * Update GitLab projects data')
        projects = gitlab.get_projects()
        helpers.store_json_data(GITLAB_PROJECTS_PATH, projects)
    gitlab_projects = helpers.load_json_data(GITLAB_PROJECTS_PATH)
    # get gitlab members (and update if needed)
    if not os.path.isfile(GITLAB_USERS_PATH) or args.UPDATE:
        print(' * Update GitLab users data')
        members = gitlab.get_members(SYSTEMS_GROUP_ID)
        helpers.store_json_data(GITLAB_USERS_PATH, members)
    gitlab_members = helpers.load_json_data(GITLAB_USERS_PATH)
    # get MPEG meetings (and update if needed)
    if not os.path.isfile(MEETINGS_PATH) or args.UPDATE:
        print(' * Update MPEG meetings data')
        meetings = mdms.get_meetings()
        if len(meetings) == 0:
            print(' * ERROR, could not get meetings from MDMS. Check your password.')
            sys.exit(1)
        helpers.store_json_data(MEETINGS_PATH, meetings)
    meetings = helpers.load_json_data(MEETINGS_PATH)
    # get MPEG meeting we want to work on
    meeting = helpers.find_meeting(meetings, args.meeting)
    if not meeting:
Dimitri Podborski's avatar
Dimitri Podborski committed
530
531
        print('Warning: Could not find meeting #{}. Update MDMS database and see if it was added'.format(
            args.meeting))
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
532
533
534
535
536
537
538
539
540
541
542
543
544
        meetings = mdms.get_meetings()
        if len(meetings) == 0:
            print(' * ERROR, could not get meetings from MDMS.')
            sys.exit(1)
        helpers.store_json_data(MEETINGS_PATH, meetings)
        meetings = helpers.load_json_data(MEETINGS_PATH)
        meeting = helpers.find_meeting(meetings, args.meeting)
        if not meeting:
            print('Error: No such meeting found on MDMS. Check your input!')
            sys.exit(-1)
    print(' * Operating on MPEG#{} ({}) from {} to {}'.format(meeting['number'], meeting['name'], meeting['start_date'],
                                                              meeting['end_date']))
    # get input documents (and update if needed)
Dimitri Podborski's avatar
Dimitri Podborski committed
545
546
    input_docs_path = os.path.join(
        DATA_PATH, 'input_docs_{}.json'.format(meeting['number']))
547
548
549
550
551
552
    print(' * Update MPEG input documents data for MPEG#', meeting['number'])
    input_docs = mdms.get_input_documents(meeting['id'])
    if len(input_docs) == 0:
        print(' * ERROR, could not get input documents from MDMS. Check your password.')
        sys.exit(1)
    helpers.store_json_data(input_docs_path, input_docs)
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
553
554
555
556
557
558
559

    output_path = 'output.docx'
    table_entries = []
    if args.csv is not None:
        table_entries = parse_csv(args.csv, gitlab_projects, input_docs)
        output_path = args.csv.replace('.csv', '') + '.docx'
    elif args.documents is not None:
Dimitri Podborski's avatar
Dimitri Podborski committed
560
561
562
563
        table_entries = parse_cli(
            args.documents, args.project, args.close, gitlab_projects, input_docs)
    elif args.fileformat:
        table_entries = derive_fileformat(gitlab_projects, input_docs)
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
564

Dimitri Podborski's avatar
Dimitri Podborski committed
565
    # do some work
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
566
    if args.list:
Dimitri Podborski's avatar
Dimitri Podborski committed
567
        print_infos(table_entries, args.project, gitlab_projects)
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
568
569
570
571
572
573
574
    if args.open:
        meeting_start = helpers.try_parsing_date(meeting['start_date'])
        open_issues(table_entries, args.test, gitlab_members, meeting_start)
    if args.close:
        close_issues(table_entries, args.test, args.CLOSE)
    if args.docx:
        create_output_doc(table_entries, output_path, args.template)
575
576
    if args.fetch:
        fetch_contributions(table_entries)
Dimitri Podborski's avatar
pip 8    
Dimitri Podborski committed
577
578
579
580


if __name__ == "__main__":
    main()