systems.py

"""
Script for MPEG Systems Subgroups

Some of the things you can do with this script are listed below:

  1.  Open issues based on the information provided in a CSV file:
        e.g.: python systems.py -o --csv Contribs.csv
  2.  Open issues based on CLI options:
        e.g.: python systems.py -o -m 55958,55959,56121 -p http://...
  3.  Generate an output document based on the information provided in a CSV file:
        e.g.: python systems.py -d --csv Contribs.csv --template template.docx
  4.  Close issues based on the information provided in a CSV file:
        e.g.: python systems.py -c --csv Contribs.csv
  5.  Close issues based on CLI options:
        e.g.: python systems.py -c -m m55958,m55959,m56121 -p http://...
  6.  Print information about input documents on MDMS and GitLab:
        e.g.: python systems.py -l -m m55958,m55959,m56121 -p http://...

NOTE 1: The script stores data from mdms and gitlab to ./data folder to minimize the number of
        requests to both systems and to impove the performance. You can use the -U option to update all data.
NOTE 2: You can use --test option to 'test run' the script. It will not create or close any issues on GitLab.
"""

import argparse
import csv
import os
import sys
from datetime import datetime

import requests

from automation import gitlab, helpers, mdms

__version__ = '1.4'

DATA_PATH = './data'
GITLAB_PROJECTS_PATH = os.path.join(DATA_PATH, 'gitlab_projects.json')
GITLAB_USERS_PATH = os.path.join(DATA_PATH, 'gitlab_users.json')
MEETINGS_PATH = os.path.join(DATA_PATH, 'meetings.json')
SYSTEMS_GROUP_ID = 727  # GitLab Group ID for Systems Subgroup

PROJECTS_FF = ['isobmff', 'HEIF', 'NALuFF', 'FFConformanceRefSoft', 'rawvideo', 'Text',
               'eventmessage', 'General', 'DerivedVis', 'CENC', 'Metrics', 'PartialFF', 'MP4FF', 'Audio']


def download_url(url, save_path, chunk_size=128):
    r = requests.get(url, auth=(mdms.MPEG_LOGIN, mdms.MPEG_PWD), stream=True)
    with open(save_path, 'wb') as fd:
        for chunk in r.iter_content(chunk_size=chunk_size):
            fd.write(chunk)


def fetch_contributions(table_entries):
    print('\nDownload contributions')
    for entry in table_entries:
        path = os.path.join(DATA_PATH, 'contributions')
        document = entry['document']
        project = entry['project']
        if not document:
            print('WARNING: Document not found. Try updating the database (-U) or select another meeting (--meeting).')
            continue
        url = document['latest_version_url']
        if url is None:
            print(document['document'], 'Skip [no document]')
            continue
        if project is not None:
            path = os.path.join(path, project['path_with_namespace'])
        if not os.path.exists(path) and len(path) > 0:
            os.makedirs(path)
        folder, filename = os.path.split(url)
        filename = os.path.join(path, filename)
        print(document['document'], ' -> ', filename)
        download_url(url, filename)


def print_infos(table_entries, project_url, gitlab_projects):
    print('\nDump information')
    for entry in table_entries:
        document = entry['document']
        project = entry['project']
        if not document:
            print('WARNING: Document not found. Try updating the database (-U) or select another meeting (--meeting).')
            continue
        print('-' * 51 + '\n' + document['document'] + '\n' + '. ' * 25 + '.')
        print('MDMS metadata')
        details = mdms.get_document_details(document['mdms_id'])
        if details is None:
            print('  Skip', document['document'])
            continue
        print('  Title:', document['title'])
        if details['organizations']:
            print('  Organizations:', details['organizations'])
        authors = ''
        for author in document['authors']:
            authors += author['name']
            if author['email']:
                authors += ' (' + author['email'] + ')'
            authors += ', '
        print('  Authors:', authors)
        if len(details['documents']) > 0:
            last_doc = max(details['documents'], key=lambda x: x['version'])
            print('  Last version: version',
                  last_doc['version'], 'from', last_doc['timestamp'])
            print('  URL:', last_doc['path'])

        if project is not None:
            print('GitLab metadata')
            issues = gitlab.get_issues(project['id'])
            issue_with_title, issue_with_meta, meta_last_doc_version = helpers.find_issue(
                issues, document)
            if issue_with_title is not None:
                print('  User opened an issue:', issue_with_title.web_url)
                print('  Labels:', issue_with_title.labels)
                comments = issue_with_title.notes.list(lazy=True)
                print('  Comments:', len(comments))
                if len(comments) > 0:
                    last_comment = max(comments, key=lambda x: x.id)
                    print('  Last comment from',
                          last_comment.author['username'], 'at', last_comment.updated_at)
            if issue_with_meta is not None:
                print('  Automatically generated issue:',
                      issue_with_meta.web_url)
                print('  Last document version found in the meta tag:',
                      meta_last_doc_version)
                print('  Labels:', issue_with_meta.labels)
                comments = issue_with_meta.notes.list(lazy=True)
                print('  Comments:', len(comments))
                if len(comments) > 0:
                    last_comment = max(comments, key=lambda x: x.id)
                    print('  Last comment from',
                          last_comment.author['username'], 'at', last_comment.updated_at)
    project = helpers.find_project(gitlab_projects, project_url)
    if project is not None:
        print('Auto-generated contribution list')
        issues = gitlab.get_issues(project['id'])
        contribution_list = []
        for issue in issues:
            meta = helpers.get_issue_metadata(issue.description)
            if meta is not None:
                contribution_list.append(meta['document'])
        print(','.join(contribution_list))


def open_new_issue(project_id, document, test, meeting_start, gitlab_members):
    usernames = helpers.find_gitlab_users(gitlab_members, document)
    issue_title = helpers.create_issue_title(document)
    document_details = mdms.get_document_details(document['mdms_id'])
    if document_details is None:
        print(' No document details found. Skip', document['document'])
        return False
    issue_description = helpers.create_issue_description(
        document, document_details)
    if len(usernames) > 0:
        issue_description += '\n_for:_ ' + \
            ''.join('@' + str(u) + ', ' for u in usernames)
    issue_lables = []
    timestamp = datetime.now()
    if len(document_details['documents']) > 0:
        issue_lables.append(gitlab.Label.DocAvailable.value)
        first_doc = min(
            document_details['documents'], key=lambda x: x['version'])
        timestamp = first_doc['timestamp']
    if helpers.is_document_late(meeting_start, timestamp):
        issue_lables.append(gitlab.Label.Late.value)

    if not test:
        print(' * {}: Open issue with title "{}" | Lables={}'.format(
            document['document'], issue_title, issue_lables))
        gitlab.open_issue(project_id, issue_title,
                          issue_description, issue_lables)
        return True
    else:
        print(' * {}: Test open issue with title "{}" | Lables={}'.format(document['document'], issue_title,
                                                                          issue_lables))
    return False


def close_issue(issue, test, document):
    if not test:
        print(
            ' * {}: Close issue: {}'.format(document['document'], issue.web_url))
        gitlab.close_issue(issue)
        return True
    else:
        print(
            ' * {}: Test close issue: {}'.format(document['document'], issue.web_url))
    return False


def open_issues(table_entries, test, gitlab_members, meeting_start):
    print('\nOpen {} issues. TestMode={}'.format(len(table_entries), test))
    counter = 0
    for entry in table_entries:
        document = entry['document']
        project_id = entry['project']['id']

        issues = gitlab.get_issues(project_id)
        issue_with_title, issue_with_meta, meta_last_doc_version = helpers.find_issue(
            issues, document)

        if issue_with_title is None and issue_with_meta is None:
            was_opened = open_new_issue(
                project_id, document, test, meeting_start, gitlab_members)
            if was_opened:
                counter += 1
        elif issue_with_title is not None and issue_with_meta is None:
            print(
                ' * {}: ATTENTION Another issue with the same document number in the title was found on GitLab.'.format(
                    document['document']))
            print('   - Issue URL:', issue_with_title.web_url)
            print('   - Issue Title:', issue_with_title.title)
            print('   - Should we still open a new one?')
            user_input = input('   Type y or n: ')
            if 'y' in user_input:
                was_opened = open_new_issue(
                    project_id, document, test, meeting_start, gitlab_members)
                if was_opened:
                    counter += 1
            else:
                print(
                    ' * {} Skip "{}"'.format(document['document'], document['title']))
        else:
            if issue_with_title is not None:
                print('* {0}: ATTENTION We found multiple issues with "{0}" in the title. One with, and one without '
                      'metadata tag.'.format(document['document']))
                print('   - Issue without metadata tag:',
                      issue_with_title.web_url)
            document_details = mdms.get_document_details(document['mdms_id'])
            if document_details is None:
                print(
                    ' * {}: Skip. Could not get document details from MDMS.'.format(document['document']))
                continue
            last_doc_version = 0
            if len(document_details['documents']) > 0:
                last_doc = max(
                    document_details['documents'], key=lambda x: x['version'])
                last_doc_version = last_doc['version']
            if last_doc_version > meta_last_doc_version:
                print(' * {}: A new version of the document was added to MDMS '
                      'after the issue was opened on GitLab.'.format(document['document']))
                print('   - Issue URL:', issue_with_meta.web_url)
                print('   - Issue Title:', issue_with_meta.title)
                print(
                    '   - Should we update the metadata table and add the new version to it?')
                user_input = input('   Type y or n: ')
                if 'y' in user_input:
                    new_description = helpers.get_updated_issue_description(issue_with_meta.description, document,
                                                                            document_details)
                    if 'DocAvailable' not in issue_with_meta.labels:
                        issue_with_meta.labels.append('DocAvailable')
                    if not test:
                        print(
                            '   - Update issue description of {}.'.format(document['document']))
                        issue_with_meta.description = new_description
                        issue_with_meta.save()
                    else:
                        print(
                            '   - Test update issue description of {}.'.format(document['document']))
                else:
                    print('   - Skip "{}"'.format(document['title']))
            else:
                print(
                    ' * {}: No update required for "{}"'.format(document['document'], document['title']))
    print('Opened issues:', counter)


def close_issues(table_entries, test, force):
    print('\nClose {} issues. TestMode={}'.format(len(table_entries), test))
    counter = 0
    for entry in table_entries:
        document = entry['document']
        project_id = entry['project']['id']
        close_flag = entry['close']

        if force:
            close_flag = True
        if not close_flag:
            continue

        issues = gitlab.get_issues(project_id)
        issue_with_title, issue_with_meta, _meta_last_doc_version = helpers.find_issue(
            issues, document)
        if issue_with_meta is None and issue_with_title is None:
            print(
                ' * {}: No issue to close in: {}'.format(document['document'], entry['project']['url']))
        if issue_with_meta is not None:
            was_closed = close_issue(issue_with_meta, test, document)
            if was_closed:
                counter += 1
        if issue_with_title is not None:
            print(' * {}: ATTENTION User created issue with the same document number in the title '
                  'was found on GitLab.'.format(document['document']))
            print('   - Issue URL:', issue_with_title.web_url)
            print('   - Issue Title:', issue_with_title.title)
            print('   - Should we also close it?')
            user_input = input('   Type y or n: ')
            if 'y' in user_input:
                was_closed = close_issue(issue_with_title, test, document)
                if was_closed:
                    counter += 1
            else:
                print(' * Skip "{}"'.format(issue_with_title.title))
    print('Closed issues:', counter)


def create_output_doc(table_entries, output_path, template_path):
    print('\nCreate Output Document')
    # iterate over the CSV table and gather all the data
    projects = {}
    projects_data = {}
    for entry in table_entries:
        document = entry['document']
        print(' * Process document', document['document'], document['title'])
        details = mdms.get_document_details(document['mdms_id'])
        if details is None:
            print(' * Skip', document['document'])
            continue
        project = entry['project']
        issues = gitlab.get_issues(project['id'])
        issue_with_title, issue_with_meta, _meta_last_doc_version = helpers.find_issue(
            issues, document)
        if not project['id'] in projects:
            projects[project['id']] = project
        if project['id'] in projects_data:
            projects_data[project['id']].append(
                {'document': document, 'details': details, 'issue_meta': issue_with_meta,
                 'issue_title': issue_with_title})
        else:
            projects_data[project['id']] = [{'document': document, 'details': details, 'issue_meta': issue_with_meta,
                                             'issue_title': issue_with_title}]
    # now iterate over all projects and write the docment
    formatter = helpers.DocumentFormatter(template_path)
    for project_id in projects_data:
        formatter.add_project(projects[project_id])
        # formatter.add_paragraph()
        for contribution in projects_data[project_id]:
            formatter.add_contribution(contribution)
    # save the file
    print(' * Save output document:', output_path)
    formatter.save(output_path)


def parse_csv(csv_file, projects, docs):
    table_entries = []
    with open(csv_file, 'r', encoding='utf-8-sig') as f:
        sample = f.readline()
        f.seek(0)
        dialect = csv.Sniffer().sniff(sample)  # find out the delimeter type
        has_header = csv.Sniffer().has_header(sample)
        reader = csv.reader(f, dialect)
        if not has_header:
            print('Error: Input CSV file has no header.')
            sys.exit(-1)
        header = next(reader)
        doc_number_idx = None
        project_url_idx = None
        subgroup_idx = None
        project_name_idx = None
        close_idx = None
        for n in range(len(header)):
            if 'number' == header[n].replace(' ', '').strip().lower():
                doc_number_idx = n
            elif 'projecturl' == header[n].replace(' ', '').strip().lower():
                project_url_idx = n
            elif 'subgroup' == header[n].replace(' ', '').strip().lower():
                subgroup_idx = n
            elif 'projectname' == header[n].replace(' ', '').strip().lower():
                project_name_idx = n
            elif 'closeissue' == header[n].replace(' ', '').strip().lower():
                close_idx = n
        if doc_number_idx is None or not (
                (subgroup_idx is not None and project_name_idx is not None) or project_url_idx is not None):
            print('Error: CSV header should have, "number" and ("project url" or ("sub group" and "project name")) '
                  'fields')
            sys.exit(-1)

        for row in reader:
            # skip empty lines
            if len(' '.join(row).strip()) == 0:
                continue
            project_url = None
            if project_url_idx is not None:
                if len(row[project_url_idx]) > 0:
                    project_url = row[project_url_idx]
            if project_url is None:
                print('NOTE: No project URL found:', row)
                continue
            close_flag = False
            if close_idx is not None:
                if 'true' in row[close_idx].lower() or '1' == row[close_idx]:
                    close_flag = True
            project = helpers.find_project(projects, project_url)
            doc = helpers.find_document(docs, row[doc_number_idx])
            if not project:
                print('WARNING: No project on GitLab for:', row)
            elif not doc:
                print('WARNING: Document not found:', row)
            else:
                table_entries.append({
                    'project': project,
                    'document': doc,
                    'close': close_flag
                })
    return table_entries


def parse_cli(docs, project_url, close_flag, gitlab_projects, input_docs):
    table_entries = []
    docs = docs.replace('m', '').replace('M', '').strip().split(',')
    project = helpers.find_project(gitlab_projects, project_url)
    if not project:
        print(
            '   - Could not find a GitLab project with project URL "{}"'.format(project_url))
        return table_entries
    for doc in docs:
        document = helpers.find_document(input_docs, 'm' + doc)
        if not document:
            print('WARNING: Document "m{}" not found. Try updating the database (-U) or select another meeting ('
                  '--meeting).'.format(doc))
            continue
        table_entries.append({
            'project': project,
            'document': document,
            'close': close_flag
        })
    return table_entries


def derive_fileformat(gitlab_projects, input_docs):
    """return table_entries based on already opened issues in the FIleFormat group"""
    proj_urls = []
    table_entries = []
    for proj in PROJECTS_FF:
        proj_urls.append(os.path.join(
            'http://mpegx.int-evry.fr/software/MPEG/Systems/FileFormat', proj))

    for project_url in proj_urls:
        print(f'gather contributions from {project_url}')
        project = helpers.find_project(gitlab_projects, project_url)
        if project is not None:
            issues = gitlab.get_issues(project['id'])
            for issue in issues:
                meta = helpers.get_issue_metadata(issue.description)
                if meta is not None:
                    document = helpers.find_document(
                        input_docs, meta['document'])
                    if not document:
                        print(
                            f'WARNING: Document "{meta["document"]}" not found. Try updating the database (-U) or select another meeting (--meeting).')
                        continue
                    table_entries.append({
                        'project': project,
                        'document': document,
                        'close': False
                    })
    return table_entries


def main():
    print('*' * 35)
    print('* MPEG Systems script version', __version__, '*')
    print('*' * 35 + '\n')
    # program options
    usage_examples = '''Examples:
      python systems.py -l -m m55958,m55959,m56121 -p FileFormat/CENC --meeting 133
      python systems.py -o -d --csv Contribs.csv
      python systems.py -c --csv Contribs.csv
      python systems.py -d -m m55958,m55959,m56121 -p FileFormat/CENC --meeting 133
    '''
    parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter,
                                     description='A tool for managing the GitLab issues for MPEG Systems Subgroups.',
                                     epilog=usage_examples)
    parser.add_argument(
        '-o', '--open', help='Mode: Open GitLab issues.', action='store_true')
    parser.add_argument(
        '-c', '--close', help='Mode: Close GitLab issues.', action='store_true')
    parser.add_argument(
        '-d', '--docx', help='Mode: Generate output word document.', action='store_true')
    parser.add_argument(
        '-l', '--list', help='Mode: List information about the contribution(s).', action='store_true')
    parser.add_argument(
        '-f', '--fetch', help='Mode: Download contributions.', action='store_true')
    parser.add_argument(
        '-C', '--CLOSE', help='Force closing GitLab issues.', action='store_true')
    parser.add_argument(
        '-U', '--UPDATE', help='Update all databases.', action='store_true')
    parser.add_argument(
        '-i', '--csv', help='Input CSV file. Header row shall include "Number" and "Project URL"')
    parser.add_argument(
        '-m', '--documents', help='Comma separated MDMS document number(s). e.g.: m12345,...', type=str)
    parser.add_argument(
        '--fileformat', help='Derive input from already opened issues in FileFormat group', action='store_true')
    parser.add_argument(
        '-p', '--project', help='GitLab project URL or "SubGroup/ProjectName".', type=str)
    parser.add_argument('--meeting', help='MPEG meeting number. If not set, the latest meeting is used.', default=-1,
                        type=int)
    parser.add_argument('-t', '--template', help='Document template path if you want to use your .docx template for '
                                                 'output document.', type=str)
    parser.add_argument(
        '--test', help='Test mode. If set, no issues will be opened or closed.', action='store_true')
    args = parser.parse_args()

    if not args.open and not args.docx and not args.close and not args.list and not args.fetch:
        print('There is no work to be done. Check the CLI options. Use -h for help.')
        sys.exit(1)
    # get gitlab projects (and update if needed)
    if not os.path.isfile(GITLAB_PROJECTS_PATH) or args.UPDATE:
        print(' * Update GitLab projects data')
        projects = gitlab.get_projects()
        helpers.store_json_data(GITLAB_PROJECTS_PATH, projects)
    gitlab_projects = helpers.load_json_data(GITLAB_PROJECTS_PATH)
    # get gitlab members (and update if needed)
    if not os.path.isfile(GITLAB_USERS_PATH) or args.UPDATE:
        print(' * Update GitLab users data')
        members = gitlab.get_members(SYSTEMS_GROUP_ID)
        helpers.store_json_data(GITLAB_USERS_PATH, members)
    gitlab_members = helpers.load_json_data(GITLAB_USERS_PATH)
    # get MPEG meetings (and update if needed)
    if not os.path.isfile(MEETINGS_PATH) or args.UPDATE:
        print(' * Update MPEG meetings data')
        meetings = mdms.get_meetings()
        if len(meetings) == 0:
            print(' * ERROR, could not get meetings from MDMS. Check your password.')
            sys.exit(1)
        helpers.store_json_data(MEETINGS_PATH, meetings)
    meetings = helpers.load_json_data(MEETINGS_PATH)
    # get MPEG meeting we want to work on
    meeting = helpers.find_meeting(meetings, args.meeting)
    if not meeting:
        print('Warning: Could not find meeting #{}. Update MDMS database and see if it was added'.format(
            args.meeting))
        meetings = mdms.get_meetings()
        if len(meetings) == 0:
            print(' * ERROR, could not get meetings from MDMS.')
            sys.exit(1)
        helpers.store_json_data(MEETINGS_PATH, meetings)
        meetings = helpers.load_json_data(MEETINGS_PATH)
        meeting = helpers.find_meeting(meetings, args.meeting)
        if not meeting:
            print('Error: No such meeting found on MDMS. Check your input!')
            sys.exit(-1)
    print(' * Operating on MPEG#{} ({}) from {} to {}'.format(meeting['number'], meeting['name'], meeting['start_date'],
                                                              meeting['end_date']))
    # get input documents (and update if needed)
    input_docs_path = os.path.join(
        DATA_PATH, 'input_docs_{}.json'.format(meeting['number']))
    print(' * Update MPEG input documents data for MPEG#', meeting['number'])
    input_docs = mdms.get_input_documents(meeting['id'])
    if len(input_docs) == 0:
        print(' * ERROR, could not get input documents from MDMS. Check your password.')
        sys.exit(1)
    helpers.store_json_data(input_docs_path, input_docs)

    output_path = 'output.docx'
    table_entries = []
    if args.csv is not None:
        table_entries = parse_csv(args.csv, gitlab_projects, input_docs)
        output_path = args.csv.replace('.csv', '') + '.docx'
    elif args.documents is not None:
        table_entries = parse_cli(
            args.documents, args.project, args.close, gitlab_projects, input_docs)
    elif args.fileformat:
        table_entries = derive_fileformat(gitlab_projects, input_docs)

    # do some work
    if args.list:
        print_infos(table_entries, args.project, gitlab_projects)
    if args.open:
        meeting_start = helpers.try_parsing_date(meeting['start_date'])
        open_issues(table_entries, args.test, gitlab_members, meeting_start)
    if args.close:
        close_issues(table_entries, args.test, args.CLOSE)
    if args.docx:
        create_output_doc(table_entries, output_path, args.template)
    if args.fetch:
        fetch_contributions(table_entries)


if __name__ == "__main__":
    main()