add close issue option

we also change CLI options. -c - close issues, -o - open issues, -d
create output document

README.md

@@ -32,7 +32,7 @@ source venv/bin/activate
 pip install -r requirements.txt
 
 (run your scripts)
-(e.g.: python systems.py --csv Contribs.csv -o ... )
+(e.g.: python systems.py --csv Contribs.csv -d ... )
 
 deactivate
 ```
@@ -44,18 +44,21 @@ deactivate
 1. Print information about input documents on MDMS and GitLab (`-p` is optional):  
 e.g.: `python systems.py -m m55958,m55959,m56121 -p FileFormat/CENC --meeting 133`
 2. Open issues based on the information provided in a CSV file:  
-e.g.: `python systems.py --csv Contribs.csv -i`
+e.g.: `python systems.py --csv Contribs.csv -o`
 3. Generate an output document based on the information provided in a CSV file. Use a template as a basis (`--template` is optional):  
-e.g.: `python systems.py --csv Contribs.csv -o --template templates/WG03_input_template_dimitri.docx`
+e.g.: `python systems.py --csv Contribs.csv -d --template templates/WG03_input_template_dimitri.docx`
+4. Close issues basd on the information provided in a CSV file:  
+e.g.: `python systems.py --csv Contribs.csv -c`
 
 The CSV file must have a header row with the folowing entries:
 
-- `Number` - MPEG document number e.g. m12345
+- **Number** - MPEG document number with entries like `m12345`
 - To determine which GitLab project needs to be used make sure that your CSV file has either:
-  - `Project URL` - a full URL to your GitLab project (**recommended**)
-  - or `Sub Group` and `Project Name` - two last elements of the Project URL. (non-case-sensitive)  
+  - **Project URL** - a full URL to your GitLab project (*recommended*)
+  - or **Sub Group** and **Project Name** - two last elements of the Project URL. (non-case-sensitive)  
   e.g.: http://mpegx.int-evry.fr/software/MPEG/Systems/PCC-SYS/V-PCC →  `Sub Group = PCC-SYS` and `Project Name = V-PCC`.  
   e.g.: http://mpegx.int-evry.fr/software/MPEG/Systems/FileFormat/isobmff →  `Sub Group = FileFormat` and `Project Name = ISOBMFF`
+- **Close issue** - is required if you want to close multiple issues at once. Supported values are `0`, `1`, `TRUE`, `FALSE`, `true` and `false`.
 
 The example CSV below has both `Project URL` and (`Sub Group` with `Project Name`) but you can also have one of these in your CSV. The CSV delimiter is determined automatically.
 

automation/gitlab.py

@@ -100,3 +100,8 @@ def open_issue(project_id, title, description, labels=[]):
     return
   issue = project.issues.create({'title': title, 'description': description, 'labels': labels})
   issue.save()
+
+def close_issue(issue):
+  if isinstance(issue, gitlab.v4.objects.ProjectIssue):
+    issue.state_event = 'close'
+    issue.save()

systems.py

@@ -6,9 +6,9 @@ This script allows you to do the following things:
   1.  Print information about input documents on MDMS and GitLab (optional):
         e.g.: python systems.py -m m55958,m55959,m56121 -p FileFormat/CENC
   2.  Open issues based on the information provided in a CSV file:
-        e.g.: python systems.py --csv Contribs.csv -i
+        e.g.: python systems.py --csv Contribs.csv -o
   3.  Generate an output document based on the information provided in a CSV file:
-        e.g.: python systems.py --csv Contribs.csv -o --template template.docx
+        e.g.: python systems.py --csv Contribs.csv -d --template template.docx
 
 NOTE 1: The script stores data from mdms and gitlab to ./data folder to minimize the number of 
         requests to both systems and to impove the performance. You can use the -u option to update 
@@ -23,7 +23,7 @@ import os
 import sys
 from automation import gitlab, mdms, helpers
 
-__version__ = '1.1'
+__version__ = '1.2'
 
 DATA_PATH = './data'
 GITLAB_PROJECTS_PATH = os.path.join(DATA_PATH, 'gitlab_projects.json')
@@ -107,9 +107,17 @@ def open_new_issue(project_id, document, test):
   else:
     print(' * Test open issue', issue_title, '| Labels', issue_lables)
 
+def close_issue(issue, test):
+  if not test:
+    print(' * Close issue:', issue.web_url)
+    gitlab.close_issue(issue)
+  else:
+    print(' * Test close issue:', issue.web_url)
+
 def open_issues(csv_path, test, gitlab_projects, input_docs, gitlab_members, meeting_start):
   print(' * Open issues. TestMode =', test)
   table_entries = parse_csv(args.csv, gitlab_projects, input_docs)
+  counter = 0
   for entry in table_entries:
     document = entry['document']
     project_id = entry['project']['id']
@@ -120,6 +128,7 @@ def open_issues(csv_path, test, gitlab_projects, input_docs, gitlab_members, mee
     # TODO find gitlab_members and add them to the issue description, this will automatically notify the contributors
     if issue_with_title is None and issue_with_meta is None:
       open_new_issue(project_id, document, test)
+      counter += 1
     elif issue_with_title is not None and issue_with_meta is None:
       print(' * *** ATTENTION *** Issue with the same document number in the title aleady found on GitLab.')
       print('   - Issue:', issue_with_title.web_url)
@@ -127,6 +136,7 @@ def open_issues(csv_path, test, gitlab_projects, input_docs, gitlab_members, mee
       user_input = input('Type y or n: ')
       if 'y' in user_input:
         open_new_issue(project_id, document, test)
+        counter += 1
       else:
         print(' * Skip', document['document'], document['title'])
     else:
@@ -157,6 +167,38 @@ def open_issues(csv_path, test, gitlab_projects, input_docs, gitlab_members, mee
           print(' * Skip', document['document'], document['title'])
       else:
         print(' * No update required for', document['document'], document['title'])
+  print(' * Opened issues:', counter)
+
+def close_issues(csv_path, test, gitlab_projects, input_docs):
+  print(' * Close issues. TestMode =', test)
+  table_entries = parse_csv(args.csv, gitlab_projects, input_docs)
+  counter = 0
+  for entry in table_entries:
+    document = entry['document']
+    project_id = entry['project']['id']
+    close_flag = entry['close']
+
+    if not close_flag == True:
+      continue
+    
+    issues = gitlab.get_issues(project_id)
+    issue_with_title, issue_with_meta, meta_last_doc_version = helpers.find_issue(issues, document)
+    if issue_with_meta is None and issue_with_title is None:
+      print(' * No issue to close for:', document['document'], 'in', entry['project']['url'])
+    if issue_with_meta is not None:
+      close_issue(issue_with_meta, test)
+      counter += 1
+    if issue_with_title is not None:
+      print(' * *** ATTENTION *** User created issue with the same document number in the title was found on GitLab.')
+      print('   - Issue:', issue_with_title.web_url)
+      print('   - Should we also close it?')
+      user_input = input('Type y or n: ')
+      if 'y' in user_input:
+        close_issue(issue_with_title, test)
+        counter += 1
+      else:
+        print(' * Skip', document['document'], document['title'])
+  print(' * Closed issues:', counter)
 
 def create_output_doc(csv_path, gitlab_projects, input_docs, template_path):
   print(' * Create Output Document')
@@ -208,6 +250,7 @@ def parse_csv(csv_file, projects, docs):
     project_url_idx = None
     subgroup_idx = None
     project_name_idx = None
+    close_idx = None
     for n in range(len(header)):
       if 'number' == header[n].replace(' ', '').strip().lower():
         doc_number_idx = n
@@ -217,6 +260,8 @@ def parse_csv(csv_file, projects, docs):
         subgroup_idx = n
       elif 'projectname' == header[n].replace(' ', '').strip().lower():
         project_name_idx = n
+      elif 'closeissue' == header[n].replace(' ', '').strip().lower():
+        close_idx = n
     if doc_number_idx is None or not ((subgroup_idx is not None and project_name_idx is not None) or project_url_idx is not None):
       print('Error: CSV header should have, "number" and ("project url" or ("sub group" and "project name")) fields')
       sys.exit(-1)
@@ -234,6 +279,13 @@ def parse_csv(csv_file, projects, docs):
       if project_url_or_path is None and subgroup_idx is not None and project_name_idx is not None:
         if len(row[subgroup_idx]) > 0 and len(row[project_name_idx]) > 0:
           project_url_or_path = row[subgroup_idx] + '/' + row[project_name_idx]
+      if project_url_or_path is None:
+        print('NOTE: No project or path found:', row)
+        continue
+      close_flag = False
+      if close_idx is not None:
+        if 'true' in row[close_idx].lower() or '1' == row[close_idx]:
+          close_flag = True
       project = helpers.find_project(projects, project_url_or_path, 'MPEG/Systems')
       doc = helpers.find_document(docs, row[doc_number_idx])
       if not project:
@@ -243,7 +295,8 @@ def parse_csv(csv_file, projects, docs):
       else:
         issues.append({
           'project': project,
-          'document': doc
+          'document': doc,
+          'close': close_flag
         })
   return issues
 
@@ -252,9 +305,9 @@ print('* MPEG Systems script version', __version__, '*')
 print('*'*35 + '\n')
 # program options
 usage_examples= '''Examples:
-  python3 systems.py -m m55958,m55959,m56121 -p FileFormat/CENC
-  python3 systems.py --csv Contribs.csv -i
+  python3 systems.py -m m55958,m55959,m56121 -p FileFormat/CENC --meeting 133
   python3 systems.py --csv Contribs.csv -o
+  python3 systems.py --csv Contribs.csv -d
 '''
 parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, 
   description='A tool for managing the GitLab issues for MPEG Systems Subgroups.',
@@ -263,20 +316,21 @@ parser.add_argument('-u', help='Update MDMS documents database.', action='store_
 parser.add_argument('-U', help='Update all databases.', action='store_true')
 parser.add_argument('--meeting', help='MPEG meeting number. If not set, the latest meeting is used.', default=-1, type=int)
 parser.add_argument('--csv', help='Input CSV file. Header row shall include "Number" and ("Project URL" or "Sub Group" and "Project Name").')
-parser.add_argument('-i', help='Input mode. Open issues based on CSV file.', action='store_true')
-parser.add_argument('-o', help='Output mode. Generate output document based on CSV file.', action='store_true')
+parser.add_argument('-o', help='Open issues mode. Open issues based on CSV file.', action='store_true')
+parser.add_argument('-d', help='Generate output document mode. Generate output document based on CSV file.', action='store_true')
+parser.add_argument('-c', help='Close issues mode. Close issues based on CSV file.', action='store_true')
 parser.add_argument('--template', help='Document template path.', type=str)
 parser.add_argument('--test', help='Test mode. If set, no issues will be opened.', action='store_true')
 parser.add_argument('-m', help='Documents (comma separated values). List information about the contribution(s).', type=str)
 parser.add_argument('-p', help='GitLab project URL or "SubGroup/ProjectName". This option is used with -m option.', type=str)
 args = parser.parse_args()
 # check command line options
-if not (args.csv is not None and (args.i or args.o) or args.m):
+if not (args.csv is not None and (args.o or args.d or args.c) or args.m):
   print('Wrong arguments. Use option -h for help.')
   parser.print_usage()
   sys.exit(-1)
-if (args.i or args.o) and args.csv is None:
-  print('Wrong arguments. Both -i and -o shall be used with --csv option. Use option -h for help.')
+if (args.o or args.d or args.c) and args.csv is None:
+  print('Wrong arguments. Options -o, -d and -c require --csv option. Use option -h for help.')
   sys.exit(-1)
 # get gitlab projects (and update if needed)
 if not os.path.isfile(GITLAB_PROJECTS_PATH) or args.U:
@@ -328,8 +382,10 @@ input_docs = helpers.load_json_data(input_docs_path)
 # do some action
 if args.m:
   print_infos(args.m, args.p, gitlab_projects, input_docs)
-if args.i:
+if args.o:
   meeting_start = helpers.try_parsing_date(meeting['start_date'])
   open_issues(args.csv, args.test, gitlab_projects, input_docs, gitlab_members, meeting_start)
-if args.o:
+if args.d:
   create_output_doc(args.csv, gitlab_projects, input_docs, args.template)
+if args.c:
+  close_issues(args.csv, args.test, gitlab_projects, input_docs)