add option [mode] to download contributions

systems.py

@@ -25,6 +25,7 @@ import argparse
 import csv
 import os
 import sys
+import requests
 from datetime import datetime
 from automation import gitlab, mdms, helpers
 
@@ -37,6 +38,36 @@ MEETINGS_PATH = os.path.join(DATA_PATH, 'meetings.json')
 SYSTEMS_GROUP_ID = 727  # GitLab Group ID for Systems Subgroup
 
 
+def download_url(url, save_path, chunk_size=128):
+    r = requests.get(url, stream=True)
+    with open(save_path, 'wb') as fd:
+        for chunk in r.iter_content(chunk_size=chunk_size):
+            fd.write(chunk)
+
+
+def fetch_contributions(table_entries):
+    print('\nDownload contributions')
+    for entry in table_entries:
+        path = os.path.join(DATA_PATH, 'contributions')
+        document = entry['document']
+        project = entry['project']
+        if not document:
+            print('WARNING: Document not found. Try updating the database (-u) or select another meeting (--meeting).')
+            continue
+        url = document['latest_version_url']
+        if url is None:
+            print(document['document'], 'Skip [no document]')
+            continue
+        if project is not None:
+            path = os.path.join(path, project['path_with_namespace'])
+        if not os.path.exists(path) and len(path) > 0:
+            os.makedirs(path)
+        folder, filename = os.path.split(url)
+        filename = os.path.join(path, filename)
+        print(document['document'], ' -> ', filename)
+        download_url(url, filename)
+
+
 def print_infos(table_entries):
     print('\nDump information')
     for entry in table_entries:
@@ -371,10 +402,10 @@ def main():
     parser.add_argument('-c', '--close', help='Mode: Close GitLab issues.', action='store_true')
     parser.add_argument('-d', '--docx', help='Mode: Generate output word document.', action='store_true')
     parser.add_argument('-l', '--list', help='Mode: List information about the contribution(s).', action='store_true')
+    parser.add_argument('-f', '--fetch', help='Mode: Download contributions.', action='store_true')
     parser.add_argument('-C', '--CLOSE', help='Force closing GitLab issues.', action='store_true')
     parser.add_argument('-U', '--UPDATE', help='Update all databases.', action='store_true')
-    parser.add_argument('-i', '--csv', help='Input CSV file. Header row shall include "Number" and ("Project URL" or '
-                                            '"Sub Group" and "Project Name").')
+    parser.add_argument('-i', '--csv', help='Input CSV file. Header row shall include "Number" and "Project URL"')
     parser.add_argument('-m', '--documents', help='Comma separated MDMS document number(s). e.g.: m12345,...', type=str)
     parser.add_argument('-p', '--project', help='GitLab project URL or "SubGroup/ProjectName".', type=str)
     parser.add_argument('--meeting', help='MPEG meeting number. If not set, the latest meeting is used.', default=-1,
@@ -385,7 +416,7 @@ def main():
     parser.add_argument('-n', '--notify', help='Notify (@mention) authors in opened issues.', action='store_true')
     args = parser.parse_args()
 
-    if not args.open and not args.docx and not args.close and not args.list:
+    if not args.open and not args.docx and not args.close and not args.list and not args.fetch:
         print('There is no work to be done. Check the CLI options. Use -h for help.')
         sys.exit(1)
     # get gitlab projects (and update if needed)
@@ -454,6 +485,8 @@ def main():
         close_issues(table_entries, args.test, args.CLOSE)
     if args.docx:
         create_output_doc(table_entries, output_path, args.template)
+    if args.fetch:
+        fetch_contributions(table_entries)
 
 
 if __name__ == "__main__":