Commit 094c7503 authored by Dimitri Podborski's avatar Dimitri Podborski
Browse files

add option [mode] to download contributions

parent 176b065a
...@@ -25,6 +25,7 @@ import argparse ...@@ -25,6 +25,7 @@ import argparse
import csv import csv
import os import os
import sys import sys
import requests
from datetime import datetime from datetime import datetime
from automation import gitlab, mdms, helpers from automation import gitlab, mdms, helpers
...@@ -37,6 +38,36 @@ MEETINGS_PATH = os.path.join(DATA_PATH, 'meetings.json') ...@@ -37,6 +38,36 @@ MEETINGS_PATH = os.path.join(DATA_PATH, 'meetings.json')
SYSTEMS_GROUP_ID = 727 # GitLab Group ID for Systems Subgroup SYSTEMS_GROUP_ID = 727 # GitLab Group ID for Systems Subgroup
def download_url(url, save_path, chunk_size=128):
r = requests.get(url, stream=True)
with open(save_path, 'wb') as fd:
for chunk in r.iter_content(chunk_size=chunk_size):
fd.write(chunk)
def fetch_contributions(table_entries):
print('\nDownload contributions')
for entry in table_entries:
path = os.path.join(DATA_PATH, 'contributions')
document = entry['document']
project = entry['project']
if not document:
print('WARNING: Document not found. Try updating the database (-u) or select another meeting (--meeting).')
continue
url = document['latest_version_url']
if url is None:
print(document['document'], 'Skip [no document]')
continue
if project is not None:
path = os.path.join(path, project['path_with_namespace'])
if not os.path.exists(path) and len(path) > 0:
os.makedirs(path)
folder, filename = os.path.split(url)
filename = os.path.join(path, filename)
print(document['document'], ' -> ', filename)
download_url(url, filename)
def print_infos(table_entries): def print_infos(table_entries):
print('\nDump information') print('\nDump information')
for entry in table_entries: for entry in table_entries:
...@@ -371,10 +402,10 @@ def main(): ...@@ -371,10 +402,10 @@ def main():
parser.add_argument('-c', '--close', help='Mode: Close GitLab issues.', action='store_true') parser.add_argument('-c', '--close', help='Mode: Close GitLab issues.', action='store_true')
parser.add_argument('-d', '--docx', help='Mode: Generate output word document.', action='store_true') parser.add_argument('-d', '--docx', help='Mode: Generate output word document.', action='store_true')
parser.add_argument('-l', '--list', help='Mode: List information about the contribution(s).', action='store_true') parser.add_argument('-l', '--list', help='Mode: List information about the contribution(s).', action='store_true')
parser.add_argument('-f', '--fetch', help='Mode: Download contributions.', action='store_true')
parser.add_argument('-C', '--CLOSE', help='Force closing GitLab issues.', action='store_true') parser.add_argument('-C', '--CLOSE', help='Force closing GitLab issues.', action='store_true')
parser.add_argument('-U', '--UPDATE', help='Update all databases.', action='store_true') parser.add_argument('-U', '--UPDATE', help='Update all databases.', action='store_true')
parser.add_argument('-i', '--csv', help='Input CSV file. Header row shall include "Number" and ("Project URL" or ' parser.add_argument('-i', '--csv', help='Input CSV file. Header row shall include "Number" and "Project URL"')
'"Sub Group" and "Project Name").')
parser.add_argument('-m', '--documents', help='Comma separated MDMS document number(s). e.g.: m12345,...', type=str) parser.add_argument('-m', '--documents', help='Comma separated MDMS document number(s). e.g.: m12345,...', type=str)
parser.add_argument('-p', '--project', help='GitLab project URL or "SubGroup/ProjectName".', type=str) parser.add_argument('-p', '--project', help='GitLab project URL or "SubGroup/ProjectName".', type=str)
parser.add_argument('--meeting', help='MPEG meeting number. If not set, the latest meeting is used.', default=-1, parser.add_argument('--meeting', help='MPEG meeting number. If not set, the latest meeting is used.', default=-1,
...@@ -385,7 +416,7 @@ def main(): ...@@ -385,7 +416,7 @@ def main():
parser.add_argument('-n', '--notify', help='Notify (@mention) authors in opened issues.', action='store_true') parser.add_argument('-n', '--notify', help='Notify (@mention) authors in opened issues.', action='store_true')
args = parser.parse_args() args = parser.parse_args()
if not args.open and not args.docx and not args.close and not args.list: if not args.open and not args.docx and not args.close and not args.list and not args.fetch:
print('There is no work to be done. Check the CLI options. Use -h for help.') print('There is no work to be done. Check the CLI options. Use -h for help.')
sys.exit(1) sys.exit(1)
# get gitlab projects (and update if needed) # get gitlab projects (and update if needed)
...@@ -454,6 +485,8 @@ def main(): ...@@ -454,6 +485,8 @@ def main():
close_issues(table_entries, args.test, args.CLOSE) close_issues(table_entries, args.test, args.CLOSE)
if args.docx: if args.docx:
create_output_doc(table_entries, output_path, args.template) create_output_doc(table_entries, output_path, args.template)
if args.fetch:
fetch_contributions(table_entries)
if __name__ == "__main__": if __name__ == "__main__":
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment