From 72b17fa36b22a82e51886c4635f80952bb80fc79 Mon Sep 17 00:00:00 2001 From: Georg Teufelberger Date: Fri, 8 Apr 2022 17:25:10 +0200 Subject: [PATCH 1/8] Use f-strings instead of %-formatting --- vo-scraper.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vo-scraper.py b/vo-scraper.py index f0a1bc1..314dc09 100755 --- a/vo-scraper.py +++ b/vo-scraper.py @@ -312,7 +312,7 @@ def pretty_print_episodes(vo_json_data, selected): for episode_nr in selected: episode = vo_json_data['episodes'][episode_nr] print_information( - "%3d".ljust(nr_length) % episode_nr + f"{episode_nr:3d}".ljust(nr_length) + " | " + episode['createdAt'][:10].ljust(max_date_length) + " | " + @@ -547,7 +547,7 @@ def vo_scrapper(vo_link, video_quality, user, passw): # The lecture requires a login print_information("Received 401 response. The following lecture requires a valid login cookie:", type='error') item = vo_json_data['episodes'][item_nr] - print_information("%2d" % item_nr + " " + item['title'] + " " + str(item['createdBy']) + " " + item['createdAt'][:10], type='error') + print_information(f"{item_nr:2d} {item['title']} {str(item['createdBy'])} {item['createdAt'][:10]}", type='error') print_information("Make sure your token is valid. See README.md on how to acquire it.", type='error') print() continue @@ -641,7 +641,7 @@ def downloader(file_name, video_src_link, episode_name): response = requests.get(video_src_link, stream=True) total_length = response.headers.get('content-length') - print_information("Downloading " + episode_name + " (%.2f" % (int(total_length) / 1024 / 1024) + " MiB)") + print_information(f"Downloading {episode_name} ({int(total_length) / 1024 / 1024:.2f} MiB)") if total_length is None or HIDE_PROGRESS_BAR: # We received no content length header... @@ -656,7 +656,7 @@ def downloader(file_name, video_src_link, episode_name): f.write(data) progressbar_width = shutil.get_terminal_size().columns - 2 done = int(progressbar_width * dl / total_length) - sys.stdout.write("\r[%s%s]" % ('=' * done, ' ' * (progressbar_width - done))) + sys.stdout.write(f"\r[{'=' * done}{' ' * (progressbar_width - done)}]") sys.stdout.flush() print() -- GitLab From 2094cf9e966b92d3dbee9313b5ebfdc2e30a2337 Mon Sep 17 00:00:00 2001 From: Georg Teufelberger Date: Fri, 8 Apr 2022 21:21:18 +0200 Subject: [PATCH 2/8] Remove special handling of webbrowser import The webbrowser package is in the python standard library so it should always exist --- vo-scraper.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/vo-scraper.py b/vo-scraper.py index 314dc09..6c008de 100755 --- a/vo-scraper.py +++ b/vo-scraper.py @@ -26,7 +26,7 @@ import argparse # For parsing commandline arguments import getpass # For getting the user password import random # For selecting a random hint import shutil # For getting terminal size - +import webbrowser # only used to open the user's browser when reporting a bug # Check whether `requests` is installed try: @@ -35,12 +35,6 @@ except: print_information("Required package `requests` is missing, try installing with `pip3 install requests`", type='error') sys.exit(1) -# Check whether `webbrowser` is installed -try: - import webbrowser # only used to open the user's browser when reporting a bug -except: - print_information("Failed to import `webbrowser`. It is however not required for downloading videos", type='warning') - # ======================================================================== # ____ _ _ _ # / ___| | | ___ | |__ __ _ | | __ __ __ _ _ __ ___ -- GitLab From f40d08d28213c86308b320b54658dac79a170243 Mon Sep 17 00:00:00 2001 From: Georg Teufelberger Date: Fri, 8 Apr 2022 21:26:26 +0200 Subject: [PATCH 3/8] Remove unused import --- vo-scraper.py | 1 - 1 file changed, 1 deletion(-) diff --git a/vo-scraper.py b/vo-scraper.py index 6c008de..6227d7b 100755 --- a/vo-scraper.py +++ b/vo-scraper.py @@ -20,7 +20,6 @@ import urllib.request import os import sys from urllib.request import Request, urlopen -from sys import platform import json # For handling json files import argparse # For parsing commandline arguments import getpass # For getting the user password -- GitLab From be9f30155780f69f368dc11328632239bab92bef Mon Sep 17 00:00:00 2001 From: Georg Teufelberger Date: Fri, 8 Apr 2022 21:38:50 +0200 Subject: [PATCH 4/8] Replace print_information with standard print The function isn't defined at that point yet which would cause a separate exception if the requests package isn't installed. --- vo-scraper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vo-scraper.py b/vo-scraper.py index 6227d7b..1672ad7 100755 --- a/vo-scraper.py +++ b/vo-scraper.py @@ -31,8 +31,8 @@ import webbrowser # only used to open the user's browser when reporting a bug try: import requests except: - print_information("Required package `requests` is missing, try installing with `pip3 install requests`", type='error') - sys.exit(1) + print("(\033[91mERR\033[0m) Required package `requests` is missing, try installing with `pip3 install requests`") + sys.exit(-1) # ======================================================================== # ____ _ _ _ -- GitLab From 38e71ed8fb2fe55522f7450a8de1c659643757c1 Mon Sep 17 00:00:00 2001 From: Georg Teufelberger Date: Fri, 8 Apr 2022 22:41:40 +0200 Subject: [PATCH 5/8] Re-order instructions In anticipation of addtional progressbar logic --- vo-scraper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vo-scraper.py b/vo-scraper.py index 1672ad7..8c03caf 100755 --- a/vo-scraper.py +++ b/vo-scraper.py @@ -642,8 +642,8 @@ def downloader(file_name, video_src_link, episode_name): f.write(response.content) else: # Download file and show progress bar - dl = 0 total_length = int(total_length) + dl = 0 for data in response.iter_content(chunk_size=4096): dl += len(data) f.write(data) -- GitLab From b954af93d4b388b6feb06eda640edf3c212060c5 Mon Sep 17 00:00:00 2001 From: Georg Teufelberger Date: Fri, 8 Apr 2022 22:42:57 +0200 Subject: [PATCH 6/8] Add tqdm as additional progressbar Checks if tqdm module is installed and if so, defaults to it for displaying progress Otherwise uses old, self-made progressbar --- vo-scraper.py | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/vo-scraper.py b/vo-scraper.py index 8c03caf..12fd4de 100755 --- a/vo-scraper.py +++ b/vo-scraper.py @@ -643,14 +643,33 @@ def downloader(file_name, video_src_link, episode_name): else: # Download file and show progress bar total_length = int(total_length) - dl = 0 - for data in response.iter_content(chunk_size=4096): - dl += len(data) - f.write(data) - progressbar_width = shutil.get_terminal_size().columns - 2 - done = int(progressbar_width * dl / total_length) - sys.stdout.write(f"\r[{'=' * done}{' ' * (progressbar_width - done)}]") - sys.stdout.flush() + + try: + # Module with better progressbar + from tqdm import tqdm + + # Setup progressbar + pbar = tqdm(unit="B", unit_scale=True, unit_divisor=1024, total=total_length) + pbar.clear() + + # Download to file and update progressbar + for data in response.iter_content(chunk_size=4096): + pbar.update(len(data)) + f.write(data) + # Close it + pbar.close() + + # If tqdm is not installed, fallback to self-made version + except ModuleNotFoundError: + print_information("Optionally dependency tqdm not installed, falling back to built-in progressbar", type='warning', verbose_only=True) + dl = 0 + for data in response.iter_content(chunk_size=4096): + dl += len(data) + f.write(data) + progressbar_width = shutil.get_terminal_size().columns - 2 + done = int(progressbar_width * dl / total_length) + sys.stdout.write(f"\r[{'=' * done}{' ' * (progressbar_width - done)}]") + sys.stdout.flush() print() # Remove `.part` suffix from file name -- GitLab From 5fd186a7e72ef4ad59cde18c05e84546355d7426 Mon Sep 17 00:00:00 2001 From: Georg Teufelberger Date: Fri, 8 Apr 2022 22:45:05 +0200 Subject: [PATCH 7/8] Auto-format --- vo-scraper.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/vo-scraper.py b/vo-scraper.py index 12fd4de..c444508 100755 --- a/vo-scraper.py +++ b/vo-scraper.py @@ -82,6 +82,7 @@ file_to_print_src_to = "" history_file = "" PARAMETER_FILE = "parameters.txt" + class bcolors: INFO = '\033[94m' ERROR = '\033[91m' @@ -369,6 +370,7 @@ def get_user_choice(max_episode_number): return choice + def resolution_from_input(resolution): # Turn named resolution into number if resolution.lower() == "4k": resolution = "2160p" @@ -419,7 +421,7 @@ def get_video_src_link_for_resolution(video_json_data, video_quality): list_of_quality_diff = [(x[0], abs(video_quality_parsed - x[2])) for x in resolutions] # Get the resolution closest to the requested one - min_value = min(list_of_quality_diff, key = lambda t: t[1]) + min_value = min(list_of_quality_diff, key=lambda t: t[1]) quality_index = min_value[0] # Show a warning if the we cannot return the requested resolution @@ -427,7 +429,7 @@ def get_video_src_link_for_resolution(video_json_data, video_quality): print_information(f"Requested quality {video_quality} not available, downloading {video_json_data['streams'][0]['sources']['mp4'][quality_index]['res']['h']}p instead", type='warning') # Save actual quality of video for filename - video_quality = str(video_json_data['streams'][0]['sources']['mp4'][resolutions[quality_index][0]]['res']['h'])+'p' + video_quality = str(video_json_data['streams'][0]['sources']['mp4'][resolutions[quality_index][0]]['res']['h']) + 'p' video_src_link = video_json_data['streams'][0]['sources']['mp4'][resolutions[quality_index][0]]['src'] return video_src_link, video_quality @@ -475,7 +477,7 @@ def vo_scrapper(vo_link, video_quality, user, passw): vo_json_data = json.loads(r.text) except json.decoder.JSONDecodeError: print_information(f"Could not get metadata for {vo_link}.html, skipping", type='warning') - return list() # Return an empty list + return list() # Return an empty list # Increase counter for stats link_counter += len(vo_json_data['episodes']) -- GitLab From 6d35f17310f0a079b718ae5b0fbd28409d41f07d Mon Sep 17 00:00:00 2001 From: Georg Teufelberger Date: Tue, 19 Apr 2022 21:26:26 +0200 Subject: [PATCH 8/8] Bump version number --- VERSION | 2 +- vo-scraper.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index a0cd9f0..a4f52a5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.0 \ No newline at end of file +3.2.0 \ No newline at end of file diff --git a/vo-scraper.py b/vo-scraper.py index c444508..f39c38e 100755 --- a/vo-scraper.py +++ b/vo-scraper.py @@ -48,7 +48,7 @@ gitlab_repo_page = "https://gitlab.ethz.ch/tgeorg/vo-scraper/" gitlab_issue_page = gitlab_repo_page + "issues" gitlab_changelog_page = gitlab_repo_page + "-/tags/v" remote_version_link = gitlab_repo_page + "raw/master/VERSION" -program_version = '3.1.0' +program_version = '3.2.0' # For web requests user_agent = 'Mozilla/5.0' -- GitLab