From e6482c6deafeeb0d708b8b57e75b3b780b7ae960 Mon Sep 17 00:00:00 2001 From: Andrey Nikiforov Date: Sat, 15 Jul 2023 19:08:03 -0700 Subject: [PATCH] support --dry-run parameter (#663) --- CHANGELOG.md | 1 + src/icloudpd/authentication.py | 7 +- src/icloudpd/autodelete.py | 24 ++- src/icloudpd/base.py | 179 +++++++++++++-------- src/icloudpd/download.py | 99 ++++++++---- src/icloudpd/exif_datetime.py | 12 +- tests/test_authentication.py | 7 +- tests/test_autodelete_photos.py | 133 ++++++++++++--- tests/test_download_live_photos.py | 10 +- tests/test_download_photos.py | 249 +++++++++++++++++++++++++---- 10 files changed, 554 insertions(+), 167 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1fcdc303f..506cc8f2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- feature: `--dry-run` parameter to run icloudpd without changes to local files and iCloud - fix: pypi.org license and description ## 1.14.5 (2023-07-06) diff --git a/src/icloudpd/authentication.py b/src/icloudpd/authentication.py index ff82d74e6..aa94cdf2f 100644 --- a/src/icloudpd/authentication.py +++ b/src/icloudpd/authentication.py @@ -1,9 +1,9 @@ """Handles username/password authentication and two-step authentication""" +import logging import sys import click import pyicloud_ipd -from icloudpd.logger import setup_logger class TwoStepAuthRequiredError(Exception): @@ -13,7 +13,7 @@ class TwoStepAuthRequiredError(Exception): """ -def authenticator(domain): +def authenticator(logger, domain): """Wraping authentication with domain context""" def authenticate_( username, @@ -23,8 +23,7 @@ def authenticate_( client_id=None, ): """Authenticate with iCloud username and password""" - logger = setup_logger() - logger.debug("Authenticating...") + logger.tqdm_write("Authenticating...", logging.DEBUG) while True: try: # If password not provided on command line variable will be set to None diff --git a/src/icloudpd/autodelete.py b/src/icloudpd/autodelete.py index 024f95f64..4a97b24e4 100644 --- a/src/icloudpd/autodelete.py +++ b/src/icloudpd/autodelete.py @@ -4,18 +4,27 @@ import os import logging from tzlocal import get_localzone -from icloudpd.logger import setup_logger from icloudpd.paths import local_download_path -def autodelete_photos(icloud, folder_structure, directory): +def delete_file(logger, path) -> bool: + """ Actual deletion of files """ + os.remove(path) + logger.tqdm_write(f"Deleted {path}", logging.INFO) + return True + +def delete_file_dry_run(logger, path) -> bool: + """ Dry run deletion of files """ + logger.tqdm_write(f"[DRY RUN] Would delete {path}", logging.INFO) + return True + +def autodelete_photos(logger, dry_run, icloud, folder_structure, directory): """ Scans the "Recently Deleted" folder and deletes any matching files from the download directory. (I.e. If you delete a photo on your phone, it's also deleted on your computer.) """ - logger = setup_logger() - logger.info("Deleting any files found in 'Recently Deleted'...") + logger.tqdm_write("Deleting any files found in 'Recently Deleted'...", logging.INFO) recently_deleted = icloud.photos.albums["Recently Deleted"] @@ -23,7 +32,7 @@ def autodelete_photos(icloud, folder_structure, directory): try: created_date = media.created.astimezone(get_localzone()) except (ValueError, OSError): - logger.set_tqdm_description( + logger.tqdm_write( f"Could not convert media created date to local timezone {media.created}", logging.ERROR) created_date = media.created @@ -36,5 +45,6 @@ def autodelete_photos(icloud, folder_structure, directory): local_download_path( media, size, download_dir)) if os.path.exists(path): - logger.info("Deleting %s!", path) - os.remove(path) + logger.tqdm_write(f"Deleting {path}...", logging.DEBUG) + delete_local = delete_file_dry_run if dry_run else delete_file + delete_local(logger, path) diff --git a/src/icloudpd/base.py b/src/icloudpd/base.py index 924cdfa18..630abda5c 100644 --- a/src/icloudpd/base.py +++ b/src/icloudpd/base.py @@ -194,7 +194,7 @@ is_flag=True, ) @click.option("--threads-num", - help="Number of cpu threads -- deprecated. To be removed in future version", + help="Number of cpu threads - deprecated & always 1. To be removed in future version", type=click.IntRange(1), default=1, ) @@ -215,6 +215,11 @@ help="Run downloading in a infinite cycle, waiting specified seconds between runs", type=click.IntRange(1), ) +@click.option("--dry-run", + help="Do not modify local system or iCloud", + is_flag=True, + default=False, + ) # a hacky way to get proper version because automatic detection does not work for some reason @click.version_option(version="1.14.5") # pylint: disable-msg=too-many-arguments,too-many-statements @@ -250,7 +255,8 @@ def main( threads_num, # pylint: disable=W0613 delete_after_download, domain, - watch_with_interval + watch_with_interval, + dry_run ): """Download all iCloud photos to a local directory""" @@ -277,7 +283,7 @@ def main( print('--auto-delete and --delete-after-download are mutually exclusive') sys.exit(2) - if watch_with_interval and (list_albums or only_print_filenames): # pragma: no cover + if watch_with_interval and (list_albums or only_print_filenames): # pragma: no cover print('--watch_with_interval is not compatible with --list_albums, --only_print_filenames') sys.exit(2) @@ -293,7 +299,8 @@ def main( only_print_filenames, set_exif_datetime, skip_live_photos, - live_photo_size), + live_photo_size, + dry_run), directory, username, password, @@ -319,7 +326,8 @@ def main( delete_after_download, domain, logger, - watch_with_interval + watch_with_interval, + dry_run ) ) @@ -337,26 +345,31 @@ def download_builder( only_print_filenames, set_exif_datetime, skip_live_photos, - live_photo_size): + live_photo_size, + dry_run): """factory for downloader""" def state_(icloud): - def download_photo_(counter, photo): + def download_photo_(counter, photo) -> bool: """internal function for actually downloading the photos""" filename = clean_filename(photo.filename) if skip_videos and photo.item_type != "image": - logger.set_tqdm_description( - f"Skipping {filename}, only downloading photos." + logger.tqdm_write( + (f"Skipping {filename}, only downloading photos." + f"(Item type was: {photo.item_type})"), + logging.DEBUG ) return False if photo.item_type not in ("image", "movie"): - logger.set_tqdm_description( - f"Skipping {filename}, only downloading photos and videos. " - f"(Item type was: {photo.item_type})") + logger.tqdm_write( + (f"Skipping {filename}, only downloading photos and videos. " + f"(Item type was: {photo.item_type})"), + logging.DEBUG + ) return False try: created_date = photo.created.astimezone(get_localzone()) except (ValueError, OSError): - logger.set_tqdm_description( + logger.tqdm_write( f"Could not convert photo created date to local timezone ({photo.created})", logging.ERROR) created_date = photo.created @@ -368,7 +381,7 @@ def download_photo_(counter, photo): date_path = folder_structure.format(created_date) except ValueError: # pragma: no cover # This error only seems to happen in Python 2 - logger.set_tqdm_description( + logger.tqdm_write( f"Photo created date was not valid ({photo.created})", logging.ERROR) # e.g. ValueError: year=5 is before 1900 # (https://github.com/icloud-photos-downloader/icloud_photos_downloader/issues/122) @@ -407,8 +420,10 @@ def download_photo_(counter, photo): if size not in versions and size != "original": if force_size: - logger.set_tqdm_description( - f"{size} size does not exist for {filename}. Skipping...", logging.ERROR, ) + logger.tqdm_write( + f"{size} size does not exist for {filename}. Skipping...", + logging.ERROR + ) return False download_size = "original" @@ -436,14 +451,16 @@ def download_photo_(counter, photo): download_path = (f"-{photo_size}.").join( download_path.rsplit(".", 1) ) - logger.set_tqdm_description( - f"{truncate_middle(download_path, 96)} deduplicated." + logger.tqdm_write( + f"{truncate_middle(download_path, 96)} deduplicated.", + logging.INFO ) file_exists = os.path.isfile(download_path) if file_exists: counter.increment() - logger.set_tqdm_description( - f"{truncate_middle(download_path, 96)} already exists." + logger.tqdm_write( + f"{truncate_middle(download_path, 96)} already exists.", + logging.INFO ) if not file_exists: @@ -452,34 +469,41 @@ def download_photo_(counter, photo): print(download_path) else: truncated_path = truncate_middle(download_path, 96) - logger.set_tqdm_description( - f"Downloading {truncated_path}" + logger.tqdm_write( + f"Downloading {truncated_path}...", + logging.DEBUG ) download_result = download.download_media( - icloud, photo, download_path, download_size + logger, dry_run, icloud, photo, download_path, download_size ) success = download_result if download_result: - if set_exif_datetime and \ + if not dry_run and \ + set_exif_datetime and \ clean_filename(photo.filename) \ .lower() \ .endswith((".jpg", ".jpeg")) and \ - not exif_datetime.get_photo_exif(download_path): + not exif_datetime.get_photo_exif(logger, download_path): # %Y:%m:%d looks wrong, but it's the correct format date_str = created_date.strftime( "%Y-%m-%d %H:%M:%S%z") - logger.debug( - "Setting EXIF timestamp for %s: %s", - download_path, - date_str, + logger.tqdm_write( + f"Setting EXIF timestamp for {download_path}: {date_str}", + logging.DEBUG ) exif_datetime.set_photo_exif( + logger, download_path, created_date.strftime("%Y:%m:%d %H:%M:%S"), ) - download.set_utime(download_path, created_date) + if not dry_run: + download.set_utime(download_path, created_date) + logger.tqdm_write( + f"Downloaded {truncated_path}", + logging.INFO + ) # Also download the live photo if present if not skip_live_photos: @@ -506,24 +530,33 @@ def download_photo_(counter, photo): lp_download_path = (f"-{lp_photo_size}.").join( lp_download_path.rsplit(".", 1) ) - logger.set_tqdm_description( - f"{truncate_middle(lp_download_path, 96)} deduplicated." + logger.tqdm_write( + f"{truncate_middle(lp_download_path, 96)} deduplicated.", + logging.DEBUG ) lp_file_exists = os.path.isfile( lp_download_path) if lp_file_exists: - logger.set_tqdm_description( - f"{truncate_middle(lp_download_path, 96)} already exists." - + logger.tqdm_write( + f"{truncate_middle(lp_download_path, 96)} already exists.", + logging.INFO ) if not lp_file_exists: truncated_path = truncate_middle( lp_download_path, 96) - logger.set_tqdm_description( - f"Downloading {truncated_path}") - success = download.download_media( - icloud, photo, lp_download_path, lp_size - ) and success + logger.tqdm_write( + f"Downloading {truncated_path}...", + logging.DEBUG + ) + download_result = download.download_media( + logger, dry_run, icloud, photo, lp_download_path, lp_size + ) + success = download_result and success + if download_result: + logger.tqdm_write( + f"Downloaded {truncated_path}", + logging.INFO + ) return success return download_photo_ return state_ @@ -531,7 +564,9 @@ def download_photo_(counter, photo): def delete_photo(logger, icloud, photo): """Delete a photo from the iCloud account.""" - logger.info("Deleting %s", clean_filename(photo.filename)) + clean_filename_local = clean_filename(photo.filename) + logger.tqdm_write( + f"Deleting {clean_filename_local} in iCloud...", logging.DEBUG) # pylint: disable=W0212 url = f"{icloud.photos._service_endpoint}/records/modify?"\ f"{urllib.parse.urlencode(icloud.photos.params)}" @@ -554,9 +589,20 @@ def delete_photo(logger, icloud, photo): icloud.photos.session.post( url, data=post_data, headers={ "Content-type": "application/json"}) + logger.tqdm_write( + f"Deleted {clean_filename_local} in iCloud", logging.INFO) + + +def delete_photo_dry_run(logger, _icloud, photo): + """Dry run for deleting a photo from the iCloud""" + logger.tqdm_write( + f"[DRY RUN] Would delete {clean_filename(photo.filename)} in iCloud", + logging.INFO) + RetrierT = TypeVar('RetrierT') + def retrier( func: Callable[[], RetrierT], error_handler: Callable[[Exception, int], None]) -> RetrierT: @@ -652,7 +698,8 @@ def core( delete_after_download, domain, logger, - watch_interval + watch_interval, + dry_run ): """Download all iCloud photos to a local directory""" @@ -662,7 +709,7 @@ def core( or notification_script is not None ) try: - icloud = authenticator(domain)( + icloud = authenticator(logger, domain)( username, password, cookie_directory, @@ -709,10 +756,11 @@ def core( directory = os.path.normpath(directory) - logger.debug( - "Looking up all photos%s from album %s...", - "" if skip_videos else " and videos", - album) + videos_phrase = "" if skip_videos else " and videos" + logger.tqdm_write( + f"Looking up all photos{videos_phrase} from album {album}...", + logging.DEBUG + ) session_exception_handler = session_error_handle_builder( logger, icloud) @@ -738,20 +786,6 @@ def core( # ensure photos iterator doesn't have a known length photos = (p for p in photos) - plural_suffix = "" if photos_count == 1 else "s" - video_suffix = "" - photos_count_str = "the first" if photos_count == 1 else photos_count - if not skip_videos: - video_suffix = " or video" if photos_count == 1 else " and videos" - logger.info( - "Downloading %s %s photo%s%s to %s ...", - photos_count_str, - size, - plural_suffix, - video_suffix, - directory, - ) - # Use only ASCII characters in progress bar tqdm_kwargs["ascii"] = True @@ -767,6 +801,17 @@ def core( photos_enumerator = tqdm(photos, **tqdm_kwargs) logger.set_tqdm(photos_enumerator) + plural_suffix = "" if photos_count == 1 else "s" + video_suffix = "" + photos_count_str = "the first" if photos_count == 1 else photos_count + if not skip_videos: + video_suffix = " or video" if photos_count == 1 else " and videos" + logger.tqdm_write( + (f"Downloading {photos_count_str} {size}" + f" photo{plural_suffix}{video_suffix} to {directory} ..."), + logging.INFO + ) + consecutive_files_found = Counter(0) def should_break(counter): @@ -778,17 +823,18 @@ def should_break(counter): try: if should_break(consecutive_files_found): logger.tqdm_write( - f"Found {until_found} consecutive previously downloaded photos. Exiting" + f"Found {until_found} consecutive previously downloaded photos. Exiting", + logging.INFO ) break item = next(photos_iterator) if download_photo( consecutive_files_found, item) and delete_after_download: - # delete_photo(logger, icloud, item) def delete_cmd(): - delete_photo(logger, icloud, item) + delete_local = delete_photo_dry_run if dry_run else delete_photo + delete_local(logger, icloud, item) retrier(delete_cmd, error_handler) @@ -801,9 +847,10 @@ def delete_cmd(): logger.info("All photos have been downloaded!") if auto_delete: - autodelete_photos(icloud, folder_structure, directory) + autodelete_photos(logger, dry_run, icloud, + folder_structure, directory) - if watch_interval: # pragma: no cover + if watch_interval: # pragma: no cover logger.info(f"Waiting for {watch_interval} sec...") interval = range(1, watch_interval) for _ in interval if skip_bar else tqdm( diff --git a/src/icloudpd/download.py b/src/icloudpd/download.py index 7f40b9c1e..4ff466950 100644 --- a/src/icloudpd/download.py +++ b/src/icloudpd/download.py @@ -4,21 +4,20 @@ import socket import time import logging +import datetime from tzlocal import get_localzone from requests.exceptions import ConnectionError # pylint: disable=redefined-builtin from pyicloud_ipd.exceptions import PyiCloudAPIResponseError -from icloudpd.logger import setup_logger # Import the constants object so that we can mock WAIT_SECONDS in tests from icloudpd import constants - -def update_mtime(photo, download_path): +def update_mtime(created: datetime.datetime, download_path): """Set the modification time of the downloaded file to the photo creation date""" - if photo.created: + if created: created_date = None try: - created_date = photo.created.astimezone( + created_date = created.astimezone( get_localzone()) except (ValueError, OSError): # We already show the timezone conversion error in base.py, @@ -33,33 +32,73 @@ def set_utime(download_path, created_date): ctime = time.mktime(created_date.timetuple()) os.utime(download_path, (ctime, ctime)) +def mkdirs_for_path(logger, download_path: str) -> bool: + """ Creates hierarchy of folders for file path if it needed """ + try: + # get back the directory for the file to be downloaded and create it if + # not there already + download_dir = os.path.dirname(download_path) + os.makedirs(name = download_dir, exist_ok=True) + return True + except OSError: + logger.tqdm_write( + f"Could not create folder {download_dir}", + logging.ERROR, + ) + return False -def download_media(icloud, photo, download_path, size): +def mkdirs_for_path_dry_run(logger, download_path: str) -> bool: + """ DRY Run for Creating hierarchy of folders for file path """ + download_dir = os.path.dirname(download_path) + if not os.path.exists(download_dir): + logger.tqdm_write( + f"[DRY RUN] Would create folder hierarchy {download_dir}", + logging.DEBUG, + ) + return True + +def download_response_to_path( + _logger, + response, + download_path: str, + created_date: datetime.datetime) -> bool: + """ Saves response content into file with desired created date """ + temp_download_path = download_path + ".part" + with open(temp_download_path, "wb") as file_obj: + for chunk in response.iter_content(chunk_size=1024): + if chunk: + file_obj.write(chunk) + os.rename(temp_download_path, download_path) + update_mtime(created_date, download_path) + return True + +def download_response_to_path_dry_run( + logger, + _response, + download_path: str, + _created_date: datetime.datetime) -> bool: + """ Pretends to save response content into a file with desired created date """ + logger.tqdm_write( + f"[DRY RUN] Would download {download_path}", + logging.INFO, + ) + return True + +# pylint: disable-msg=too-many-arguments +def download_media(logger, dry_run, icloud, photo, download_path, size) -> bool: """Download the photo to path, with retries and error handling""" - logger = setup_logger() - # get back the directory for the file to be downloaded and create it if - # not there already - download_dir = os.path.dirname(download_path) + mkdirs_local = mkdirs_for_path_dry_run if dry_run else mkdirs_for_path + download_local = download_response_to_path_dry_run if dry_run else download_response_to_path - if not os.path.exists(download_dir): - try: - os.makedirs(download_dir) - except OSError: # pragma: no cover - pass # pragma: no cover + if not mkdirs_local(logger, download_path): + return False for retries in range(constants.MAX_RETRIES): try: photo_response = photo.download(size) if photo_response: - temp_download_path = download_path + ".part" - with open(temp_download_path, "wb") as file_obj: - for chunk in photo_response.iter_content(chunk_size=1024): - if chunk: - file_obj.write(chunk) - os.rename(temp_download_path, download_path) - update_mtime(photo, download_path) - return True + return download_local(logger, photo_response, download_path, photo.created) logger.tqdm_write( f"Could not find URL to download {photo.filename} for size {size}!", @@ -89,16 +128,18 @@ def download_media(icloud, photo, download_path, size): time.sleep(wait_time) except IOError: - logger.error( - "IOError while writing file to %s! " - "You might have run out of disk space, or the file " - "might be too large for your OS. " - "Skipping this file...", download_path + logger.tqdm_write( + f"IOError while writing file to {download_path}! " + + "You might have run out of disk space, or the file " + + "might be too large for your OS. " + + "Skipping this file...", + logging.ERROR ) break else: logger.tqdm_write( - f"Could not download {photo.filename}! Please try again later." + f"Could not download {photo.filename}! Please try again later.", + logging.ERROR, ) return False diff --git a/src/icloudpd/exif_datetime.py b/src/icloudpd/exif_datetime.py index c050c43ec..22eec5fb7 100644 --- a/src/icloudpd/exif_datetime.py +++ b/src/icloudpd/exif_datetime.py @@ -1,22 +1,21 @@ """Get/set EXIF dates from photos""" +import logging import piexif from piexif._exceptions import InvalidImageDataError -from icloudpd.logger import setup_logger -def get_photo_exif(path): +def get_photo_exif(logger, path): """Get EXIF date for a photo, return nothing if there is an error""" try: exif_dict = piexif.load(path) return exif_dict.get("Exif").get(36867) except (ValueError, InvalidImageDataError): - logger = setup_logger() - logger.debug("Error fetching EXIF data for %s", path) + logger.tqdm_write(f"Error fetching EXIF data for {path}", logging.DEBUG) return None -def set_photo_exif(path, date): +def set_photo_exif(logger, path, date): """Set EXIF date on a photo, do nothing if there is an error""" try: exif_dict = piexif.load(path) @@ -26,6 +25,5 @@ def set_photo_exif(path, date): exif_bytes = piexif.dump(exif_dict) piexif.insert(exif_bytes, path) except (ValueError, InvalidImageDataError): - logger = setup_logger() - logger.debug("Error setting EXIF data for %s", path) + logger.tqdm_write(f"Error setting EXIF data for {path}", logging.DEBUG) return diff --git a/tests/test_authentication.py b/tests/test_authentication.py index 253f0e762..3166b2cbe 100644 --- a/tests/test_authentication.py +++ b/tests/test_authentication.py @@ -3,6 +3,7 @@ from vcr import VCR import pytest from click.testing import CliRunner +from icloudpd.logger import setup_logger import pyicloud_ipd from icloudpd.base import main from icloudpd.authentication import authenticator, TwoStepAuthRequiredError @@ -28,7 +29,7 @@ def test_failed_auth(self): with self.assertRaises( pyicloud_ipd.exceptions.PyiCloudFailedLoginException ) as context: - authenticator("com")( + authenticator(setup_logger(), "com")( "bad_username", "bad_password", client_id="EC5646DE-9423-11E8-BF21-14109FE0B321", @@ -44,7 +45,7 @@ def test_2sa_required(self): # delete ./tests/vcr_cassettes/auth_requires_2sa.yml, # put your actual credentials in here, run the test, # and then replace with dummy credentials. - authenticator("com")( + authenticator(setup_logger(), "com")( "jdoe@gmail.com", "password1", raise_error_on_2sa=True, @@ -58,7 +59,7 @@ def test_2sa_required(self): def test_successful_auth(self): with vcr.use_cassette(os.path.join(self.vcr_path, "successful_auth.yml")): - authenticator("com")( + authenticator(setup_logger(), "com")( "jdoe@gmail.com", "password1", client_id="EC5646DE-9423-11E8-BF21-14109FE0B321", diff --git a/tests/test_autodelete_photos.py b/tests/test_autodelete_photos.py index 73d7b0084..e19399d03 100644 --- a/tests/test_autodelete_photos.py +++ b/tests/test_autodelete_photos.py @@ -76,11 +76,11 @@ def astimezone(self, tz=None): self._caplog.text, ) self.assertIn( - f"INFO Downloading {os.path.join(base_dir, os.path.normpath('2018/01/01/IMG_3589.JPG'))}", + f"INFO Downloaded {os.path.join(base_dir, os.path.normpath('2018/01/01/IMG_3589.JPG'))}", self._caplog.text, ) self.assertIn( - f"INFO Deleting IMG_3589.JPG", + f"INFO Deleted IMG_3589.JPG", self._caplog.text, ) self.assertIn( @@ -123,7 +123,7 @@ def astimezone(self, tz=None): ) self.assertIn( - f"INFO Deleting {os.path.join(base_dir, os.path.normpath('2018/01/01/IMG_3589.JPG'))}", + f"INFO Deleted {os.path.join(base_dir, os.path.normpath('2018/01/01/IMG_3589.JPG'))}", self._caplog.text, ) @@ -166,11 +166,11 @@ def test_download_autodelete_photos(self): self._caplog.text, ) self.assertIn( - f"INFO Downloading {os.path.join(base_dir, os.path.normpath(files[0]))}", + f"DEBUG Downloading {os.path.join(base_dir, os.path.normpath(files[0]))}", self._caplog.text, ) self.assertIn( - f"INFO Deleting IMG_3589.JPG", + f"INFO Deleted IMG_3589.JPG", self._caplog.text, ) self.assertIn( @@ -212,7 +212,7 @@ def test_download_autodelete_photos(self): ) self.assertIn( - f"INFO Deleting {os.path.join(base_dir, os.path.normpath(files[0]))}", + f"INFO Deleted {os.path.join(base_dir, os.path.normpath(files[0]))}", self._caplog.text, ) @@ -282,24 +282,19 @@ def test_autodelete_photos(self): ) self.assertIn( - "INFO Deleting any files found in 'Recently Deleted'...", - self._caplog.text, - ) - - self.assertIn( - f"INFO Deleting {os.path.join(base_dir, os.path.normpath(files_to_delete[0]))}", + f"INFO Deleted {os.path.join(base_dir, os.path.normpath(files_to_delete[0]))}", self._caplog.text, ) self.assertIn( - f"INFO Deleting {os.path.join(base_dir, os.path.normpath(files_to_delete[1]))}", + f"INFO Deleted {os.path.join(base_dir, os.path.normpath(files_to_delete[1]))}", self._caplog.text, ) self.assertIn( - f"INFO Deleting {os.path.join(base_dir, os.path.normpath(files_to_delete[2]))}", + f"INFO Deleted {os.path.join(base_dir, os.path.normpath(files_to_delete[2]))}", self._caplog.text, ) self.assertIn( - f"INFO Deleting {os.path.join(base_dir, os.path.normpath(files_to_delete[3]))}", + f"INFO Deleted {os.path.join(base_dir, os.path.normpath(files_to_delete[3]))}", self._caplog.text, ) @@ -381,7 +376,7 @@ def mocked_authenticate(self): self._caplog.text, ) self.assertIn( - f"INFO Downloading {os.path.join(base_dir, os.path.normpath(files[0]))}", + f"DEBUG Downloading {os.path.join(base_dir, os.path.normpath(files[0]))}", self._caplog.text, ) @@ -465,7 +460,7 @@ def mocked_authenticate(self): self._caplog.text, ) self.assertIn( - f"INFO Downloading {os.path.join(base_dir, os.path.normpath(files[0]))}", + f"DEBUG Downloading {os.path.join(base_dir, os.path.normpath(files[0]))}", self._caplog.text, ) @@ -540,7 +535,7 @@ def mock_raise_response_error(a0_, a1_, a2_): self._caplog.text, ) self.assertIn( - f"INFO Downloading {os.path.join(base_dir, os.path.normpath(files[0]))}", + f"DEBUG Downloading {os.path.join(base_dir, os.path.normpath(files[0]))}", self._caplog.text, ) @@ -612,7 +607,7 @@ def mock_raise_response_error(a0_, a1_, a2_): self._caplog.text, ) self.assertIn( - f"INFO Downloading {os.path.join(base_dir, os.path.normpath(files[0]))}", + f"DEBUG Downloading {os.path.join(base_dir, os.path.normpath(files[0]))}", self._caplog.text, ) @@ -639,3 +634,103 @@ def mock_raise_response_error(a0_, a1_, a2_): base_dir, "**/*.*"), recursive=True) assert sum(1 for _ in files_in_result) == 1 + + def test_autodelete_photos_dry_run(self): + base_dir = os.path.join(self.fixtures_path, inspect.stack()[0][3]) + recreate_path(base_dir) + + files_to_create = [ + "2018/07/30/IMG_7407.JPG", + "2018/07/30/IMG_7407-original.JPG" + ] + + files_to_delete = [ + f"{'{:%Y/%m/%d}'.format(datetime.datetime.fromtimestamp(1532940539000.0 / 1000.0, tz=pytz.utc).astimezone(get_localzone()))}/IMG_7406.MOV", + f"{'{:%Y/%m/%d}'.format(datetime.datetime.fromtimestamp(1532618424000.0 / 1000.0, tz=pytz.utc).astimezone(get_localzone()))}/IMG_7383.PNG", + f"{'{:%Y/%m/%d}'.format(datetime.datetime.fromtimestamp(1531371164630.0 / 1000.0, tz=pytz.utc).astimezone(get_localzone()))}/IMG_7190.JPG", + f"{'{:%Y/%m/%d}'.format(datetime.datetime.fromtimestamp(1531371164630.0 / 1000.0, tz=pytz.utc).astimezone(get_localzone()))}/IMG_7190-medium.JPG" + ] + + os.makedirs(os.path.join( + base_dir, f"{'{:%Y/%m/%d}'.format(datetime.datetime.fromtimestamp(1532940539000.0 / 1000.0, tz=pytz.utc).astimezone(get_localzone()))}/")) + os.makedirs(os.path.join( + base_dir, f"{'{:%Y/%m/%d}'.format(datetime.datetime.fromtimestamp(1532618424000.0 / 1000.0, tz=pytz.utc).astimezone(get_localzone()))}/")) + os.makedirs(os.path.join( + base_dir, f"{'{:%Y/%m/%d}'.format(datetime.datetime.fromtimestamp(1531371164630.0 / 1000.0, tz=pytz.utc).astimezone(get_localzone()))}/")) + + # create some empty files + for file_name in files_to_create + files_to_delete: + open(os.path.join(base_dir, file_name), "a").close() + + with vcr.use_cassette(os.path.join(self.vcr_path, "autodelete_photos.yml")): + # Pass fixed client ID via environment variable + runner = CliRunner(env={ + "CLIENT_ID": "DE309E26-942E-11E8-92F5-14109FE0B321" + }) + result = runner.invoke( + main, + [ + "--username", + "jdoe@gmail.com", + "--password", + "password1", + "--dry-run", + "--recent", + "0", + "--skip-videos", + "--auto-delete", + "-d", + base_dir, + ], + ) + print_result_exception(result) + + self.assertIn( + "DEBUG Looking up all photos from album All Photos...", self._caplog.text) + self.assertIn( + f"INFO Downloading 0 original photos to {base_dir} ...", + self._caplog.text, + ) + self.assertIn( + "INFO All photos have been downloaded!", self._caplog.text + ) + self.assertIn( + "INFO Deleting any files found in 'Recently Deleted'...", + self._caplog.text, + ) + + self.assertIn( + f"INFO [DRY RUN] Would delete {os.path.join(base_dir, os.path.normpath(files_to_delete[0]))}", + self._caplog.text, + ) + self.assertIn( + f"INFO [DRY RUN] Would delete {os.path.join(base_dir, os.path.normpath(files_to_delete[1]))}", + self._caplog.text, + ) + self.assertIn( + f"INFO [DRY RUN] Would delete {os.path.join(base_dir, os.path.normpath(files_to_delete[2]))}", + self._caplog.text, + ) + self.assertIn( + f"INFO [DRY RUN] Would delete {os.path.join(base_dir, os.path.normpath(files_to_delete[3]))}", + self._caplog.text, + ) + + self.assertNotIn("IMG_7407.JPG", self._caplog.text) + self.assertNotIn("IMG_7407-original.JPG", self._caplog.text) + + self.assertEqual(result.exit_code, 0, "Exit code") + + files_in_result = glob.glob(os.path.join( + base_dir, "**/*.*"), recursive=True) + + self.assertEqual(sum(1 for _ in files_in_result), len(files_to_create) + len(files_to_delete), "Files in the result") + + # check files + for file_name in files_to_create: + assert os.path.exists(os.path.join( + base_dir, file_name)), f"{file_name} expected, but missing" + + for file_name in files_to_delete: + assert os.path.exists(os.path.join( + base_dir, file_name)), f"{file_name} expected to stay, but missing" diff --git a/tests/test_download_live_photos.py b/tests/test_download_live_photos.py index fe0062081..54843720f 100644 --- a/tests/test_download_live_photos.py +++ b/tests/test_download_live_photos.py @@ -65,15 +65,15 @@ def test_skip_existing_downloads_for_live_photos(self): print_result_exception(result) self.assertIn( - f"INFO Downloading {os.path.join(base_dir, os.path.normpath('2020/11/04/IMG_0514_HEVC.MOV'))}", + f"DEBUG Downloading {os.path.join(base_dir, os.path.normpath('2020/11/04/IMG_0514_HEVC.MOV'))}", self._caplog.text, ) self.assertIn( - f"INFO Downloading {os.path.join(base_dir, os.path.normpath('2020/11/04/IMG_0514.HEIC'))}", + f"DEBUG Downloading {os.path.join(base_dir, os.path.normpath('2020/11/04/IMG_0514.HEIC'))}", self._caplog.text, ) self.assertIn( - f"INFO Downloading {os.path.join(base_dir, os.path.normpath('2020/11/04/IMG_0516.HEIC'))}", + f"DEBUG Downloading {os.path.join(base_dir, os.path.normpath('2020/11/04/IMG_0516.HEIC'))}", self._caplog.text, ) self.assertIn( @@ -142,7 +142,7 @@ def test_skip_existing_live_photodownloads(self): self._caplog.text, ) self.assertIn( - f"INFO Downloading {os.path.join(base_dir, os.path.normpath('2020/11/04/IMG_0514.HEIC'))}", + f"DEBUG Downloading {os.path.join(base_dir, os.path.normpath('2020/11/04/IMG_0514.HEIC'))}", self._caplog.text, ) self.assertIn( @@ -150,7 +150,7 @@ def test_skip_existing_live_photodownloads(self): self._caplog.text, ) self.assertIn( - f"INFO Downloading {os.path.join(base_dir, os.path.normpath('2020/11/04/IMG_0514.HEIC'))}", + f"DEBUG Downloading {os.path.join(base_dir, os.path.normpath('2020/11/04/IMG_0514.HEIC'))}", self._caplog.text, ) self.assertIn( diff --git a/tests/test_download_photos.py b/tests/test_download_photos.py index 8af99847c..11270ea38 100644 --- a/tests/test_download_photos.py +++ b/tests/test_download_photos.py @@ -82,7 +82,7 @@ def test_download_and_skip_existing_photos(self): self._caplog.text, ) self.assertIn( - f"INFO Downloading {os.path.join(base_dir, os.path.normpath('2018/07/31/IMG_7409.JPG'))}", + f"DEBUG Downloading {os.path.join(base_dir, os.path.normpath('2018/07/31/IMG_7409.JPG'))}", self._caplog.text, ) self.assertNotIn( @@ -98,11 +98,11 @@ def test_download_and_skip_existing_photos(self): self._caplog.text, ) self.assertIn( - "INFO Skipping IMG_7405.MOV, only downloading photos.", + "DEBUG Skipping IMG_7405.MOV, only downloading photos.", self._caplog.text, ) self.assertIn( - "INFO Skipping IMG_7404.MOV, only downloading photos.", + "DEBUG Skipping IMG_7404.MOV, only downloading photos.", self._caplog.text, ) self.assertIn( @@ -202,7 +202,7 @@ def mocked_download(self, size): self._caplog.text, ) self.assertIn( - f"INFO Downloading {os.path.join(base_dir, os.path.normpath('2018/07/31/IMG_7409.JPG'))}", + f"DEBUG Downloading {os.path.join(base_dir, os.path.normpath('2018/07/31/IMG_7409.JPG'))}", self._caplog.text, ) # 2018:07:31 07:22:24 utc @@ -271,7 +271,7 @@ def test_download_photos_and_get_exif_exceptions(self): self._caplog.text, ) self.assertIn( - f"INFO Downloading {os.path.join(base_dir, os.path.normpath('2018/07/31/IMG_7409.JPG'))}", + f"DEBUG Downloading {os.path.join(base_dir, os.path.normpath('2018/07/31/IMG_7409.JPG'))}", self._caplog.text, ) self.assertIn( @@ -435,7 +435,7 @@ def test_until_found(self): expected_calls = list( map( lambda f: call( - ANY, ANY, os.path.join( + ANY, False, ANY, ANY, os.path.join( base_dir, os.path.normpath(f[0])), "mediumVideo" if ( f[1] == 'photo' and f[0].endswith('.MOV') @@ -588,7 +588,7 @@ def mocked_authenticate(self): ) self.assertIn( - "INFO Could not download IMG_7409.JPG! Please try again later.", + "ERROR Could not download IMG_7409.JPG! Please try again later.", self._caplog.text, ) @@ -730,7 +730,7 @@ def mocked_authenticate(self): ) self.assertIn( - "INFO Could not download IMG_7409.JPG! Please try again later.", + "ERROR Could not download IMG_7409.JPG! Please try again later.", self._caplog.text, ) assert result.exit_code == 0 @@ -910,13 +910,15 @@ def test_size_fallback_to_original(self): self._caplog.text, ) self.assertIn( - f"INFO Downloading {os.path.join(base_dir, os.path.normpath('2018/07/31/IMG_7409.JPG'))}", + f"DEBUG Downloading {os.path.join(base_dir, os.path.normpath('2018/07/31/IMG_7409.JPG'))}", self._caplog.text, ) self.assertIn( "INFO All photos have been downloaded!", self._caplog.text ) dp_patched.assert_called_once_with( + ANY, + False, ANY, ANY, f"{os.path.join(base_dir, os.path.normpath('2018/07/31/IMG_7409.JPG'))}", @@ -1043,7 +1045,7 @@ def astimezone(self, tz=None): self._caplog.text, ) self.assertIn( - f"INFO Downloading {os.path.join(base_dir, os.path.normpath('2018/01/01/IMG_7409.JPG'))}", + f"DEBUG Downloading {os.path.join(base_dir, os.path.normpath('2018/01/01/IMG_7409.JPG'))}", self._caplog.text, ) self.assertIn( @@ -1117,7 +1119,7 @@ def astimezone(self, tz=None): self._caplog.text, ) self.assertIn( - f"INFO Downloading {os.path.join(base_dir, os.path.normpath('5/01/01/IMG_7409.JPG'))}", + f"DEBUG Downloading {os.path.join(base_dir, os.path.normpath('5/01/01/IMG_7409.JPG'))}", self._caplog.text, ) self.assertIn( @@ -1176,7 +1178,7 @@ def test_unknown_item_type(self): self._caplog.text, ) self.assertIn( - "INFO Skipping IMG_7409.JPG, only downloading photos and videos. (Item type was: unknown)", + "DEBUG Skipping IMG_7409.JPG, only downloading photos and videos. (Item type was: unknown)", self._caplog.text, ) self.assertIn( @@ -1253,15 +1255,15 @@ def mocked_download(self, size): self._caplog.text, ) self.assertIn( - f"INFO Downloading {os.path.join(base_dir, os.path.normpath('2018/07/31/IMG_7409-1884695.JPG'))}", + f"DEBUG Downloading {os.path.join(base_dir, os.path.normpath('2018/07/31/IMG_7409-1884695.JPG'))}", self._caplog.text, ) self.assertIn( - f"INFO {os.path.join(base_dir, os.path.normpath('2018/07/31/IMG_7409-3294075.MOV'))} deduplicated.", + f"DEBUG {os.path.join(base_dir, os.path.normpath('2018/07/31/IMG_7409-3294075.MOV'))} deduplicated.", self._caplog.text, ) self.assertIn( - f"INFO Downloading {os.path.join(base_dir, os.path.normpath('2018/07/31/IMG_7409-3294075.MOV'))}", + f"DEBUG Downloading {os.path.join(base_dir, os.path.normpath('2018/07/31/IMG_7409-3294075.MOV'))}", self._caplog.text, ) self.assertIn( @@ -1273,10 +1275,10 @@ def mocked_download(self, size): self._caplog.text, ) self.assertIn( - "INFO Skipping IMG_7405.MOV, only downloading photos.", self._caplog.text + "DEBUG Skipping IMG_7405.MOV, only downloading photos.", self._caplog.text ) self.assertIn( - "INFO Skipping IMG_7404.MOV, only downloading photos.", self._caplog.text + "DEBUG Skipping IMG_7404.MOV, only downloading photos.", self._caplog.text ) self.assertIn( "INFO All photos have been downloaded!", self._caplog.text @@ -1352,7 +1354,7 @@ def test_download_photos_and_set_exif_exceptions(self): self._caplog.text, ) self.assertIn( - f"INFO Downloading {os.path.join(base_dir, os.path.normpath('2018/07/31/IMG_7409.JPG'))}", + f"DEBUG Downloading {os.path.join(base_dir, os.path.normpath('2018/07/31/IMG_7409.JPG'))}", self._caplog.text, ) # 2018:07:31 07:22:24 utc @@ -1421,7 +1423,7 @@ def test_download_chinese(self): self._caplog.text, ) self.assertIn( - f"INFO Downloading {os.path.join(base_dir, os.path.normpath('2018/07/31/IMG_7409.JPG'))}", + f"DEBUG Downloading {os.path.join(base_dir, os.path.normpath('2018/07/31/IMG_7409.JPG'))}", self._caplog.text, ) self.assertNotIn( @@ -1502,11 +1504,11 @@ def test_download_after_delete(self): self._caplog.text, ) self.assertIn( - f"INFO Downloading {os.path.join(base_dir, os.path.normpath('2018/07/31/IMG_7409.JPG'))}", + f"DEBUG Downloading {os.path.join(base_dir, os.path.normpath('2018/07/31/IMG_7409.JPG'))}", self._caplog.text, ) self.assertIn( - "INFO Deleting IMG_7409.JPG", self._caplog.text + "INFO Deleted IMG_7409.JPG in iCloud", self._caplog.text ) self.assertIn( "INFO All photos have been downloaded!", self._caplog.text @@ -1560,11 +1562,11 @@ def test_download_after_delete_fail(self): self._caplog.text, ) self.assertIn( - f"INFO Downloading {os.path.join(base_dir, os.path.normpath('2018/07/31/IMG_7409.JPG'))}", + f"DEBUG Downloading {os.path.join(base_dir, os.path.normpath('2018/07/31/IMG_7409.JPG'))}", self._caplog.text, ) self.assertNotIn( - "INFO Deleting IMG_7409.JPG", self._caplog.text + "INFO Deleted IMG_7409.JPG in iCloud", self._caplog.text ) self.assertIn( "INFO All photos have been downloaded!", self._caplog.text @@ -1628,7 +1630,7 @@ def test_download_over_old_original_photos(self): self._caplog.text, ) self.assertIn( - f"INFO Downloading {os.path.join(base_dir, os.path.normpath('2018/07/31/IMG_7409.JPG'))}", + f"DEBUG Downloading {os.path.join(base_dir, os.path.normpath('2018/07/31/IMG_7409.JPG'))}", self._caplog.text, ) self.assertNotIn( @@ -1644,11 +1646,11 @@ def test_download_over_old_original_photos(self): self._caplog.text, ) self.assertIn( - "INFO Skipping IMG_7405.MOV, only downloading photos.", + "DEBUG Skipping IMG_7405.MOV, only downloading photos.", self._caplog.text, ) self.assertIn( - "INFO Skipping IMG_7404.MOV, only downloading photos.", + "DEBUG Skipping IMG_7404.MOV, only downloading photos.", self._caplog.text, ) self.assertIn( @@ -1856,7 +1858,7 @@ def mock_raise_response_error(arg): # ) self.assertIn( - "INFO Could not download IMG_7409.JPG! Please try again later.", + "ERROR Could not download IMG_7409.JPG! Please try again later.", self._caplog.text, ) @@ -1928,3 +1930,196 @@ def mock_raise_response_error(offset): base_dir, "**/*.*"), recursive=True) assert sum(1 for _ in files_in_result) == 0 + + def test_handle_io_error_mkdir(self): + base_dir = os.path.join(self.fixtures_path, inspect.stack()[0][3]) + recreate_path(base_dir) + + with vcr.use_cassette(os.path.join(self.vcr_path, "listing_photos.yml")): + with mock.patch("os.makedirs", create=True) as m: + # Raise IOError when we try to write to the destination file + m.side_effect = IOError + + runner = CliRunner(env={ + "CLIENT_ID": "DE309E26-942E-11E8-92F5-14109FE0B321" + }) + result = runner.invoke( + main, + [ + "--username", + "jdoe@gmail.com", + "--password", + "password1", + "--recent", + "1", + "--skip-videos", + "--skip-live-photos", + "--no-progress-bar", + "--threads-num", + 1, + "-d", + base_dir, + ], + ) + print_result_exception(result) + + self.assertIn( + "DEBUG Looking up all photos from album All Photos...", self._caplog.text) + self.assertIn( + f"INFO Downloading the first original photo to {base_dir} ...", + self._caplog.text, + ) + self.assertIn( + f"ERROR Could not create folder {base_dir}", + self._caplog.text, + ) + self.assertEqual(result.exit_code, 0, "Exit code") + + files_in_result = glob.glob(os.path.join( + base_dir, "**/*.*"), recursive=True) + + self.assertEqual(sum(1 for _ in files_in_result), 0, "Files at the end") + + def test_dry_run(self): + base_dir = os.path.join(self.fixtures_path, inspect.stack()[0][3]) + recreate_path(base_dir) + + files_to_download = [ + '2018/07/31/IMG_7409.JPG', + # "2018/07/30/IMG_7408.JPG", + # "2018/07/30/IMG_7407.JPG", + ] + + with vcr.use_cassette(os.path.join(self.vcr_path, "listing_photos.yml")): + # Pass fixed client ID via environment variable + runner = CliRunner(env={ + "CLIENT_ID": "DE309E26-942E-11E8-92F5-14109FE0B321" + }) + result = runner.invoke( + main, + [ + "--username", + "jdoe@gmail.com", + "--password", + "password1", + "--recent", + "1", + "--skip-videos", + "--skip-live-photos", + "--set-exif-datetime", + "--no-progress-bar", + "--dry-run", + "--threads-num", + 1, + "-d", + base_dir, + ], + ) + print_result_exception(result) + + self.assertIn( + "DEBUG Looking up all photos from album All Photos...", self._caplog.text) + # self.assertIn( + # f"INFO Downloading 2 original photos to {base_dir} ...", + # self._caplog.text, + # ) + for f in files_to_download: + self.assertIn( + f"DEBUG Downloading {os.path.join(base_dir, os.path.normpath(f))}", + self._caplog.text, + ) + self.assertNotIn( + "IMG_7409.MOV", + self._caplog.text, + ) + self.assertNotIn( + "ERROR", + self._caplog.text, + ) + # self.assertIn( + # "DEBUG Skipping IMG_7405.MOV, only downloading photos.", + # self._caplog.text, + # ) + # self.assertIn( + # "DEBUG Skipping IMG_7404.MOV, only downloading photos.", + # self._caplog.text, + # ) + self.assertIn( + "INFO All photos have been downloaded!", self._caplog.text + ) + + assert result.exit_code == 0 + + files_in_result = glob.glob(os.path.join( + base_dir, "**/*.*"), recursive=True) + + self.assertEqual(sum(1 for _ in files_in_result), 0, "Files in the result") + + def test_download_after_delete_dry_run(self): + base_dir = os.path.join(self.fixtures_path, inspect.stack()[0][3]) + recreate_path(base_dir) + + def raise_response_error(a0_, a1_, a2_): + raise Exception("Unexpected call to delete_photo") + + with mock.patch.object(piexif, "insert") as piexif_patched: + piexif_patched.side_effect = InvalidImageDataError + with mock.patch( + "icloudpd.exif_datetime.get_photo_exif" + ) as get_exif_patched: + get_exif_patched.return_value = False + with mock.patch( + "icloudpd.base.delete_photo" + ) as df_patched: + df_patched.side_effect = raise_response_error + + with vcr.use_cassette(os.path.join(self.vcr_path, "listing_photos.yml")) as cass: + # Pass fixed client ID via environment variable + runner = CliRunner(env={ + "CLIENT_ID": "DE309E26-942E-11E8-92F5-14109FE0B321" + }) + result = runner.invoke( + main, + [ + "--username", + "jdoe@gmail.com", + "--password", + "password1", + "--recent", + "1", + "--skip-videos", + "--skip-live-photos", + "--no-progress-bar", + "--dry-run", + "--threads-num", + 1, + "--delete-after-download", + "-d", + base_dir, + ], + ) + print_result_exception(result) + + self.assertIn( + "DEBUG Looking up all photos from album All Photos...", self._caplog.text) + self.assertIn( + f"INFO Downloading the first original photo to {base_dir} ...", + self._caplog.text, + ) + self.assertIn( + f"DEBUG Downloading {os.path.join(base_dir, os.path.normpath('2018/07/31/IMG_7409.JPG'))}", + self._caplog.text, + ) + self.assertIn( + "INFO [DRY RUN] Would delete IMG_7409.JPG in iCloud", self._caplog.text + ) + self.assertIn( + "INFO All photos have been downloaded!", self._caplog.text + ) + self.assertEqual(cass.all_played, False, "All mocks played") + self.assertEqual(result.exit_code, 0, "Exit code") + + files_in_result = glob.glob(os.path.join( + base_dir, "**/*.*"), recursive=True) + + self.assertEqual( sum(1 for _ in files_in_result), 0, "Files in the result")