diff --git a/diycrate/cache_utils.py b/diycrate/cache_utils.py index 31a78dd..0b7258d 100644 --- a/diycrate/cache_utils.py +++ b/diycrate/cache_utils.py @@ -17,7 +17,7 @@ crate_logger = logging.getLogger(__name__) -def redis_key(key): +def redis_key(key: str) -> str: """ :param key: @@ -26,6 +26,11 @@ def redis_key(key): return "diy_crate.version.{}".format(key) +def redis_path_for_object_id_key(key: PathLike): + key_str = key.as_posix() if isinstance(key, Path) else key + return f"diy_crate.path_for_object_id:{key_str}" + + def redis_set( cache_client: redis.Redis, cloud_item: Union[File, Folder], @@ -81,12 +86,16 @@ def redis_set( cache_client.set(key, json.dumps(item_info)) last_save_time_stamp = int(time.time()) cache_client.set("diy_crate.last_save_time_stamp", last_save_time_stamp) + cache_client.set( + redis_path_for_object_id_key((path / cloud_item["name"])), cloud_item.object_id + ) + crate_logger.debug(f"Storing/updating info to redis: {key=} {item_info=}") # assert redis_get(obj) return {key: item_info, "diy_crate.last_save_time_stamp": last_save_time_stamp} -def redis_get(cache_client, obj): +def redis_get(cache_client, obj: Union[File, Folder]): """ :param cache_client: @@ -104,6 +113,9 @@ def id_for_file_path(cache_client, file_path): :param file_path: :return: """ + maybe_obj_id = cache_client.get(redis_path_for_object_id_key(file_path)) + if maybe_obj_id: + return str(maybe_obj_id) for key in cache_client.keys("diy_crate.version.*"): value = json.loads( str(cache_client.get(key), encoding="utf-8", errors="strict") diff --git a/diycrate/file_operations.py b/diycrate/file_operations.py index e7f7979..45b379e 100644 --- a/diycrate/file_operations.py +++ b/diycrate/file_operations.py @@ -30,6 +30,7 @@ r_c, local_or_box_file_m_time_key_func, redis_set, + redis_path_for_object_id_key, ) from .item_queue_io import ( UploadQueueItem, @@ -111,7 +112,7 @@ def operation_coalesce(self): ] # keep a local copy for this loop-run # operations list could have changed since the previous two instructions # pycharm complained that I was re-assigning the instance - # variable outside of the __init__. + # variable outside the __init__. self.operations.clear() self.operations.extend( self.operations[cur_num_operations:] @@ -659,7 +660,7 @@ def process_move_event(self, event: pyinotify.Event): is_dir and entry["name"] == src_event.name and entry["type"] == "folder" ) if did_find_src_file: - src_file = client.file(file_id=entry["id"]).get() + src_file: File = client.file(file_id=entry["id"]).get() if is_rename: last_modified_time = ( datetime.fromtimestamp( @@ -682,25 +683,40 @@ def process_move_event(self, event: pyinotify.Event): .timestamp(), ) src_file.rename(dest_event.name) - file_obj = client.file(file_id=src_file.object_id).get() + file_obj: File = client.file(file_id=src_file.object_id).get() version_info["file_path"] = dest_event.pathname version_info["etag"] = file_obj["etag"] + r_c.set( + redis_path_for_object_id_key(dest_event.pathname), + src_file.object_id, + ) + r_c.delete( + redis_path_for_object_id_key( + Path(src_event.path) / src_file.name + ) + ) r_c.set(redis_key(src_file.object_id), json.dumps(version_info)) r_c.set("diy_crate.last_save_time_stamp", int(time.time())) path_builder = BOX_DIR oauth = setup_remote_oauth(r_c, conf=conf_obj) client = Client(oauth) - for updated_entry in ( + parent_folders: list[Folder] = ( client.file(file_obj.object_id) .get(fields=["path_collection"]) .path_collection["entries"] - ): + ) + for updated_entry in parent_folders: if updated_entry.id == "0": continue path_builder /= updated_entry.name - folder_entry = client.folder(updated_entry.id).get( + folder_entry: Folder = client.folder(updated_entry.id).get( fields=["modified_at"] ) + if not r_c.exists(redis_path_for_object_id_key(path_builder)): + r_c.set( + redis_path_for_object_id_key(path_builder), + updated_entry.object_id, + ) r_c.set( local_or_box_file_m_time_key_func(path_builder, True), parse(folder_entry.modified_at) @@ -713,9 +729,10 @@ def process_move_event(self, event: pyinotify.Event): dest_event.pathname ) # should check box instead cur_offset = 0 - for cur_entry in cur_box_folder.get_items( - offset=cur_offset, limit=limit - ): + cur_box_folder_items: list[ + Union[File, Folder] + ] = cur_box_folder.get_items(offset=cur_offset, limit=limit) + for cur_entry in cur_box_folder_items: matching_name = cur_entry["name"] == os.path.basename( dest_event.pathname ) @@ -1020,7 +1037,7 @@ def process_IN_MOVED_TO(self, event: pyinotify.Event): self.incoming_operations.put( [event, "modify"] ) # "close"/"modify" seems appropriate - # allow moving from a ~.lock file...i guess that may be okay + # allow moving from a ~.lock file...I guess that may be okay crate_logger.debug("Moved to: {}".format(event.pathname)) # noinspection PyPep8Naming,PyMethodMayBeStatic diff --git a/diycrate/item_queue_io.py b/diycrate/item_queue_io.py index c8e52f3..62cbbe9 100644 --- a/diycrate/item_queue_io.py +++ b/diycrate/item_queue_io.py @@ -27,6 +27,7 @@ redis_get, r_c, local_or_box_file_m_time_key_func, + redis_path_for_object_id_key, ) from .gui import notify_user_with_gui from .log_utils import setup_logger @@ -111,7 +112,9 @@ def perform_upload( item = ret_val # is the new/updated item if isinstance(item, File): client = Client(oauth) - file_obj: File = client.file(file_id=item.object_id).get() + file_obj: File = client.file(file_id=item.object_id).get( + fields=["path_collection", "name", "etag"] + ) redis_set(r_c, file_obj, last_modified_time, box_dir_path=BOX_DIR) r_c.set( local_or_box_file_m_time_key_func(path_name, False), @@ -119,15 +122,12 @@ def perform_upload( .astimezone(dateutil.tz.tzutc()) .timestamp(), ) + r_c.set(redis_path_for_object_id_key(path_name), item.object_id) path_builder = BOX_DIR - oauth = setup_remote_oauth(r_c, conf=conf_obj) - client = Client(oauth) - for entry in ( - client.file(item.object_id) - .get(fields=["path_collection"]) - .path_collection["entries"] - ): + # oauth = setup_remote_oauth(r_c, conf=conf_obj) + # client = Client(oauth) + for entry in file_obj.path_collection["entries"]: if entry.id == "0": continue path_builder /= entry.name @@ -140,6 +140,10 @@ def perform_upload( .astimezone(dateutil.tz.tzutc()) .timestamp(), ) + r_c.set( + redis_path_for_object_id_key(path_builder), + folder_entry.object_id, + ) break except BoxAPIException as e: @@ -296,6 +300,7 @@ def perform_download(item: File, path: Union[str, Path], retry_limit=15): local_or_box_file_m_time_key_func(path / item.name, True), parse(item.modified_at).astimezone(dateutil.tz.tzutc()).timestamp(), ) + r_c.set(redis_path_for_object_id_key(path / item.name), item.object_id) path_builder = BOX_DIR oauth = setup_remote_oauth(r_c, conf=conf_obj) client = Client(oauth) @@ -312,12 +317,25 @@ def perform_download(item: File, path: Union[str, Path], retry_limit=15): .astimezone(dateutil.tz.tzutc()) .timestamp(), ) + r_c.set( + redis_path_for_object_id_key(path_builder), + folder_entry.object_id, + ) except boxsdk.exception.BoxAPIException as box_exc: if box_exc.status == 404 and box_exc.code == "trashed": crate_logger.debug( f"Object {item.object_id=} was previously deleted from Box. " f"{(path / item.name)=}" ) + + object_id_lookup = r_c.get( + redis_path_for_object_id_key(path / item.name) + ) + object_id_lookup = ( + str(object_id_lookup) if object_id_lookup else object_id_lookup + ) + if object_id_lookup == item.object_id: + r_c.delete(redis_path_for_object_id_key(path / item.name)) r_c.delete(redis_key(item.object_id)) r_c.set("diy_crate.last_save_time_stamp", int(time.time())) else: diff --git a/diycrate/long_poll_processing.py b/diycrate/long_poll_processing.py index 7025a10..a2e84cc 100644 --- a/diycrate/long_poll_processing.py +++ b/diycrate/long_poll_processing.py @@ -29,6 +29,7 @@ redis_key, redis_get, local_or_box_file_m_time_key_func, + redis_path_for_object_id_key, ) from diycrate.gui import notify_user_with_gui from diycrate.item_queue_io import ( @@ -119,6 +120,9 @@ def process_item_create_long_poll(client: Client, event: Union[Event, Mapping]): local_or_box_file_m_time_key_func(path / box_item.name, True), parse(box_item.modified_at).astimezone(tzutc()).timestamp(), ) + r_c.set( + redis_path_for_object_id_key(path / box_item.name), box_item.object_id + ) r_c.setex( f"diy_crate:event_ids:{event.event_id}", timedelta(days=32), @@ -218,6 +222,9 @@ def process_item_copy_long_poll(client: Client, event: Union[Event, Mapping]): local_or_box_file_m_time_key_func(path / box_item.name, True), parse(box_item.modified_at).astimezone(tzutc()).timestamp(), ) + r_c.set( + redis_path_for_object_id_key(path / box_item.name), box_item.object_id + ) r_c.setex( f"diy_crate:event_ids:{event.event_id}", timedelta(days=32), @@ -304,7 +311,7 @@ def process_item_trash_file(event: Union[Event, Mapping], obj_id): if file_path.exists(): send2trash(file_path.as_posix()) if r_c.exists(redis_key(obj_id)): - r_c.delete(redis_key(obj_id)) + r_c.delete(redis_key(obj_id), redis_path_for_object_id_key(file_path)) r_c.set("diy_crate.last_save_time_stamp", int(time.time())) notify_user_with_gui( "Box message: Deleted:", @@ -337,14 +344,20 @@ def process_item_trash_folder(event: Union[Event, Mapping], obj_id): file_path = path / item_info["file_path"] if file_path.is_dir(): # still need to get the parent_id - for box_id in get_sub_ids(obj_id): - r_c.delete(redis_key(box_id)) + for sub_box_id in get_sub_ids(obj_id): + cur_sub_box_redis_data = r_c.get(redis_key(sub_box_id)) + if cur_sub_box_redis_data: + cur_sub_box_item_info = json.loads( + str(cur_sub_box_redis_data, encoding="utf-8", errors="strict") + ) + r_c.delete(cur_sub_box_item_info["file_name"]) + r_c.delete(redis_key(sub_box_id)) r_c.delete(redis_key(obj_id)) shutil.rmtree(file_path) - obj_cache_data = r_c.get(redis_key(obj_id)) + obj_cache_data = json.loads(str(r_c.get(redis_key(obj_id)))) parent_id = obj_cache_data.get("parent_id") if obj_cache_data else None if parent_id: - parent_folder = r_c.get(redis_key(parent_id)) + parent_folder = json.loads(str(r_c.get(redis_key(parent_id)))) sub_ids = parent_folder.get("sub_ids", []) if sub_ids: sub_ids.remove(obj_id) @@ -514,8 +527,15 @@ def process_item_rename_long_poll(client: Client, event: Union[Event, Mapping]): local_or_box_file_m_time_key_func(path / file_obj.name, True), parse(file_obj.modified_at).astimezone(tzutc()).timestamp(), ) - r_c.delete(local_or_box_file_m_time_key_func(src_file_path, False)) - r_c.delete(local_or_box_file_m_time_key_func(src_file_path, True)) + r_c.set( + redis_path_for_object_id_key(path / file_obj.name), + file_obj.object_id, + ) + r_c.delete( + redis_path_for_object_id_key(src_file_path), + local_or_box_file_m_time_key_func(src_file_path, False), + local_or_box_file_m_time_key_func(src_file_path, True), + ) r_c.setex( f"diy_crate:event_ids:{event.event_id}", timedelta(days=32), @@ -603,6 +623,10 @@ def process_item_rename_long_poll(client: Client, event: Union[Event, Mapping]): local_or_box_file_m_time_key_func(path / folder_obj.name, True), parse(folder_obj.modified_at).astimezone(tzutc()).timestamp(), ) + r_c.set( + redis_path_for_object_id_key(path / folder_obj.name), + folder_obj.object_id, + ) r_c.delete(local_or_box_file_m_time_key_func(src_file_path, False)) r_c.delete(local_or_box_file_m_time_key_func(src_file_path, True)) r_c.setex( @@ -677,6 +701,13 @@ def process_item_move_long_poll(event: Union[Event, Mapping]): json.dumps(sub_item_info), ) ) + to_set.append( + partial( + r_c.set, + redis_path_for_object_id_key(new_sub_path), + sub_id, + ) + ) shutil.move(src_file_path, file_path) for item in to_set: item() @@ -698,6 +729,8 @@ def process_item_move_long_poll(event: Union[Event, Mapping]): .astimezone(tzutc()) .timestamp(), ) + r_c.set(redis_path_for_object_id_key(file_path), obj_id) + r_c.delete(redis_path_for_object_id_key(src_file_path)) r_c.delete(local_or_box_file_m_time_key_func(src_file_path, False)) r_c.delete(local_or_box_file_m_time_key_func(src_file_path, True)) r_c.setex( diff --git a/diycrate/path_utils.py b/diycrate/path_utils.py index bbad046..b5f6bed 100644 --- a/diycrate/path_utils.py +++ b/diycrate/path_utils.py @@ -31,6 +31,7 @@ redis_set, redis_get, local_or_box_file_m_time_key_func, + redis_path_for_object_id_key, ) from .iter_utils import SafeIter from .log_utils import setup_logger @@ -143,6 +144,7 @@ def walk_and_notify_and_download_tree( local_or_box_file_m_time_key_func(path / box_item.name, True), parse(box_item.modified_at).astimezone(tzutc()).timestamp(), ) + r_c.set(redis_path_for_object_id_key(path / box_item.name), box_item.object_id) if not local_only: redis_set( cache_client=r_c,