From 50961be2fb369661a29003c945ef1badf216f320 Mon Sep 17 00:00:00 2001 From: Brian Duncan Date: Tue, 3 Dec 2024 05:58:36 -0600 Subject: [PATCH] add script to look for invalid robot configs (#615) --- .../_scripts/fix-and-report-robot-config.py | 194 ++++++++++++++++++ 1 file changed, 194 insertions(+) create mode 100644 castle/cms/_scripts/fix-and-report-robot-config.py diff --git a/castle/cms/_scripts/fix-and-report-robot-config.py b/castle/cms/_scripts/fix-and-report-robot-config.py new file mode 100644 index 000000000..96e3bb9a6 --- /dev/null +++ b/castle/cms/_scripts/fix-and-report-robot-config.py @@ -0,0 +1,194 @@ +import argparse +import transaction +import os +import plone.api as api + +from zope.component.hooks import setSite +import csv +from decimal import Decimal +from castle.cms.behaviors.search import ISearch + + +parser = argparse.ArgumentParser(description='Fix any issues with None values in existing robot configs, and run a report on current values at the same time') # noqa: E501 +parser.add_argument('--site-id', dest='site_id', required=True) +parser.add_argument('--dir-path', dest='dir_path', required=True) +parser.add_argument('--file-name', dest='file_name', default='robot-report.csv') +parser.add_argument('--commit', dest='should_commit', type=bool, default=False) +parser.add_argument('--batch-size', dest='batch_size', type=int, default=50) + +parsed_args, _ = parser.parse_known_args() + +NO_VALUE = '' + + +class FileExistsError(Exception): + pass + + +def get_physical_path(content_object): + path_segments = [ + segment + for segment in content_object.getPhysicalPath() + ] + public_url = api.portal.get_registry_record('plone.public_url', None) + if public_url: + path_segments = [public_url] + path_segments[2:] + return '/'.join(path_segments) + + +def get_status_info(content_object): + return { + 'physical_path': get_physical_path(content_object), + 'workflow_state': api.content.get_state(content_object), + 'robot_configuration': getattr( + content_object, + 'robot_configuration', + NO_VALUE, + ), + 'robot_configuration_reset': False, + 'backend_robot_configuration': getattr( + content_object, + 'backend_robot_configuration', + NO_VALUE, + ), + 'backend_robot_configuration_reset': False, + } + + +def fix_robot_configs(): + report_data = [] + catalog = api.portal.get_tool('portal_catalog') + content_object_brains = catalog.unrestrictedSearchResults( + object_provides=ISearch.__identifier__, + ) + content_object_count = len(content_object_brains) + print('Found {} content objects to process'.format(content_object_count)) + current_index = 0 + for content_object_brain in content_object_brains: + try: + content_object = content_object_brain.getObject() + content_status_info = get_status_info(content_object) + + for robot_key in ['robot_configuration', 'backend_robot_configuration']: + current_value = content_status_info[robot_key] + + # this means the value is present but is not a list - that's the only scenario that's bad + if not isinstance(current_value, list) and current_value != NO_VALUE: + try: + delattr(content_object, robot_key) + content_status_info[robot_key] = NO_VALUE + content_status_info[robot_key + '_reset'] = True + + # now show what the new rendered value on the object is + if content_status_info[robot_key] == NO_VALUE: + content_status_info[robot_key] = getattr( + content_object, + robot_key, + '', + ), + print('Fixed {robot_key} for {obj} with value {configuration_value}'.format( + robot_key=robot_key, + obj=content_object, + configuration_value=content_status_info[robot_key], + )) + except Exception: + print( + 'Could not delete {robot_key} for {obj} with value {configuration_value}'.format( + robot_key=robot_key, + obj=content_object, + configuration_value=content_status_info[robot_key], + ) + ) + continue + except Exception: + print('Something very unexpected went wrong handling {}. Continuing'.format( + content_object_brain.getPath() + )) + continue + + report_data.append(content_status_info) + current_index += 1 + commit_action = 'Committing' if parsed_args.should_commit is True else 'Not committing' + if current_index % parsed_args.batch_size == 0: + print( + '{percentage}% Complete ({current_index} of {total_count} objects processed). {commit_action}.'.format( # noqa: E501 + percentage=round(Decimal(current_index) / Decimal(content_object_count) * 100, 2), + current_index=current_index, + total_count=content_object_count, + commit_action=commit_action, + ) + ) + + if parsed_args.should_commit is True: + transaction.commit() + print('100% Complete ({total_count} of {total_count} objects processed). {commit_action}.'.format( + total_count=content_object_count, + commit_action=commit_action, + )) + + if parsed_args.should_commit is True: + transaction.commit() + + reset_item_paths = [ + item_info['physical_path'] + for item_info in report_data + if item_info['robot_configuration_reset'] or item_info['backend_robot_configuration_reset'] + ] + + print( + 'Done fixing robot configs for {total_count} objects. {reset_count} objects reset: {reset_item_paths}'.format( # noqa: E501 + total_count=content_object_count, + reset_count=len(reset_item_paths), + reset_item_paths=reset_item_paths, + ) + ) + return report_data + + +def get_csv_report_row(report_row_data): + return [ + report_row_data['physical_path'], + report_row_data['workflow_state'], + report_row_data['robot_configuration'], + report_row_data.get('robot_configuration_reset', False), + report_row_data['backend_robot_configuration'], + report_row_data.get('backend_robot_configuration_reset', False), + ] + + +def get_target_file_path(): + return os.path.join(parsed_args.dir_path, parsed_args.file_name) + + +def verify_target_file_does_not_exist(): + if os.path.exists(get_target_file_path()): + raise FileExistsError + + +def write_report(report_data): + file_path = get_target_file_path() + print('Writing report to ' + file_path) + with open(file_path, 'wb') as csvfile: + csv_writer = csv.DictWriter( + csvfile, + fieldnames=[ + 'physical_path', + 'workflow_state', + 'robot_configuration', + 'robot_configuration_reset', + 'backend_robot_configuration', + 'backend_robot_configuration_reset', + ], + ) + csv_writer.writeheader() + csv_writer.writerows(report_data) + print('Finished writing report') + + +if __name__ == '__main__': + verify_target_file_does_not_exist() + site = app[parsed_args.site_id] # noqa: F821 + setSite(site) + report_data = fix_robot_configs() + write_report(report_data) + print("That's all folks! We hope you've enjoyed your scripting experience.")