From 98ba74a96a018c77be5f8da7186479f7785d9235 Mon Sep 17 00:00:00 2001 From: Pablo Olivares <65406121+pab1s@users.noreply.github.com> Date: Mon, 15 Jul 2024 22:39:03 +0200 Subject: [PATCH] Final cleaning closes #2 --- config/config_efficientnet.yaml | 24 ++++- config/mm_config_densenet.yaml | 33 +++++-- environment.yaml | 2 +- fine_tuning.py | 91 ------------------ homology_times.py | 121 ------------------------ main.py | 163 -------------------------------- train.py | 81 ---------------- 7 files changed, 47 insertions(+), 468 deletions(-) delete mode 100644 fine_tuning.py delete mode 100644 homology_times.py delete mode 100644 main.py delete mode 100644 train.py diff --git a/config/config_efficientnet.yaml b/config/config_efficientnet.yaml index 72d539c..763d594 100644 --- a/config/config_efficientnet.yaml +++ b/config/config_efficientnet.yaml @@ -8,15 +8,15 @@ model: pretrained: true training: - batch_size: 32 + batch_size: 64 epochs: - initial: 200 - fine_tuning: 200 + initial: 300 + fine_tuning: 300 loss_function: type: "CrossEntropyLoss" parameters: {} optimizer: - type: "Adam" + type: "SGD" parameters: learning_rate: 0.005 learning_rates: @@ -62,10 +62,24 @@ data: parameters: mean: [0.485, 0.456, 0.406] std: [0.229, 0.224, 0.225] + - type: "ColorJitter" + parameters: + brightness: 0.3 + contrast: 0.3 + saturation: 0.3 + hue: 0.1 + - type: "RandomResizedCrop" + parameters: + size: [224, 224] + scale: [0.8, 1.0] + ratio: [0.75, 1.33] + - type: "RandomRotation" + parameters: + degrees: [-10, 10] eval_transforms: - type: "Resize" parameters: - size: [240, 240] + size: [224, 224] - type: "ToTensor" parameters: {} - type: "Normalize" diff --git a/config/mm_config_densenet.yaml b/config/mm_config_densenet.yaml index abef724..564a31a 100644 --- a/config/mm_config_densenet.yaml +++ b/config/mm_config_densenet.yaml @@ -10,13 +10,13 @@ model: training: batch_size: 32 epochs: - initial: 200 - fine_tuning: 200 + initial: 300 + fine_tuning: 300 loss_function: type: "CrossEntropyLoss" parameters: {} optimizer: - type: "Adam" + type: "SGD" parameters: learning_rate: 0.005 learning_rates: @@ -41,7 +41,7 @@ callbacks: EarlyStopping: parameters: monitor: "val_loss" - patience: 5 + patience: 15 delta: 0 verbose: true @@ -53,7 +53,7 @@ data: transforms: - type: "Resize" parameters: - size: [240, 240] + size: [224, 224] #- type: "TrivialAugmentWide" # parameters: {} - type: "ToTensor" @@ -62,10 +62,31 @@ data: parameters: mean: [0.485, 0.456, 0.406] std: [0.229, 0.224, 0.225] + - type: "ColorJitter" + parameters: + brightness: 0.3 + contrast: 0.3 + saturation: 0.3 + hue: 0.1 + - type: "RandomResizedCrop" + parameters: + size: [224, 224] + scale: [0.8, 1.0] + ratio: [0.75, 1.33] + - type: "RandomRotation" + parameters: + degrees: [-10, 10] + - type: "GaussianBlur" + parameters: + kernel_size: [5, 5] + sigma: [0.1, 2.0] + - type: "RandomHorizontalFlip" + parameters: + p: 0.5 eval_transforms: - type: "Resize" parameters: - size: [240, 240] + size: [224, 224] - type: "ToTensor" parameters: {} - type: "Normalize" diff --git a/environment.yaml b/environment.yaml index c43506d..f4a6b61 100644 --- a/environment.yaml +++ b/environment.yaml @@ -328,4 +328,4 @@ dependencies: - torch-topological==0.1.7 - wheel==0.43.0 - widgetsnbextension==4.0.11 -prefix: /home/pab1s/.conda/envs/tda-nn-separability +prefix: /home/pab1s/.conda/envs/tda-nn-analysis diff --git a/fine_tuning.py b/fine_tuning.py deleted file mode 100644 index 9185764..0000000 --- a/fine_tuning.py +++ /dev/null @@ -1,91 +0,0 @@ -import torch -import yaml -from datetime import datetime -from torch.utils.data import DataLoader, random_split -from datasets.dataset import get_dataset -from datasets.transformations import get_transforms -from utils.metrics import Accuracy, Precision -from factories.model_factory import ModelFactory -from factories.loss_factory import LossFactory -from factories.optimizer_factory import OptimizerFactory -from trainers import get_trainer -from os import path - -def main(config_path): - with open(config_path, 'r') as file: - config = yaml.safe_load(file) - - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - - # Load and transform data - transforms = get_transforms(config['data']['transforms']) - data = get_dataset(config['data']['name'], config['data']['dataset_path'], train=True, transform=transforms) - - # Split data - total_size = len(data) - test_size = int(total_size * config['data']['test_size']) - val_size = int((total_size - test_size) * config['data']['val_size']) - train_size = total_size - test_size - val_size - - data_train, data_test = random_split(data, [train_size + val_size, test_size], generator=torch.Generator().manual_seed(config['random_seed'])) - data_train, data_val = random_split(data_train, [train_size, val_size], generator=torch.Generator().manual_seed(config['random_seed'])) - - # Data loaders - train_loader = DataLoader(data_train, batch_size=config['training']['batch_size'], shuffle=True) - valid_loader = DataLoader(data_val, batch_size=config['training']['batch_size'], shuffle=False) - test_loader = DataLoader(data_test, batch_size=config['training']['batch_size'], shuffle=False) - - # Model setup - model_factory = ModelFactory() - model = model_factory.create(config['model']['type'], **config['model']['parameters']).to(device) - - # Loss and optimizer setup - loss_factory = LossFactory() - criterion = loss_factory.create(config['training']['loss_function']['type']) - - optimizer_factory = OptimizerFactory() - optimizer = optimizer_factory.create(config['training']['optimizer']['type'], params=model.parameters(), **config['training']['optimizer']['parameters']) - - # Metrics and trainer setup - metrics = [Accuracy(), Precision()] - trainer = get_trainer(config['trainer'], model=model, device=device) - - # Training stages setup - current_time = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") - model_dataset_time = f"{config['model']['type']}_{config['data']['name']}_{current_time}" - log_filename = path.join(config['paths']['log_path'], f"log_finetuning_{model_dataset_time}.csv") - plot_filename = path.join(config['paths']['plot_path'], f"plot_finetuning_{model_dataset_time}.png") - - # Initial training stage - print("Starting initial training stage with frozen layers...") - trainer.build( - criterion=criterion, - optimizer=optimizer, - freeze_until_layer=config['training']['freeze_until_layer'], - metrics=metrics - ) - trainer.train( - train_loader=train_loader, - valid_loader=valid_loader, - num_epochs=config['training']['epochs']['initial'], - plot_path=plot_filename - ) - - # Fine-tuning stage - print("Unfreezing all layers for fine-tuning...") - trainer.unfreeze_all_layers() - optimizer_factory.update(optimizer, lr=config['training']['learning_rates']['final_fine_tuning']) - - print("Starting full model fine-tuning...") - trainer.train( - train_loader=train_loader, - valid_loader=valid_loader, - num_epochs=config['training']['epochs']['fine_tuning'], - plot_path=plot_filename - ) - - # Evaluate - trainer.evaluate(data_loader=test_loader) - -if __name__ == "__main__": - main("config/fine_tuning_config.yaml") diff --git a/homology_times.py b/homology_times.py deleted file mode 100644 index 148aad5..0000000 --- a/homology_times.py +++ /dev/null @@ -1,121 +0,0 @@ -import torch -import yaml -import numpy as np -import scipy.spatial -import time -import logging -from scipy.cluster.hierarchy import linkage -from torch.utils.data import DataLoader, random_split -from datasets.dataset import get_dataset -from datasets.transformations import get_transforms -from factories.model_factory import ModelFactory -from gtda.homology import VietorisRipsPersistence -from sklearn.manifold import LocallyLinearEmbedding -import matplotlib.pyplot as plt - -# Setup basic logging -logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') - -def load_pretrained_model(model_path, config, device): - model_factory = ModelFactory() - model = model_factory.create(config['model']['type'], num_classes=config['model']['parameters']['num_classes'], pretrained=False).to(device) - model.load_state_dict(torch.load(model_path)) - return model - -def load_config(config_path: str): - try: - with open(config_path, 'r') as file: - return yaml.safe_load(file) - except FileNotFoundError: - logging.error(f"The configuration file at {config_path} was not found.") - raise - except yaml.YAMLError as exc: - logging.error(f"Error parsing YAML file: {exc}") - raise - -def register_all_hooks(model, activations): - def get_activation(name): - def hook(model, input, output): - # Flattening the output for analysis, reshape as required - activations[name] = output.detach().cpu().numpy().reshape(output.size(0), -1) - return hook - - for name, layer in model.named_modules(): - if isinstance(layer, torch.nn.ReLU) or isinstance(layer, torch.nn.Linear): - layer.register_forward_hook(get_activation(name)) - -def incremental_processing(test_loader, model, device, activations): - with torch.no_grad(): - for inputs, labels in test_loader: - inputs = inputs.to(device) - model(inputs) # Triggers hooks and populates activations - for name, feature_array in activations.items(): - if feature_array is not None: - process_feature_layer(name, feature_array) - activations[name] = None # Clear memory after processing - -def process_feature_layer(layer_name, feature_array): - if feature_array.size > 0: - # Ensure the array is 2-dimensional - if len(feature_array.shape) == 1: - feature_array = feature_array.reshape(-1, 1) # Reshape if it's a single dimension - elif len(feature_array.shape) > 2: - feature_array = feature_array.reshape(feature_array.shape[0], -1) # Flatten multi-dimensional arrays - - logging.info(f"Computing persistence diagrams for layer {layer_name}") - distance_matrix = scipy.spatial.distance.pdist(feature_array) - square_distance_matrix = scipy.spatial.distance.squareform(distance_matrix) - persistence_diagram = compute_persistence_diagrams_using_giotto(square_distance_matrix) - - # Sum of persistence diagram values - diagram_sum = persistence_diagram[:,1].sum() - logging.info(f"Layer {layer_name}: Persistence Diagram Shape: {persistence_diagram.shape}") - logging.info(f"Layer {layer_name}: Sum of Persistence Diagram Values: {diagram_sum}") - logging.info(f"Actual diagram: {persistence_diagram}") - else: - logging.info(f"Layer {layer_name}: No features to process.") - -def compute_persistence_diagram_using_single_linkage(distance_matrix): - condensed_matrix = scipy.spatial.distance.squareform(distance_matrix) - deaths = linkage(condensed_matrix, method='single')[:, 2] - return np.sort(np.array([[0, d] for d in deaths])) - -def compute_persistence_diagrams_using_giotto(distance_matrix, dimensions=[0,1]): - vr_computator = VietorisRipsPersistence(homology_dimensions=dimensions, metric="precomputed") - diagrams = vr_computator.fit_transform([distance_matrix])[0] - # return np.sort(diagrams[diagrams[:, 2] == 0][:, :2]) # Filter zero-dimensional features - return np.sort(diagrams[:, :2]) - -def perform_lle_and_plot(features, n_neighbors=10, n_components=2, title="LLE Embedding"): - lle = LocallyLinearEmbedding(n_neighbors=n_neighbors, n_components=n_components, method='standard') - transformed_features = lle.fit_transform(features) - - plt.figure(figsize=(8, 6)) - plt.scatter(transformed_features[:, 0], transformed_features[:, 1], c='blue', marker='o', edgecolor='k') - plt.title(title) - plt.xlabel('Component 1') - plt.ylabel('Component 2') - plt.grid(True) - plt.show() - -def main(config_path: str, model_path: str): - config = load_config(config_path) - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - - transforms = get_transforms(config['data']['eval_transforms']) - data = get_dataset(config['data']['name'], config['data']['dataset_path'], train=True, transform=transforms) - - total_size = len(data) - test_size = int(total_size * config['data']['test_size']) - _, data_test = random_split(data, [total_size - test_size, test_size], generator=torch.Generator().manual_seed(config['random_seed'])) - test_loader = DataLoader(data_test, batch_size=len(data_test), shuffle=False) # Reduced batch size to manage memory better - - model = load_pretrained_model(model_path, config, device) - print(model) - activations = {} - register_all_hooks(model, activations) - - incremental_processing(test_loader, model, device, activations) - -if __name__ == "__main__": - main("config/config_resnet.yaml", "outputs/models/resnet18_CarDataset_SGD_64_2024-05-30_21-44-58.pth") diff --git a/main.py b/main.py deleted file mode 100644 index 59d928c..0000000 --- a/main.py +++ /dev/null @@ -1,163 +0,0 @@ -import torch -import yaml -import argparse -from datetime import datetime -from torch.utils.data import DataLoader, random_split, WeightedRandomSampler -from datasets.dataset import get_dataset -from datasets.transformations import get_transforms -from utils.metrics import Accuracy, Precision, Recall, F1Score -from factories.model_factory import ModelFactory -from factories.loss_factory import LossFactory -from factories.optimizer_factory import OptimizerFactory -from factories.callback_factory import CallbackFactory -from trainers import get_trainer -from os import path - -def main(config_path, optimizer_type, optimizer_params, batch_size): - with open(config_path, 'r') as file: - config = yaml.safe_load(file) - - # If CUDA not available, finish execution - if not torch.cuda.is_available(): - print("CUDA is not available. Exiting...") - exit() - device = torch.device("cuda") - - # Load and transform data - transforms = get_transforms(config['data']['transforms']) - eval_transforms = get_transforms(config['data']['eval_transforms']) - data = get_dataset(config['data']['name'], config['data']['dataset_path'], train=True, transform=transforms) - - # Split data - total_size = len(data) - test_size = int(total_size * config['data']['test_size']) - val_size = int(total_size * config['data']['val_size']) - train_size = total_size - test_size - val_size - assert train_size > 0 and val_size > 0 and test_size > 0, "One of the splits has zero or negative size." - data_train, data_test = random_split(data, [train_size + val_size, test_size], generator=torch.Generator().manual_seed(config['random_seed'])) - data_train, data_val = random_split(data_train, [train_size, val_size], generator=torch.Generator().manual_seed(config['random_seed'])) - - # Apply evaluation transforms to validation and test datasets - data_test.dataset.transform = eval_transforms - data_val.dataset.transform = eval_transforms - - # Data loaders using the given batch_size - train_loader = DataLoader(data_train, batch_size=batch_size, shuffle=True) - valid_loader = DataLoader(data_val, batch_size=batch_size, shuffle=False) - test_loader = DataLoader(data_test, batch_size=batch_size, shuffle=False) - - # Model setup - model_factory = ModelFactory() - model = model_factory.create(config['model']['type'], num_classes=config['model']['parameters']['num_classes'], pretrained=config['model']['parameters']['pretrained']).to(device) - print(model) - - # Loss setup - class_weights = data.get_class_weights().to(device) - loss_factory = LossFactory() - criterion = loss_factory.create(config['training']['loss_function']['type'] ) #, weight=class_weights) - - # Optimizer setup with given parameters - optimizer_factory = OptimizerFactory() - optimizer = optimizer_factory.create(optimizer_type) - print("Using optimizer: ", optimizer, " with params: ", optimizer_params) - print("Batch size: ", batch_size) - - # Training stages setup - current_time = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") - model_dataset_time = f"{config['model']['type']}_{config['data']['name']}_{optimizer_type}_{batch_size}_{current_time}" - log_filename = path.join(config['paths']['log_path'], f"log_finetuning_{model_dataset_time}.csv") - - # Callbacks setup - callbacks_config = config['callbacks'] - if "CSVLogging" in callbacks_config: - callbacks_config["CSVLogging"]["parameters"]["csv_path"] = log_filename - - # Metrics and trainer setup - metrics = [Accuracy(), Precision(), Recall(), F1Score()] - trainer = get_trainer(config['trainer'], model=model, device=device) - - # Initial training stage - print("Starting initial training stage with frozen layers...") - trainer.build( - criterion=criterion, - optimizer_class=optimizer, - optimizer_params=optimizer_params, - # freeze_until_layer=config['training']['freeze_until_layer'], - metrics=metrics - ) - - callback_factory = CallbackFactory() - callbacks = [] - for name, params in callbacks_config.items(): - if name == "Checkpoint": - params["parameters"]["checkpoint_dir"] = path.join(config['paths']['checkpoint_path'], model_dataset_time) - params["parameters"]["model"] = model - params["parameters"]["optimizer"] = trainer.optimizer - params["parameters"]["scheduler"] = trainer.scheduler - - callback = callback_factory.create(name, **params["parameters"]) - - if name == "EarlyStopping": - callback.set_model_and_optimizer(model, trainer.optimizer) - - callbacks.append(callback) - - #trainer.train( - # train_loader=train_loader, - # valid_loader=valid_loader, - # num_epochs=config['training']['epochs']['initial'], - # callbacks=callbacks - #) - - # Fine-tuning stage with all layers unfrozen - #print("Unfreezing all layers for fine-tuning...") - #trainer.unfreeze_all_layers() - - #optimizer_instance = trainer.optimizer - #optimizer_factory.update(optimizer_instance, config['training']['learning_rates']['initial']) - - print("Starting full model fine-tuning...") - trainer.train( - train_loader=train_loader, - valid_loader=valid_loader, - num_epochs=config['training']['epochs']['fine_tuning'], - callbacks=callbacks - ) - - # Save model - model_path = path.join(config['paths']['model_path'], f"{model_dataset_time}.pth") - torch.save(model.state_dict(), model_path) - - # Evaluate - trainer.evaluate(data_loader=test_loader) - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description='Process some optimizer, batch size, and configuration file.') - parser.add_argument('config_filename', type=str, help='Filename of the configuration file within the "config" directory') - parser.add_argument('optimizer_type', type=str, help='Optimizer type ("SGD" or "Adam")') - parser.add_argument('batch_size', type=int, help='Batch size for training') - parser.add_argument('learning_rate', type=float, help='Learning rate for the optimizer') - - args = parser.parse_args() - - optimizer_types = ["SGD", "Adam"] - if args.optimizer_type not in optimizer_types: - raise ValueError("Optimizer type must be 'SGD' or 'Adam'") - - adam_params = { - "lr": 0.001, - } - sgd_params = { - "lr": 0.01, - "momentum": 0.9, - "weight_decay": 0, - "nesterov": False - } - - adam_params['lr'] = args.learning_rate - sgd_params['lr'] = args.learning_rate - - config_path = f"config/{args.config_filename}" - optimizer_params = adam_params if args.optimizer_type == "Adam" else sgd_params - - main(config_path, args.optimizer_type, optimizer_params, args.batch_size) diff --git a/train.py b/train.py deleted file mode 100644 index 03cbc21..0000000 --- a/train.py +++ /dev/null @@ -1,81 +0,0 @@ -import torch -import yaml -from datetime import datetime -from torch.utils.data import DataLoader, random_split -from datasets.dataset import get_dataset -from datasets.transformations import get_transforms -from utils.metrics import Accuracy, Precision, Recall, F1Score -from trainers.basic_trainer import BasicTrainer -from factories.model_factory import ModelFactory -from factories.optimizer_factory import OptimizerFactory -from factories.loss_factory import LossFactory -from factories.callback_factory import CallbackFactory -from os import path - -def main(config_path): - with open(config_path, 'r') as file: - config = yaml.safe_load(file) - - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - - transforms = get_transforms(config['data']['transforms']) - data = get_dataset(config['data']['name'], config['data']['dataset_path'], train=True, transform=transforms) - - total_size = len(data) - test_size = int(total_size * config['data']['test_size']) - val_size = int((total_size - test_size) * config['data']['val_size']) - train_size = total_size - test_size - val_size - - data_train, data_test = random_split(data, [train_size + val_size, test_size], generator=torch.Generator().manual_seed(config['random_seed'])) - data_train, data_val = random_split(data_train, [train_size, val_size], generator=torch.Generator().manual_seed(config['random_seed'])) - - train_loader = DataLoader(data_train, batch_size=config['training']['batch_size'], shuffle=True) - valid_loader = DataLoader(data_val, batch_size=config['training']['batch_size'], shuffle=False) - test_loader = DataLoader(data_test, batch_size=config['training']['batch_size'], shuffle=False) - - model_factory = ModelFactory() - model = model_factory.create(config['model']['type'], num_classes=config['model']['parameters']['num_classes'], pretrained=config['model']['parameters']['pretrained']).to(device) - - loss_factory = LossFactory() - criterion = loss_factory.create(config['training']['loss_function']['type']) - - optimizer_factory = OptimizerFactory() - optimizer = optimizer_factory.create(config['training']['optimizer']['type']) - optimizer_params = {'lr': config['training']['optimizer']['parameters']['learning_rate']} - - current_time = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") - model_dataset_time = f"{config['model']['type']}_{config['data']['name']}_{current_time}" - log_filename = path.join(config['paths']['log_path'], f"log_{model_dataset_time}.csv") - - callbacks_config = config['callbacks'] - if "CSVLogging" in callbacks_config: - callbacks_config["CSVLogging"]["parameters"]["csv_path"] = log_filename - - trainer = BasicTrainer(model=model, device=device) - - metrics = [Accuracy(), Precision(), Recall(), F1Score()] - - trainer.build( - criterion=criterion, - optimizer_class=optimizer, - optimizer_params=optimizer_params, - metrics=metrics - ) - - callback_factory = CallbackFactory() - callbacks = [] - for name, params in callbacks_config.items(): - callback = callback_factory.create(name, **params["parameters"]) - callbacks.append(callback) - - trainer.train( - train_loader=train_loader, - valid_loader=valid_loader, - num_epochs=config['training']['num_epochs'], - callbacks=callbacks, - ) - - trainer.evaluate(data_loader=test_loader) - -if __name__ == "__main__": - main("config/config.yaml")