From 775843220c9a84fb71dfc192dc16ff2938e6338c Mon Sep 17 00:00:00 2001 From: Evan Smothers Date: Thu, 12 Dec 2024 12:36:37 -0800 Subject: [PATCH] Small readme, config updates --- README.md | 2 +- recipes/configs/llama3/8B_dora.yaml | 2 +- recipes/configs/llama3/8B_dora_single_device.yaml | 2 +- recipes/configs/llama3/8B_qdora_single_device.yaml | 2 +- recipes/configs/llama3_1/405B_qlora.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c9980a6c0f..289d433426 100644 --- a/README.md +++ b/README.md @@ -140,7 +140,7 @@ loss=torchtune.modules.loss.CEWithChunkedOutputLoss \ enable_activation_checkpointing=True \ optimizer_in_bwd=False \ enable_activation_offloading=True \ -optimizer._component_=torch.optim.AdamW \ +optimizer=torch.optim.AdamW \ tokenizer.max_seq_len=4096 \ gradient_accumulation_steps=1 \ epochs=1 \ diff --git a/recipes/configs/llama3/8B_dora.yaml b/recipes/configs/llama3/8B_dora.yaml index 98bd75b08e..a68f5cf7ff 100644 --- a/recipes/configs/llama3/8B_dora.yaml +++ b/recipes/configs/llama3/8B_dora.yaml @@ -61,7 +61,7 @@ lr_scheduler: num_warmup_steps: 100 loss: - _component_: torch.nn.CrossEntropyLoss + _component_: torchtune.modules.loss.CEWithChunkedOutputLoss # Training epochs: 1 diff --git a/recipes/configs/llama3/8B_dora_single_device.yaml b/recipes/configs/llama3/8B_dora_single_device.yaml index 4258cc08a4..f27acc3a12 100644 --- a/recipes/configs/llama3/8B_dora_single_device.yaml +++ b/recipes/configs/llama3/8B_dora_single_device.yaml @@ -63,7 +63,7 @@ lr_scheduler: num_warmup_steps: 100 loss: - _component_: torch.nn.CrossEntropyLoss + _component_: torchtune.modules.loss.CEWithChunkedOutputLoss # Training epochs: 1 diff --git a/recipes/configs/llama3/8B_qdora_single_device.yaml b/recipes/configs/llama3/8B_qdora_single_device.yaml index 8a5a39b58b..45b515476e 100644 --- a/recipes/configs/llama3/8B_qdora_single_device.yaml +++ b/recipes/configs/llama3/8B_qdora_single_device.yaml @@ -64,7 +64,7 @@ lr_scheduler: num_warmup_steps: 100 loss: - _component_: torch.nn.CrossEntropyLoss + _component_: torchtune.modules.loss.CEWithChunkedOutputLoss # Training epochs: 1 diff --git a/recipes/configs/llama3_1/405B_qlora.yaml b/recipes/configs/llama3_1/405B_qlora.yaml index 749b16717f..4a15d8b25f 100644 --- a/recipes/configs/llama3_1/405B_qlora.yaml +++ b/recipes/configs/llama3_1/405B_qlora.yaml @@ -61,7 +61,7 @@ lr_scheduler: num_warmup_steps: 100 loss: - _component_: torch.nn.CrossEntropyLoss + _component_: torchtune.modules.loss.CEWithChunkedOutputLoss fsdp: cpu_offload: False