diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000..7eeb55620 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,34 @@ +--- +name: Bug report +about: Create a report to help us improve +title: "" +labels: "" +assignees: "" +--- + +## Describe the bug + +A clear and concise description of what the bug is. + +## Platform + +Please provide details about the environment you are using, including the following: + +- Interpreter version: +- Library version: + +## Sample Code + +Please include a minimal sample of the code that will (if possible) reproduce the bug in isolation + +## Expected behavior + +A clear and concise description of what you expected to happen. + +## Observed behavior + +What you see happening (error messages, stack traces, etc...) + +## Additional context + +Add any other context about the problem here. \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 000000000..96b857215 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,23 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: "" +labels: "" +assignees: "" +--- + +## Is your feature request related to a problem? Please describe. + +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +## Describe the solution you'd like + +A clear and concise description of what you want to happen. + +## Describe alternatives you've considered + +A clear and concise description of any alternative solutions or features you've considered. + +## Additional context + +Add any other context about the feature request here. \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/user_story.md b/.github/ISSUE_TEMPLATE/user_story.md new file mode 100644 index 000000000..4b62a291d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/user_story.md @@ -0,0 +1,23 @@ +--- +name: User story +about: A user-oriented story describing a piece of work to do +title: "" +labels: "" +assignees: "" +--- + +## Description + +As a , I want to , so that I can + +## Discussion + +Provide detailed discussion here + +## Acceptance Criteria + + + +- [ ] Unit tests cover new/changed code +- [ ] Examples build against new/changed code +- [ ] READMEs are updated \ No newline at end of file diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..467b55498 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,19 @@ + + +### Description of the change + + + +### Related issue number + + + +### How to verify the PR + + + +### Was the PR tested + + +- [ ] I have added >=1 unit test(s) for every new method I have added. +- [ ] I have ensured all unit tests pass \ No newline at end of file diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index d926b1220..f8cb15e72 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -35,4 +35,6 @@ jobs: python -m pip install -r setup_requirements.txt - name: Check Formatting run: tox -e fmt + - name: Run pylint + run: tox -e lint diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 000000000..1d8ee828e --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,22 @@ +name: Test + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.9 + uses: actions/setup-python@v4 + with: + python-version: 3.9 + - name: Install dependencies + run: | + python -m pip install -r setup_requirements.txt + - name: Run unit tests + run: tox -e py \ No newline at end of file diff --git a/.gitignore b/.gitignore index 61f39d73a..178b160fc 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,6 @@ durations/* coverage*.xml dist htmlcov -build test # IDEs diff --git a/.pylintrc b/.pylintrc index 5c7b4676e..e94869511 100644 --- a/.pylintrc +++ b/.pylintrc @@ -443,7 +443,9 @@ disable=raw-checker-failed, attribute-defined-outside-init, abstract-method, pointless-statement, - wrong-import-order + wrong-import-order, + duplicate-code, + unbalanced-tuple-unpacking # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 000000000..a28fcff97 --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,11 @@ +##################################################### +# +# List of approvers for fms-hf-tuning repository +# +##################################################### +# +# Learn about CODEOWNERS file format: +# https://help.github.com/en/articles/about-code-owners +# + +* @anhuong @Ssukriti @alex-jw-brooks diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..375096ed7 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,141 @@ +# Contributing + +👍🎉 First off, thank you for taking the time to contribute! 🎉👍 + +The following is a set of guidelines for contributing. These are just guidelines, not rules. Use your best judgment, and feel free to propose changes to this document in a pull request. + +## What Should I Know Before I Get Started? + +### Code of Conduct + +This project adheres to the [Contributor Covenant](./code-of-conduct.md). By participating, you are expected to uphold this code. + +Please report unacceptable behavior to one of the [Code Owners](./CODEOWNERS). + +### How Do I Start Contributing? + +The below workflow is designed to help you begin your first contribution journey. It will guide you through creating and picking up issues, working through them, having your work reviewed, and then merging. + +Help on open source projects is always welcome and there is always something that can be improved. For example, documentation (like the text you are reading now) can always use improvement, code can always be clarified, variables or functions can always be renamed or commented on, and there is always a need for more test coverage. If you see something that you think should be fixed, take ownership! Here is how you get started: + +## How Can I Contribute? +TODO: Add link to ADR and add template to this repository + +For any contributions that need design changes/API changes, first contribute an ADR. Reason for ADR: teams agree on the design, to avoid back and forth after writing code. An ADR gives context on the code being written. + +When contributing, it's useful to start by looking at [issues](https://github.com/foundation-model-stack/fms-hf-tuning/issues). After picking up an issue, writing code, or updating a document, make a pull request and your work will be reviewed and merged. If you're adding a new feature or find a bug, it's best to [write an issue](https://github.com/foundation-model-stack/fms-hf-tuning/issues/new) first to discuss it with maintainers. + +To contribute to this repo, you'll use the Fork and Pull model common in many open source repositories. For details on this process, check out [The GitHub Workflow +Guide](https://github.com/kubernetes/community/blob/master/contributors/guide/github-workflow.md) +from Kubernetes. + +When your contribution is ready, you can create a pull request. Pull requests are often referred to as "PR". In general, we follow the standard [GitHub pull request](https://help.github.com/en/articles/about-pull-requests) process. Follow the template to provide details about your pull request to the maintainers. It's best to break your contribution into smaller PRs with incremental changes, and include a good description of the changes. + +Before sending pull requests, make sure your changes pass formatting, linting and unit tests. + +#### Code Review + +Once you've [created a pull request](#how-can-i-contribute), maintainers will review your code and may make suggestions to fix before merging. It will be easier for your pull request to receive reviews if you consider the criteria the reviewers follow while working. Remember to: + +- Run tests locally and ensure they pass +- Follow the project coding conventions +- Write detailed commit messages +- Break large changes into a logical series of smaller patches, which are easy to understand individually and combine to solve a broader issue + +Maintainers will perform "squash and merge" actions on PRs in this repo, so it doesn't matter how many commits your PR has, as they will end up being a single commit after merging. + +### Reporting Bugs + +This section guides you through submitting a bug report. Following these guidelines helps maintainers and the community understand your report ✏️, reproduce the behavior 💻, and find related reports 🔎. + +#### How Do I Submit A (Good) Bug Report? + +Bugs are tracked as [GitHub issues using the Bug Report template]((https://github.com/foundation-model-stack/fms-hf-tuning/issues/new?template=bug_report.md). Create an issue on that and provide the information suggested in the bug report issue template. + +### Suggesting Enhancements + +This section guides you through submitting an enhancement suggestion, including completely new features, tools, and minor improvements to existing functionality. Following these guidelines helps maintainers and the community understand your suggestion ✏️ and find related suggestions 🔎 + +#### How Do I Submit A (Good) Enhancement Suggestion? + +Enhancement suggestions are tracked as [GitHub issues using the Feature Request template](https://github.com/foundation-model-stack/fms-hf-tuning/issues/new?template=feature_request.md). Create an issue and provide the information suggested in the feature requests or user story issue template. + +#### How Do I Submit A (Good) Improvement Item? + +Improvements to existing functionality are tracked as [GitHub issues using the User Story template](https://github.com/foundation-model-stack/fms-hf-tuning/issues/new?template=user_story.md). Create an issue and provide the information suggested in the feature requests or user story issue template. + +## Development + +### Set up your dev environment + +The following tools are required: + +- [git](https://git-scm.com) +- [python](https://www.python.org) (v3.8+) +- [pip](https://pypi.org/project/pip/) (v23.0+) + +Installation: +``` +pip install -r requirements.txt +pip install -U datasets +pip install -e . +``` +
+Linting + +To lint your code: +```shell +tox -e lint +``` + +We use Pylint to checks your Python code for errors, coding standards, code convention and refactoring suggestions. + +Pylint emits [messages](https://pylint.pycqa.org/en/latest/user_guide/messages/index.html) that provides explanations of the failed checks. + +You should fix all message in the following order: +1. Fix each message provided. Select a message [description](https://pylint.pycqa.org/en/latest/user_guide/messages/messages_overview.html#messages-overview) to fix a message. +2. Disable a message (i.e: unbalanced-tuple-unpacking) caused by a particular line of code: + ```python + a, b = ... # pylint: disable=unbalanced-tuple-unpacking + ``` + Please see [here](https://pylint.pycqa.org/en/latest/user_guide/messages/message_control.html#block-disables) for the progma syntax. + +3. Disable a checker globally. Please extend the `disable=` list in the [pylintrc](.pylintrc) file. + > Note: Disable checkers only if there is good reason. +
+ +
+Formatting + +To format your code: +```shell +tox -e fmt +``` +We use [black](https://github.com/psf/black) formatter to format the code. + +You could optionally install the git pre-commit hooks if you would like to format the code automatically for each commit: +``` +brew install pre-commit +pre-commit install +``` +
+ +
+Unit tests + +To run unit tests: +```shell +tox -e py +``` +Running unit tests ensures your contributions do not break exiting code. +We use [pytest](https://docs.pytest.org/) framework to run unit tests. The framework is setup to run all run all test_*.py or *_test.py in the [tests](./tests) directory. + +> Optionally, run `make test` command to do formatting, linting, and testing at once. +
+ +## Your First Code Contribution + +Unsure where to begin contributing? You can start by looking through these issues: + +- Issues with the [`good first issue` label](https://github.com/foundation-model-stack/fms-hf-tuning/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) - these should only require a few lines of code and are good targets if you're just starting contributing. +- Issues with the [`help wanted` label](https://github.com/foundation-model-stack/fms-hf-tuning/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22) - these range from simple to more complex, but are generally things we want but can't get to in a short time frame. diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..1541e8e6c --- /dev/null +++ b/Makefile @@ -0,0 +1,14 @@ +# Run unit tests +.PHONY: test +test: fmt lint + tox -e py + +# Format python code +.PHONY: fmt +fmt: + tox -e fmt + +# Run pylint to check code +..PHONY: lint +lint: + tox -e lint \ No newline at end of file diff --git a/README.md b/README.md index b15b69815..d839a132e 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,102 @@ tuning/sft_trainer.py \ For `GPTBigCode` models, Hugging Face has enabled Flash v2 and one can simply replace the `'LlamaDecoderLayer'` with `'GPTBigCodeBlock'` in `tuning/config/fsdp_config.json` for proper sharding of the model. +### LoRA Tuning Example + +```bash +python tuning/sft_trainer.py \ +--model_name_or_path $MODEL_PATH \ +--data_path $DATA_PATH \ +--output_dir $OUTPUT_PATH \ +--num_train_epochs 40 \ +--per_device_train_batch_size 4 \ +--per_device_eval_batch_size 4 \ +--gradient_accumulation_steps 4 \ +--save_strategy "epoch" \ +--learning_rate 1e-4 \ +--weight_decay 0. \ +--warmup_ratio 0.03 \ +--lr_scheduler_type "cosine" \ +--logging_steps 1 \ +--include_tokens_per_second \ +--packing False \ +--response_template "\n### Label:" \ +--dataset_text_field "output" \ +--use_flash_attn False \ +--tokenizer_name_or_path $MODEL_PATH \ +--torch_dtype float32 \ +--peft_method "lora" \ +--logging_strategy "epoch" \ +--r 8 \ +--lora_dropout 0.05 \ +--lora_alpha 16 +``` + +where [`LoraConfig`](https://github.com/foundation-model-stack/fms-hf-tuning/blob/main/tuning/config/peft_config.py#L7) that is being set looks like: +```py +LoraConfig( + r=8, + lora_alpha=16, + target_modules=['q_proj', 'v_proj'], + lora_dropout=0.05 +) +``` + +Notice the `target_modules` that are set are the default values. `target_modules` are the names of the modules to apply the adapter to. If this is specified, only the modules with the specified names will be replaced. When passing a list of strings, either an exact match will be performed or it is checked if the name of the module ends with any of the passed strings. If this is specified as `all-linear`, then all linear/Conv1D modules are chosen, excluding the output layer. If this is not specified, modules will be chosen according to the model architecture. If the architecture is not known, an error will be raised — in this case, you should specify the target modules manually. See [HuggingFace docs](https://huggingface.co/docs/peft/en/package_reference/lora#peft.LoraConfig) for more details. + +For each model, the `target_modules` will depend on the type of model architecture. You can specify linear or attention layers to `target_modules`. To obtain list of `target_modules` for a model: + +```py +from transformers import AutoModelForCausalLM +# load the model +model = AutoModelForCausalLM.from_pretrained(MODEL_PATH) +# see the module list +model.modules + +# to get just linear layers +import re +model_modules = str(model.modules) +pattern = r'\((\w+)\): Linear' +linear_layer_names = re.findall(pattern, model_modules) + +names = [] +for name in linear_layer_names: + names.append(name) +target_modules = list(set(names)) +``` + +For example for LLaMA model the modules look like: +``` + +``` + +You can specify attention or linear layers. With the CLI, you can specify layers with `--target_modules "q_proj" "v_proj" "k_proj" "o_proj"` or `--target_modules "all-linear"`. + ## Inference Currently, we do *not* offer inference support as part of the library, but we provide a standalone script for running inference on tuned models for testing purposes. For a full list of options run `python scripts/run_inference.py --help`. Note that no data formatting / templating is applied at inference time. diff --git a/build/Dockerfile b/build/Dockerfile new file mode 100644 index 000000000..b41f5ff7c --- /dev/null +++ b/build/Dockerfile @@ -0,0 +1,132 @@ +# Copyright The IBM Tuning Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM registry.access.redhat.com/ubi9/ubi AS release + +ARG CUDA_VERSION=11.8.0 +ARG USER=tuning +ARG USER_UID=1000 + +USER root + +RUN dnf remove -y --disableplugin=subscription-manager \ + subscription-manager \ + # we install newer version of requests via pip + python3.11-requests \ + && dnf install -y make \ + # to help with debugging + procps \ + && dnf clean all + +ENV LANG=C.UTF-8 \ + LC_ALL=C.UTF-8 + +ENV CUDA_VERSION=$CUDA_VERSION \ + NV_CUDA_LIB_VERSION=11.8.0-1 \ + NVIDIA_VISIBLE_DEVICES=all \ + NVIDIA_DRIVER_CAPABILITIES=compute,utility \ + NV_CUDA_CUDART_VERSION=11.8.89-1 \ + NV_CUDA_COMPAT_VERSION=520.61.05-1 + +RUN dnf config-manager \ + --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo \ + && dnf install -y \ + cuda-cudart-11-8-${NV_CUDA_CUDART_VERSION} \ + cuda-compat-11-8-${NV_CUDA_COMPAT_VERSION} \ + && echo "/usr/local/nvidia/lib" >> /etc/ld.so.conf.d/nvidia.conf \ + && echo "/usr/local/nvidia/lib64" >> /etc/ld.so.conf.d/nvidia.conf \ + && dnf clean all + +ENV CUDA_HOME="/usr/local/cuda" \ + PATH="/usr/local/nvidia/bin:${CUDA_HOME}/bin:${PATH}" \ + LD_LIBRARY_PATH="/usr/local/nvidia/lib:/usr/local/nvidia/lib64:$CUDA_HOME/lib64:$CUDA_HOME/extras/CUPTI/lib64:${LD_LIBRARY_PATH}" + + +ENV NV_NVTX_VERSION=11.8.86-1 \ + NV_LIBNPP_VERSION=11.8.0.86-1 \ + NV_LIBCUBLAS_VERSION=11.11.3.6-1 \ + NV_LIBNCCL_PACKAGE_VERSION=2.15.5-1+cuda11.8 + +RUN dnf config-manager \ + --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo \ + && dnf install -y \ + cuda-libraries-11-8-${NV_CUDA_LIB_VERSION} \ + cuda-nvtx-11-8-${NV_NVTX_VERSION} \ + libnpp-11-8-${NV_LIBNPP_VERSION} \ + libcublas-11-8-${NV_LIBCUBLAS_VERSION} \ + libnccl-${NV_LIBNCCL_PACKAGE_VERSION} \ + && dnf clean all + +ENV NV_CUDA_CUDART_DEV_VERSION=11.8.89-1 \ + NV_NVML_DEV_VERSION=11.8.86-1 \ + NV_LIBCUBLAS_DEV_VERSION=11.11.3.6-1 \ + NV_LIBNPP_DEV_VERSION=11.8.0.86-1 \ + NV_LIBNCCL_DEV_PACKAGE_VERSION=2.15.5-1+cuda11.8 + +RUN dnf config-manager \ + --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo \ + && dnf install -y \ + cuda-command-line-tools-11-8-${NV_CUDA_LIB_VERSION} \ + cuda-libraries-devel-11-8-${NV_CUDA_LIB_VERSION} \ + cuda-minimal-build-11-8-${NV_CUDA_LIB_VERSION} \ + cuda-cudart-devel-11-8-${NV_CUDA_CUDART_DEV_VERSION} \ + cuda-nvml-devel-11-8-${NV_NVML_DEV_VERSION} \ + libcublas-devel-11-8-${NV_LIBCUBLAS_DEV_VERSION} \ + libnpp-devel-11-8-${NV_LIBNPP_DEV_VERSION} \ + libnccl-devel-${NV_LIBNCCL_DEV_PACKAGE_VERSION} \ + && dnf clean all + +ENV LIBRARY_PATH="$CUDA_HOME/lib64/stubs" + +RUN dnf install -y python3.11 git && \ + ln -s /usr/bin/python3.11 /bin/python && \ + python -m ensurepip --upgrade + +RUN mkdir /app + +WORKDIR /tmp +RUN python -m pip install packaging && \ + python -m pip install --upgrade pip && \ + python -m pip install torch && \ + python -m pip install wheel + +# TODO Move to installing wheel once we have proper releases setup instead of cloning the repo +RUN git clone https://github.com/foundation-model-stack/fms-hf-tuning.git && \ + cd fms-hf-tuning && \ + python -m pip install -r requirements.txt && \ + python -m pip install -r flashattn_requirements.txt && \ + python -m pip install -U datasets && \ + python -m pip install /tmp/fms-hf-tuning + +RUN mkdir -p /licenses +COPY LICENSE /licenses/ + +COPY launch_training.py /app +RUN chmod +x /app/launch_training.py + +# Need a better way to address this hack +RUN touch /.aim_profile && \ + chmod -R 777 /.aim_profile && \ + mkdir /.cache && \ + chmod -R 777 /.cache + +# create tuning user and give ownership to dirs +RUN useradd -u $USER_UID tuning -m -g 0 --system && \ + chown -R $USER:0 /app && \ + chmod -R g+rwX /app + +WORKDIR /app +USER ${USER} + +CMD [ "tail", "-f", "/dev/null" ] \ No newline at end of file diff --git a/build/launch_training.py b/build/launch_training.py new file mode 100644 index 000000000..5592d5888 --- /dev/null +++ b/build/launch_training.py @@ -0,0 +1,182 @@ +# Copyright The IBM Tuning Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Script wraps SFT Trainer to run for Train Conductor. +Read SFTTrainer configuration via environment variable `SFT_TRAINER_CONFIG_JSON_PATH` +for the path to the JSON config file with parameters or `SFT_TRAINER_CONFIG_JSON_ENV_VAR` +for the encoded config string to parse. +""" + +# Standard +import base64 +import os +import pickle +import json +import tempfile +import shutil +import glob + +# First Party +import logging +from tuning import sft_trainer +from tuning.config import configs, peft_config +from tuning.utils.merge_model_utils import create_merged_model + +# Third Party +import transformers + + +def txt_to_obj(txt): + base64_bytes = txt.encode("ascii") + message_bytes = base64.b64decode(base64_bytes) + try: + # If the bytes represent JSON string + return json.loads(message_bytes) + except UnicodeDecodeError: + # Otherwise the bytes are a pickled python dictionary + return pickle.loads(message_bytes) + + +def get_highest_checkpoint(dir_path): + checkpoint_dir = "" + for curr_dir in os.listdir(dir_path): + if curr_dir.startswith("checkpoint"): + if checkpoint_dir: + curr_dir_num = int(checkpoint_dir.rsplit("-", maxsplit=1)[-1]) + new_dir_num = int(curr_dir.split("-")[-1]) + if new_dir_num > curr_dir_num: + checkpoint_dir = curr_dir + else: + checkpoint_dir = curr_dir + + return checkpoint_dir + + +def main(): + LOGLEVEL = os.environ.get("LOG_LEVEL", "WARNING").upper() + logging.basicConfig(level=LOGLEVEL) + + logging.info("Attempting to launch training script") + parser = transformers.HfArgumentParser( + dataclass_types=( + configs.ModelArguments, + configs.DataArguments, + configs.TrainingArguments, + peft_config.LoraConfig, + peft_config.PromptTuningConfig, + ) + ) + peft_method_parsed = "pt" + json_path = os.getenv("SFT_TRAINER_CONFIG_JSON_PATH") + json_env_var = os.getenv("SFT_TRAINER_CONFIG_JSON_ENV_VAR") + + # accepts either path to JSON file or encoded string config + if json_path: + ( + model_args, + data_args, + training_args, + lora_config, + prompt_tuning_config, + ) = parser.parse_json_file(json_path, allow_extra_keys=True) + + contents = "" + with open(json_path, "r", encoding="utf-8") as f: + contents = json.load(f) + peft_method_parsed = contents.get("peft_method") + logging.debug("Input params parsed: %s", contents) + elif json_env_var: + job_config_dict = txt_to_obj(json_env_var) + logging.debug("Input params parsed: %s", job_config_dict) + + ( + model_args, + data_args, + training_args, + lora_config, + prompt_tuning_config, + ) = parser.parse_dict(job_config_dict, allow_extra_keys=True) + + peft_method_parsed = job_config_dict.get("peft_method") + else: + raise ValueError( + "Must set environment variable 'SFT_TRAINER_CONFIG_JSON_PATH' \ + or 'SFT_TRAINER_CONFIG_JSON_ENV_VAR'." + ) + + tune_config = None + merge_model = False + if peft_method_parsed == "lora": + tune_config = lora_config + merge_model = True + elif peft_method_parsed == "pt": + tune_config = prompt_tuning_config + + logging.debug( + "Parameters used to launch training: \ + model_args %s, data_args %s, training_args %s, tune_config %s", + model_args, + data_args, + training_args, + tune_config, + ) + + original_output_dir = training_args.output_dir + with tempfile.TemporaryDirectory() as tempdir: + training_args.output_dir = tempdir + sft_trainer.train(model_args, data_args, training_args, tune_config) + + if merge_model: + export_path = os.getenv( + "LORA_MERGE_MODELS_EXPORT_PATH", original_output_dir + ) + + # get the highest checkpoint dir (last checkpoint) + lora_checkpoint_dir = get_highest_checkpoint(training_args.output_dir) + full_checkpoint_dir = os.path.join( + training_args.output_dir, lora_checkpoint_dir + ) + + logging.info( + "Merging lora tuned checkpoint %s with base model into output path: %s", + lora_checkpoint_dir, + export_path, + ) + + create_merged_model( + checkpoint_models=full_checkpoint_dir, + export_path=export_path, + base_model=model_args.model_name_or_path, + save_tokenizer=True, + ) + else: + # copy last checkpoint into mounted output dir + pt_checkpoint_dir = get_highest_checkpoint(training_args.output_dir) + logging.info( + "Copying last checkpoint %s into output dir %s", + pt_checkpoint_dir, + original_output_dir, + ) + shutil.copytree( + os.path.join(training_args.output_dir, pt_checkpoint_dir), + original_output_dir, + dirs_exist_ok=True, + ) + + # copy over any loss logs + for file in glob.glob(f"{training_args.output_dir}/*loss.jsonl"): + shutil.copy(file, original_output_dir) + + +if __name__ == "__main__": + main() diff --git a/code-of-conduct.md b/code-of-conduct.md new file mode 100644 index 000000000..b2e6e4b6d --- /dev/null +++ b/code-of-conduct.md @@ -0,0 +1,76 @@ +# Community Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +- Using welcoming and inclusive language +- Being respectful of differing viewpoints and experiences +- Gracefully accepting constructive criticism +- Focusing on what is best for the community +- Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +- The use of sexualized language or imagery and unwelcome sexual attention or + advances +- Trolling, insulting/derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information, such as a physical or electronic + address, without explicit permission +- Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the [project team](./CODEOWNERS). All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq diff --git a/scripts/run_inference.py b/scripts/run_inference.py index b67dd1cc0..40739c888 100644 --- a/scripts/run_inference.py +++ b/scripts/run_inference.py @@ -1,3 +1,16 @@ +# Copyright The IBM Tuning Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """CLI for running loading a tuned model and running one or more inference calls on it. NOTE: For the moment, this script is intentionally written to contain all dependencies for two @@ -78,11 +91,11 @@ def _apply_config_changes(self, overrides: dict) -> dict: # If we have no overrides, this context manager is a noop; no need to do anything if not overrides: return {} - with open(self.config_path, "r") as config_file: + with open(self.config_path, "r", encoding="utf-8") as config_file: adapter_config = json.load(config_file) overridden_values = self._get_old_config_values(adapter_config, overrides) adapter_config = {**adapter_config, **overrides} - with open(self.config_path, "w") as config_file: + with open(self.config_path, "w", encoding="utf-8") as config_file: json.dump(adapter_config, config_file, indent=4) return overridden_values @@ -214,7 +227,8 @@ def main(): ) parser.add_argument( "--base_model_name_or_path", - help="Override for base model to be used for non-merged models [default: value in model adapter_config.json]", + help="Override for base model to be used for non-merged models \ + [default: value in model adapter_config.json]", default=None, ) parser.add_argument( @@ -244,7 +258,7 @@ def main(): if args.text: texts = [args.text] else: - with open(args.text_file, "r") as text_file: + with open(args.text_file, "r", encoding="utf-8") as text_file: texts = [line.strip() for line in text_file.readlines()] # TODO: we should add batch inference support @@ -257,7 +271,7 @@ def main(): ] # Export the results to a file - with open(args.out_file, "w") as out_file: + with open(args.out_file, "w", encoding="utf-8") as out_file: json.dump(results, out_file, sort_keys=True, indent=4) print(f"Exported results to: {args.out_file}") diff --git a/setup.py b/setup.py index ae71369c0..1fd2df909 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,17 @@ +# Copyright The IBM Tuning Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Third Party from setuptools import find_packages, setup diff --git a/tests/utils/test_data_type_utils.py b/tests/utils/test_data_type_utils.py new file mode 100644 index 000000000..6a3c5bfd0 --- /dev/null +++ b/tests/utils/test_data_type_utils.py @@ -0,0 +1,49 @@ +# Copyright The IBM Tuning Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# SPDX-License-Identifier: Apache-2.0 +# https://spdx.dev/learn/handling-license-info/ + +# Third Party +import pytest +import torch + +# Local +from tuning.utils import data_type_utils + +dtype_dict = { + "bool": torch.bool, + "double": torch.double, + "float32": torch.float32, + "int64": torch.int64, + "long": torch.long, +} + + +def test_str_to_torch_dtype(): + for t in dtype_dict.keys(): + assert data_type_utils.str_to_torch_dtype(t) == dtype_dict.get(t) + + +def test_str_to_torch_dtype_exit(): + with pytest.raises(ValueError): + data_type_utils.str_to_torch_dtype("foo") + + +def test_get_torch_dtype(): + for t in dtype_dict.keys(): + # When passed a string, it gets converted to torch.dtype + assert data_type_utils.get_torch_dtype(t) == dtype_dict.get(t) + # When passed a torch.dtype, we get the same torch.dtype returned + assert data_type_utils.get_torch_dtype(dtype_dict.get(t)) == dtype_dict.get(t) diff --git a/tox.ini b/tox.ini index bbcbba9b0..8485dfc8b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,13 @@ [tox] -envlist = lint, fmt +envlist = py, lint, fmt + +[testenv] +description = run unit tests +deps = + pytest>=7 + -r requirements.txt +commands = + pytest {posargs:tests} [testenv:fmt] description = format with pre-commit @@ -8,5 +16,7 @@ allowlist_externals = ./scripts/fmt.sh [testenv:lint] description = lint with pylint -commands = pylint tuning scripts/*.py +deps = pylint>=2.16.2,<=3.1.0 + -r requirements.txt +commands = pylint tuning scripts/*.py build/*.py allowlist_externals = pylint diff --git a/tuning/__init__.py b/tuning/__init__.py index e69de29bb..a211ad5c2 100644 --- a/tuning/__init__.py +++ b/tuning/__init__.py @@ -0,0 +1,13 @@ +# Copyright The IBM Tuning Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tuning/config/__init__.py b/tuning/config/__init__.py index e69de29bb..a211ad5c2 100644 --- a/tuning/config/__init__.py +++ b/tuning/config/__init__.py @@ -0,0 +1,13 @@ +# Copyright The IBM Tuning Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tuning/config/configs.py b/tuning/config/configs.py index 279b006f4..8bd81865b 100644 --- a/tuning/config/configs.py +++ b/tuning/config/configs.py @@ -1,6 +1,20 @@ +# Copyright The IBM Tuning Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Standard from dataclasses import dataclass, field -from typing import Dict, Optional, Union +from typing import Optional, Union # Third Party import torch @@ -50,7 +64,8 @@ class TrainingArguments(transformers.TrainingArguments): model_max_length: int = field( default=DEFAULT_CONTEXT_LENGTH, metadata={ - "help": "Maximum sequence length. Sequences will be right padded (and possibly truncated)." + "help": "Maximum sequence length. Sequences will be right padded \ + (and possibly truncated)." }, ) packing: bool = field( diff --git a/tuning/config/peft_config.py b/tuning/config/peft_config.py index a3d30c763..773a57eca 100644 --- a/tuning/config/peft_config.py +++ b/tuning/config/peft_config.py @@ -1,3 +1,17 @@ +# Copyright The IBM Tuning Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Standard from dataclasses import dataclass, field from typing import List @@ -10,8 +24,10 @@ class LoraConfig: target_modules: List[str] = field( default_factory=lambda: ["q_proj", "v_proj"], metadata={ - "help": "The names of the modules to apply LORA to. LORA selects modules which either completely match or " - 'end with one of the strings. If the value is ["all-linear"], then LORA selects all linear and Conv1D ' + "help": "The names of the modules to apply LORA to. LORA selects modules which either \ + completely match or " + 'end with one of the strings. If the value is ["all-linear"], \ + then LORA selects all linear and Conv1D ' "modules except for the output layer." }, ) diff --git a/tuning/config/tracker_configs.py b/tuning/config/tracker_configs.py index 49135adbf..5ce243458 100644 --- a/tuning/config/tracker_configs.py +++ b/tuning/config/tracker_configs.py @@ -1,7 +1,20 @@ +# Copyright The IBM Tuning Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Standard from dataclasses import dataclass - @dataclass class AimConfig: # Name of the experiment diff --git a/tuning/data/__init__.py b/tuning/data/__init__.py index e69de29bb..a211ad5c2 100644 --- a/tuning/data/__init__.py +++ b/tuning/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright The IBM Tuning Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tuning/data/tokenizer_data_utils.py b/tuning/data/tokenizer_data_utils.py index 3a8a288f3..337def4ea 100644 --- a/tuning/data/tokenizer_data_utils.py +++ b/tuning/data/tokenizer_data_utils.py @@ -1,17 +1,23 @@ +# Copyright The IBM Tuning Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Standard -from typing import Dict, Sequence -import copy -import json -import logging +from typing import Dict # Third Party -from torch.utils.data import Dataset -import torch import transformers -# Local -from tuning.config import configs - def tokenizer_and_embedding_resize( special_tokens_dict: Dict, diff --git a/tuning/sft_trainer.py b/tuning/sft_trainer.py index cba25a67a..be5f35a4a 100644 --- a/tuning/sft_trainer.py +++ b/tuning/sft_trainer.py @@ -1,9 +1,24 @@ +# Copyright The IBM Tuning Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Standard from datetime import datetime from typing import Dict, List, Optional, Union import json import os import time +import sys # Third Party from peft.utils.other import fsdp_auto_wrap_policy @@ -33,17 +48,6 @@ logger = logging.get_logger("sft_trainer") -class PeftSavingCallback(TrainerCallback): - def on_save(self, args, state, control, **kwargs): - checkpoint_path = os.path.join( - args.output_dir, f"checkpoint-{state.global_step}" - ) - kwargs["model"].save_pretrained(checkpoint_path) - - if "pytorch_model.bin" in os.listdir(checkpoint_path): - os.remove(os.path.join(checkpoint_path, "pytorch_model.bin")) - - class FileLoggingCallback(TrainerCallback): """Exports metrics, e.g., training loss to a file in the checkpoint directory.""" @@ -84,7 +88,7 @@ def _track_loss(self, loss_key, log_file, logs, state): return # append the current log to the jsonl file - with open(log_file, "a") as f: + with open(log_file, "a", encoding="utf-8") as f: f.write(f"{json.dumps(log_obj, sort_keys=True)}\n") @@ -92,7 +96,7 @@ def train( model_args: configs.ModelArguments, data_args: configs.DataArguments, train_args: configs.TrainingArguments, - peft_config: Optional[ + peft_config: Optional[ # pylint: disable=redefined-outer-name Union[peft_config.LoraConfig, peft_config.PromptTuningConfig] ] = None, callbacks: Optional[List[TrainerCallback]] = None, @@ -115,7 +119,6 @@ def train( Using configs in tuning.config.tracker_configs exp_metadata: Dict of key value pairs passed to train to be recoreded by the tracker. """ - run_distributed = int(os.environ.get("WORLD_SIZE", "1")) > 1 # Validate parameters if (not isinstance(train_args.num_train_epochs, float)) or ( @@ -127,11 +130,6 @@ def train( ): raise ValueError("gradient_accumulation_steps has to be an integer >= 1") - # make sure to unset FSDP args when running on single gpu - if not run_distributed: - train_args.fsdp = "" - train_args.fsdp_config = {"xla": False} - task_type = "CAUSAL_LM" additional_metrics = {} @@ -146,17 +144,13 @@ def train( peft_config = get_hf_peft_config(task_type, peft_config) - model.gradient_checkpointing_enable() - # TODO: Move these to a config as well tokenizer = AutoTokenizer.from_pretrained( model_args.model_name_or_path, cache_dir=train_args.cache_dir, use_fast=True ) # TODO: understand if we need to hardcode these here or just use defaults in model - if isinstance(tokenizer, LlamaTokenizer) or isinstance( - tokenizer, LlamaTokenizerFast - ): + if isinstance(tokenizer, (LlamaTokenizer, LlamaTokenizerFast)): tokenizer.add_special_tokens( { "bos_token": "", @@ -165,33 +159,36 @@ def train( "pad_token": "", } ) - elif isinstance(tokenizer, GPTNeoXTokenizerFast) or isinstance( - tokenizer, GPT2Tokenizer - ): + elif isinstance(tokenizer, (GPT2Tokenizer, GPTNeoXTokenizerFast)): tokenizer.add_special_tokens( { "pad_token": "", } ) - """TODO: near term - how response template ids are parsed out needs to be cleaned. - The [2:] here applies if response template has \n prefix, it is needed to strip \n, otherwise template is not found. - We will create issue to clean this out after we discuss data formats and collators we will support - """ + # TODO: near term - how response template ids are parsed out needs to be cleaned. + # The [2:] here applies if response template has \n prefix, it is needed to strip \n, + # otherwise template is not found. We will create issue to clean this out after we discuss + # data formats and collators we will support. response_template_ids = tokenizer.encode( data_args.response_template, add_special_tokens=False )[2:] - # TODO: This is actually max_seq_length and not model_max_length. we should not override model_max_length - # as in current main. We need to change name of this parameter we expose to users. + # TODO: This is actually max_seq_length and not model_max_length. we should not override + # model_max_length as in current main. We need to change name of this parameter we expose + # to users. model_max_length = min(train_args.model_max_length, tokenizer.model_max_length) - logger.info(f"Model max length {model_max_length}") + logger.info("Model max length %s, model_max_length") if train_args.model_max_length > tokenizer.model_max_length: logger.warning( - f"model_max_length {train_args.model_max_length} exceeds tokenizer.model_max_length {tokenizer.model_max_length}, using tokenizer.model_max_length {tokenizer.model_max_length}" + "model_max_length %s exceeds tokenizer.model_max_length \ + %s, using tokenizer.model_max_length %s", + train_args.model_max_length, + tokenizer.model_max_length, + tokenizer.model_max_length, ) # TODO: we need to change this, perhaps follow what open instruct does? - special_tokens_dict = dict() + special_tokens_dict = {} if tokenizer.pad_token is None: logger.warning("PAD token set to default, missing in tokenizer") special_tokens_dict["pad_token"] = configs.DEFAULT_PAD_TOKEN @@ -219,19 +216,21 @@ def train( if data_args.validation_data_path: data_files["validation"] = data_args.validation_data_path - format_dataset = lambda example: { + format_dataset = lambda example: { # pylint: disable=unnecessary-lambda-assignment f"{data_args.dataset_text_field}": example[f"{data_args.dataset_text_field}"] + tokenizer.eos_token } json_dataset = datasets.load_dataset("json", data_files=data_files) formatted_train_dataset = json_dataset["train"].map(format_dataset) - logger.info(f"Training dataset length is {len(formatted_train_dataset)}") + logger.info("Training dataset length is %s", len(formatted_train_dataset)) formatted_validation_dataset = None if data_args.validation_data_path: formatted_validation_dataset = json_dataset["validation"].map(format_dataset) - logger.info(f"Validation dataset length is {len(formatted_validation_dataset)}") + logger.info( + "Validation dataset length is %s", len(formatted_validation_dataset) + ) if train_args.packing: logger.info("Packing is set to True") @@ -243,13 +242,13 @@ def train( logger.error( "Error, response template is None, needs to be set for training" ) - exit(-1) + sys.exit(-1) if data_args.dataset_text_field is None: logger.error( "Error, dataset_text_field is None, needs to be set for training" ) - exit(-1) + sys.exit(-1) data_collator = DataCollatorForCompletionOnlyLM( response_template_ids, @@ -282,14 +281,14 @@ def train( tracker.track(metric=v, name=k, stage="additional_metrics") tracker.set_params(params=exp_metadata, name="experiment_metadata") - if run_distributed and peft_config is not None: + if trainer.is_fsdp_enabled and peft_config is not None: trainer.accelerator.state.fsdp_plugin.auto_wrap_policy = fsdp_auto_wrap_policy( model ) trainer.train() -def main(**kwargs): +def main(**kwargs): # pylint: disable=unused-argument parser = transformers.HfArgumentParser( dataclass_types=( configs.ModelArguments, @@ -339,8 +338,7 @@ def main(**kwargs): # Initialize callbacks file_logger_callback = FileLoggingCallback(logger) - peft_saving_callback = PeftSavingCallback() - callbacks = [peft_saving_callback, file_logger_callback] + callbacks = [file_logger_callback] # Initialize the tracker tracker = get_tracker(tracker_name, tracker_config) diff --git a/tuning/trackers/__init__.py b/tuning/trackers/__init__.py index e69de29bb..55b377746 100644 --- a/tuning/trackers/__init__.py +++ b/tuning/trackers/__init__.py @@ -0,0 +1,13 @@ +# Copyright The IBM Tuning Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. \ No newline at end of file diff --git a/tuning/trackers/aimstack_tracker.py b/tuning/trackers/aimstack_tracker.py index 82f424f36..21e66e5fe 100644 --- a/tuning/trackers/aimstack_tracker.py +++ b/tuning/trackers/aimstack_tracker.py @@ -1,3 +1,17 @@ +# Copyright The IBM Tuning Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Standard import os @@ -8,7 +22,6 @@ from .tracker import Tracker from tuning.config.tracker_configs import AimConfig - class CustomAimCallback(AimCallback): # A path to export run hash generated by Aim diff --git a/tuning/trackers/tracker.py b/tuning/trackers/tracker.py index 71ad60183..162d793c9 100644 --- a/tuning/trackers/tracker.py +++ b/tuning/trackers/tracker.py @@ -1,6 +1,18 @@ -# Generic Tracker API - +# Copyright The IBM Tuning Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Generic Tracker API class Tracker: def __init__(self, name=None, tracker_config=None) -> None: if tracker_config is not None: diff --git a/tuning/trackers/tracker_factory.py b/tuning/trackers/tracker_factory.py index a9c32d641..917d1741d 100644 --- a/tuning/trackers/tracker_factory.py +++ b/tuning/trackers/tracker_factory.py @@ -1,3 +1,17 @@ +# Copyright The IBM Tuning Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Local from .aimstack_tracker import AimStackTracker from .tracker import Tracker diff --git a/tuning/utils/__init__.py b/tuning/utils/__init__.py index e69de29bb..a211ad5c2 100644 --- a/tuning/utils/__init__.py +++ b/tuning/utils/__init__.py @@ -0,0 +1,13 @@ +# Copyright The IBM Tuning Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tuning/utils/config_utils.py b/tuning/utils/config_utils.py index 58896c1f9..0674b209e 100644 --- a/tuning/utils/config_utils.py +++ b/tuning/utils/config_utils.py @@ -1,3 +1,17 @@ +# Copyright The IBM Tuning Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Standard from dataclasses import asdict diff --git a/tuning/utils/data_type_utils.py b/tuning/utils/data_type_utils.py index 42b058cde..ea82d9af3 100644 --- a/tuning/utils/data_type_utils.py +++ b/tuning/utils/data_type_utils.py @@ -1,3 +1,17 @@ +# Copyright The IBM Tuning Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Standard from typing import Union @@ -22,7 +36,7 @@ def str_to_torch_dtype(dtype_str: str) -> torch.dtype: dt = getattr(torch, dtype_str, None) if not isinstance(dt, torch.dtype): logger.error(" ValueError: Unrecognized data type of a torch.Tensor") - exit(-1) + raise ValueError("Unrecognized data type of a torch.Tensor") return dt diff --git a/tuning/utils/merge_model_utils.py b/tuning/utils/merge_model_utils.py index a8a41fecb..981382349 100644 --- a/tuning/utils/merge_model_utils.py +++ b/tuning/utils/merge_model_utils.py @@ -1,6 +1,19 @@ +# Copyright The IBM Tuning Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Standard from typing import Union -import argparse import json import os @@ -27,7 +40,7 @@ def create_merged_model( References: - https://github.com/huggingface/peft/issues/1040 - https://github.com/huggingface/peft/issues/280#issuecomment-1500805831 - - https://huggingface.co/docs/peft/main/en/package_reference/lora#peft.LoraModel.add_weighted_adapter + - https://huggingface.co/docs/peft/main/en/package_reference/lora#peft.LoraModel.add_weighted_adapter # pylint: disable=line-too-long Args: checkpoint_model: Union[str, list[str]] @@ -82,7 +95,7 @@ def fetch_base_model_from_checkpoint(checkpoint_model: str) -> str: if not os.path.isfile(adapter_config): raise FileNotFoundError("Unable to locate adapter config to infer base model!") - with open(adapter_config, "r") as cfg: + with open(adapter_config, "r", encoding="utf-8") as cfg: adapter_dict = json.load(cfg) if "base_model_name_or_path" not in adapter_dict: raise KeyError(