From ea9ff46615b838754663fcc7ab20f3215433ea0f Mon Sep 17 00:00:00 2001 From: Can Balioglu Date: Sun, 10 Dec 2023 15:30:47 -0800 Subject: [PATCH] Update --- .github/workflows/_build_doc.yaml | 10 ++++-- .github/workflows/_build_wheel-linux.yaml | 20 ++++++------ .github/workflows/_build_wheel-macos.yaml | 16 +++++----- .github/workflows/_lint_cc.yaml | 13 +++++--- .github/workflows/_lint_py.yaml | 13 +++++--- .github/workflows/_publish_pypi.yaml | 2 +- .github/workflows/_publish_s3.yaml | 2 +- .gitmodules | 14 ++++----- doc/conf.py | 4 +-- fairseq2n/python/src/fairseq2n/config.py | 13 -------- {fairseq2n => native}/.clang-format | 0 {fairseq2n => native}/.clang-tidy | 0 {fairseq2n => native}/CMakeLists.txt | 0 {fairseq2n => native}/LSan.supp | 0 {fairseq2n => native}/README.md | 0 {fairseq2n => native}/cmake/base.cmake | 0 .../cmake/modules/FindClangTidy.cmake | 0 .../cmake/modules/FindSndFile.cmake | 0 .../cmake/modules/FindTBB.cmake | 0 .../cmake/modules/FindTorch.cmake | 0 {fairseq2n => native}/cmake/summary.cmake | 0 .../python/requirements-build.txt | 0 {fairseq2n => native}/python/setup.py | 0 .../python/src/fairseq2n/.gitignore | 0 .../python/src/fairseq2n/CMakeLists.txt | 4 +++ .../python/src/fairseq2n/__init__.py | 31 +++++++++++-------- .../src/fairseq2n/bindings/CMakeLists.txt | 0 .../src/fairseq2n/bindings/data/audio.cc | 0 .../fairseq2n/bindings/data/data_pipeline.cc | 0 .../src/fairseq2n/bindings/data/image.cc | 0 .../src/fairseq2n/bindings/data/init.cc | 0 .../src/fairseq2n/bindings/data/string.cc | 0 .../bindings/data/text/converters.cc | 0 .../src/fairseq2n/bindings/data/text/init.cc | 0 .../bindings/data/text/sentencepiece.cc | 0 .../bindings/data/text/text_reader.cc | 0 .../python/src/fairseq2n/bindings/init.cc | 18 ----------- .../python/src/fairseq2n/bindings/memory.cc | 0 .../python/src/fairseq2n/bindings/module.h | 0 .../fairseq2n/bindings/type_casters/data.cc | 0 .../fairseq2n/bindings/type_casters/data.h | 0 .../fairseq2n/bindings/type_casters/map_fn.cc | 0 .../fairseq2n/bindings/type_casters/map_fn.h | 0 .../src/fairseq2n/bindings/type_casters/py.h | 0 .../fairseq2n/bindings/type_casters/string.cc | 0 .../fairseq2n/bindings/type_casters/string.h | 0 .../fairseq2n/bindings/type_casters/torch.cc | 0 .../fairseq2n/bindings/type_casters/torch.h | 0 .../python/src/fairseq2n/config.py.in | 4 +-- native/python/src/fairseq2n/py.typed | 0 .../src/fairseq2n-config.cmake.in | 0 .../src/fairseq2n/CMakeLists.txt | 0 {fairseq2n => native}/src/fairseq2n/api.h | 0 .../src/fairseq2n/config.h.in | 0 .../src/fairseq2n/data/audio/audio_decoder.cc | 0 .../src/fairseq2n/data/audio/audio_decoder.h | 0 .../data/audio/detail/kaldi_fbank.cc | 0 .../fairseq2n/data/audio/detail/kaldi_fbank.h | 0 .../fairseq2n/data/audio/detail/sndfile.cc | 0 .../src/fairseq2n/data/audio/detail/sndfile.h | 0 .../data/audio/waveform_to_fbank_converter.cc | 0 .../data/audio/waveform_to_fbank_converter.h | 0 .../data/bucket_by_length_data_source.cc | 0 .../data/bucket_by_length_data_source.h | 0 .../src/fairseq2n/data/bucket_data_source.cc | 0 .../src/fairseq2n/data/bucket_data_source.h | 0 .../src/fairseq2n/data/byte_stream.cc | 0 .../src/fairseq2n/data/byte_stream.h | 0 .../src/fairseq2n/data/collater.cc | 0 .../src/fairseq2n/data/collater.h | 0 .../fairseq2n/data/composite_data_source.cc | 0 .../fairseq2n/data/composite_data_source.h | 0 .../src/fairseq2n/data/concat_data_source.cc | 0 .../src/fairseq2n/data/concat_data_source.h | 0 .../fairseq2n/data/constant_data_source.cc | 0 .../src/fairseq2n/data/constant_data_source.h | 0 .../src/fairseq2n/data/count_data_source.cc | 0 .../src/fairseq2n/data/count_data_source.h | 0 .../src/fairseq2n/data/data.cc | 0 .../src/fairseq2n/data/data.h | 0 .../fairseq2n/data/data_length_extractor.cc | 0 .../fairseq2n/data/data_length_extractor.h | 0 .../src/fairseq2n/data/data_pipeline.cc | 0 .../src/fairseq2n/data/data_pipeline.h | 0 .../src/fairseq2n/data/data_source.cc | 0 .../src/fairseq2n/data/data_source.h | 0 .../src/fairseq2n/data/detail/exception.h | 0 .../src/fairseq2n/data/detail/file.cc | 0 .../src/fairseq2n/data/detail/file.h | 0 .../src/fairseq2n/data/detail/file_system.cc | 0 .../src/fairseq2n/data/detail/file_system.h | 0 .../src/fairseq2n/data/detail/lru_cache.h | 0 .../fairseq2n/data/detail/tensor_helpers.h | 0 .../src/fairseq2n/data/detail/thread.h | 0 .../src/fairseq2n/data/element_mapper.cc | 0 .../src/fairseq2n/data/element_mapper.h | 0 .../src/fairseq2n/data/element_selector.cc | 0 .../src/fairseq2n/data/element_selector.h | 0 .../src/fairseq2n/data/file.cc | 0 .../src/fairseq2n/data/file.h | 0 .../src/fairseq2n/data/file_mapper.cc | 0 .../src/fairseq2n/data/file_mapper.h | 0 .../src/fairseq2n/data/file_stream.cc | 0 .../src/fairseq2n/data/file_stream.h | 0 .../src/fairseq2n/data/filter_data_source.cc | 0 .../src/fairseq2n/data/filter_data_source.h | 0 .../image/detail/jpeg_decompress_struct.cc | 0 .../image/detail/jpeg_decompress_struct.h | 0 .../data/image/detail/png_read_struct.cc | 0 .../data/image/detail/png_read_struct.h | 0 .../src/fairseq2n/data/image/image_decoder.cc | 0 .../src/fairseq2n/data/image/image_decoder.h | 0 .../src/fairseq2n/data/immutable_string.cc | 0 .../src/fairseq2n/data/immutable_string.h | 0 .../src/fairseq2n/data/list_data_source.cc | 0 .../src/fairseq2n/data/list_data_source.h | 0 .../src/fairseq2n/data/map_data_source.cc | 0 .../src/fairseq2n/data/map_data_source.h | 0 .../src/fairseq2n/data/memory_stream.cc | 0 .../src/fairseq2n/data/memory_stream.h | 0 .../fairseq2n/data/prefetch_data_source.cc | 0 .../src/fairseq2n/data/prefetch_data_source.h | 0 .../src/fairseq2n/data/py.cc | 0 {fairseq2n => native}/src/fairseq2n/data/py.h | 0 .../src/fairseq2n/data/record_reader.cc | 0 .../src/fairseq2n/data/record_reader.h | 0 .../fairseq2n/data/round_robin_data_source.cc | 0 .../fairseq2n/data/round_robin_data_source.h | 0 .../src/fairseq2n/data/sample_data_source.cc | 0 .../src/fairseq2n/data/sample_data_source.h | 0 .../src/fairseq2n/data/shard_data_source.cc | 0 .../src/fairseq2n/data/shard_data_source.h | 0 .../src/fairseq2n/data/shuffle_data_source.cc | 0 .../src/fairseq2n/data/shuffle_data_source.h | 0 .../src/fairseq2n/data/skip_data_source.cc | 0 .../src/fairseq2n/data/skip_data_source.h | 0 .../src/fairseq2n/data/take_data_source.cc | 0 .../src/fairseq2n/data/take_data_source.h | 0 .../src/fairseq2n/data/tape-inl.h | 0 .../src/fairseq2n/data/tape.cc | 0 .../src/fairseq2n/data/tape.h | 0 .../src/fairseq2n/data/text/detail/utf.cc | 0 .../src/fairseq2n/data/text/detail/utf.h | 0 .../data/text/sentencepiece/sentencepiece.h | 0 .../data/text/sentencepiece/sp_decoder.cc | 0 .../data/text/sentencepiece/sp_decoder.h | 0 .../data/text/sentencepiece/sp_encoder.cc | 0 .../data/text/sentencepiece/sp_encoder.h | 0 .../data/text/sentencepiece/sp_model.cc | 0 .../data/text/sentencepiece/sp_model.h | 0 .../data/text/sentencepiece/sp_processor.cc | 0 .../data/text/sentencepiece/sp_processor.h | 0 .../fairseq2n/data/text/string_splitter.cc | 0 .../src/fairseq2n/data/text/string_splitter.h | 0 .../data/text/string_to_int_converter.cc | 0 .../data/text/string_to_int_converter.h | 0 .../data/text/string_to_tensor_converter.cc | 0 .../data/text/string_to_tensor_converter.h | 0 .../fairseq2n/data/text/text_data_source.cc | 0 .../fairseq2n/data/text/text_data_source.h | 0 .../fairseq2n/data/text/text_line_reader.cc | 0 .../fairseq2n/data/text/text_line_reader.h | 0 .../src/fairseq2n/data/text/text_reader.cc | 0 .../src/fairseq2n/data/text/text_reader.h | 0 .../src/fairseq2n/data/text/utf8_stream.cc | 0 .../src/fairseq2n/data/text/utf8_stream.h | 0 .../fairseq2n/data/yield_from_data_source.cc | 0 .../fairseq2n/data/yield_from_data_source.h | 0 .../src/fairseq2n/data/zip_data_source.cc | 0 .../src/fairseq2n/data/zip_data_source.h | 0 .../fairseq2n/data/zip_file_data_source.cc | 0 .../src/fairseq2n/data/zip_file_data_source.h | 0 .../src/fairseq2n/detail/error.h | 0 .../src/fairseq2n/detail/exception.h | 0 .../src/fairseq2n/detail/parallel.h | 0 .../src/fairseq2n/exception.cc | 0 .../src/fairseq2n/exception.h | 0 {fairseq2n => native}/src/fairseq2n/float.h | 0 {fairseq2n => native}/src/fairseq2n/fmt.h | 0 .../generation/ngram_repeat_block.cu | 0 {fairseq2n => native}/src/fairseq2n/memory.cc | 0 {fairseq2n => native}/src/fairseq2n/memory.h | 0 {fairseq2n => native}/src/fairseq2n/span.h | 0 .../src/fairseq2n/utils/cast.h | 0 .../src/fairseq2n/utils/string.h | 0 .../src/fairseq2n/utils/tensor.h | 0 {fairseq2n => native}/tests/CMakeLists.txt | 0 .../tests/data/detail/test_lru_cache.cc | 0 .../tests/data/test_immutable_string.cc | 0 {fairseq2n => native}/tests/data/test_tape.cc | 0 {fairseq2n => native}/tests/test_float.cc | 0 {fairseq2n => native}/tests/test_memory.cc | 0 {fairseq2n => native}/tests/test_span.cc | 0 .../tests/utils/test_cast.cc | 0 .../third-party/CMakeLists.txt | 0 {fairseq2n => native}/third-party/fmt | 0 {fairseq2n => native}/third-party/fmt.cmake | 0 {fairseq2n => native}/third-party/gtest | 0 {fairseq2n => native}/third-party/gtest.cmake | 0 .../third-party/kaldi-native-fbank | 0 .../third-party/kaldi-native-fbank.cmake | 0 .../third-party/kaldi-native-fbank.patch | 0 .../third-party/libjpeg-turbo.cmake | 0 {fairseq2n => native}/third-party/libpng | 0 .../third-party/libpng.cmake | 2 +- .../third-party/natsort.cmake | 0 .../third-party/natsort/CMakeLists.txt | 0 .../third-party/natsort/strnatcmp.c | 0 .../third-party/natsort/strnatcmp.h | 0 {fairseq2n => native}/third-party/pybind11 | 0 .../third-party/pybind11.cmake | 0 .../third-party/sentencepiece | 0 .../third-party/sentencepiece.cmake | 0 {fairseq2n => native}/third-party/zip | 0 {fairseq2n => native}/third-party/zip.cmake | 0 pyproject.toml | 2 +- src/fairseq2/data/audio.py | 2 +- src/fairseq2/data/cstring.py | 2 +- src/fairseq2/data/data_pipeline.py | 2 +- src/fairseq2/data/image.py | 2 +- src/fairseq2/data/text/converters.py | 2 +- src/fairseq2/data/text/sentencepiece.py | 2 +- src/fairseq2/data/text/text_reader.py | 2 +- src/fairseq2/memory.py | 2 +- tests/unit/data/image/test_image_decoder.py | 2 +- tools/set-project-version.sh | 6 ++-- 226 files changed, 91 insertions(+), 101 deletions(-) delete mode 100644 fairseq2n/python/src/fairseq2n/config.py rename {fairseq2n => native}/.clang-format (100%) rename {fairseq2n => native}/.clang-tidy (100%) rename {fairseq2n => native}/CMakeLists.txt (100%) rename {fairseq2n => native}/LSan.supp (100%) rename {fairseq2n => native}/README.md (100%) rename {fairseq2n => native}/cmake/base.cmake (100%) rename {fairseq2n => native}/cmake/modules/FindClangTidy.cmake (100%) rename {fairseq2n => native}/cmake/modules/FindSndFile.cmake (100%) rename {fairseq2n => native}/cmake/modules/FindTBB.cmake (100%) rename {fairseq2n => native}/cmake/modules/FindTorch.cmake (100%) rename {fairseq2n => native}/cmake/summary.cmake (100%) rename {fairseq2n => native}/python/requirements-build.txt (100%) rename {fairseq2n => native}/python/setup.py (100%) rename {fairseq2n => native}/python/src/fairseq2n/.gitignore (100%) rename {fairseq2n => native}/python/src/fairseq2n/CMakeLists.txt (85%) rename {fairseq2n => native}/python/src/fairseq2n/__init__.py (84%) rename {fairseq2n => native}/python/src/fairseq2n/bindings/CMakeLists.txt (100%) rename {fairseq2n => native}/python/src/fairseq2n/bindings/data/audio.cc (100%) rename {fairseq2n => native}/python/src/fairseq2n/bindings/data/data_pipeline.cc (100%) rename {fairseq2n => native}/python/src/fairseq2n/bindings/data/image.cc (100%) rename {fairseq2n => native}/python/src/fairseq2n/bindings/data/init.cc (100%) rename {fairseq2n => native}/python/src/fairseq2n/bindings/data/string.cc (100%) rename {fairseq2n => native}/python/src/fairseq2n/bindings/data/text/converters.cc (100%) rename {fairseq2n => native}/python/src/fairseq2n/bindings/data/text/init.cc (100%) rename {fairseq2n => native}/python/src/fairseq2n/bindings/data/text/sentencepiece.cc (100%) rename {fairseq2n => native}/python/src/fairseq2n/bindings/data/text/text_reader.cc (100%) rename {fairseq2n => native}/python/src/fairseq2n/bindings/init.cc (50%) rename {fairseq2n => native}/python/src/fairseq2n/bindings/memory.cc (100%) rename {fairseq2n => native}/python/src/fairseq2n/bindings/module.h (100%) rename {fairseq2n => native}/python/src/fairseq2n/bindings/type_casters/data.cc (100%) rename {fairseq2n => native}/python/src/fairseq2n/bindings/type_casters/data.h (100%) rename {fairseq2n => native}/python/src/fairseq2n/bindings/type_casters/map_fn.cc (100%) rename {fairseq2n => native}/python/src/fairseq2n/bindings/type_casters/map_fn.h (100%) rename {fairseq2n => native}/python/src/fairseq2n/bindings/type_casters/py.h (100%) rename {fairseq2n => native}/python/src/fairseq2n/bindings/type_casters/string.cc (100%) rename {fairseq2n => native}/python/src/fairseq2n/bindings/type_casters/string.h (100%) rename {fairseq2n => native}/python/src/fairseq2n/bindings/type_casters/torch.cc (100%) rename {fairseq2n => native}/python/src/fairseq2n/bindings/type_casters/torch.h (100%) rename {fairseq2n => native}/python/src/fairseq2n/config.py.in (89%) create mode 100644 native/python/src/fairseq2n/py.typed rename {fairseq2n => native}/src/fairseq2n-config.cmake.in (100%) rename {fairseq2n => native}/src/fairseq2n/CMakeLists.txt (100%) rename {fairseq2n => native}/src/fairseq2n/api.h (100%) rename {fairseq2n => native}/src/fairseq2n/config.h.in (100%) rename {fairseq2n => native}/src/fairseq2n/data/audio/audio_decoder.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/audio/audio_decoder.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/audio/detail/kaldi_fbank.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/audio/detail/kaldi_fbank.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/audio/detail/sndfile.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/audio/detail/sndfile.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/audio/waveform_to_fbank_converter.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/audio/waveform_to_fbank_converter.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/bucket_by_length_data_source.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/bucket_by_length_data_source.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/bucket_data_source.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/bucket_data_source.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/byte_stream.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/byte_stream.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/collater.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/collater.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/composite_data_source.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/composite_data_source.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/concat_data_source.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/concat_data_source.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/constant_data_source.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/constant_data_source.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/count_data_source.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/count_data_source.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/data.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/data.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/data_length_extractor.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/data_length_extractor.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/data_pipeline.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/data_pipeline.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/data_source.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/data_source.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/detail/exception.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/detail/file.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/detail/file.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/detail/file_system.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/detail/file_system.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/detail/lru_cache.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/detail/tensor_helpers.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/detail/thread.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/element_mapper.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/element_mapper.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/element_selector.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/element_selector.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/file.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/file.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/file_mapper.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/file_mapper.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/file_stream.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/file_stream.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/filter_data_source.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/filter_data_source.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/image/detail/jpeg_decompress_struct.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/image/detail/jpeg_decompress_struct.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/image/detail/png_read_struct.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/image/detail/png_read_struct.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/image/image_decoder.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/image/image_decoder.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/immutable_string.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/immutable_string.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/list_data_source.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/list_data_source.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/map_data_source.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/map_data_source.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/memory_stream.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/memory_stream.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/prefetch_data_source.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/prefetch_data_source.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/py.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/py.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/record_reader.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/record_reader.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/round_robin_data_source.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/round_robin_data_source.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/sample_data_source.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/sample_data_source.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/shard_data_source.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/shard_data_source.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/shuffle_data_source.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/shuffle_data_source.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/skip_data_source.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/skip_data_source.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/take_data_source.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/take_data_source.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/tape-inl.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/tape.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/tape.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/text/detail/utf.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/text/detail/utf.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/text/sentencepiece/sentencepiece.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/text/sentencepiece/sp_decoder.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/text/sentencepiece/sp_decoder.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/text/sentencepiece/sp_encoder.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/text/sentencepiece/sp_encoder.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/text/sentencepiece/sp_model.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/text/sentencepiece/sp_model.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/text/sentencepiece/sp_processor.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/text/sentencepiece/sp_processor.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/text/string_splitter.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/text/string_splitter.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/text/string_to_int_converter.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/text/string_to_int_converter.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/text/string_to_tensor_converter.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/text/string_to_tensor_converter.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/text/text_data_source.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/text/text_data_source.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/text/text_line_reader.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/text/text_line_reader.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/text/text_reader.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/text/text_reader.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/text/utf8_stream.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/text/utf8_stream.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/yield_from_data_source.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/yield_from_data_source.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/zip_data_source.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/zip_data_source.h (100%) rename {fairseq2n => native}/src/fairseq2n/data/zip_file_data_source.cc (100%) rename {fairseq2n => native}/src/fairseq2n/data/zip_file_data_source.h (100%) rename {fairseq2n => native}/src/fairseq2n/detail/error.h (100%) rename {fairseq2n => native}/src/fairseq2n/detail/exception.h (100%) rename {fairseq2n => native}/src/fairseq2n/detail/parallel.h (100%) rename {fairseq2n => native}/src/fairseq2n/exception.cc (100%) rename {fairseq2n => native}/src/fairseq2n/exception.h (100%) rename {fairseq2n => native}/src/fairseq2n/float.h (100%) rename {fairseq2n => native}/src/fairseq2n/fmt.h (100%) rename {fairseq2n => native}/src/fairseq2n/generation/ngram_repeat_block.cu (100%) rename {fairseq2n => native}/src/fairseq2n/memory.cc (100%) rename {fairseq2n => native}/src/fairseq2n/memory.h (100%) rename {fairseq2n => native}/src/fairseq2n/span.h (100%) rename {fairseq2n => native}/src/fairseq2n/utils/cast.h (100%) rename {fairseq2n => native}/src/fairseq2n/utils/string.h (100%) rename {fairseq2n => native}/src/fairseq2n/utils/tensor.h (100%) rename {fairseq2n => native}/tests/CMakeLists.txt (100%) rename {fairseq2n => native}/tests/data/detail/test_lru_cache.cc (100%) rename {fairseq2n => native}/tests/data/test_immutable_string.cc (100%) rename {fairseq2n => native}/tests/data/test_tape.cc (100%) rename {fairseq2n => native}/tests/test_float.cc (100%) rename {fairseq2n => native}/tests/test_memory.cc (100%) rename {fairseq2n => native}/tests/test_span.cc (100%) rename {fairseq2n => native}/tests/utils/test_cast.cc (100%) rename {fairseq2n => native}/third-party/CMakeLists.txt (100%) rename {fairseq2n => native}/third-party/fmt (100%) rename {fairseq2n => native}/third-party/fmt.cmake (100%) rename {fairseq2n => native}/third-party/gtest (100%) rename {fairseq2n => native}/third-party/gtest.cmake (100%) rename {fairseq2n => native}/third-party/kaldi-native-fbank (100%) rename {fairseq2n => native}/third-party/kaldi-native-fbank.cmake (100%) rename {fairseq2n => native}/third-party/kaldi-native-fbank.patch (100%) rename {fairseq2n => native}/third-party/libjpeg-turbo.cmake (100%) rename {fairseq2n => native}/third-party/libpng (100%) rename {fairseq2n => native}/third-party/libpng.cmake (100%) rename {fairseq2n => native}/third-party/natsort.cmake (100%) rename {fairseq2n => native}/third-party/natsort/CMakeLists.txt (100%) rename {fairseq2n => native}/third-party/natsort/strnatcmp.c (100%) rename {fairseq2n => native}/third-party/natsort/strnatcmp.h (100%) rename {fairseq2n => native}/third-party/pybind11 (100%) rename {fairseq2n => native}/third-party/pybind11.cmake (100%) rename {fairseq2n => native}/third-party/sentencepiece (100%) rename {fairseq2n => native}/third-party/sentencepiece.cmake (100%) rename {fairseq2n => native}/third-party/zip (100%) rename {fairseq2n => native}/third-party/zip.cmake (100%) diff --git a/.github/workflows/_build_doc.yaml b/.github/workflows/_build_doc.yaml index 67264322a..21804ff5c 100644 --- a/.github/workflows/_build_doc.yaml +++ b/.github/workflows/_build_doc.yaml @@ -30,6 +30,8 @@ jobs: steps: - name: Check-out the repository uses: actions/checkout@v3 + with: + submodules: recursive - name: Create the Python virtual environment run: | python${{ inputs.py }} -m venv ~/venv @@ -41,7 +43,7 @@ jobs: torch==${{ inputs.torch }} - name: Install requirements run: | - pip install --requirement fairseq2n/python/requirements-build.txt + pip install --requirement native/python/requirements-build.txt - name: Install Sphinx run: | pip install --requirement doc/requirements.txt @@ -49,9 +51,13 @@ jobs: if: inputs.version_override run: | tools/set-project-version.sh ${{ inputs.version_override }} + - name: Configure fairseq2n + working-directory: native + run: | + cmake -GNinja -B build - name: Install fairseq2n run: | - pip install --editable fairseq2n/python + pip install --editable native/python - name: Install fairseq2 run: | pip install --editable . diff --git a/.github/workflows/_build_wheel-linux.yaml b/.github/workflows/_build_wheel-linux.yaml index 10a39a581..a1056349c 100644 --- a/.github/workflows/_build_wheel-linux.yaml +++ b/.github/workflows/_build_wheel-linux.yaml @@ -64,7 +64,7 @@ jobs: torch==${{ inputs.torch }} - name: Install requirements run: | - pip install --requirement fairseq2n/python/requirements-build.txt + pip install --requirement native/python/requirements-build.txt - name: Override project version if: inputs.version_override run: | @@ -83,7 +83,7 @@ jobs: tools/set-project-version.sh $version+$VARIANT fi - name: Build fairseq2n - working-directory: fairseq2n + working-directory: native env: VARIANT: ${{ inputs.variant }} SANITIZERS: ${{ inputs.sanitizers }} @@ -122,7 +122,7 @@ jobs: cmake --build build - name: Package fairseq2n - working-directory: fairseq2n/python + working-directory: native/python run: | pip wheel\ --use-pep517\ @@ -141,8 +141,8 @@ jobs: name: ${{ inputs.artifacts_name }}-pt${{ inputs.torch }}-py${{ inputs.py }}-linux_${{ inputs.arch }}-${{ inputs.variant }}-${{ inputs.sanitizers }} path: | build/wheelhouse/*.whl - fairseq2n/build/tests/run-tests - fairseq2n/python/build/wheelhouse/*.whl + native/build/tests/run-tests + native/python/build/wheelhouse/*.whl retention-days: 1 test: @@ -185,9 +185,7 @@ jobs: fi - name: Install fairseq2n run: | - whl=$(ls ~/artifacts/fairseq2n/python/build/wheelhouse/*.whl) - - pip install --no-cache-dir "fairseq2n@file://$whl" + pip install --no-cache-dir ~/artifacts/native/python/build/wheelhouse/*.whl - name: Install fairseq2 run: | whl=$(ls ~/artifacts/build/wheelhouse/*.whl) @@ -204,7 +202,7 @@ jobs: # in non-instrumented dependencies. echo ASAN_OPTIONS=detect_container_overflow=0 - echo LSAN_OPTIONS=suppressions=fairseq2n/LSan.supp,exitcode=0,log_path=$HOME/lsan.out + echo LSAN_OPTIONS=suppressions=native/LSan.supp,exitcode=0,log_path=$HOME/lsan.out for sanitizer in ${SANITIZERS//_/ }; do # LIBASAN and LIBTSAN environment variables are defined in the @@ -225,12 +223,12 @@ jobs: } >> "$GITHUB_ENV" - name: Run native tests run: | - chmod 755 ~/artifacts/fairseq2n/build/tests/run-tests + chmod 755 ~/artifacts/native/build/tests/run-tests site_packages=~/venv/lib/python${{ inputs.py }}/site-packages LD_LIBRARY_PATH=~/venv/lib:$site_packages/torch/lib:$site_packages/fairseq2n/lib:$LD_LIBRARY_PATH\ - ~/artifacts/fairseq2n/build/tests/run-tests + ~/artifacts/native/build/tests/run-tests - name: Run Python tests env: RUN_INTEGRATION_TESTS: ${{ inputs.run_integration_tests }} diff --git a/.github/workflows/_build_wheel-macos.yaml b/.github/workflows/_build_wheel-macos.yaml index 1137d4878..23d83da19 100644 --- a/.github/workflows/_build_wheel-macos.yaml +++ b/.github/workflows/_build_wheel-macos.yaml @@ -48,13 +48,13 @@ jobs: pip install torch==${{ inputs.torch }} - name: Install requirements run: | - pip install --requirement fairseq2n/python/requirements-build.txt + pip install --requirement native/python/requirements-build.txt - name: Override project version if: inputs.version_override run: | tools/set-project-version.sh ${{ inputs.version_override }} - name: Build fairseq2n - working-directory: fairseq2n + working-directory: native run: | cmake\ -GNinja\ @@ -69,7 +69,7 @@ jobs: cmake --build build - name: Package fairseq2n - working-directory: fairseq2n/python + working-directory: native/python run: | pip wheel\ --use-pep517\ @@ -86,8 +86,8 @@ jobs: name: ${{ inputs.artifacts_name }}-pt${{ inputs.torch }}-py${{ inputs.py }}-macos_${{ inputs.arch }}-cpu path: | build/wheelhouse/*.whl - fairseq2n/build/tests/run-tests - fairseq2n/python/build/wheelhouse/*.whl + native/build/tests/run-tests + native/python/build/wheelhouse/*.whl retention-days: 1 test: @@ -121,16 +121,16 @@ jobs: pip install --requirement requirements-devel.txt - name: Install fairseq2n run: | - pip install --no-cache-dir ~/artifacts/fairseq2n/python/build/wheelhouse/*.whl + pip install --no-cache-dir ~/artifacts/native/python/build/wheelhouse/*.whl - name: Install fairseq2 run: | pip install --no-cache-dir ~/artifacts/build/wheelhouse/*.whl - name: Run native tests run: | - chmod 755 ~/artifacts/fairseq2n/build/tests/run-tests + chmod 755 ~/artifacts/native/build/tests/run-tests DYLD_LIBRARY_PATH=~/venv/lib:~/venv/lib/python${{ inputs.py }}/site-packages/torch/lib:~/venv/lib/python${{ inputs.py }}/site-packages/fairseq2n/lib\ - ~/artifacts/fairseq2n/build/tests/run-tests + ~/artifacts/native/build/tests/run-tests - name: Run Python tests run: | pytest -rP --verbose diff --git a/.github/workflows/_lint_cc.yaml b/.github/workflows/_lint_cc.yaml index 6d5b9110f..6b20b49dd 100644 --- a/.github/workflows/_lint_cc.yaml +++ b/.github/workflows/_lint_cc.yaml @@ -29,7 +29,6 @@ jobs: uses: actions/checkout@v3 with: submodules: recursive - fetch-depth: 2 - name: Create the Python virtual environment run: | python${{ inputs.py }} -m venv ~/venv @@ -41,19 +40,23 @@ jobs: torch==${{ inputs.torch }}\ - name: Install requirements run: | - pip install --requirement fairseq2n/python/requirements-build.txt + pip install --requirement native/python/requirements-build.txt - name: Build the compilation database id: setup - working-directory: fairseq2n + working-directory: native run: | # Since we are only building the compilation database, we can ignore # CUDA compatibility. export CUDAFLAGS="--allow-unsupported-compiler" - CC=clang CXX=clang++ cmake -GNinja -DCMAKE_BUILD_TYPE=Release -B build + # TODO: set FAIRSEQ2N_USE_CUDA once lint issues are fixed. + CC=clang CXX=clang++ cmake\ + -GNinja -DFAIRSEQ2N_PYTHON_DEVEL=OFF -B build + + cmake --build build - name: Run clang-tidy if: success() || (failure() && steps.setup.outcome == 'success') - working-directory: fairseq2n + working-directory: native run: | echo "::add-matcher::./ci/problem-matchers/gcc.json" diff --git a/.github/workflows/_lint_py.yaml b/.github/workflows/_lint_py.yaml index e2ba5e6ce..a54f63240 100644 --- a/.github/workflows/_lint_py.yaml +++ b/.github/workflows/_lint_py.yaml @@ -27,6 +27,8 @@ jobs: steps: - name: Check-out the repository uses: actions/checkout@v3 + with: + submodules: recursive - name: Create the Python virtual environment run: | python${{ inputs.py }} -m venv ~/venv @@ -38,12 +40,16 @@ jobs: torch==${{ inputs.torch }} - name: Install requirements run: | - pip install --requirement fairseq2n/python/requirements-build.txt + pip install --requirement native/python/requirements-build.txt pip install --requirement requirements-devel.txt + - name: Configure fairseq2n + working-directory: native + run: | + cmake -GNinja -B build - name: Install fairseq2n run: | - pip install --editable fairseq2n/python + pip install --editable native/python - name: Install fairseq2 id: install_fairseq2 run: | @@ -100,7 +106,6 @@ jobs: trap remove_matcher EXIT mypy - mypy setup.py - mypy fairseq2n/python/setup.py + mypy native/python mypy recipes/llama mypy recipes/mistral diff --git a/.github/workflows/_publish_pypi.yaml b/.github/workflows/_publish_pypi.yaml index 2222fc551..99e3ace2c 100644 --- a/.github/workflows/_publish_pypi.yaml +++ b/.github/workflows/_publish_pypi.yaml @@ -42,7 +42,7 @@ jobs: - name: Publish fairseq2n uses: pypa/gh-action-pypi-publish@release/v1 with: - packages-dir: artifacts/fairseq2n/python/build/wheelhouse + packages-dir: artifacts/native/python/build/wheelhouse - name: Publish fairseq2 uses: pypa/gh-action-pypi-publish@release/v1 with: diff --git a/.github/workflows/_publish_s3.yaml b/.github/workflows/_publish_s3.yaml index 8e7977462..1e8dee25b 100644 --- a/.github/workflows/_publish_s3.yaml +++ b/.github/workflows/_publish_s3.yaml @@ -68,7 +68,7 @@ jobs: echo S3_PREFIX=$s3_prefix >> "$GITHUB_ENV" - name: Upload wheel run: | - for pkg in ~/artifacts/fairseq2n/python/build/wheelhouse/*.whl; do + for pkg in ~/artifacts/native/python/build/wheelhouse/*.whl; do aws s3 cp "$pkg" s3://$S3_BUCKET/$S3_PREFIX/fairseq2n/ done diff --git a/.gitmodules b/.gitmodules index a24eaad1b..b140f38b1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,28 +1,28 @@ [submodule "third-party/pybind11"] - path = fairseq2n/third-party/pybind11 + path = native/third-party/pybind11 url = https://github.com/pybind/pybind11.git ignore = dirty [submodule "third-party/fmt"] - path = fairseq2n/third-party/fmt + path = native/third-party/fmt url = https://github.com/fmtlib/fmt.git ignore = dirty [submodule "third-party/gtest"] - path = fairseq2n/third-party/gtest + path = native/third-party/gtest url = https://github.com/google/googletest.git ignore = dirty [submodule "third-party/sentencepiece"] - path = fairseq2n/third-party/sentencepiece + path = native/third-party/sentencepiece url = https://github.com/google/sentencepiece.git ignore = dirty [submodule "third-party/zip"] - path = fairseq2n/third-party/zip + path = native/third-party/zip url = https://github.com/kuba--/zip.git ignore = dirty [submodule "third-party/kaldi-native-fbank"] - path = fairseq2n/third-party/kaldi-native-fbank + path = native/third-party/kaldi-native-fbank url = https://github.com/csukuangfj/kaldi-native-fbank.git ignore = dirty [submodule "fairseq2n/third-party/libpng"] - path = fairseq2n/third-party/libpng + path = native/third-party/libpng url = https://github.com/glennrp/libpng.git ignore = dirty diff --git a/doc/conf.py b/doc/conf.py index 125812919..346755ee9 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -4,9 +4,9 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -import fairseq2n.config +import fairseq2n -fairseq2n.config.DOC_MODE = True +fairseq2n.DOC_MODE = True import fairseq2 diff --git a/fairseq2n/python/src/fairseq2n/config.py b/fairseq2n/python/src/fairseq2n/config.py deleted file mode 100644 index 53fbfcf16..000000000 --- a/fairseq2n/python/src/fairseq2n/config.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -from typing import Final - -DOC_MODE: Final = False - -_SUPPORTS_IMAGE: Final = False - -_SUPPORTS_CUDA: Final = False diff --git a/fairseq2n/.clang-format b/native/.clang-format similarity index 100% rename from fairseq2n/.clang-format rename to native/.clang-format diff --git a/fairseq2n/.clang-tidy b/native/.clang-tidy similarity index 100% rename from fairseq2n/.clang-tidy rename to native/.clang-tidy diff --git a/fairseq2n/CMakeLists.txt b/native/CMakeLists.txt similarity index 100% rename from fairseq2n/CMakeLists.txt rename to native/CMakeLists.txt diff --git a/fairseq2n/LSan.supp b/native/LSan.supp similarity index 100% rename from fairseq2n/LSan.supp rename to native/LSan.supp diff --git a/fairseq2n/README.md b/native/README.md similarity index 100% rename from fairseq2n/README.md rename to native/README.md diff --git a/fairseq2n/cmake/base.cmake b/native/cmake/base.cmake similarity index 100% rename from fairseq2n/cmake/base.cmake rename to native/cmake/base.cmake diff --git a/fairseq2n/cmake/modules/FindClangTidy.cmake b/native/cmake/modules/FindClangTidy.cmake similarity index 100% rename from fairseq2n/cmake/modules/FindClangTidy.cmake rename to native/cmake/modules/FindClangTidy.cmake diff --git a/fairseq2n/cmake/modules/FindSndFile.cmake b/native/cmake/modules/FindSndFile.cmake similarity index 100% rename from fairseq2n/cmake/modules/FindSndFile.cmake rename to native/cmake/modules/FindSndFile.cmake diff --git a/fairseq2n/cmake/modules/FindTBB.cmake b/native/cmake/modules/FindTBB.cmake similarity index 100% rename from fairseq2n/cmake/modules/FindTBB.cmake rename to native/cmake/modules/FindTBB.cmake diff --git a/fairseq2n/cmake/modules/FindTorch.cmake b/native/cmake/modules/FindTorch.cmake similarity index 100% rename from fairseq2n/cmake/modules/FindTorch.cmake rename to native/cmake/modules/FindTorch.cmake diff --git a/fairseq2n/cmake/summary.cmake b/native/cmake/summary.cmake similarity index 100% rename from fairseq2n/cmake/summary.cmake rename to native/cmake/summary.cmake diff --git a/fairseq2n/python/requirements-build.txt b/native/python/requirements-build.txt similarity index 100% rename from fairseq2n/python/requirements-build.txt rename to native/python/requirements-build.txt diff --git a/fairseq2n/python/setup.py b/native/python/setup.py similarity index 100% rename from fairseq2n/python/setup.py rename to native/python/setup.py diff --git a/fairseq2n/python/src/fairseq2n/.gitignore b/native/python/src/fairseq2n/.gitignore similarity index 100% rename from fairseq2n/python/src/fairseq2n/.gitignore rename to native/python/src/fairseq2n/.gitignore diff --git a/fairseq2n/python/src/fairseq2n/CMakeLists.txt b/native/python/src/fairseq2n/CMakeLists.txt similarity index 85% rename from fairseq2n/python/src/fairseq2n/CMakeLists.txt rename to native/python/src/fairseq2n/CMakeLists.txt index 085b50545..94a92fd30 100644 --- a/fairseq2n/python/src/fairseq2n/CMakeLists.txt +++ b/native/python/src/fairseq2n/CMakeLists.txt @@ -18,8 +18,12 @@ endif() if(FAIRSEQ2N_USE_CUDA) set(USES_CUDA "True") + + set(CUDA_VERSION "(${CUDAToolkit_VERSION_MAJOR}, ${CUDAToolkit_VERSION_MINOR})") else() set(USES_CUDA "False") + + set(CUDA_VERSION "None") endif() configure_file(config.py.in ${CMAKE_CURRENT_SOURCE_DIR}/config.py @ONLY) diff --git a/fairseq2n/python/src/fairseq2n/__init__.py b/native/python/src/fairseq2n/__init__.py similarity index 84% rename from fairseq2n/python/src/fairseq2n/__init__.py rename to native/python/src/fairseq2n/__init__.py index beb92c298..011c53345 100644 --- a/fairseq2n/python/src/fairseq2n/__init__.py +++ b/native/python/src/fairseq2n/__init__.py @@ -13,10 +13,13 @@ from pathlib import Path from typing import TYPE_CHECKING, List, Optional, Tuple -from fairseq2n.config import _SUPPORTS_CUDA, _SUPPORTS_IMAGE, DOC_MODE +from fairseq2n.config import _CUDA_VERSION, _SUPPORTS_CUDA, _SUPPORTS_IMAGE __version__ = "0.2.1.dev0" +# Indicates whether we are run under Sphinx. +DOC_MODE = False + # Keeps the shared libraries that we load using our own extended lookup logic # in memory. @@ -116,10 +119,16 @@ def _check_cuda_runtime() -> None: if not _SUPPORTS_CUDA: return + assert _CUDA_VERSION is not None + + major_cuda_ver, minor_cuda_ver = _CUDA_VERSION + libcudart = _load_shared_library("libcudart.so") if libcudart is None: + cuda = f"CUDA {major_cuda_ver}.{minor_cuda_ver}" + raise OSError( - "fairseq2 is built with CUDA, but the CUDA runtime cannot be found. Either install the CUDA Toolkit or a CPU-only version of fairseq2 (see https://github.com/facebookresearch/fairseq2#variants)." + f"fairseq2 is built with {cuda}, but {cuda} runtime cannot be found on your system. Either install {cuda} Toolkit or a CPU-only version of fairseq2 (see https://github.com/facebookresearch/fairseq2#variants)." ) @@ -138,7 +147,7 @@ def get_include() -> Path: def get_cmake_prefix_path() -> Path: """Return the directory that contains fairseq2n CMake package.""" - return Path(__file__).parent.joinpath("lib", "cmake") + return Path(__file__).parent.joinpath("lib/cmake") def supports_image() -> bool: @@ -151,14 +160,10 @@ def supports_cuda() -> bool: return _SUPPORTS_CUDA -if TYPE_CHECKING or DOC_MODE: - - def cuda_version() -> Optional[Tuple[int, int]]: - """Return the version of CUDA that fairseq2n supports. - - :returns: - The major and minor version segments. - """ +def cuda_version() -> Optional[Tuple[int, int]]: + """Return the version of CUDA that fairseq2n supports. -else: - from fairseq2n.bindings import cuda_version as cuda_version + :returns: + The major and minor version segments. + """ + return _CUDA_VERSION diff --git a/fairseq2n/python/src/fairseq2n/bindings/CMakeLists.txt b/native/python/src/fairseq2n/bindings/CMakeLists.txt similarity index 100% rename from fairseq2n/python/src/fairseq2n/bindings/CMakeLists.txt rename to native/python/src/fairseq2n/bindings/CMakeLists.txt diff --git a/fairseq2n/python/src/fairseq2n/bindings/data/audio.cc b/native/python/src/fairseq2n/bindings/data/audio.cc similarity index 100% rename from fairseq2n/python/src/fairseq2n/bindings/data/audio.cc rename to native/python/src/fairseq2n/bindings/data/audio.cc diff --git a/fairseq2n/python/src/fairseq2n/bindings/data/data_pipeline.cc b/native/python/src/fairseq2n/bindings/data/data_pipeline.cc similarity index 100% rename from fairseq2n/python/src/fairseq2n/bindings/data/data_pipeline.cc rename to native/python/src/fairseq2n/bindings/data/data_pipeline.cc diff --git a/fairseq2n/python/src/fairseq2n/bindings/data/image.cc b/native/python/src/fairseq2n/bindings/data/image.cc similarity index 100% rename from fairseq2n/python/src/fairseq2n/bindings/data/image.cc rename to native/python/src/fairseq2n/bindings/data/image.cc diff --git a/fairseq2n/python/src/fairseq2n/bindings/data/init.cc b/native/python/src/fairseq2n/bindings/data/init.cc similarity index 100% rename from fairseq2n/python/src/fairseq2n/bindings/data/init.cc rename to native/python/src/fairseq2n/bindings/data/init.cc diff --git a/fairseq2n/python/src/fairseq2n/bindings/data/string.cc b/native/python/src/fairseq2n/bindings/data/string.cc similarity index 100% rename from fairseq2n/python/src/fairseq2n/bindings/data/string.cc rename to native/python/src/fairseq2n/bindings/data/string.cc diff --git a/fairseq2n/python/src/fairseq2n/bindings/data/text/converters.cc b/native/python/src/fairseq2n/bindings/data/text/converters.cc similarity index 100% rename from fairseq2n/python/src/fairseq2n/bindings/data/text/converters.cc rename to native/python/src/fairseq2n/bindings/data/text/converters.cc diff --git a/fairseq2n/python/src/fairseq2n/bindings/data/text/init.cc b/native/python/src/fairseq2n/bindings/data/text/init.cc similarity index 100% rename from fairseq2n/python/src/fairseq2n/bindings/data/text/init.cc rename to native/python/src/fairseq2n/bindings/data/text/init.cc diff --git a/fairseq2n/python/src/fairseq2n/bindings/data/text/sentencepiece.cc b/native/python/src/fairseq2n/bindings/data/text/sentencepiece.cc similarity index 100% rename from fairseq2n/python/src/fairseq2n/bindings/data/text/sentencepiece.cc rename to native/python/src/fairseq2n/bindings/data/text/sentencepiece.cc diff --git a/fairseq2n/python/src/fairseq2n/bindings/data/text/text_reader.cc b/native/python/src/fairseq2n/bindings/data/text/text_reader.cc similarity index 100% rename from fairseq2n/python/src/fairseq2n/bindings/data/text/text_reader.cc rename to native/python/src/fairseq2n/bindings/data/text/text_reader.cc diff --git a/fairseq2n/python/src/fairseq2n/bindings/init.cc b/native/python/src/fairseq2n/bindings/init.cc similarity index 50% rename from fairseq2n/python/src/fairseq2n/bindings/init.cc rename to native/python/src/fairseq2n/bindings/init.cc index 6548f15a0..c354b0ccf 100644 --- a/fairseq2n/python/src/fairseq2n/bindings/init.cc +++ b/native/python/src/fairseq2n/bindings/init.cc @@ -6,8 +6,6 @@ #include "fairseq2n/bindings/module.h" -#include - namespace py = pybind11; namespace fairseq2n { @@ -17,22 +15,6 @@ PYBIND11_MODULE(bindings, m) py::options opts{}; opts.disable_function_signatures(); -// See https://github.com/llvm/llvm-project/issues/57123. -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wunreachable-code-return" - - m.def( - "cuda_version", - [] - { - if constexpr (cuda_version_major) - return py::make_tuple(*cuda_version_major, *cuda_version_minor); - else - return py::none(); - }); - -#pragma clang diagnostic pop - def_data(m); def_memory(m); diff --git a/fairseq2n/python/src/fairseq2n/bindings/memory.cc b/native/python/src/fairseq2n/bindings/memory.cc similarity index 100% rename from fairseq2n/python/src/fairseq2n/bindings/memory.cc rename to native/python/src/fairseq2n/bindings/memory.cc diff --git a/fairseq2n/python/src/fairseq2n/bindings/module.h b/native/python/src/fairseq2n/bindings/module.h similarity index 100% rename from fairseq2n/python/src/fairseq2n/bindings/module.h rename to native/python/src/fairseq2n/bindings/module.h diff --git a/fairseq2n/python/src/fairseq2n/bindings/type_casters/data.cc b/native/python/src/fairseq2n/bindings/type_casters/data.cc similarity index 100% rename from fairseq2n/python/src/fairseq2n/bindings/type_casters/data.cc rename to native/python/src/fairseq2n/bindings/type_casters/data.cc diff --git a/fairseq2n/python/src/fairseq2n/bindings/type_casters/data.h b/native/python/src/fairseq2n/bindings/type_casters/data.h similarity index 100% rename from fairseq2n/python/src/fairseq2n/bindings/type_casters/data.h rename to native/python/src/fairseq2n/bindings/type_casters/data.h diff --git a/fairseq2n/python/src/fairseq2n/bindings/type_casters/map_fn.cc b/native/python/src/fairseq2n/bindings/type_casters/map_fn.cc similarity index 100% rename from fairseq2n/python/src/fairseq2n/bindings/type_casters/map_fn.cc rename to native/python/src/fairseq2n/bindings/type_casters/map_fn.cc diff --git a/fairseq2n/python/src/fairseq2n/bindings/type_casters/map_fn.h b/native/python/src/fairseq2n/bindings/type_casters/map_fn.h similarity index 100% rename from fairseq2n/python/src/fairseq2n/bindings/type_casters/map_fn.h rename to native/python/src/fairseq2n/bindings/type_casters/map_fn.h diff --git a/fairseq2n/python/src/fairseq2n/bindings/type_casters/py.h b/native/python/src/fairseq2n/bindings/type_casters/py.h similarity index 100% rename from fairseq2n/python/src/fairseq2n/bindings/type_casters/py.h rename to native/python/src/fairseq2n/bindings/type_casters/py.h diff --git a/fairseq2n/python/src/fairseq2n/bindings/type_casters/string.cc b/native/python/src/fairseq2n/bindings/type_casters/string.cc similarity index 100% rename from fairseq2n/python/src/fairseq2n/bindings/type_casters/string.cc rename to native/python/src/fairseq2n/bindings/type_casters/string.cc diff --git a/fairseq2n/python/src/fairseq2n/bindings/type_casters/string.h b/native/python/src/fairseq2n/bindings/type_casters/string.h similarity index 100% rename from fairseq2n/python/src/fairseq2n/bindings/type_casters/string.h rename to native/python/src/fairseq2n/bindings/type_casters/string.h diff --git a/fairseq2n/python/src/fairseq2n/bindings/type_casters/torch.cc b/native/python/src/fairseq2n/bindings/type_casters/torch.cc similarity index 100% rename from fairseq2n/python/src/fairseq2n/bindings/type_casters/torch.cc rename to native/python/src/fairseq2n/bindings/type_casters/torch.cc diff --git a/fairseq2n/python/src/fairseq2n/bindings/type_casters/torch.h b/native/python/src/fairseq2n/bindings/type_casters/torch.h similarity index 100% rename from fairseq2n/python/src/fairseq2n/bindings/type_casters/torch.h rename to native/python/src/fairseq2n/bindings/type_casters/torch.h diff --git a/fairseq2n/python/src/fairseq2n/config.py.in b/native/python/src/fairseq2n/config.py.in similarity index 89% rename from fairseq2n/python/src/fairseq2n/config.py.in rename to native/python/src/fairseq2n/config.py.in index 031f35a80..39a1a0584 100644 --- a/fairseq2n/python/src/fairseq2n/config.py.in +++ b/native/python/src/fairseq2n/config.py.in @@ -6,8 +6,8 @@ from typing import Final -DOC_MODE: Final = False - _SUPPORTS_IMAGE: Final = @SUPPORTS_IMAGE@ _SUPPORTS_CUDA: Final = @USES_CUDA@ + +_CUDA_VERSION: Final = @CUDA_VERSION@ diff --git a/native/python/src/fairseq2n/py.typed b/native/python/src/fairseq2n/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/fairseq2n/src/fairseq2n-config.cmake.in b/native/src/fairseq2n-config.cmake.in similarity index 100% rename from fairseq2n/src/fairseq2n-config.cmake.in rename to native/src/fairseq2n-config.cmake.in diff --git a/fairseq2n/src/fairseq2n/CMakeLists.txt b/native/src/fairseq2n/CMakeLists.txt similarity index 100% rename from fairseq2n/src/fairseq2n/CMakeLists.txt rename to native/src/fairseq2n/CMakeLists.txt diff --git a/fairseq2n/src/fairseq2n/api.h b/native/src/fairseq2n/api.h similarity index 100% rename from fairseq2n/src/fairseq2n/api.h rename to native/src/fairseq2n/api.h diff --git a/fairseq2n/src/fairseq2n/config.h.in b/native/src/fairseq2n/config.h.in similarity index 100% rename from fairseq2n/src/fairseq2n/config.h.in rename to native/src/fairseq2n/config.h.in diff --git a/fairseq2n/src/fairseq2n/data/audio/audio_decoder.cc b/native/src/fairseq2n/data/audio/audio_decoder.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/audio/audio_decoder.cc rename to native/src/fairseq2n/data/audio/audio_decoder.cc diff --git a/fairseq2n/src/fairseq2n/data/audio/audio_decoder.h b/native/src/fairseq2n/data/audio/audio_decoder.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/audio/audio_decoder.h rename to native/src/fairseq2n/data/audio/audio_decoder.h diff --git a/fairseq2n/src/fairseq2n/data/audio/detail/kaldi_fbank.cc b/native/src/fairseq2n/data/audio/detail/kaldi_fbank.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/audio/detail/kaldi_fbank.cc rename to native/src/fairseq2n/data/audio/detail/kaldi_fbank.cc diff --git a/fairseq2n/src/fairseq2n/data/audio/detail/kaldi_fbank.h b/native/src/fairseq2n/data/audio/detail/kaldi_fbank.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/audio/detail/kaldi_fbank.h rename to native/src/fairseq2n/data/audio/detail/kaldi_fbank.h diff --git a/fairseq2n/src/fairseq2n/data/audio/detail/sndfile.cc b/native/src/fairseq2n/data/audio/detail/sndfile.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/audio/detail/sndfile.cc rename to native/src/fairseq2n/data/audio/detail/sndfile.cc diff --git a/fairseq2n/src/fairseq2n/data/audio/detail/sndfile.h b/native/src/fairseq2n/data/audio/detail/sndfile.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/audio/detail/sndfile.h rename to native/src/fairseq2n/data/audio/detail/sndfile.h diff --git a/fairseq2n/src/fairseq2n/data/audio/waveform_to_fbank_converter.cc b/native/src/fairseq2n/data/audio/waveform_to_fbank_converter.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/audio/waveform_to_fbank_converter.cc rename to native/src/fairseq2n/data/audio/waveform_to_fbank_converter.cc diff --git a/fairseq2n/src/fairseq2n/data/audio/waveform_to_fbank_converter.h b/native/src/fairseq2n/data/audio/waveform_to_fbank_converter.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/audio/waveform_to_fbank_converter.h rename to native/src/fairseq2n/data/audio/waveform_to_fbank_converter.h diff --git a/fairseq2n/src/fairseq2n/data/bucket_by_length_data_source.cc b/native/src/fairseq2n/data/bucket_by_length_data_source.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/bucket_by_length_data_source.cc rename to native/src/fairseq2n/data/bucket_by_length_data_source.cc diff --git a/fairseq2n/src/fairseq2n/data/bucket_by_length_data_source.h b/native/src/fairseq2n/data/bucket_by_length_data_source.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/bucket_by_length_data_source.h rename to native/src/fairseq2n/data/bucket_by_length_data_source.h diff --git a/fairseq2n/src/fairseq2n/data/bucket_data_source.cc b/native/src/fairseq2n/data/bucket_data_source.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/bucket_data_source.cc rename to native/src/fairseq2n/data/bucket_data_source.cc diff --git a/fairseq2n/src/fairseq2n/data/bucket_data_source.h b/native/src/fairseq2n/data/bucket_data_source.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/bucket_data_source.h rename to native/src/fairseq2n/data/bucket_data_source.h diff --git a/fairseq2n/src/fairseq2n/data/byte_stream.cc b/native/src/fairseq2n/data/byte_stream.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/byte_stream.cc rename to native/src/fairseq2n/data/byte_stream.cc diff --git a/fairseq2n/src/fairseq2n/data/byte_stream.h b/native/src/fairseq2n/data/byte_stream.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/byte_stream.h rename to native/src/fairseq2n/data/byte_stream.h diff --git a/fairseq2n/src/fairseq2n/data/collater.cc b/native/src/fairseq2n/data/collater.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/collater.cc rename to native/src/fairseq2n/data/collater.cc diff --git a/fairseq2n/src/fairseq2n/data/collater.h b/native/src/fairseq2n/data/collater.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/collater.h rename to native/src/fairseq2n/data/collater.h diff --git a/fairseq2n/src/fairseq2n/data/composite_data_source.cc b/native/src/fairseq2n/data/composite_data_source.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/composite_data_source.cc rename to native/src/fairseq2n/data/composite_data_source.cc diff --git a/fairseq2n/src/fairseq2n/data/composite_data_source.h b/native/src/fairseq2n/data/composite_data_source.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/composite_data_source.h rename to native/src/fairseq2n/data/composite_data_source.h diff --git a/fairseq2n/src/fairseq2n/data/concat_data_source.cc b/native/src/fairseq2n/data/concat_data_source.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/concat_data_source.cc rename to native/src/fairseq2n/data/concat_data_source.cc diff --git a/fairseq2n/src/fairseq2n/data/concat_data_source.h b/native/src/fairseq2n/data/concat_data_source.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/concat_data_source.h rename to native/src/fairseq2n/data/concat_data_source.h diff --git a/fairseq2n/src/fairseq2n/data/constant_data_source.cc b/native/src/fairseq2n/data/constant_data_source.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/constant_data_source.cc rename to native/src/fairseq2n/data/constant_data_source.cc diff --git a/fairseq2n/src/fairseq2n/data/constant_data_source.h b/native/src/fairseq2n/data/constant_data_source.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/constant_data_source.h rename to native/src/fairseq2n/data/constant_data_source.h diff --git a/fairseq2n/src/fairseq2n/data/count_data_source.cc b/native/src/fairseq2n/data/count_data_source.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/count_data_source.cc rename to native/src/fairseq2n/data/count_data_source.cc diff --git a/fairseq2n/src/fairseq2n/data/count_data_source.h b/native/src/fairseq2n/data/count_data_source.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/count_data_source.h rename to native/src/fairseq2n/data/count_data_source.h diff --git a/fairseq2n/src/fairseq2n/data/data.cc b/native/src/fairseq2n/data/data.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/data.cc rename to native/src/fairseq2n/data/data.cc diff --git a/fairseq2n/src/fairseq2n/data/data.h b/native/src/fairseq2n/data/data.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/data.h rename to native/src/fairseq2n/data/data.h diff --git a/fairseq2n/src/fairseq2n/data/data_length_extractor.cc b/native/src/fairseq2n/data/data_length_extractor.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/data_length_extractor.cc rename to native/src/fairseq2n/data/data_length_extractor.cc diff --git a/fairseq2n/src/fairseq2n/data/data_length_extractor.h b/native/src/fairseq2n/data/data_length_extractor.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/data_length_extractor.h rename to native/src/fairseq2n/data/data_length_extractor.h diff --git a/fairseq2n/src/fairseq2n/data/data_pipeline.cc b/native/src/fairseq2n/data/data_pipeline.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/data_pipeline.cc rename to native/src/fairseq2n/data/data_pipeline.cc diff --git a/fairseq2n/src/fairseq2n/data/data_pipeline.h b/native/src/fairseq2n/data/data_pipeline.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/data_pipeline.h rename to native/src/fairseq2n/data/data_pipeline.h diff --git a/fairseq2n/src/fairseq2n/data/data_source.cc b/native/src/fairseq2n/data/data_source.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/data_source.cc rename to native/src/fairseq2n/data/data_source.cc diff --git a/fairseq2n/src/fairseq2n/data/data_source.h b/native/src/fairseq2n/data/data_source.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/data_source.h rename to native/src/fairseq2n/data/data_source.h diff --git a/fairseq2n/src/fairseq2n/data/detail/exception.h b/native/src/fairseq2n/data/detail/exception.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/detail/exception.h rename to native/src/fairseq2n/data/detail/exception.h diff --git a/fairseq2n/src/fairseq2n/data/detail/file.cc b/native/src/fairseq2n/data/detail/file.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/detail/file.cc rename to native/src/fairseq2n/data/detail/file.cc diff --git a/fairseq2n/src/fairseq2n/data/detail/file.h b/native/src/fairseq2n/data/detail/file.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/detail/file.h rename to native/src/fairseq2n/data/detail/file.h diff --git a/fairseq2n/src/fairseq2n/data/detail/file_system.cc b/native/src/fairseq2n/data/detail/file_system.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/detail/file_system.cc rename to native/src/fairseq2n/data/detail/file_system.cc diff --git a/fairseq2n/src/fairseq2n/data/detail/file_system.h b/native/src/fairseq2n/data/detail/file_system.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/detail/file_system.h rename to native/src/fairseq2n/data/detail/file_system.h diff --git a/fairseq2n/src/fairseq2n/data/detail/lru_cache.h b/native/src/fairseq2n/data/detail/lru_cache.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/detail/lru_cache.h rename to native/src/fairseq2n/data/detail/lru_cache.h diff --git a/fairseq2n/src/fairseq2n/data/detail/tensor_helpers.h b/native/src/fairseq2n/data/detail/tensor_helpers.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/detail/tensor_helpers.h rename to native/src/fairseq2n/data/detail/tensor_helpers.h diff --git a/fairseq2n/src/fairseq2n/data/detail/thread.h b/native/src/fairseq2n/data/detail/thread.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/detail/thread.h rename to native/src/fairseq2n/data/detail/thread.h diff --git a/fairseq2n/src/fairseq2n/data/element_mapper.cc b/native/src/fairseq2n/data/element_mapper.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/element_mapper.cc rename to native/src/fairseq2n/data/element_mapper.cc diff --git a/fairseq2n/src/fairseq2n/data/element_mapper.h b/native/src/fairseq2n/data/element_mapper.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/element_mapper.h rename to native/src/fairseq2n/data/element_mapper.h diff --git a/fairseq2n/src/fairseq2n/data/element_selector.cc b/native/src/fairseq2n/data/element_selector.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/element_selector.cc rename to native/src/fairseq2n/data/element_selector.cc diff --git a/fairseq2n/src/fairseq2n/data/element_selector.h b/native/src/fairseq2n/data/element_selector.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/element_selector.h rename to native/src/fairseq2n/data/element_selector.h diff --git a/fairseq2n/src/fairseq2n/data/file.cc b/native/src/fairseq2n/data/file.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/file.cc rename to native/src/fairseq2n/data/file.cc diff --git a/fairseq2n/src/fairseq2n/data/file.h b/native/src/fairseq2n/data/file.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/file.h rename to native/src/fairseq2n/data/file.h diff --git a/fairseq2n/src/fairseq2n/data/file_mapper.cc b/native/src/fairseq2n/data/file_mapper.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/file_mapper.cc rename to native/src/fairseq2n/data/file_mapper.cc diff --git a/fairseq2n/src/fairseq2n/data/file_mapper.h b/native/src/fairseq2n/data/file_mapper.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/file_mapper.h rename to native/src/fairseq2n/data/file_mapper.h diff --git a/fairseq2n/src/fairseq2n/data/file_stream.cc b/native/src/fairseq2n/data/file_stream.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/file_stream.cc rename to native/src/fairseq2n/data/file_stream.cc diff --git a/fairseq2n/src/fairseq2n/data/file_stream.h b/native/src/fairseq2n/data/file_stream.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/file_stream.h rename to native/src/fairseq2n/data/file_stream.h diff --git a/fairseq2n/src/fairseq2n/data/filter_data_source.cc b/native/src/fairseq2n/data/filter_data_source.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/filter_data_source.cc rename to native/src/fairseq2n/data/filter_data_source.cc diff --git a/fairseq2n/src/fairseq2n/data/filter_data_source.h b/native/src/fairseq2n/data/filter_data_source.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/filter_data_source.h rename to native/src/fairseq2n/data/filter_data_source.h diff --git a/fairseq2n/src/fairseq2n/data/image/detail/jpeg_decompress_struct.cc b/native/src/fairseq2n/data/image/detail/jpeg_decompress_struct.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/image/detail/jpeg_decompress_struct.cc rename to native/src/fairseq2n/data/image/detail/jpeg_decompress_struct.cc diff --git a/fairseq2n/src/fairseq2n/data/image/detail/jpeg_decompress_struct.h b/native/src/fairseq2n/data/image/detail/jpeg_decompress_struct.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/image/detail/jpeg_decompress_struct.h rename to native/src/fairseq2n/data/image/detail/jpeg_decompress_struct.h diff --git a/fairseq2n/src/fairseq2n/data/image/detail/png_read_struct.cc b/native/src/fairseq2n/data/image/detail/png_read_struct.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/image/detail/png_read_struct.cc rename to native/src/fairseq2n/data/image/detail/png_read_struct.cc diff --git a/fairseq2n/src/fairseq2n/data/image/detail/png_read_struct.h b/native/src/fairseq2n/data/image/detail/png_read_struct.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/image/detail/png_read_struct.h rename to native/src/fairseq2n/data/image/detail/png_read_struct.h diff --git a/fairseq2n/src/fairseq2n/data/image/image_decoder.cc b/native/src/fairseq2n/data/image/image_decoder.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/image/image_decoder.cc rename to native/src/fairseq2n/data/image/image_decoder.cc diff --git a/fairseq2n/src/fairseq2n/data/image/image_decoder.h b/native/src/fairseq2n/data/image/image_decoder.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/image/image_decoder.h rename to native/src/fairseq2n/data/image/image_decoder.h diff --git a/fairseq2n/src/fairseq2n/data/immutable_string.cc b/native/src/fairseq2n/data/immutable_string.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/immutable_string.cc rename to native/src/fairseq2n/data/immutable_string.cc diff --git a/fairseq2n/src/fairseq2n/data/immutable_string.h b/native/src/fairseq2n/data/immutable_string.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/immutable_string.h rename to native/src/fairseq2n/data/immutable_string.h diff --git a/fairseq2n/src/fairseq2n/data/list_data_source.cc b/native/src/fairseq2n/data/list_data_source.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/list_data_source.cc rename to native/src/fairseq2n/data/list_data_source.cc diff --git a/fairseq2n/src/fairseq2n/data/list_data_source.h b/native/src/fairseq2n/data/list_data_source.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/list_data_source.h rename to native/src/fairseq2n/data/list_data_source.h diff --git a/fairseq2n/src/fairseq2n/data/map_data_source.cc b/native/src/fairseq2n/data/map_data_source.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/map_data_source.cc rename to native/src/fairseq2n/data/map_data_source.cc diff --git a/fairseq2n/src/fairseq2n/data/map_data_source.h b/native/src/fairseq2n/data/map_data_source.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/map_data_source.h rename to native/src/fairseq2n/data/map_data_source.h diff --git a/fairseq2n/src/fairseq2n/data/memory_stream.cc b/native/src/fairseq2n/data/memory_stream.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/memory_stream.cc rename to native/src/fairseq2n/data/memory_stream.cc diff --git a/fairseq2n/src/fairseq2n/data/memory_stream.h b/native/src/fairseq2n/data/memory_stream.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/memory_stream.h rename to native/src/fairseq2n/data/memory_stream.h diff --git a/fairseq2n/src/fairseq2n/data/prefetch_data_source.cc b/native/src/fairseq2n/data/prefetch_data_source.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/prefetch_data_source.cc rename to native/src/fairseq2n/data/prefetch_data_source.cc diff --git a/fairseq2n/src/fairseq2n/data/prefetch_data_source.h b/native/src/fairseq2n/data/prefetch_data_source.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/prefetch_data_source.h rename to native/src/fairseq2n/data/prefetch_data_source.h diff --git a/fairseq2n/src/fairseq2n/data/py.cc b/native/src/fairseq2n/data/py.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/py.cc rename to native/src/fairseq2n/data/py.cc diff --git a/fairseq2n/src/fairseq2n/data/py.h b/native/src/fairseq2n/data/py.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/py.h rename to native/src/fairseq2n/data/py.h diff --git a/fairseq2n/src/fairseq2n/data/record_reader.cc b/native/src/fairseq2n/data/record_reader.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/record_reader.cc rename to native/src/fairseq2n/data/record_reader.cc diff --git a/fairseq2n/src/fairseq2n/data/record_reader.h b/native/src/fairseq2n/data/record_reader.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/record_reader.h rename to native/src/fairseq2n/data/record_reader.h diff --git a/fairseq2n/src/fairseq2n/data/round_robin_data_source.cc b/native/src/fairseq2n/data/round_robin_data_source.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/round_robin_data_source.cc rename to native/src/fairseq2n/data/round_robin_data_source.cc diff --git a/fairseq2n/src/fairseq2n/data/round_robin_data_source.h b/native/src/fairseq2n/data/round_robin_data_source.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/round_robin_data_source.h rename to native/src/fairseq2n/data/round_robin_data_source.h diff --git a/fairseq2n/src/fairseq2n/data/sample_data_source.cc b/native/src/fairseq2n/data/sample_data_source.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/sample_data_source.cc rename to native/src/fairseq2n/data/sample_data_source.cc diff --git a/fairseq2n/src/fairseq2n/data/sample_data_source.h b/native/src/fairseq2n/data/sample_data_source.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/sample_data_source.h rename to native/src/fairseq2n/data/sample_data_source.h diff --git a/fairseq2n/src/fairseq2n/data/shard_data_source.cc b/native/src/fairseq2n/data/shard_data_source.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/shard_data_source.cc rename to native/src/fairseq2n/data/shard_data_source.cc diff --git a/fairseq2n/src/fairseq2n/data/shard_data_source.h b/native/src/fairseq2n/data/shard_data_source.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/shard_data_source.h rename to native/src/fairseq2n/data/shard_data_source.h diff --git a/fairseq2n/src/fairseq2n/data/shuffle_data_source.cc b/native/src/fairseq2n/data/shuffle_data_source.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/shuffle_data_source.cc rename to native/src/fairseq2n/data/shuffle_data_source.cc diff --git a/fairseq2n/src/fairseq2n/data/shuffle_data_source.h b/native/src/fairseq2n/data/shuffle_data_source.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/shuffle_data_source.h rename to native/src/fairseq2n/data/shuffle_data_source.h diff --git a/fairseq2n/src/fairseq2n/data/skip_data_source.cc b/native/src/fairseq2n/data/skip_data_source.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/skip_data_source.cc rename to native/src/fairseq2n/data/skip_data_source.cc diff --git a/fairseq2n/src/fairseq2n/data/skip_data_source.h b/native/src/fairseq2n/data/skip_data_source.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/skip_data_source.h rename to native/src/fairseq2n/data/skip_data_source.h diff --git a/fairseq2n/src/fairseq2n/data/take_data_source.cc b/native/src/fairseq2n/data/take_data_source.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/take_data_source.cc rename to native/src/fairseq2n/data/take_data_source.cc diff --git a/fairseq2n/src/fairseq2n/data/take_data_source.h b/native/src/fairseq2n/data/take_data_source.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/take_data_source.h rename to native/src/fairseq2n/data/take_data_source.h diff --git a/fairseq2n/src/fairseq2n/data/tape-inl.h b/native/src/fairseq2n/data/tape-inl.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/tape-inl.h rename to native/src/fairseq2n/data/tape-inl.h diff --git a/fairseq2n/src/fairseq2n/data/tape.cc b/native/src/fairseq2n/data/tape.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/tape.cc rename to native/src/fairseq2n/data/tape.cc diff --git a/fairseq2n/src/fairseq2n/data/tape.h b/native/src/fairseq2n/data/tape.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/tape.h rename to native/src/fairseq2n/data/tape.h diff --git a/fairseq2n/src/fairseq2n/data/text/detail/utf.cc b/native/src/fairseq2n/data/text/detail/utf.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/text/detail/utf.cc rename to native/src/fairseq2n/data/text/detail/utf.cc diff --git a/fairseq2n/src/fairseq2n/data/text/detail/utf.h b/native/src/fairseq2n/data/text/detail/utf.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/text/detail/utf.h rename to native/src/fairseq2n/data/text/detail/utf.h diff --git a/fairseq2n/src/fairseq2n/data/text/sentencepiece/sentencepiece.h b/native/src/fairseq2n/data/text/sentencepiece/sentencepiece.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/text/sentencepiece/sentencepiece.h rename to native/src/fairseq2n/data/text/sentencepiece/sentencepiece.h diff --git a/fairseq2n/src/fairseq2n/data/text/sentencepiece/sp_decoder.cc b/native/src/fairseq2n/data/text/sentencepiece/sp_decoder.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/text/sentencepiece/sp_decoder.cc rename to native/src/fairseq2n/data/text/sentencepiece/sp_decoder.cc diff --git a/fairseq2n/src/fairseq2n/data/text/sentencepiece/sp_decoder.h b/native/src/fairseq2n/data/text/sentencepiece/sp_decoder.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/text/sentencepiece/sp_decoder.h rename to native/src/fairseq2n/data/text/sentencepiece/sp_decoder.h diff --git a/fairseq2n/src/fairseq2n/data/text/sentencepiece/sp_encoder.cc b/native/src/fairseq2n/data/text/sentencepiece/sp_encoder.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/text/sentencepiece/sp_encoder.cc rename to native/src/fairseq2n/data/text/sentencepiece/sp_encoder.cc diff --git a/fairseq2n/src/fairseq2n/data/text/sentencepiece/sp_encoder.h b/native/src/fairseq2n/data/text/sentencepiece/sp_encoder.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/text/sentencepiece/sp_encoder.h rename to native/src/fairseq2n/data/text/sentencepiece/sp_encoder.h diff --git a/fairseq2n/src/fairseq2n/data/text/sentencepiece/sp_model.cc b/native/src/fairseq2n/data/text/sentencepiece/sp_model.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/text/sentencepiece/sp_model.cc rename to native/src/fairseq2n/data/text/sentencepiece/sp_model.cc diff --git a/fairseq2n/src/fairseq2n/data/text/sentencepiece/sp_model.h b/native/src/fairseq2n/data/text/sentencepiece/sp_model.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/text/sentencepiece/sp_model.h rename to native/src/fairseq2n/data/text/sentencepiece/sp_model.h diff --git a/fairseq2n/src/fairseq2n/data/text/sentencepiece/sp_processor.cc b/native/src/fairseq2n/data/text/sentencepiece/sp_processor.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/text/sentencepiece/sp_processor.cc rename to native/src/fairseq2n/data/text/sentencepiece/sp_processor.cc diff --git a/fairseq2n/src/fairseq2n/data/text/sentencepiece/sp_processor.h b/native/src/fairseq2n/data/text/sentencepiece/sp_processor.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/text/sentencepiece/sp_processor.h rename to native/src/fairseq2n/data/text/sentencepiece/sp_processor.h diff --git a/fairseq2n/src/fairseq2n/data/text/string_splitter.cc b/native/src/fairseq2n/data/text/string_splitter.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/text/string_splitter.cc rename to native/src/fairseq2n/data/text/string_splitter.cc diff --git a/fairseq2n/src/fairseq2n/data/text/string_splitter.h b/native/src/fairseq2n/data/text/string_splitter.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/text/string_splitter.h rename to native/src/fairseq2n/data/text/string_splitter.h diff --git a/fairseq2n/src/fairseq2n/data/text/string_to_int_converter.cc b/native/src/fairseq2n/data/text/string_to_int_converter.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/text/string_to_int_converter.cc rename to native/src/fairseq2n/data/text/string_to_int_converter.cc diff --git a/fairseq2n/src/fairseq2n/data/text/string_to_int_converter.h b/native/src/fairseq2n/data/text/string_to_int_converter.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/text/string_to_int_converter.h rename to native/src/fairseq2n/data/text/string_to_int_converter.h diff --git a/fairseq2n/src/fairseq2n/data/text/string_to_tensor_converter.cc b/native/src/fairseq2n/data/text/string_to_tensor_converter.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/text/string_to_tensor_converter.cc rename to native/src/fairseq2n/data/text/string_to_tensor_converter.cc diff --git a/fairseq2n/src/fairseq2n/data/text/string_to_tensor_converter.h b/native/src/fairseq2n/data/text/string_to_tensor_converter.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/text/string_to_tensor_converter.h rename to native/src/fairseq2n/data/text/string_to_tensor_converter.h diff --git a/fairseq2n/src/fairseq2n/data/text/text_data_source.cc b/native/src/fairseq2n/data/text/text_data_source.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/text/text_data_source.cc rename to native/src/fairseq2n/data/text/text_data_source.cc diff --git a/fairseq2n/src/fairseq2n/data/text/text_data_source.h b/native/src/fairseq2n/data/text/text_data_source.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/text/text_data_source.h rename to native/src/fairseq2n/data/text/text_data_source.h diff --git a/fairseq2n/src/fairseq2n/data/text/text_line_reader.cc b/native/src/fairseq2n/data/text/text_line_reader.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/text/text_line_reader.cc rename to native/src/fairseq2n/data/text/text_line_reader.cc diff --git a/fairseq2n/src/fairseq2n/data/text/text_line_reader.h b/native/src/fairseq2n/data/text/text_line_reader.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/text/text_line_reader.h rename to native/src/fairseq2n/data/text/text_line_reader.h diff --git a/fairseq2n/src/fairseq2n/data/text/text_reader.cc b/native/src/fairseq2n/data/text/text_reader.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/text/text_reader.cc rename to native/src/fairseq2n/data/text/text_reader.cc diff --git a/fairseq2n/src/fairseq2n/data/text/text_reader.h b/native/src/fairseq2n/data/text/text_reader.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/text/text_reader.h rename to native/src/fairseq2n/data/text/text_reader.h diff --git a/fairseq2n/src/fairseq2n/data/text/utf8_stream.cc b/native/src/fairseq2n/data/text/utf8_stream.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/text/utf8_stream.cc rename to native/src/fairseq2n/data/text/utf8_stream.cc diff --git a/fairseq2n/src/fairseq2n/data/text/utf8_stream.h b/native/src/fairseq2n/data/text/utf8_stream.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/text/utf8_stream.h rename to native/src/fairseq2n/data/text/utf8_stream.h diff --git a/fairseq2n/src/fairseq2n/data/yield_from_data_source.cc b/native/src/fairseq2n/data/yield_from_data_source.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/yield_from_data_source.cc rename to native/src/fairseq2n/data/yield_from_data_source.cc diff --git a/fairseq2n/src/fairseq2n/data/yield_from_data_source.h b/native/src/fairseq2n/data/yield_from_data_source.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/yield_from_data_source.h rename to native/src/fairseq2n/data/yield_from_data_source.h diff --git a/fairseq2n/src/fairseq2n/data/zip_data_source.cc b/native/src/fairseq2n/data/zip_data_source.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/zip_data_source.cc rename to native/src/fairseq2n/data/zip_data_source.cc diff --git a/fairseq2n/src/fairseq2n/data/zip_data_source.h b/native/src/fairseq2n/data/zip_data_source.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/zip_data_source.h rename to native/src/fairseq2n/data/zip_data_source.h diff --git a/fairseq2n/src/fairseq2n/data/zip_file_data_source.cc b/native/src/fairseq2n/data/zip_file_data_source.cc similarity index 100% rename from fairseq2n/src/fairseq2n/data/zip_file_data_source.cc rename to native/src/fairseq2n/data/zip_file_data_source.cc diff --git a/fairseq2n/src/fairseq2n/data/zip_file_data_source.h b/native/src/fairseq2n/data/zip_file_data_source.h similarity index 100% rename from fairseq2n/src/fairseq2n/data/zip_file_data_source.h rename to native/src/fairseq2n/data/zip_file_data_source.h diff --git a/fairseq2n/src/fairseq2n/detail/error.h b/native/src/fairseq2n/detail/error.h similarity index 100% rename from fairseq2n/src/fairseq2n/detail/error.h rename to native/src/fairseq2n/detail/error.h diff --git a/fairseq2n/src/fairseq2n/detail/exception.h b/native/src/fairseq2n/detail/exception.h similarity index 100% rename from fairseq2n/src/fairseq2n/detail/exception.h rename to native/src/fairseq2n/detail/exception.h diff --git a/fairseq2n/src/fairseq2n/detail/parallel.h b/native/src/fairseq2n/detail/parallel.h similarity index 100% rename from fairseq2n/src/fairseq2n/detail/parallel.h rename to native/src/fairseq2n/detail/parallel.h diff --git a/fairseq2n/src/fairseq2n/exception.cc b/native/src/fairseq2n/exception.cc similarity index 100% rename from fairseq2n/src/fairseq2n/exception.cc rename to native/src/fairseq2n/exception.cc diff --git a/fairseq2n/src/fairseq2n/exception.h b/native/src/fairseq2n/exception.h similarity index 100% rename from fairseq2n/src/fairseq2n/exception.h rename to native/src/fairseq2n/exception.h diff --git a/fairseq2n/src/fairseq2n/float.h b/native/src/fairseq2n/float.h similarity index 100% rename from fairseq2n/src/fairseq2n/float.h rename to native/src/fairseq2n/float.h diff --git a/fairseq2n/src/fairseq2n/fmt.h b/native/src/fairseq2n/fmt.h similarity index 100% rename from fairseq2n/src/fairseq2n/fmt.h rename to native/src/fairseq2n/fmt.h diff --git a/fairseq2n/src/fairseq2n/generation/ngram_repeat_block.cu b/native/src/fairseq2n/generation/ngram_repeat_block.cu similarity index 100% rename from fairseq2n/src/fairseq2n/generation/ngram_repeat_block.cu rename to native/src/fairseq2n/generation/ngram_repeat_block.cu diff --git a/fairseq2n/src/fairseq2n/memory.cc b/native/src/fairseq2n/memory.cc similarity index 100% rename from fairseq2n/src/fairseq2n/memory.cc rename to native/src/fairseq2n/memory.cc diff --git a/fairseq2n/src/fairseq2n/memory.h b/native/src/fairseq2n/memory.h similarity index 100% rename from fairseq2n/src/fairseq2n/memory.h rename to native/src/fairseq2n/memory.h diff --git a/fairseq2n/src/fairseq2n/span.h b/native/src/fairseq2n/span.h similarity index 100% rename from fairseq2n/src/fairseq2n/span.h rename to native/src/fairseq2n/span.h diff --git a/fairseq2n/src/fairseq2n/utils/cast.h b/native/src/fairseq2n/utils/cast.h similarity index 100% rename from fairseq2n/src/fairseq2n/utils/cast.h rename to native/src/fairseq2n/utils/cast.h diff --git a/fairseq2n/src/fairseq2n/utils/string.h b/native/src/fairseq2n/utils/string.h similarity index 100% rename from fairseq2n/src/fairseq2n/utils/string.h rename to native/src/fairseq2n/utils/string.h diff --git a/fairseq2n/src/fairseq2n/utils/tensor.h b/native/src/fairseq2n/utils/tensor.h similarity index 100% rename from fairseq2n/src/fairseq2n/utils/tensor.h rename to native/src/fairseq2n/utils/tensor.h diff --git a/fairseq2n/tests/CMakeLists.txt b/native/tests/CMakeLists.txt similarity index 100% rename from fairseq2n/tests/CMakeLists.txt rename to native/tests/CMakeLists.txt diff --git a/fairseq2n/tests/data/detail/test_lru_cache.cc b/native/tests/data/detail/test_lru_cache.cc similarity index 100% rename from fairseq2n/tests/data/detail/test_lru_cache.cc rename to native/tests/data/detail/test_lru_cache.cc diff --git a/fairseq2n/tests/data/test_immutable_string.cc b/native/tests/data/test_immutable_string.cc similarity index 100% rename from fairseq2n/tests/data/test_immutable_string.cc rename to native/tests/data/test_immutable_string.cc diff --git a/fairseq2n/tests/data/test_tape.cc b/native/tests/data/test_tape.cc similarity index 100% rename from fairseq2n/tests/data/test_tape.cc rename to native/tests/data/test_tape.cc diff --git a/fairseq2n/tests/test_float.cc b/native/tests/test_float.cc similarity index 100% rename from fairseq2n/tests/test_float.cc rename to native/tests/test_float.cc diff --git a/fairseq2n/tests/test_memory.cc b/native/tests/test_memory.cc similarity index 100% rename from fairseq2n/tests/test_memory.cc rename to native/tests/test_memory.cc diff --git a/fairseq2n/tests/test_span.cc b/native/tests/test_span.cc similarity index 100% rename from fairseq2n/tests/test_span.cc rename to native/tests/test_span.cc diff --git a/fairseq2n/tests/utils/test_cast.cc b/native/tests/utils/test_cast.cc similarity index 100% rename from fairseq2n/tests/utils/test_cast.cc rename to native/tests/utils/test_cast.cc diff --git a/fairseq2n/third-party/CMakeLists.txt b/native/third-party/CMakeLists.txt similarity index 100% rename from fairseq2n/third-party/CMakeLists.txt rename to native/third-party/CMakeLists.txt diff --git a/fairseq2n/third-party/fmt b/native/third-party/fmt similarity index 100% rename from fairseq2n/third-party/fmt rename to native/third-party/fmt diff --git a/fairseq2n/third-party/fmt.cmake b/native/third-party/fmt.cmake similarity index 100% rename from fairseq2n/third-party/fmt.cmake rename to native/third-party/fmt.cmake diff --git a/fairseq2n/third-party/gtest b/native/third-party/gtest similarity index 100% rename from fairseq2n/third-party/gtest rename to native/third-party/gtest diff --git a/fairseq2n/third-party/gtest.cmake b/native/third-party/gtest.cmake similarity index 100% rename from fairseq2n/third-party/gtest.cmake rename to native/third-party/gtest.cmake diff --git a/fairseq2n/third-party/kaldi-native-fbank b/native/third-party/kaldi-native-fbank similarity index 100% rename from fairseq2n/third-party/kaldi-native-fbank rename to native/third-party/kaldi-native-fbank diff --git a/fairseq2n/third-party/kaldi-native-fbank.cmake b/native/third-party/kaldi-native-fbank.cmake similarity index 100% rename from fairseq2n/third-party/kaldi-native-fbank.cmake rename to native/third-party/kaldi-native-fbank.cmake diff --git a/fairseq2n/third-party/kaldi-native-fbank.patch b/native/third-party/kaldi-native-fbank.patch similarity index 100% rename from fairseq2n/third-party/kaldi-native-fbank.patch rename to native/third-party/kaldi-native-fbank.patch diff --git a/fairseq2n/third-party/libjpeg-turbo.cmake b/native/third-party/libjpeg-turbo.cmake similarity index 100% rename from fairseq2n/third-party/libjpeg-turbo.cmake rename to native/third-party/libjpeg-turbo.cmake diff --git a/fairseq2n/third-party/libpng b/native/third-party/libpng similarity index 100% rename from fairseq2n/third-party/libpng rename to native/third-party/libpng diff --git a/fairseq2n/third-party/libpng.cmake b/native/third-party/libpng.cmake similarity index 100% rename from fairseq2n/third-party/libpng.cmake rename to native/third-party/libpng.cmake index bcd7c3597..6fd3bc561 100644 --- a/fairseq2n/third-party/libpng.cmake +++ b/native/third-party/libpng.cmake @@ -29,8 +29,8 @@ macro(fairseq2n_add_libpng) target_include_directories(png_static SYSTEM PUBLIC - ${PROJECT_SOURCE_DIR}/third-party/libpng ${PROJECT_BINARY_DIR}/third-party/libpng + ${PROJECT_SOURCE_DIR}/third-party/libpng ) unset(SKIP_INSTALL_ALL) diff --git a/fairseq2n/third-party/natsort.cmake b/native/third-party/natsort.cmake similarity index 100% rename from fairseq2n/third-party/natsort.cmake rename to native/third-party/natsort.cmake diff --git a/fairseq2n/third-party/natsort/CMakeLists.txt b/native/third-party/natsort/CMakeLists.txt similarity index 100% rename from fairseq2n/third-party/natsort/CMakeLists.txt rename to native/third-party/natsort/CMakeLists.txt diff --git a/fairseq2n/third-party/natsort/strnatcmp.c b/native/third-party/natsort/strnatcmp.c similarity index 100% rename from fairseq2n/third-party/natsort/strnatcmp.c rename to native/third-party/natsort/strnatcmp.c diff --git a/fairseq2n/third-party/natsort/strnatcmp.h b/native/third-party/natsort/strnatcmp.h similarity index 100% rename from fairseq2n/third-party/natsort/strnatcmp.h rename to native/third-party/natsort/strnatcmp.h diff --git a/fairseq2n/third-party/pybind11 b/native/third-party/pybind11 similarity index 100% rename from fairseq2n/third-party/pybind11 rename to native/third-party/pybind11 diff --git a/fairseq2n/third-party/pybind11.cmake b/native/third-party/pybind11.cmake similarity index 100% rename from fairseq2n/third-party/pybind11.cmake rename to native/third-party/pybind11.cmake diff --git a/fairseq2n/third-party/sentencepiece b/native/third-party/sentencepiece similarity index 100% rename from fairseq2n/third-party/sentencepiece rename to native/third-party/sentencepiece diff --git a/fairseq2n/third-party/sentencepiece.cmake b/native/third-party/sentencepiece.cmake similarity index 100% rename from fairseq2n/third-party/sentencepiece.cmake rename to native/third-party/sentencepiece.cmake diff --git a/fairseq2n/third-party/zip b/native/third-party/zip similarity index 100% rename from fairseq2n/third-party/zip rename to native/third-party/zip diff --git a/fairseq2n/third-party/zip.cmake b/native/third-party/zip.cmake similarity index 100% rename from fairseq2n/third-party/zip.cmake rename to native/third-party/zip.cmake diff --git a/pyproject.toml b/pyproject.toml index bfb66eb43..72746098e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ profile = "black" disable_error_code = "type-abstract,typeddict-unknown-key" disallow_untyped_calls = false disallow_untyped_decorators = false -files = "src,fairseq2n/python/src,tests" +files = "setup.py,src,tests" ignore_missing_imports = true python_version = 3.8 show_error_codes = true diff --git a/src/fairseq2/data/audio.py b/src/fairseq2/data/audio.py index 0fd775289..83329dce0 100644 --- a/src/fairseq2/data/audio.py +++ b/src/fairseq2/data/audio.py @@ -8,7 +8,7 @@ from typing import TYPE_CHECKING, Optional, TypedDict, Union -from fairseq2n.config import DOC_MODE +from fairseq2n import DOC_MODE from torch import Tensor from typing_extensions import NotRequired diff --git a/src/fairseq2/data/cstring.py b/src/fairseq2/data/cstring.py index 9d0dd84b1..dd874acca 100644 --- a/src/fairseq2/data/cstring.py +++ b/src/fairseq2/data/cstring.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, List, Optional, overload -from fairseq2n.config import DOC_MODE +from fairseq2n import DOC_MODE if TYPE_CHECKING or DOC_MODE: diff --git a/src/fairseq2/data/data_pipeline.py b/src/fairseq2/data/data_pipeline.py index 278e843a4..3ec8cb9cc 100644 --- a/src/fairseq2/data/data_pipeline.py +++ b/src/fairseq2/data/data_pipeline.py @@ -21,7 +21,7 @@ Union, ) -from fairseq2n.config import DOC_MODE +from fairseq2n import DOC_MODE from torch import Tensor from typing_extensions import Self diff --git a/src/fairseq2/data/image.py b/src/fairseq2/data/image.py index c99d941d3..d4b76d369 100644 --- a/src/fairseq2/data/image.py +++ b/src/fairseq2/data/image.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Optional, TypedDict -from fairseq2n.config import DOC_MODE +from fairseq2n import DOC_MODE from torch import Tensor from fairseq2.memory import MemoryBlock diff --git a/src/fairseq2/data/text/converters.py b/src/fairseq2/data/text/converters.py index 682169c61..8278f8729 100644 --- a/src/fairseq2/data/text/converters.py +++ b/src/fairseq2/data/text/converters.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Dict, List, Optional, Sequence, Union -from fairseq2n.config import DOC_MODE +from fairseq2n import DOC_MODE from torch import Tensor from fairseq2.data.typing import StringLike diff --git a/src/fairseq2/data/text/sentencepiece.py b/src/fairseq2/data/text/sentencepiece.py index 0d8071d9e..46bcd3163 100644 --- a/src/fairseq2/data/text/sentencepiece.py +++ b/src/fairseq2/data/text/sentencepiece.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, List, Optional, Sequence, final -from fairseq2n.config import DOC_MODE +from fairseq2n import DOC_MODE from torch import Tensor from fairseq2.data.text.text_tokenizer import ( diff --git a/src/fairseq2/data/text/text_reader.py b/src/fairseq2/data/text/text_reader.py index 7fedcded4..4a00f82fa 100644 --- a/src/fairseq2/data/text/text_reader.py +++ b/src/fairseq2/data/text/text_reader.py @@ -7,7 +7,7 @@ from enum import Enum from typing import TYPE_CHECKING, Optional -from fairseq2n.config import DOC_MODE +from fairseq2n import DOC_MODE from fairseq2.data.data_pipeline import DataPipelineBuilder from fairseq2.data.typing import PathLike, StringLike diff --git a/src/fairseq2/memory.py b/src/fairseq2/memory.py index ce49f0d53..0c7eb5015 100644 --- a/src/fairseq2/memory.py +++ b/src/fairseq2/memory.py @@ -7,7 +7,7 @@ from array import array from typing import TYPE_CHECKING, Optional, Union, overload -from fairseq2n.config import DOC_MODE +from fairseq2n import DOC_MODE from typing_extensions import TypeAlias Buffer: TypeAlias = Union[bytes, bytearray, memoryview, array] diff --git a/tests/unit/data/image/test_image_decoder.py b/tests/unit/data/image/test_image_decoder.py index 560af619b..c18d5b3e7 100644 --- a/tests/unit/data/image/test_image_decoder.py +++ b/tests/unit/data/image/test_image_decoder.py @@ -9,10 +9,10 @@ import pytest import torch +from fairseq2n import supports_image from fairseq2.data.image import ImageDecoder from fairseq2.memory import MemoryBlock -from fairseq2n import supports_image from tests.common import assert_close, device TEST_PNG_PATH: Final = Path(__file__).parent.joinpath("test.png") diff --git a/tools/set-project-version.sh b/tools/set-project-version.sh index c28d33adf..468931b51 100755 --- a/tools/set-project-version.sh +++ b/tools/set-project-version.sh @@ -68,17 +68,17 @@ replace_match\ # Update fairseq2n CMake project. replace_match\ - "$base/fairseq2n/CMakeLists.txt"\ + "$base/native/CMakeLists.txt"\ "s/VERSION .* LANGUAGES/VERSION $mmm_ver LANGUAGES/" # Update fairseq2n Python distribution. replace_match\ - "$base/fairseq2n/python/setup.py"\ + "$base/native/python/setup.py"\ "s/ version=\".*\",$/ version=\"$pep_ver\",/" # Update fairseq2n Python package. replace_match\ - "$base/fairseq2n/python/src/fairseq2n/__init__.py"\ + "$base/native/python/src/fairseq2n/__init__.py"\ "s/^__version__ = \".*\"$/__version__ = \"$pep_ver\"/" # Update VERSION file.