Update

.github/workflows/_build_doc.yaml

@@ -30,6 +30,8 @@ jobs:
     steps:
       - name: Check-out the repository
         uses: actions/checkout@v3
+        with:
+          submodules: recursive
       - name: Create the Python virtual environment
         run: |
           python${{ inputs.py }} -m venv ~/venv
@@ -41,17 +43,21 @@ jobs:
             torch==${{ inputs.torch }}
       - name: Install requirements
         run: |
-          pip install --requirement fairseq2n/python/requirements-build.txt
+          pip install --requirement native/python/requirements-build.txt
       - name: Install Sphinx
         run: |
           pip install --requirement doc/requirements.txt
       - name: Override project version
         if: inputs.version_override
         run: |
           tools/set-project-version.sh ${{ inputs.version_override }}
+      - name: Configure fairseq2n
+        working-directory: native
+        run: |
+          cmake -GNinja -B build
       - name: Install fairseq2n
         run: |
-          pip install --editable fairseq2n/python
+          pip install --editable native/python
       - name: Install fairseq2
         run: |
           pip install --editable .

.github/workflows/_build_wheel-linux.yaml

@@ -64,7 +64,7 @@ jobs:
             torch==${{ inputs.torch }}
       - name: Install requirements
         run: |
-          pip install --requirement fairseq2n/python/requirements-build.txt
+          pip install --requirement native/python/requirements-build.txt
       - name: Override project version
         if: inputs.version_override
         run: |
@@ -83,7 +83,7 @@ jobs:
               tools/set-project-version.sh $version+$VARIANT
           fi
       - name: Build fairseq2n
-        working-directory: fairseq2n
+        working-directory: native
         env:
           VARIANT: ${{ inputs.variant }}
           SANITIZERS: ${{ inputs.sanitizers }}
@@ -122,7 +122,7 @@ jobs:
 
           cmake --build build
       - name: Package fairseq2n
-        working-directory: fairseq2n/python
+        working-directory: native/python
         run: |
           pip wheel\
             --use-pep517\
@@ -141,8 +141,8 @@ jobs:
           name: ${{ inputs.artifacts_name }}-pt${{ inputs.torch }}-py${{ inputs.py }}-linux_${{ inputs.arch }}-${{ inputs.variant }}-${{ inputs.sanitizers }}
           path: |
             build/wheelhouse/*.whl
-            fairseq2n/build/tests/run-tests
-            fairseq2n/python/build/wheelhouse/*.whl
+            native/build/tests/run-tests
+            native/python/build/wheelhouse/*.whl
           retention-days: 1
 
   test:
@@ -185,9 +185,7 @@ jobs:
           fi
       - name: Install fairseq2n
         run: |
-          whl=$(ls ~/artifacts/fairseq2n/python/build/wheelhouse/*.whl)
-
-          pip install --no-cache-dir "fairseq2n@file://$whl"
+          pip install --no-cache-dir ~/artifacts/native/python/build/wheelhouse/*.whl
       - name: Install fairseq2
         run: |
           whl=$(ls ~/artifacts/build/wheelhouse/*.whl)
@@ -204,7 +202,7 @@ jobs:
             # in non-instrumented dependencies.
             echo ASAN_OPTIONS=detect_container_overflow=0
 
-            echo LSAN_OPTIONS=suppressions=fairseq2n/LSan.supp,exitcode=0,log_path=$HOME/lsan.out
+            echo LSAN_OPTIONS=suppressions=native/LSan.supp,exitcode=0,log_path=$HOME/lsan.out
 
             for sanitizer in ${SANITIZERS//_/ }; do
               # LIBASAN and LIBTSAN environment variables are defined in the
@@ -225,12 +223,12 @@ jobs:
           } >> "$GITHUB_ENV"
       - name: Run native tests
         run: |
-          chmod 755 ~/artifacts/fairseq2n/build/tests/run-tests
+          chmod 755 ~/artifacts/native/build/tests/run-tests
 
           site_packages=~/venv/lib/python${{ inputs.py }}/site-packages
 
           LD_LIBRARY_PATH=~/venv/lib:$site_packages/torch/lib:$site_packages/fairseq2n/lib:$LD_LIBRARY_PATH\
-            ~/artifacts/fairseq2n/build/tests/run-tests
+            ~/artifacts/native/build/tests/run-tests
       - name: Run Python tests
         env:
           RUN_INTEGRATION_TESTS: ${{ inputs.run_integration_tests }}

.github/workflows/_build_wheel-macos.yaml

@@ -48,13 +48,13 @@ jobs:
           pip install torch==${{ inputs.torch }}
       - name: Install requirements
         run: |
-          pip install --requirement fairseq2n/python/requirements-build.txt
+          pip install --requirement native/python/requirements-build.txt
       - name: Override project version
         if: inputs.version_override
         run: |
           tools/set-project-version.sh ${{ inputs.version_override }}
       - name: Build fairseq2n
-        working-directory: fairseq2n
+        working-directory: native
         run: |
           cmake\
             -GNinja\
@@ -69,7 +69,7 @@ jobs:
 
           cmake --build build
       - name: Package fairseq2n
-        working-directory: fairseq2n/python
+        working-directory: native/python
         run: |
           pip wheel\
             --use-pep517\
@@ -86,8 +86,8 @@ jobs:
           name: ${{ inputs.artifacts_name }}-pt${{ inputs.torch }}-py${{ inputs.py }}-macos_${{ inputs.arch }}-cpu
           path: |
             build/wheelhouse/*.whl
-            fairseq2n/build/tests/run-tests
-            fairseq2n/python/build/wheelhouse/*.whl
+            native/build/tests/run-tests
+            native/python/build/wheelhouse/*.whl
           retention-days: 1
 
   test:
@@ -121,16 +121,16 @@ jobs:
           pip install --requirement requirements-devel.txt
       - name: Install fairseq2n
         run: |
-          pip install --no-cache-dir ~/artifacts/fairseq2n/python/build/wheelhouse/*.whl
+          pip install --no-cache-dir ~/artifacts/native/python/build/wheelhouse/*.whl
       - name: Install fairseq2
         run: |
           pip install --no-cache-dir ~/artifacts/build/wheelhouse/*.whl
       - name: Run native tests
         run: |
-          chmod 755 ~/artifacts/fairseq2n/build/tests/run-tests
+          chmod 755 ~/artifacts/native/build/tests/run-tests
 
           DYLD_LIBRARY_PATH=~/venv/lib:~/venv/lib/python${{ inputs.py }}/site-packages/torch/lib:~/venv/lib/python${{ inputs.py }}/site-packages/fairseq2n/lib\
-            ~/artifacts/fairseq2n/build/tests/run-tests
+            ~/artifacts/native/build/tests/run-tests
       - name: Run Python tests
         run: |
           pytest -rP --verbose

.github/workflows/_lint_cc.yaml

@@ -29,7 +29,6 @@ jobs:
         uses: actions/checkout@v3
         with:
           submodules: recursive
-          fetch-depth: 2
       - name: Create the Python virtual environment
         run: |
           python${{ inputs.py }} -m venv ~/venv
@@ -41,10 +40,10 @@ jobs:
             torch==${{ inputs.torch }}\
       - name: Install requirements
         run: |
-          pip install --requirement fairseq2n/python/requirements-build.txt
+          pip install --requirement native/python/requirements-build.txt
       - name: Build the compilation database
         id: setup
-        working-directory: fairseq2n
+        working-directory: native
         run: |
           # Since we are only building the compilation database, we can ignore
           # CUDA compatibility.
@@ -53,7 +52,7 @@ jobs:
           CC=clang CXX=clang++ cmake -GNinja -DCMAKE_BUILD_TYPE=Release -B build
       - name: Run clang-tidy
         if: success() || (failure() && steps.setup.outcome == 'success')
-        working-directory: fairseq2n
+        working-directory: native
         run: |
           echo "::add-matcher::./ci/problem-matchers/gcc.json"
 

.github/workflows/_lint_py.yaml

@@ -27,6 +27,8 @@ jobs:
     steps:
       - name: Check-out the repository
         uses: actions/checkout@v3
+        with:
+          submodules: recursive
       - name: Create the Python virtual environment
         run: |
           python${{ inputs.py }} -m venv ~/venv
@@ -38,12 +40,16 @@ jobs:
             torch==${{ inputs.torch }}
       - name: Install requirements
         run: |
-          pip install --requirement fairseq2n/python/requirements-build.txt
+          pip install --requirement native/python/requirements-build.txt
 
           pip install --requirement requirements-devel.txt
+      - name: Configure fairseq2n
+        working-directory: native
+        run: |
+          cmake -GNinja -B build
       - name: Install fairseq2n
         run: |
-          pip install --editable fairseq2n/python
+          pip install --editable native/python
       - name: Install fairseq2
         id: install_fairseq2
         run: |
@@ -100,7 +106,6 @@ jobs:
           trap remove_matcher EXIT
 
           mypy
-          mypy setup.py
-          mypy fairseq2n/python/setup.py
+          mypy native/python
           mypy recipes/llama
           mypy recipes/mistral

.github/workflows/_publish_pypi.yaml

@@ -42,7 +42,7 @@ jobs:
       - name: Publish fairseq2n
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
-            packages-dir: artifacts/fairseq2n/python/build/wheelhouse
+            packages-dir: artifacts/native/python/build/wheelhouse
       - name: Publish fairseq2
         uses: pypa/gh-action-pypi-publish@release/v1
         with:

.github/workflows/_publish_s3.yaml

@@ -68,7 +68,7 @@ jobs:
           echo S3_PREFIX=$s3_prefix >> "$GITHUB_ENV"
       - name: Upload wheel
         run: |
-          for pkg in ~/artifacts/fairseq2n/python/build/wheelhouse/*.whl; do
+          for pkg in ~/artifacts/native/python/build/wheelhouse/*.whl; do
             aws s3 cp "$pkg" s3://$S3_BUCKET/$S3_PREFIX/fairseq2n/
           done
 

.gitmodules

@@ -1,28 +1,28 @@
 [submodule "third-party/pybind11"]
-	path = fairseq2n/third-party/pybind11
+	path = native/third-party/pybind11
 	url = https://github.com/pybind/pybind11.git
 	ignore = dirty
 [submodule "third-party/fmt"]
-	path = fairseq2n/third-party/fmt
+	path = native/third-party/fmt
 	url = https://github.com/fmtlib/fmt.git
 	ignore = dirty
 [submodule "third-party/gtest"]
-	path = fairseq2n/third-party/gtest
+	path = native/third-party/gtest
 	url = https://github.com/google/googletest.git
 	ignore = dirty
 [submodule "third-party/sentencepiece"]
-	path = fairseq2n/third-party/sentencepiece
+	path = native/third-party/sentencepiece
 	url = https://github.com/google/sentencepiece.git
 	ignore = dirty
 [submodule "third-party/zip"]
-	path = fairseq2n/third-party/zip
+	path = native/third-party/zip
 	url = https://github.com/kuba--/zip.git
 	ignore = dirty
 [submodule "third-party/kaldi-native-fbank"]
-	path = fairseq2n/third-party/kaldi-native-fbank
+	path = native/third-party/kaldi-native-fbank
 	url = https://github.com/csukuangfj/kaldi-native-fbank.git
 	ignore = dirty
 [submodule "fairseq2n/third-party/libpng"]
-	path = fairseq2n/third-party/libpng
+	path = native/third-party/libpng
 	url = https://github.com/glennrp/libpng.git
 	ignore = dirty

doc/conf.py

@@ -4,9 +4,9 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
-import fairseq2n.config
+import fairseq2n
 
-fairseq2n.config.DOC_MODE = True
+fairseq2n.DOC_MODE = True
 
 import fairseq2
 

fairseq2n/python/src/fairseq2n/__init__.py → native/python/src/fairseq2n/__init__.py

@@ -13,10 +13,13 @@
 from pathlib import Path
 from typing import TYPE_CHECKING, List, Optional, Tuple
 
-from fairseq2n.config import _SUPPORTS_CUDA, _SUPPORTS_IMAGE, DOC_MODE
+from fairseq2n.config import _SUPPORTS_CUDA, _SUPPORTS_IMAGE
 
 __version__ = "0.2.1.dev0"
 
+# Indicates whether we are run under Sphinx.
+DOC_MODE = False
+
 
 # Keeps the shared libraries that we load using our own extended lookup logic
 # in memory.
@@ -151,14 +154,12 @@ def supports_cuda() -> bool:
     return _SUPPORTS_CUDA
 
 
-if TYPE_CHECKING or DOC_MODE:
-
-    def cuda_version() -> Optional[Tuple[int, int]]:
-        """Return the version of CUDA that fairseq2n supports.
+def cuda_version() -> Optional[Tuple[int, int]]:
+    """Return the version of CUDA that fairseq2n supports.
 
-        :returns:
-            The major and minor version segments.
-        """
+    :returns:
+        The major and minor version segments.
+    """
+    from fairseq2n.bindings import _cuda_version
 
-else:
-    from fairseq2n.bindings import cuda_version as cuda_version
+    return _cuda_version()

fairseq2n/python/src/fairseq2n/bindings/init.cc → native/python/src/fairseq2n/bindings/init.cc

@@ -22,7 +22,7 @@ PYBIND11_MODULE(bindings, m)
 #pragma clang diagnostic ignored "-Wunreachable-code-return"
 
     m.def(
-        "cuda_version",
+        "_cuda_version",
         []
         {
           if constexpr (cuda_version_major)

fairseq2n/python/src/fairseq2n/config.py.in → native/python/src/fairseq2n/config.py.in

@@ -6,8 +6,6 @@
 
 from typing import Final
 
-DOC_MODE: Final = False
-
 _SUPPORTS_IMAGE: Final = @SUPPORTS_IMAGE@
 
 _SUPPORTS_CUDA: Final = @USES_CUDA@

fairseq2n/third-party/libpng.cmake → native/third-party/libpng.cmake

@@ -29,8 +29,8 @@ macro(fairseq2n_add_libpng)
 
         target_include_directories(png_static SYSTEM
             PUBLIC
-                ${PROJECT_SOURCE_DIR}/third-party/libpng
                 ${PROJECT_BINARY_DIR}/third-party/libpng
+                ${PROJECT_SOURCE_DIR}/third-party/libpng
         )
 
         unset(SKIP_INSTALL_ALL)

pyproject.toml

@@ -20,7 +20,7 @@ profile = "black"
 disable_error_code = "type-abstract,typeddict-unknown-key"
 disallow_untyped_calls = false
 disallow_untyped_decorators = false
-files = "src,fairseq2n/python/src,tests"
+files = "setup.py,src,tests"
 ignore_missing_imports = true
 python_version = 3.8
 show_error_codes = true

src/fairseq2/data/audio.py

@@ -8,7 +8,7 @@
 
 from typing import TYPE_CHECKING, Optional, TypedDict, Union
 
-from fairseq2n.config import DOC_MODE
+from fairseq2n import DOC_MODE
 from torch import Tensor
 from typing_extensions import NotRequired