Update textsplitter & fix documents

.github/workflows/documentation.yml

@@ -0,0 +1,52 @@
+name: Documentation
+
+on:
+  push:
+    branches:
+      - xiaoyi_doc  # Ensure this is the branch where you commit documentation updates
+
+permissions:
+  contents: write
+  actions: read
+
+jobs:
+  build-and-deploy:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          pip install --upgrade pip
+          pip install -r ./docs/requirements.txt
+          pip freeze  # Display installed packages for debugging
+
+      - name: Build documentation
+        run: |
+          sphinx-build -b html ./docs/source/ ./docs/build/ -v
+          ls -l ./docs/build/  # List output files for debugging
+
+      - name: Create .nojekyll file
+        run: touch ./docs/build/.nojekyll
+
+      - name: Deploy to GitHub Pages
+        uses: peaceiris/actions-gh-pages@v3
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_branch: gh-pages  # Ensure this is the branch for GitHub Pages
+          publish_dir: ./docs/build/
+          user_name: github-actions[bot]  # Automated commit user name
+          user_email: github-actions[bot]@users.noreply.github.com
+
+      - name: Debug Output
+        run: |
+          pwd  # Print the current working directory
+          ls -l  # List files in the current directory
+          cat ./docs/source/conf.py  # Show Sphinx config file for debugging

docs/requirements.txt

@@ -1,4 +1,11 @@
-pydata-sphinx-theme==0.15.2
-Sphinx==7.3.7
-sphinx_design==0.6.0
-sphinx-copybutton==0.5.2
+pydata-sphinx-theme==0.15.3
+sphinx-design==0.6.0
+sphinx-copybutton==0.5.2
+sphinx==7.3.7
+nbsphinx==0.9.4
+nbconvert==7.16.4
+PyYAML
+readthedocs-sphinx-search==0.3.2
+numpy
+tqdm
+tiktoken

docs/source/apis/components/index.rst

@@ -9,9 +9,10 @@ Overview
 
    components.agent
    components.model_client
+   components.data_process
 
    .. components.reasoning
-
+   
    components.retriever
    components.output_parsers
 
@@ -37,6 +38,13 @@ Model Clients
 
    components.model_client
 
+Data Process
+----------------
+.. toctree::
+   :maxdepth: 1
+
+   components.data_process
+
 .. Embedders
 .. ---------
 .. .. toctree::

docs/source/apis/core/index.rst

@@ -7,22 +7,19 @@ Overview
 ----------
 .. autosummary::
 
-   core.base_data_class
-   core.model_client
+   core.base_data_class  
    core.component
-   core.data_components
    core.db
    core.default_prompt_template
-   core.document_splitter
    core.embedder
    core.functional
    core.generator
    core.memory
+   core.model_client
    core.parameter
    core.prompt_builder
    core.retriever
    core.string_parser
-   core.text_splitter
    core.tokenizer
    core.func_tool
    core.tool_manager
@@ -51,8 +48,6 @@ Data Handling
    core.base_data_class
    core.types
 
-
-   core.data_components
    core.db
 
 Prompts and Templates
@@ -63,10 +58,10 @@ Prompts and Templates
    core.default_prompt_template
    core.prompt_builder
 
-Document Processing
--------------------
-.. toctree::
-   :maxdepth: 1
+.. Document Processing
+.. -------------------
+.. .. toctree::
+..    :maxdepth: 1
 
    .. core.document_splitter
    core.text_splitter

docs/source/apis/index.rst

@@ -17,7 +17,6 @@ The core section of the LightRAG API documentation provides detailed information
    core.data_components
    core.db
    core.default_prompt_template
-   core.document_splitter
    core.embedder
    core.functional
    core.generator
@@ -26,7 +25,6 @@ The core section of the LightRAG API documentation provides detailed information
    core.prompt_builder
    core.retriever
    core.string_parser
-   core.text_splitter
    core.tokenizer
    core.func_tool
    core.tool_manager
@@ -42,9 +40,9 @@ The components section of the LightRAG API documentation outlines the detailed s
 
    components.agent
    components.model_client
-
+   componnets.data_process
    .. components.reasoning
-
+   
    components.retriever
    components.output_parsers
 

docs/source/conf.py

@@ -36,7 +36,6 @@
 copyright = "2024, SylphAI"
 author = "SylphAI"
 
-
 # -- General configuration ---------------------------------------------------
 
 # Add any Sphinx extension module names here, as strings. They can be