From a8f6e83ccf2262e861c56deb3ee9a211301f0574 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Mon, 9 Sep 2024 13:51:01 +0200 Subject: [PATCH 1/3] docs: soft-wrap code block --- docs/_static/theme_overrides.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/_static/theme_overrides.css b/docs/_static/theme_overrides.css index 656b8ff..a88100a 100644 --- a/docs/_static/theme_overrides.css +++ b/docs/_static/theme_overrides.css @@ -49,3 +49,7 @@ font-weight: bold; font-style: italic; } + +pre { + white-space: pre-wrap !important; +} From cea252077de25b45660d7e87d01a7e7ab0eb3907 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Mon, 9 Sep 2024 13:55:38 +0200 Subject: [PATCH 2/3] docs: add dependency-driven build feature guide --- docs/en/explanations/build.rst | 4 + .../explanations/dependency_driven_build.rst | 116 ++++++++++++++++++ docs/en/explanations/find.rst | 4 + docs/en/index.rst | 1 + docs/en/references/cli.rst | 4 + docs/en/references/manifest.md | 60 --------- 6 files changed, 129 insertions(+), 60 deletions(-) create mode 100644 docs/en/explanations/dependency_driven_build.rst diff --git a/docs/en/explanations/build.rst b/docs/en/explanations/build.rst index 47f6b42..0ec021a 100644 --- a/docs/en/explanations/build.rst +++ b/docs/en/explanations/build.rst @@ -8,6 +8,10 @@ This page explains the process of build apps, and how to use the ``idf-build-apps build`` command to build apps in the projects. +.. note:: + + For detailed list of arguments, please refer to the :class:`~idf_build_apps.args.FindArguments` reference. + ************************* Basic ``build`` Command ************************* diff --git a/docs/en/explanations/dependency_driven_build.rst b/docs/en/explanations/dependency_driven_build.rst new file mode 100644 index 0000000..8dd9c7a --- /dev/null +++ b/docs/en/explanations/dependency_driven_build.rst @@ -0,0 +1,116 @@ +######################### + Dependency-Driven Build +######################### + +In large projects or monorepos, it is often desirable to only run builds and tests which are somehow related to the changes in a pull request. + +idf-build-apps supports this by checking whether a particular app has been modified, or depends on modified components or modified files. This check is based on the knowledge of two things: the list of components/files the app depends on, and the list of components/app which are modified in the pull request. + +.. note:: + + For detailed list of arguments, please refer to the :class:`~idf_build_apps.args.DependencyDrivenBuildArguments` reference. + +.. _basic-usage: + +************* + Basic Usage +************* + +To enable this feature, the simplest way is to pass ``--modified-components`` to the ``idf-build-apps build`` command. + +While building the app, ``idf-build-apps`` will first run ``idf.py reconfigure``. ``idf.py reconfigure`` will run the first-step of the build system, which will determine the list of components the app depends on. Then, ``idf-build-apps`` will compare the list of modified components with the list of components the app depends on. If any of the modified components are present in the list of dependencies, the app will be built. + +For example, if we run + +.. code:: bash + + cd $IDF_PATH/examples/get-started/hello_world + idf-build-apps build -t esp32 --modified-components fake + +We'll see the following output: + +.. code:: text + + (cmake) App ., target esp32, sdkconfig (default), build in ./build, skipped in 4.271822s: app . depends components: {'esp_app_format', 'esp_driver_sdmmc', 'esp_driver_gpio', 'esp_common', 'esp_driver_parlio', 'esp_http_client', 'esp-tls', 'heap', 'app_trace', 'esp_driver_rmt', 'bt', 'esp_driver_ana_cmpr', 'esptool_py', 'wear_levelling', 'esp_driver_ppa', 'esp_driver_cam', 'unity', 'usb', 'app_update', 'esp_driver_spi', 'protocomm', 'esp_ringbuf', 'esp_security', 'bootloader', 'freertos', 'idf_test', 'vfs', 'hal', 'log', 'nvs_flash', 'esp_system', 'esp_driver_sdio', 'rt', 'efuse', 'esp_https_ota', 'espcoredump', 'esp_timer', 'esp_adc', 'esp_local_ctrl', 'xtensa', 'nvs_sec_provider', 'esp_pm', 'esp_gdbstub', 'lwip', 'json', 'partition_table', 'ulp', 'mbedtls', 'wifi_provisioning', 'esp_driver_sdspi', 'esp_vfs_console', 'esp_partition', 'soc', 'esp_psram', 'esp_eth', 'perfmon', 'sdmmc', 'esp_driver_usb_serial_jtag', 'esp_driver_dac', 'esp_driver_jpeg', 'esp_lcd', 'esp_driver_i2s', 'esp_driver_pcnt', 'ieee802154', 'esp_driver_i2c', 'spiffs', 'esp_driver_tsens', 'driver', 'mqtt', 'main', 'tcp_transport', 'newlib', 'openthread', 'esp_hid', 'esp_driver_gptimer', 'fatfs', 'protobuf-c', 'esp_netif', 'esp_rom', 'cxx', 'esp_bootloader_format', 'esp_wifi', 'esp_driver_ledc', 'pthread', 'esp_phy', 'esp_driver_touch_sens', 'http_parser', 'esp_https_server', 'bootloader_support', 'esp_hw_support', 'esp_event', 'esp_driver_uart', 'esp_netif_stack', 'cmock', 'spi_flash', 'esp_driver_sdm', 'esp_coex', 'esp_driver_isp', 'esp_mm', 'esp_driver_mcpwm', 'wpa_supplicant', 'esp_http_server', 'console'}, while current build modified components: ['fake'] + +The app is skipped because it does not depend on the modified component `fake`. + +************************************ + Customize the Dependency of an App +************************************ + +.. note:: + + If you're unfamiliar with the manifest file, please refer to the :doc:`Manifest File Reference <../references/manifest>`. + +To customize the dependencies of an app, `idf-build-apps` supports declaring the dependencies in the manifest files with the `depends_components` and `depends_filepatterns` fields. ``idf-build-apps`` will build the app in the following conditions: + +- any of the files under the app directory are modified, except for the ``.md`` files. +- any of the modified components are listed in the ``depends_components`` field. (if ``depends_components`` specified) +- any of the modified components are listed in the ``idf.py reconfigure`` output. (if ``depends_components`` not specified, as explained in the :ref:`previous section `) +- any of the modified files are matched by the ``depends_filepatterns`` field. + +Here is an example of a manifest file: + +.. code:: yaml + + # rules.yml + examples/foo: + depends_components: + - comp1 + - comp2 + - comp3 + depends_filepatterns: + - "common_header_files/**/*" + +The apps under folder ``examples/foo`` will be built with the following CLI options: + +- ``--manifest-files rules.yml --modified-files examples/foo/main/foo.c`` + + modified file is under the app directory + +- ``--manifest-files rules.yml --modified-components comp1`` + + modified component is listed in the ``depends_components`` field + +- ``--manifest-files rules.yml --modified-components comp2;comp4 --modified-files /tmp/foo.h`` + + modified component is listed in the ``depends_components`` field + +- ``--manifest-files rules.yml --modified-files common_header_files/foo.h`` + + modified file is matched by the ``depends_filepatterns`` field + +- ``--manifest-files rules.yml --modified-components comp4 --modified-files common_header_files/foo.h`` + + modified file is matched by the ``depends_filepatterns`` field + +The apps will not be built with the following CLI options: + +- ``--manifest-files rules.yml --modified-files examples/foo/main/foo.md`` + + only the ``.md`` files are modified + +- ``--manifest-files rules.yml --modified-components bar`` + + modified component is not listed in the ``depends_components`` field + +- ``--modified-components comp1`` + + ``--manifest-files`` is not passed + +The entries in the manifest files are relative paths. By default they are relative to the current working directory. If you want to set the root directory of the manifest files, you can use the ``--manifest-rootpath`` CLI option. + +********************************************************** + Disable the Feature When Touching Low-level Dependencies +********************************************************** + +Low-level dependencies, are components or files that are used by many others. For example, component ``freertos`` provides the operating system support for all apps, and ESP-IDF build system related cmake files are also used by all apps. When these items are modified, we definitely need to build and test all the apps. + +To disable the dependency-driven build feature, you can use the CLI option ``--deactivate-dependency-driven-build-by-components`` or ``--deactivate-dependency-driven-build-by-filepatterns``. For example: + +.. code:: bash + + idf-build-apps build -t esp32 --modified-components freertos --deactivate-dependency-driven-build-by-components freertos + +This command will build all the apps, even if the apps do not depend on the component ``freertos``. diff --git a/docs/en/explanations/find.rst b/docs/en/explanations/find.rst index 2e82839..b35c76a 100644 --- a/docs/en/explanations/find.rst +++ b/docs/en/explanations/find.rst @@ -28,6 +28,10 @@ All examples are based on the following demo projects, with the folder structure ├── CMakeLists.txt └── test-2.c +.. note:: + + For detailed list of arguments, please refer to the :class:`~idf_build_apps.args.FindArguments` reference. + ************************ Basic ``find`` Command ************************ diff --git a/docs/en/index.rst b/docs/en/index.rst index 1cb5b51..5639cc8 100644 --- a/docs/en/index.rst +++ b/docs/en/index.rst @@ -11,6 +11,7 @@ This documentation is for idf-build-apps. idf-build-apps is a tool that allows d explanations/config_rules explanations/find explanations/build + explanations/dependency_driven_build .. toctree:: :maxdepth: 1 diff --git a/docs/en/references/cli.rst b/docs/en/references/cli.rst index c59063d..3deb95b 100644 --- a/docs/en/references/cli.rst +++ b/docs/en/references/cli.rst @@ -2,6 +2,10 @@ CLI Reference ############### +.. note:: + + All CLI options could be defined in the config file. For more information, please refer to the :doc:`Config File Reference <./config_file>`. + .. argparse:: :ref: idf_build_apps.main.get_parser :prog: idf-build-apps diff --git a/docs/en/references/manifest.md b/docs/en/references/manifest.md index 0f7cf3a..490729e 100644 --- a/docs/en/references/manifest.md +++ b/docs/en/references/manifest.md @@ -146,66 +146,6 @@ examples/get-started/blink: reason: This one supports all supported targets and linux ``` -## Building Apps Only on Related Changes - -In large projects or monorepos, it is often desirable to only run builds and tests which are somehow related to the changes in a pull request. - -idf-build-apps supports this by checking whether a particular app has been modified, or depends on modified components or modified files. This check is based on the knowledge of two things: the list of components/files the app depends on, and the list of components/app which are modified in the pull request. - -### Specify the List of Modified Files and Components - -To enable this feature, you need to pass the list of modified files or modified components to idf-build-apps using the following CLI options: - -- `--modified-files` -- `--modified-components` - -For example, if the project uses Git, you can obtain the list of files modified in a branch or a pull request by calling - -```shell -git diff --name-only ${pr_branch_head} $(git merge-base ${pr_branch_head} main) -``` - -where `pr_branch_head` is the branch of the pull request, and `main` is the default branch. - -### Specifying the app dependencies - -idf-build-apps uses the following rules to determine whether to build an app or not: - -1. The app is built if any files in the app itself are modified (.md files are excluded) -2. If `depends_components` or `depends_filepatterns` are specified in the manifest file, idf-build-apps matches `--modified-components` and `--modified-files` against these two entries. If any of the modified components are in the `depends_components` list or any of the modified files are matched by `depends_filepatterns`, the app is built. -3. If `depends_components` or `depends_filepatterns` are not specified in the manifest files, idf-build-apps determines the list of components the app depends on using `BUILD_COMPONENTS` property in IDF build system. For the given app, this property contains the list of all the components included into the build. idf-build-apps runs `idf.py reconfigure` to determine the value of this property for the app. If any of the modified components are present in the `BUILD_COMPONENTS` list, the app is built. - -For example, this is an app `example/foo`, which depends on `comp1`, `comp2`, `comp3` and all files under `common_header_files`: - -```yaml -examples/foo: - depends_components: - - comp1 - - comp2 - - comp3 - depends_filepatterns: - - "common_header_files/**/*" -``` - -This app will be built with the following CLI options: - -- `--modified-files examples/foo/main/foo.c` -- `--modified-components comp1` -- `--modified-components comp2;comp4 --modified-files /tmp/foo.h` -- `--modified-files common_header_files/foo.h` -- `--modified-components comp4 --modified-files common_header_files/foo.h` - -This app will not be built with the following CLI options: - -- `--modified-files examples/foo/main/foo.md` -- `--modified-components bar` - -### Handle Low-level Dependencies - -Low-level dependencies, are components or files that are used by many others. For example, component `freertos` provides the operating system support for all apps, and ESP-IDF build system related cmake files are also used by all apps. When these items are modified, we definitely need to build and test all the apps. - -To disable the build-apps-only-on-related-changes feature, you can use the CLI option `--ignore-app-dependencies-filepatterns`. Once any of the modified files matches the specified patterns, the special rules will be disabled. All apps will be built, no exceptions. - ## Enhanced YAML Syntax ### Switch-Like Clauses From fc446d16e429e9b9e0b8f72a4ecb04892dcb6b4f Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Mon, 9 Sep 2024 13:57:46 +0200 Subject: [PATCH 3/3] chore: small fixes --- idf_build_apps/args.py | 5 +---- idf_build_apps/main.py | 14 +++++++++----- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/idf_build_apps/args.py b/idf_build_apps/args.py index a0b9908..b76ac41 100644 --- a/idf_build_apps/args.py +++ b/idf_build_apps/args.py @@ -59,7 +59,7 @@ class FieldMetadata: """ description: t.Optional[str] = None - # when deprecated, the field description will be copied from the deprecated_by field if not specified + # the field description will be copied from the deprecates field if not specified deprecates: t.Optional[t.Dict[str, t.Dict[str, t.Any]]] = None shorthand: t.Optional[str] = None # argparse_kwargs @@ -958,9 +958,6 @@ def add_arguments_to_obj_doc_as_params(argument_cls: t.Type[GlobalArguments], ob _docs_s += '\n' for f in fields(argument_cls): - if f.metadata.get('deprecated_by'): - continue - # typing generic alias is not a class _annotation = f.type.__name__ if inspect.isclass(f.type) else f.type diff --git a/idf_build_apps/main.py b/idf_build_apps/main.py index e62e732..3f70399 100644 --- a/idf_build_apps/main.py +++ b/idf_build_apps/main.py @@ -409,13 +409,17 @@ def json_to_app(json_str: str, extra_classes: t.Optional[t.List[t.Type[App]]] = You can pass extra_cls to support custom App class. A custom App class must be a subclass of App, and have a different value of `build_system`. For example, a custom CMake app - >>> class CustomApp(CMakeApp): - >>> build_system: Literal['custom_cmake'] = 'custom_cmake' + .. code:: python - Then you can pass the CustomApp class to the `extra_cls` argument + class CustomApp(CMakeApp): + build_system: Literal['custom_cmake'] = 'custom_cmake' - >>> json_str = CustomApp('.', 'esp32').to_json() - >>> json_to_app(json_str, extra_classes=[CustomApp]) + Then you can pass the :class:`CustomApp` class to the :attr:`extra_cls` argument + + .. code:: python + + json_str = CustomApp('.', 'esp32').to_json() + json_to_app(json_str, extra_classes=[CustomApp]) :param json_str: json string :param extra_classes: extra App class