Add autonomy integration rostest back with new simulator (#650)

* Make the sim be headless, refactor some nav logging, update rover parts, fix failure id warnings, add integration test

* Update dockerfile and tests

* Update

* Update

* Fix GLFW problem in headless

* Update dockerfile

* WOrk on dev ops

* Refactor cmake

* Look for dawn first

* Make imgui its own system cmake

* Pre-format

* Format

* Only style check nav for now

* Fix clang tidy

* Bruh

* Run with pr into integration

* Update names

* Clang tidy build takes too long only do it on master

* Fix gh

.gitattributes

@@ -5,4 +5,5 @@ text=auto
 *.stl filter=lfs diff=lfs merge=lfs -text
 *.glb filter=lfs diff=lfs merge=lfs -text
 *.fbx filter=lfs diff=lfs merge=lfs -text
+*.deb filter=lfs diff=lfs merge=lfs -text
 ansible/roles/esw/files/bin/fdcanusb_daemon filter=lfs diff=lfs merge=lfs -text

.github/workflows/ci.yml

@@ -3,6 +3,7 @@ on:
   pull_request:
     branches:
       - master
+      - integration
   push:
     branches:
       - master
@@ -37,9 +38,12 @@ jobs:
       - name: Initialize
         if: github.event.pull_request.draft == false
         run: . /opt/ros/noetic/setup.sh && catkin init && catkin profile set ci
-      - name: Build
-        if: github.event.pull_request.draft == false
+      - name: Build Without Clang Tidy
+        if: github.event.pull_request.draft == false && github.event.pull_request.base.ref != 'refs/heads/master'
         run: . /opt/ros/noetic/setup.sh && . /home/mrover/catkin_ws/src/mrover/venv/bin/activate && catkin build
+      - name: Build With Clang Tidy
+        if : github.event_name == 'push' && github.event.pull_request.base.ref == 'refs/heads/master'
+        run: . /opt/ros/noetic/setup.sh && . /home/mrover/catkin_ws/src/mrover/venv/bin/activate && catkin build -DCMAKE_CXX_CLANG_TIDY=clang-tidy-16
       - name: Test
         if: github.event.pull_request.draft == false
-        run: . /opt/ros/noetic/setup.sh && . /home/mrover/catkin_ws/src/mrover/venv/bin/activate && . $GITHUB_WORKSPACE/devel/setup.sh && catkin test -j1
+        run: . /opt/ros/noetic/setup.sh && . /home/mrover/catkin_ws/src/mrover/venv/bin/activate && . $GITHUB_WORKSPACE/devel/setup.sh && catkin test && rostest mrover integration.test --text

.gitmodules

@@ -1,6 +1,6 @@
 [submodule "deps/libde265"]
 	path = deps/libde265
-	url = git@github.com:strukturag/libde265.git
+	url = https://github.com/strukturag/libde265.git
 	shallow = true
 	branch = 0.1-20240106
 [submodule "deps/dawn"]

CMakeLists.txt

@@ -32,12 +32,6 @@ if (MROVER_CI)
     list(APPEND MROVER_CPP_COMPILE_OPTIONS -Werror)
 endif ()
 
-if (EXISTS ${CMAKE_CURRENT_LIST_DIR}/deps/dawn/out/Release)
-    set(MROVER_BUILD_SIM ON)
-else ()
-    set(MROVER_BUILD_SIM OFF)
-endif ()
-
 include(cmake/macros.cmake)
 
 # ROS packages list
@@ -77,29 +71,8 @@ set(MROVER_CMAKE_INCLUDES
 ### Dependencies ###
 ### ============ ###
 
-if (MROVER_BUILD_SIM)
-    # Apparently Assimp has different names on different systems
-    # find_package is case-sensitive so try both
-    find_package(Assimp QUIET)
-    find_package(assimp QUIET)
-    if (NOT Assimp_FOUND AND NOT assimp_FOUND)
-        message(FATAL_ERROR "Assimp not found")
-    endif ()
-
-    find_package(Bullet REQUIRED)
+include(cmake/deps.cmake)
 
-    find_package(glfw3 REQUIRED)
-
-    include(cmake/webgpu.cmake)
-    add_subdirectory(deps/glfw3webgpu SYSTEM)
-endif ()
-find_package(OpenCV REQUIRED)
-find_package(ZED QUIET)
-find_package(Eigen3 REQUIRED)
-add_subdirectory(deps/manif SYSTEM)
-find_package(PkgConfig REQUIRED)
-pkg_search_module(NetLink libnl-3.0)
-pkg_search_module(NetLinkRoute libnl-route-3.0)
 if (ZED_FOUND)
     # Anything newer than C++17 combined with libstdc++13 is not supported just yet by NVCC (the CUDA compiler)
     set(CMAKE_CUDA_STANDARD 17)
@@ -146,7 +119,7 @@ catkin_package()
 ## Libraries
 
 mrover_add_library(lie src/util/lie/*.cpp src/util/lie)
-target_link_libraries(lie PUBLIC manif)
+target_link_libraries(lie PUBLIC MANIF::manif)
 
 ## ESW
 
@@ -177,7 +150,7 @@ if (NOT APPLE)
     mrover_add_esw_bridge_node(led src/esw/led/*.cpp)
     mrover_add_esw_bridge_node(mast_gimbal_bridge src/esw/mast_gimbal_bridge/*.cpp)
     mrover_add_esw_bridge_node(pdb_bridge src/esw/pdb_bridge/*.cpp)
-#    mrover_add_esw_bridge_node(sa_bridge src/esw/sa_translator_bridge/*.cpp)
+    #    mrover_add_esw_bridge_node(sa_bridge src/esw/sa_translator_bridge/*.cpp)
     mrover_add_esw_bridge_node(science_bridge src/esw/science_bridge/*.cpp)
     mrover_add_esw_bridge_node(brushless_test_bridge src/esw/brushless_test_bridge/*.cpp motor_library)
     mrover_add_esw_bridge_node(brushed_test_bridge src/esw/brushed_test_bridge/*.cpp)
@@ -188,7 +161,7 @@ endif ()
 ## Perception
 
 mrover_add_nodelet(tag_detector src/perception/tag_detector/*.cpp src/perception/tag_detector src/perception/tag_detector/pch.hpp)
-mrover_nodelet_link_libraries(tag_detector opencv_core opencv_objdetect opencv_aruco opencv_imgproc opencv_highgui lie manif)
+mrover_nodelet_link_libraries(tag_detector opencv_core opencv_objdetect opencv_aruco opencv_imgproc opencv_highgui lie)
 
 if (CUDA_FOUND)
     mrover_add_library(streaming src/esw/streaming/*.c* src/esw/streaming)
@@ -201,7 +174,7 @@ endif ()
 if (ZED_FOUND)
     mrover_add_nodelet(zed src/perception/zed_wrapper/*.c* src/perception/zed_wrapper src/perception/zed_wrapper/pch.hpp)
     mrover_nodelet_include_directories(zed ${ZED_INCLUDE_DIRS} ${CUDA_INCLUDE_DIRS})
-    mrover_nodelet_link_libraries(zed ${ZED_LIBRARIES} ${SPECIAL_OS_LIBS} lie manif)
+    mrover_nodelet_link_libraries(zed ${ZED_LIBRARIES} ${SPECIAL_OS_LIBS} lie)
     mrover_nodelet_defines(zed
             ALLOW_BUILD_DEBUG # Ignore ZED warnings about Debug mode
             __CUDA_INCLUDE_COMPILER_INTERNAL_HEADERS__ # Eigen includes some files it should not, ignore
@@ -216,8 +189,8 @@ mrover_add_node(arm_controller src/teleoperation/arm_controller/*.cpp)
 
 if (MROVER_BUILD_SIM)
     mrover_add_nodelet(simulator src/simulator/*.cpp src/simulator src/simulator/pch.hpp)
-    mrover_nodelet_include_directories(simulator ${ASSIMP_INCLUDE_DIRS} ${BULLET_INCLUDE_DIRS} ${OPENCV_INCLUDE_DIRS} src/simulator/imgui)
-    mrover_nodelet_link_libraries(simulator ${ASSIMP_LIBRARIES} ${BULLET_LIBRARIES} glfw3webgpu webgpu glfw opencv_core opencv_imgcodecs opencv_imgproc lie manif)
+    mrover_nodelet_include_directories(simulator ${ASSIMP_INCLUDE_DIRS} ${BULLET_INCLUDE_DIRS} ${OPENCV_INCLUDE_DIRS})
+    mrover_nodelet_link_libraries(simulator ${ASSIMP_LIBRARIES} ${BULLET_LIBRARIES} glfw3webgpu webgpu glfw opencv_core opencv_imgcodecs opencv_imgproc webgpu_hpp imgui lie)
     mrover_nodelet_defines(simulator BOOST_THREAD_PROVIDES_FUTURE)
 endif ()
 
@@ -232,15 +205,16 @@ catkin_add_gtest(example-cpp-test test/example/cpp_test.cpp)
 
 # Python unit tests
 catkin_add_nosetests(test/navigation/drive_test.py)
-catkin_add_nosetests(test/teleop/teleop_test.py)
 catkin_add_nosetests(test/util/SE3_test.py)
 catkin_add_nosetests(test/util/SO3_test.py)
 
 # Integration tests (python and c++)
-find_package(rostest REQUIRED)
-add_rostest(test/example/basic_integration_test.test)
-add_rostest(test/integration/integration.test)
-add_rostest(test/util/SE3_tf_test.test)
+# Note(quintin): Commented these out because they ignore the Python virtual environment
+#                Most of our code relies on newer Python features so they fail to initialize
+#find_package(rostest REQUIRED)
+#add_rostest(test/example/basic_integration_test.test)
+#add_rostest(test/integration/integration.test)
+#add_rostest(test/util/SE3_tf_test.test)
 
 ## Install
 

Dockerfile

@@ -1,12 +1,14 @@
-FROM ros:noetic
+FROM ubuntu:focal
 
-# DEBIAN_FRONTEND=noninteractive and keyboard-configuration are needed to prevent stdin prompting later on
-# This was super annoying to figure out because otherwise the build would hang
+# DEBIAN_FRONTEND=noninteractive prevents apt from asking for user input
 # software-properties-common is needed for apt-add-repository
-RUN apt-get update -y && apt-get upgrade -y && DEBIAN_FRONTEND=noninteractive apt-get install software-properties-common keyboard-configuration -y
+# sudo is needed for ansible since it escalates from a normal user to root
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update -y && apt-get install software-properties-common sudo -y
 RUN apt-add-repository ppa:ansible/ansible -y
-RUN apt-add-repository ppa:git-core/ppa -y
 RUN apt-get install -y ansible git git-lfs
+ADD ./pkg /tmp/
+RUN apt-get install -f /tmp/*.deb && rm /tmp/*.deb
 
 RUN useradd --create-home --groups sudo --shell /bin/zsh mrover
 # Give mrover user sudo access with no password
@@ -17,18 +19,20 @@ RUN mkdir -p /home/mrover/catkin_ws/src/mrover
 WORKDIR /home/mrover/catkin_ws/src/mrover
 # Defines the APT packages that need to be installed
 # rosdep is called from Ansible to install them
-ADD ./package.xml .
+ADD --chown=mrover:mrover ./package.xml .
 # Defines the Python packages that need to be installed
 # pip is called from Ansible to install them
-ADD ./pyproject.toml .
+ADD --chown=mrover:mrover ./pyproject.toml .
+# Define the NPM packages that need to be installed by Bun
+ADD --chown=mrover:mrover ./src/teleoperation/frontend/package.json ./src/teleoperation/frontend/bun.lockb ./src/teleoperation/frontend/
 # Copy over all Ansible files
-ADD ./ansible ./ansible
-ADD ./ansible.sh .
-RUN ./ansible.sh build.yml
+ADD --chown=mrover:mrover ./ansible ./ansible
+ADD --chown=mrover:mrover ./ansible.sh .
+RUN ./ansible.sh ci.yml
 
 USER root
 # Remove apt cache to free up space in the image
 RUN apt-get clean && rm -rf /var/lib/apt/lists/*
 
 USER mrover
-ENTRYPOINT [ "/bin/zsh" ]
+ENTRYPOINT [ "/bin/bash" ]

ansible/ci.yml

@@ -0,0 +1,7 @@
+---
+- hosts: all
+  vars:
+    ros_distro: noetic
+    ubuntu_release: focal
+  roles:
+    - ci

ansible/roles/build/files/profiles/ci/config.yaml

@@ -7,7 +7,6 @@ cmake_args:
   - -DCMAKE_C_COMPILER=clang-16
   - -DCMAKE_CXX_FLAGS=-pipe
   - -DCMAKE_CXX_COMPILER=clang++-16
-  - -DCMAKE_CXX_CLANG_TIDY=clang-tidy-16
   - -DCMAKE_CUDA_HOST_COMPILER=clang++-16
   - -DMROVER_CI=ON
   - -Wno-dev

ansible/roles/build/tasks/main.yml

@@ -89,10 +89,11 @@
       - ros-{{ ros_distro }}-rosbash
       - libbullet-dev
       - libglfw3-dev
-      - xorg-dev
       - libx11-xcb-dev
       - libnl-3-dev
       - libnl-route-3-dev
+      - libtbb-dev
+      - libopencv-dev
 
 - name: Initialize rosdep
   become: True