retargeting

MUJOCO_LOG.TXT

@@ -10,3 +10,6 @@ ERROR: could not initialize GLFW
 Mon Oct 14 17:02:20 2024
 ERROR: could not initialize GLFW
 
+Mon Dec  9 20:38:38 2024
+ERROR: could not initialize GLFW
+

docs/retargeting.md

@@ -15,7 +15,53 @@ Then, we will find the SMPL motion parameters that best fit the humanoid. (pass
 python scripts/data_process/fit_smpl_motion.py robot=unitree_g1_fitting +amass_root=/path/to/your/amass/data 
 ```
 
+To visualize the fitted motion, you can run the following script. 
 ```
-python scripts/vis/vis_q_mj.py robot=unitree_g1_fitting
+python scripts/vis/vis_q_mj.py robot=unitree_g1_fitting +motion_name=0-Transitions_mocap_mazen_c3d_dance_stand_poses
 ```
 
+
+## Details about the YAML files:
+
+
+```
+extend_config:
+  - joint_name: "head_link"
+    parent_name: "pelvis"
+    pos: [0.0, 0.0, 0.4]
+    rot: [1.0, 0.0, 0.0, 0.0]
+
+base_link: "torso_link"
+joint_matches:
+  - ["pelvis", "Pelvis"]
+  - ["left_hip_pitch_link", "L_Hip"]
+  - ["left_knee_link", "L_Knee"]
+  - ["left_ankle_roll_link", "L_Ankle"]
+  - ["right_hip_pitch_link", "R_Hip"]
+  - ["right_knee_link", "R_Knee"]
+  - ["right_ankle_roll_link", "R_Ankle"]
+  - ["left_shoulder_roll_link", "L_Shoulder"]
+  - ["left_elbow_pitch_link", "L_Elbow"]
+  - ["left_zero_link", "L_Hand"]
+  - ["right_shoulder_roll_link", "R_Shoulder"]
+  - ["right_elbow_pitch_link", "R_Elbow"]
+  - ["right_zero_link", "R_Hand"]
+  - ["head_link", "Head"]
+
+
+smpl_pose_modifier:
+  - Pelvis: "[np.pi/2, 0, np.pi/2]"
+  - L_Shoulder: "[0, 0, -np.pi/2]"
+  - R_Shoulder: "[0, 0, np.pi/2]"
+  - L_Elbow: "[0, -np.pi/2, 0]"
+  - R_Elbow: "[0, np.pi/2, 0]"
+```
+
+The above YAML file is used to setup the retargeting process. 
+
+Extend_config: creating new joints for the humanoid such that there can be more constraints for the motion fitting process (e.g. the head joint is created to be a child of the pelvis joint)
+
+Joint matches: matching the joints between the humanoid and the SMPL model. 
+
+SMPL pose modifier: modifying the SMPL pose to fit reseting pose of the humanoid. For instance, H1's default pose has the elbow bent, so we need to modify the SMPL pose to fit the humanoid. 
+

env.txt

@@ -0,0 +1,196 @@
+absl-py==2.1.0
+addict==2.4.0
+aiohttp==3.9.3
+aiosignal==1.3.1
+antlr4-python3-runtime==4.9.3
+appdirs==1.4.4
+asttokens==2.4.1
+async-timeout==4.0.3
+attrs==23.2.0
+autograd==1.6.2
+backcall==0.2.0
+beautifulsoup4==4.12.3
+blinker==1.7.0
+bps-torch @ git+https://github.com/otaheri/bps_torch@4b01f5c4e9fbbfac08e5c37293881678ef2f1ad0
+Brotli @ file:///tmp/abs_ecyw11_7ze/croots/recipe/brotli-split_1659616059936/work
+cachetools==5.3.2
+cchardet==2.1.7
+certifi @ file:///croot/certifi_1707229174982/work/certifi
+chamfer-distance @ git+https://github.com/otaheri/chamfer_distance@f86f6f7cadd3aca642704573d1626c67ca2e2846
+chardet==5.2.0
+charset-normalizer @ file:///tmp/build/80754af9/charset-normalizer_1630003229654/work
+chumpy==0.70
+click==8.1.7
+cloudpickle==3.0.0
+comm==0.2.1
+ConfigArgParse==1.7
+configer==1.3.1
+configparser==6.0.1
+contourpy==1.1.1
+cycler==0.12.1
+dash==2.15.0
+dash-core-components==2.0.0
+dash-html-components==2.0.0
+dash-table==5.0.0
+decorator==5.1.1
+docker-pycreds==0.4.0
+easydict==1.12
+etils==1.3.0
+executing==2.0.1
+Farama-Notifications==0.0.4
+fastjsonschema==2.19.1
+filelock @ file:///croot/filelock_1700591183607/work
+Flask==3.0.2
+fonttools==4.49.0
+freetype-py==2.4.0
+frozenlist==1.4.1
+fsspec==2024.2.0
+future==1.0.0
+fvcore==0.1.5.post20221221
+gdown==5.1.0
+gitdb==4.0.11
+GitPython==3.1.42
+glfw==2.7.0
+gmpy2 @ file:///tmp/build/80754af9/gmpy2_1645455532332/work
+google-auth==2.28.1
+google-auth-oauthlib==1.0.0
+grpcio==1.62.0
+gym==0.26.2
+gym-notices==0.0.8
+gymnasium==0.29.1
+h5py==3.11.0
+human-body-prior==0.8.5.0
+hydra-core==1.3.2
+idna @ file:///croot/idna_1666125576474/work
+imageio==2.34.0
+imageio-ffmpeg==0.4.9
+importlib-metadata==7.0.1
+importlib-resources==3.0.0
+iopath==0.1.10
+ipdb==0.13.13
+ipython==8.12.3
+ipywidgets==8.1.2
+# Editable install with no version control (isaacgym==1.0rc4)
+-e /hdd/zen/dev/nv/isaacgym/python
+itsdangerous==2.1.2
+jedi==0.19.1
+Jinja2 @ file:///croot/jinja2_1706733616596/work
+joblib==1.3.2
+jsonschema==4.21.1
+jsonschema-specifications==2023.12.1
+jupyter_core==5.7.1
+jupyterlab_widgets==3.0.10
+kiwisolver==1.4.5
+lazy_loader==0.3
+lightning-utilities==0.10.1
+lxml==5.1.0
+Markdown==3.5.2
+MarkupSafe @ file:///croot/markupsafe_1704205993651/work
+matplotlib==3.6.3
+matplotlib-inline==0.1.6
+mediapy==1.2.0
+mjcf2urdf==0.0.1
+mjcf_urdf_simple_converter==0.4
+mkl-fft @ file:///croot/mkl_fft_1695058164594/work
+mkl-random @ file:///croot/mkl_random_1695059800811/work
+mkl-service==2.4.0
+mpmath @ file:///croot/mpmath_1690848262763/work
+msgpack==1.0.7
+mujoco==3.2.3
+multidict==6.0.5
+nbformat==5.9.2
+nest-asyncio==1.6.0
+networkx @ file:///croot/networkx_1690561992265/work
+ninja==1.11.1.1
+numpy==1.23.5
+numpy-stl==3.1.1
+oauthlib==3.2.2
+omegaconf==2.3.0
+open3d==0.18.0
+open3d-python==0.3.0.0
+opencv-python==4.6.0.66
+packaging==23.2
+pandas==2.0.3
+parso==0.8.3
+patchelf==0.17.2.1
+pexpect==4.9.0
+pickleshare==0.7.5
+pillow==10.4.0
+pkgutil_resolve_name==1.3.10
+platformdirs==4.2.0
+plotly==5.19.0
+portalocker==2.8.2
+prompt-toolkit==3.0.43
+protobuf==4.25.3
+psutil==5.9.8
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pyasn1==0.5.1
+pyasn1-modules==0.3.0
+pybullet==3.2.6
+pyglet==2.0.10
+Pygments==2.17.2
+PyOpenGL==3.1.0
+pyparsing==3.1.1
+pyquaternion==0.9.9
+pyrender==0.1.45
+PySocks @ file:///tmp/build/80754af9/pysocks_1605305779399/work
+python-dateutil==2.8.2
+python-utils==3.8.2
+pytorch-lightning==2.2.0.post0
+pytorch3d==0.7.6+pt2.3.0cu118
+pytz==2024.1
+PyVirtualDisplay==3.0
+PyWavelets==1.4.1
+PyYAML @ file:///croot/pyyaml_1698096049011/work
+ray==2.9.3
+referencing==0.33.0
+requests==2.28.1
+requests-oauthlib==1.3.1
+retrying==1.3.4
+rl-games==1.1.4
+rpds-py==0.18.0
+rsa==4.9
+scikit-image==0.21.0
+scikit-learn==1.3.2
+scipy==1.10.1
+sentry-sdk==1.40.5
+setproctitle==1.3.3
+shapely==2.0.4
+six==1.16.0
+smmap==5.0.1
+smpl_sim @ git+https://github.com/ZhengyiLuo/SMPLSim.git@c2701839869f320b1dc73100d131c6a160cc7d1e
+smplx @ git+https://github.com/ZhengyiLuo/smplx.git@a5b8e4ac14f79f3f33fd2cf2a16e6f507146b813
+soupsieve==2.5
+stack-data==0.6.3
+sympy @ file:///croot/sympy_1701397643339/work
+tabulate==0.9.0
+tenacity==8.2.3
+tensorboard==2.14.0
+tensorboard-data-server==0.7.2
+tensorboardX==2.6.2.2
+termcolor==2.4.0
+threadpoolctl==3.3.0
+tifffile==2023.7.10
+tomli==2.0.1
+torch==2.2.2
+torchaudio==0.13.1+cu116
+torchgeometry==0.1.2
+torchmetrics==1.3.1
+torchvision==0.14.1+cu116
+tqdm==4.66.2
+traitlets==5.14.1
+transforms3d==0.4.1
+trimesh==4.1.5
+triton==2.2.0
+typing_extensions==4.8.0
+tzdata==2024.1
+urllib3==1.26.13
+vtk==9.3.0
+wandb==0.16.3
+wcwidth==0.2.13
+Werkzeug==3.0.1
+widgetsnbextension==4.0.10
+yacs==0.1.8
+yarl==1.9.4
+zipp==3.17.0

scripts/data_process/fit_smpl_shape.py

@@ -22,7 +22,6 @@
 import math
 from smpl_sim.utils.pytorch3d_transforms import axis_angle_to_matrix
 from torch.autograd import Variable
-from tqdm.notebook import tqdm
 from smpl_sim.smpllib.smpl_joint_names import SMPL_MUJOCO_NAMES, SMPL_BONE_ORDER_NAMES, SMPLH_BONE_ORDER_NAMES, SMPLH_MUJOCO_NAMES
 from phc.utils.torch_humanoid_batch import Humanoid_Batch
 from easydict import EasyDict

scripts/vis/vis_q_mj.py

@@ -63,8 +63,7 @@ def main(cfg : DictConfig) -> None:
 
     curr_start, num_motions, motion_id, motion_acc, time_step, dt, paused = 0, 1, 0, set(), 0, 1/30, False
 
-    motion_file = f"data/{cfg.robot.humanoid_type}/v1/singles/{cfg.get('motion_name', '0-KIT_3_walking_slow08_poses')}.pkl"
-    motion_file = "data/h1/v1/amass_phc.pkl"
+    motion_file = f"data/{cfg.robot.humanoid_type}/v1/singles/{cfg.motion_name}.pkl"
     print(motion_file)
     motion_data = joblib.load(motion_file)
     motion_data_keys = list(motion_data.keys())
@@ -86,7 +85,6 @@ def main(cfg : DictConfig) -> None:
         while viewer.is_running():
             step_start = time.time()
             curr_motion_key = motion_data_keys[motion_id]
-            curr_motion_key = "0-KIT_183_walking_run08_poses"
             curr_motion = motion_data[curr_motion_key]
             curr_time = int(time_step/dt) % curr_motion['dof'].shape[0]