[Meshcat] Set object from JavaScript code

bindings/pydrake/geometry/geometry_py_visualizers.cc

@@ -282,6 +282,11 @@ void DefineMeshcat(py::module m) {
             py::arg("wireframe") = false, py::arg("wireframe_line_width") = 1.0,
             py::arg("side") = Meshcat::SideOfFaceToRender::kDoubleSide,
             cls_doc.SetObject.doc_triangle_surface_mesh)
+        .def("SetObjectFromThreeJsCode",
+            py::overload_cast<std::string_view, std::string_view>(
+                &Class::SetObjectFromThreeJsCode),
+            py::arg("path"), py::arg("three_js_lambda"),
+            cls_doc.SetObjectFromThreeJsCode.doc)
         .def("SetLine", &Class::SetLine, py::arg("path"), py::arg("vertices"),
             py::arg("line_width") = 1.0,
             py::arg("rgba") = Rgba(0.1, 0.1, 0.1, 1.0), cls_doc.SetLine.doc)

geometry/meshcat.cc

@@ -1110,6 +1110,25 @@ class Meshcat::Impl {
                     wireframe_line_width, side);
   }
 
+  // This function is public via the PIMPL.
+  void SetObjectFromThreeJsCode(std::string_view path,
+                                std::string_view three_js_lambda) {
+    DRAKE_DEMAND(IsThread(main_thread_id_));
+
+    internal::SetObjectFromThreeJsCodeData data;
+    data.path = FullPath(path);
+    data.code = three_js_lambda;
+
+    Defer([this, data = std::move(data)]() {
+      std::stringstream message_stream;
+      msgpack::pack(message_stream, data);
+      std::string message = message_stream.str();
+      app_->publish("all", message, uWS::OpCode::BINARY, false);
+      SceneTreeElement& e = scene_tree_root_[data.path];
+      e.object().emplace() = std::move(message);
+    });
+  }
+
   // This function is public via the PIMPL.
   void SetLine(std::string_view path,
                const Eigen::Ref<const Eigen::Matrix3Xd>& vertices,
@@ -2571,6 +2590,11 @@ void Meshcat::SetObject(std::string_view path,
   impl().SetObject(path, mesh, rgba, wireframe, wireframe_line_width, side);
 }
 
+void Meshcat::SetObjectFromThreeJsCode(std::string_view path,
+                                       std::string_view three_js_lambda) {
+  impl().SetObjectFromThreeJsCode(path, three_js_lambda);
+}
+
 void Meshcat::SetLine(std::string_view path,
                       const Eigen::Ref<const Eigen::Matrix3Xd>& vertices,
                       double line_width, const Rgba& rgba) {

geometry/meshcat.h

@@ -228,6 +228,21 @@ class Meshcat {
                  bool wireframe = false, double wireframe_line_width = 1.0,
                  SideOfFaceToRender side = kDoubleSide);
 
+  /** Sets the "object" at a given `path` in the scene tree by executing native
+  THREE.js JavaScript code. The code snippet is expected to be a JavaScript
+  lambda function that returns a THREE.Object3D. This object is then set at
+  the specified `path`. See @ref meshcat_path. Any objects previously set at
+  this `path` will be replaced.
+  @param path a "/"-delimited string indicating the path in the scene tree. See
+              @ref meshcat_path "Meshcat paths" for the semantics.
+  @param three_js_lambda a string containing JavaScript code that returns a
+                         THREE.Object3D instance. The code should be a lambda
+                         function,
+                         e.g. "() => { ...; return new THREE.Mesh(...); }".
+  */
+  void SetObjectFromThreeJsCode(std::string_view path,
+                                std::string_view three_js_lambda);
+
   /** Sets the "object" at `path` in the scene tree to a piecewise-linear
   interpolation between the `vertices`.
 

geometry/meshcat_types_internal.h

@@ -395,6 +395,13 @@ struct SetObjectData {
   MSGPACK_DEFINE_MAP(type, path, object);
 };
 
+struct SetObjectFromThreeJsCodeData {
+  std::string type{"set_object_from_code"};
+  std::string path;
+  std::string code;
+  MSGPACK_DEFINE_MAP(type, path, code);
+};
+
 template <typename CameraData>
 struct LumpedCameraData {
   CameraData object;

geometry/test/meshcat_manual_test.cc

@@ -103,7 +103,7 @@ int do_main() {
   // For every items we add to the initial array, decrement start_x by one half
   // to keep things centered.
   // Use ++x as the x-position of new items.
-  const double start_x = -8.5;
+  const double start_x = -9.5;
   double x = start_x;
 
   Vector3d sphere_home{++x, 0, 0};
@@ -266,6 +266,55 @@ int do_main() {
                           RigidTransformd(Vector3d{++x, -0.25, 0}));
   }
 
+  // Text using SetObjectFromThreeJsCode.
+  {
+    std::string javascript = R"""(() => {
+      // Create a plane geometry to serve as the canvas for our text
+      const geometry = new THREE.PlaneGeometry(0.5, 0.5, 1, 1);
+
+      // Create a canvas to draw the text
+      const canvas = document.createElement('canvas');
+      const context = canvas.getContext('2d');
+      canvas.width = 1024;
+      canvas.height = 1024;
+
+      // Set up the canvas with a transparent background
+      context.fillStyle = 'rgba(0, 0, 0, 0)';
+      context.fillRect(0, 0, canvas.width, canvas.height);
+
+      // Draw the text
+      context.font = '380px sans-serif';
+      context.fillStyle = 'red';
+      context.textAlign = 'center';
+      context.textBaseline = 'middle';
+
+      // Draw "Hello" slightly above the center
+      context.fillText('Hello,', canvas.width / 2, canvas.height / 2 - 180);
+
+      // Draw "world!" slightly below the center
+      context.fillText('world!', canvas.width / 2, canvas.height / 2 + 180);
+
+      // Create a texture from the canvas
+      const texture = new THREE.CanvasTexture(canvas);
+
+      // Create a material using the texture
+      const material = new THREE.MeshPhongMaterial({
+        map: texture,
+        transparent: true,
+        side: THREE.DoubleSide
+      });
+
+      // Create and return the mesh
+      return new THREE.Mesh(geometry, material);
+    })""";
+    meshcat->SetObjectFromThreeJsCode("text", javascript);
+
+    x += 1.5;  // the previous surface occupies 1 meter.
+    meshcat->SetTransform("text", RigidTransformd(
+        RotationMatrixd::MakeXRotation(M_PI/2),  // Rotate 90° around X axis
+        Vector3d{x, 0, 0.25}));
+  }
+
   std::cout << "\nDo *not* open up your browser to the URL above. Instead use "
             << "the following URL\n\n"
             << meshcat->web_url() << "?tracked_camera=on\n\n";
@@ -331,6 +380,7 @@ Ignore those for now; we'll need to circle back and fix them later.
   - the same purple triangle mesh drawn as a wireframe.
   - the same triangle mesh drawn in multicolor.
   - a blue mesh plot of the function z = y*sin(5*x).
+  - floating text that says "Hello, world!".
 )""");
   MaybePauseForUser();
 

tools/workspace/meshcat/repository.bzl

@@ -5,13 +5,13 @@ def meshcat_repository(
         mirrors = None):
     github_archive(
         name = name,
-        repository = "meshcat-dev/meshcat",
+        repository = "siddancha/meshcat",
         upgrade_advice = """
         Updating this commit requires local testing; see
         drake/tools/workspace/meshcat/README.md for details.
         """,
-        commit = "df31f9c357ba37121dc38d463ac94780204a3e37",
-        sha256 = "b0d8210e11495c1c6011927a9b0a76b1940e8d97e44044124333f1d19cb4aecd",  # noqa
+        commit = "d48277939b7654fd1f1cf62f6630d8f79e5f0faa",
+        sha256 = "adeeeb4f41be94862a2946014ee0d475489689fa03d491460b07aa43916b2722",  # noqa
         build_file = ":package.BUILD.bazel",
         mirrors = mirrors,
     )