Reduce delays in webRTC remote display. (#7496)

    Update libwebrtc to latest (M149).
    Remove duplicate redraw on input (WebRTCWindowSystem.cpp)
    Draw coalescing (BitmapWindowSystem.cpp)
    Input (mouse event) coalescing (BitmapWindowSystem.cpp)
    JS requestAnimationFrame coalescing / throttling (webrtcstreamer.js)
    Async encoder thread (PeerConnectionManager.cpp/.h)
    Data channel low-latency mode (webrtcstreamer.js)
    Reduced startup delay (500ms -> 250ms) (WebRTCWindowSystem.cpp)
    VP9 codec preference over VP8 (webrtcstreamer.js)
    Fix RGB color range: Switch default 16-235 back to 0-255.
    Improve ASSIMP export texture handling.

Also: Check textures for validity before trying to write them to gltf.

Author: ssheorey

.github/workflows/macos.yml

@@ -67,7 +67,8 @@ jobs:
       - name: Install dependencies
         run: |
           brew install ccache glslang spirv-cross
-          # Fix gfortran not found issue
+          # Fix gfortran not found issue. 
+          brew install gcc
           ln -s $(brew --prefix gcc)/bin/gfortran-* /usr/local/bin/gfortran
 
           ccache -M 2G  # See .github/workflows/readme.md for ccache strategy.
@@ -202,7 +203,9 @@ jobs:
           install_python_dependencies
 
           # Fix macos-14 arm64 runner image issues, see comments in MacOS job.
-          ln -s $(which gfortran-13) /usr/local/bin/gfortran
+          # brew install gcc is required so that `brew --prefix gcc` resolves;
+          brew install gcc
+          ln -s $(brew --prefix gcc)/bin/gfortran-* /usr/local/bin/gfortran
 
           brew install ccache glslang spirv-cross 
           ccache -M 2G  # See .github/workflows/readme.md for ccache strategy.

.github/workflows/webrtc.yml

@@ -5,13 +5,13 @@ on:
   workflow_dispatch:
     inputs:
       webrtc_commit:
-        description: 'Specify WebRTC commit to build.'
+        description: 'WebRTC src commit (full or short).'
         required: false
-        default: '60e674842ebae283cc6b2627f4b6f2f8186f3317'  # Date: Wed Apr 7 19:12:13 2021 +0200
+        default: 'e8b4d4c5952a8fb7b35c2a6cba4e8c3de2ea2e1e'
       depot_tools_commit:
-        description: 'Specify Depot Tools commit to to use for the build.'
+        description: 'depot_tools commit (override pin in webrtc_build.sh).'
         required: false
-        default: 'e1a98941d3ab10549be6d82d0686bb0fb91ec903' # Date: Wed Apr 7 21:35:29 2021 +0000
+        default: '10eda50a3fd9c34ad8d31ec74e5f4eb5823d60f6'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -20,29 +20,35 @@ concurrency:
 env:
   WEBRTC_COMMIT: ${{ github.event.inputs.webrtc_commit }}
   DEPOT_TOOLS_COMMIT: ${{ github.event.inputs.depot_tools_commit }}
+  WEBRTC_WORK_ROOT: ${{ github.workspace }}/..
+  GCLIENT_JOBS: 8
 
 jobs:
 
   Unix:
     permissions:
-      contents: write  # upload
+      contents: read
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04, macos-13]
+        include:
+          - os: ubuntu-22.04
+            package_suffix: linux_cxx-abi-1
+          - os: macos-14
+            package_suffix: macos_arm64
 
     steps:
       - name: Checkout source code
         uses: actions/checkout@v4
 
-      - name: Set up Python version
+      - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: 3.10
+          python-version: '3.10'
 
-      - name: Install dependencies
-        if: ${{ matrix.os == 'ubuntu-22.04' }}
+      - name: Install dependencies (Ubuntu)
+        if: matrix.os == 'ubuntu-22.04'
         run: |
           source 3rdparty/webrtc/webrtc_build.sh
           install_dependencies_ubuntu
@@ -60,116 +66,117 @@ jobs:
       - name: Upload WebRTC
         uses: actions/upload-artifact@v4
         with:
-          name: webrtc_release_${{ matrix.os }}
+          name: webrtc_${{ matrix.package_suffix }}
           path: |
-                 webrtc_*.tar.gz
-                 checksum_*.txt
+            webrtc_*.tar.gz
+            checksum_webrtc_*.tar.gz
           if-no-files-found: error
 
   Windows:
     permissions:
-      contents: write  # upload
-    # https://chromium.googlesource.com/chromium/src/+/HEAD/docs/windows_build_instructions.md
+      contents: read
     runs-on: windows-2022
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - config: Release
+            static_runtime: ON
+            tag: Release_mt
+          - config: Release
+            static_runtime: OFF
+            tag: Release_md
+          - config: Debug
+            static_runtime: ON
+            tag: Debug_mt
+          - config: Debug
+            static_runtime: OFF
+            tag: Debug_md
     env:
-      WORK_DIR: "C:\\WebRTC"  # Not enough space in D:
-      OPEN3D_DIR: "D:\\a\\open3d\\open3d"
-      DEPOT_TOOLS_UPDATE: 1   # Fix cannot find python3_bin_reldir.txt
-      DEPOT_TOOLS_WIN_TOOLCHAIN: 0
-      NPROC: 2
+      WORK_DIR: 'C:\WebRTC'
+      OPEN3D_DIR: ${{ github.workspace }}
+      WEBRTC_WORK_ROOT: 'C:\WebRTC'
+      DEPOT_TOOLS_UPDATE: 0        # belt-and-suspenders; also set by webrtc_setup_path
+      DEPOT_TOOLS_WIN_TOOLCHAIN: 0 # use locally installed VS, not the Chromium toolchain
 
     steps:
       - name: Checkout source code
         uses: actions/checkout@v4
 
-      - name: Set up Python version
+      - name: Set up Python
         uses: actions/setup-python@v5
         with:
           python-version: '3.10'
 
-      - name: Disk space
+      - name: Prepare work directory
+        shell: pwsh
         run: |
           Get-PSDrive
-          mkdir "$env:WORK_DIR"
+          New-Item -ItemType Directory -Force -Path $env:WORK_DIR
 
-      - name: Setup PATH for Visual Studio # Required for Ninja
+      - name: Setup PATH for Visual Studio
         uses: ilammy/msvc-dev-cmd@v1
         with:
           arch: x64
 
       - name: Download WebRTC sources
-        shell: pwsh
-        working-directory: ${{ env.WORK_DIR }}
+        shell: bash # Git Bash transparently converts Windows-style env paths
         run: |
-          $ErrorActionPreference = 'Stop'
-          echo "Get depot_tools"
-          # Checkout to a specific version
-          # Ref: https://chromium.googlesource.com/chromium/src/+/main/docs/building_old_revisions.md
-          git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git
-          git -C depot_tools checkout $env:DEPOT_TOOLS_COMMIT
-          $env:Path = (Get-Item depot_tools).FullName + ";" + $env:Path
-
-          echo "Get WebRTC"
-          mkdir webrtc
-          cd webrtc
-          fetch webrtc
-
-          git -C src checkout $env:WEBRTC_COMMIT
-          git -C src submodule update --init --recursive
-          echo "gclient sync"
-          gclient sync -D --force --reset
-          cd ..
-          echo "random.org"
-          curl "https://www.random.org/cgi-bin/randbyte?nbytes=10&format=h" -o skipcache
+          source "$OPEN3D_DIR/3rdparty/webrtc/webrtc_build.sh"
+          download_webrtc_sources
 
       - name: Patch WebRTC
+        shell: pwsh
         working-directory: ${{ env.WORK_DIR }}
         run: |
           $ErrorActionPreference = 'Stop'
-          cp "$env:OPEN3D_DIR/3rdparty/webrtc/CMakeLists.txt" webrtc/
-          cp "$env:OPEN3D_DIR/3rdparty/webrtc/webrtc_common.cmake" webrtc/
-
-      - name: Build WebRTC (Release)
-        working-directory: ${{ env.WORK_DIR }}
-        run: |
-          $ErrorActionPreference = 'Stop'
-          $env:Path = (Get-Item depot_tools).FullName + ";" + $env:Path
-          mkdir webrtc/build
-          cd webrtc/build
-          cmake -G Ninja -D CMAKE_BUILD_TYPE=Release `
-                -D CMAKE_INSTALL_PREFIX=${{ env.WORK_DIR }}/webrtc_release/Release `
-                ..
-          ninja install
-          echo "Cleanup build folder for next config build"
-          cd ..
-          rm -r build
+          Copy-Item "$env:OPEN3D_DIR/3rdparty/webrtc/CMakeLists.txt" webrtc/
+          Copy-Item "$env:OPEN3D_DIR/3rdparty/webrtc/webrtc_common.cmake" webrtc/
+          bash "$env:OPEN3D_DIR/3rdparty/webrtc/apply_webrtc_patches.sh" `
+            "$env:OPEN3D_DIR" "$env:WORK_DIR/webrtc/src"
 
-      - name: Build WebRTC (Debug)
+      - name: Build and package WebRTC
+        shell: pwsh
         working-directory: ${{ env.WORK_DIR }}
+        env:
+          BUILD_CONFIG: ${{ matrix.config }}
+          STATIC_RT: ${{ matrix.static_runtime }}
+          WIN_TAG: ${{ matrix.tag }}
         run: |
           $ErrorActionPreference = 'Stop'
-          $env:Path = (Get-Item depot_tools).FullName + ";" + $env:Path
-          mkdir webrtc/build
-          cd webrtc/build
-          cmake -G Ninja -D CMAKE_BUILD_TYPE=Debug `
-                -D CMAKE_INSTALL_PREFIX=${{ env.WORK_DIR }}/webrtc_release/Debug `
-                ..
+          # Locate the VS-installed ninja.exe *before* prepending depot_tools to
+          # PATH.  depot_tools contains a Unix wrapper script named 'ninja' (no
+          # extension) that Windows cannot execute; if depot_tools appears first
+          # in PATH cmake -G Ninja picks up that script and fails with
+          # "unknown error" when trying to run 'ninja --version'.
+          $ninjaExe = (Get-Command ninja.exe -ErrorAction Stop).Source
+          $env:Path = (Get-Item depot_tools).FullName + ';' + $env:Path
+          $installRoot = Join-Path $env:WORK_DIR "webrtc_pkg"
+          if (Test-Path $installRoot) { Remove-Item -Recurse -Force $installRoot }
+          New-Item -ItemType Directory -Force -Path webrtc/build | Out-Null
+          Push-Location webrtc/build
+          $debugFlag = if ($env:BUILD_CONFIG -eq 'Debug') { 'ON' } else { 'OFF' }
+          cmake -G Ninja `
+            -D CMAKE_MAKE_PROGRAM="$ninjaExe" `
+            -D CMAKE_BUILD_TYPE=$env:BUILD_CONFIG `
+            -D WEBRTC_IS_DEBUG=$debugFlag `
+            -D WEBRTC_STATIC_MSVC_RUNTIME=$env:STATIC_RT `
+            -D CMAKE_INSTALL_PREFIX=$installRoot `
+            ..
           ninja install
-
-      - name: Package WebRTC
-        working-directory: ${{ env.WORK_DIR }}
-        run: |
-          $ErrorActionPreference = 'Stop'
-          $env:WEBRTC_COMMIT_SHORT = (git -C webrtc/src rev-parse --short=7 HEAD)
-          cmake -E tar cv webrtc_${env:WEBRTC_COMMIT_SHORT}_win.zip `
-            --format=zip -- webrtc_release
-          cmake -E sha256sum webrtc_${env:WEBRTC_COMMIT_SHORT}_win.zip | Tee-Object -FilePath checksum_win.txt
+          Pop-Location
+          $short = (git -C webrtc/src rev-parse --short=7 HEAD)
+          $zip = "webrtc_${short}_win_$env:WIN_TAG.zip"
+          Push-Location $installRoot
+          cmake -E tar cvf (Join-Path $env:OPEN3D_DIR $zip) --format=zip .
+          Pop-Location
+          cmake -E sha256sum (Join-Path $env:OPEN3D_DIR $zip) | Tee-Object -FilePath (Join-Path $env:OPEN3D_DIR "checksum_$zip")
 
       - name: Upload WebRTC
         uses: actions/upload-artifact@v4
         with:
-          name: webrtc_release_windows
+          name: webrtc_win_${{ matrix.tag }}
           path: |
-                 ${{ env.WORK_DIR }}/webrtc_*.zip
-                 ${{ env.WORK_DIR }}/checksum_*.txt
+            webrtc_*_win_*.zip
+            checksum_webrtc_*_win_*.zip
           if-no-files-found: error

.github/workflows/windows.yml

@@ -56,7 +56,6 @@ jobs:
           - BUILD_CUDA_MODULE: ON   # FIXME
             CONFIG: Debug
     env:
-      BUILD_WEBRTC: ${{ ( matrix.BUILD_SHARED_LIBS == 'OFF' && matrix.STATIC_RUNTIME == 'ON' ) && 'ON' || 'OFF' }}
       BUILD_PYTORCH_OPS: ${{ ( matrix.BUILD_CUDA_MODULE == 'ON' || matrix.CONFIG == 'Debug' ) && 'OFF' || 'ON' }}   # FIXME
 
     steps:
@@ -145,7 +144,6 @@ jobs:
             -DSTATIC_WINDOWS_RUNTIME=${{ matrix.STATIC_RUNTIME }} `
             -DBUILD_COMMON_ISPC_ISAS=ON `
             -DBUILD_LIBREALSENSE=ON `
-            -DBUILD_WEBRTC=${{ env.BUILD_WEBRTC }} `
             -DBUILD_UNIT_TESTS=ON `
             -DBUILD_CUDA_MODULE=${{ matrix.BUILD_CUDA_MODULE }} `
             -DBUILD_PYTORCH_OPS=${{ env.BUILD_PYTORCH_OPS }} `
@@ -361,7 +359,6 @@ jobs:
             -DBUILD_COMMON_ISPC_ISAS=ON `
             -DBUILD_AZURE_KINECT=ON `
             -DBUILD_LIBREALSENSE=ON `
-            -DBUILD_WEBRTC=ON `
             -DBUILD_JUPYTER_EXTENSION=ON `
             -DBUILD_PYTORCH_OPS=${{ env.BUILD_PYTORCH_OPS }} `
             ${{ env.SRC_DIR }}

3rdparty/find_dependencies.cmake

@@ -1988,11 +1988,43 @@ if(BUILD_WEBRTC)
     open3d_import_3rdparty_library(3rdparty_webrtc
         HIDDEN
         INCLUDE_DIRS ${WEBRTC_INCLUDE_DIRS}
-        LIB_DIR      ${WEBRTC_LIB_DIR}
-        LIBRARIES    ${WEBRTC_LIBRARIES}
         DEPENDS      ext_webrtc_all
     )
+    # webrtc/webrtc_extra need custom --whole-archive handling (below), so
+    # they can't use open3d_import_3rdparty_library()'s LIBRARIES option.
+    # Install them manually and reference $<INSTALL_INTERFACE:...> paths, so
+    # examples built against an installed *static* Open3D package (i.e. not
+    # from within this build tree) still link against them; see the
+    # LIBRARIES branch of open3d_import_3rdparty_library() for reference.
+    if(NOT BUILD_SHARED_LIBS)
+        foreach(_o3d_webrtc_lib webrtc webrtc_extra)
+            install(FILES "${WEBRTC_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}${_o3d_webrtc_lib}${CMAKE_STATIC_LIBRARY_SUFFIX}"
+                DESTINATION ${Open3D_INSTALL_LIB_DIR}
+                RENAME "${CMAKE_STATIC_LIBRARY_PREFIX}${PROJECT_NAME}_3rdparty_webrtc_${_o3d_webrtc_lib}${CMAKE_STATIC_LIBRARY_SUFFIX}")
+        endforeach()
+    endif()
+    set(WEBRTC_INSTALLED_LIB
+        "$<INSTALL_PREFIX>/${Open3D_INSTALL_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}${PROJECT_NAME}_3rdparty_webrtc")
+    if(UNIX AND NOT APPLE)
+        target_link_libraries(3rdparty_webrtc INTERFACE
+            "-Wl,--whole-archive"
+            "$<BUILD_INTERFACE:${WEBRTC_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}webrtc${CMAKE_STATIC_LIBRARY_SUFFIX}>"
+            "$<INSTALL_INTERFACE:${WEBRTC_INSTALLED_LIB}_webrtc${CMAKE_STATIC_LIBRARY_SUFFIX}>"
+            "-Wl,--no-whole-archive"
+            "$<BUILD_INTERFACE:${WEBRTC_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}webrtc_extra${CMAKE_STATIC_LIBRARY_SUFFIX}>"
+            "$<INSTALL_INTERFACE:${WEBRTC_INSTALLED_LIB}_webrtc_extra${CMAKE_STATIC_LIBRARY_SUFFIX}>")
+    else()
+        target_link_libraries(3rdparty_webrtc INTERFACE
+            "$<BUILD_INTERFACE:${WEBRTC_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}webrtc${CMAKE_STATIC_LIBRARY_SUFFIX}>"
+            "$<INSTALL_INTERFACE:${WEBRTC_INSTALLED_LIB}_webrtc${CMAKE_STATIC_LIBRARY_SUFFIX}>"
+            "$<BUILD_INTERFACE:${WEBRTC_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}webrtc_extra${CMAKE_STATIC_LIBRARY_SUFFIX}>"
+            "$<INSTALL_INTERFACE:${WEBRTC_INSTALLED_LIB}_webrtc_extra${CMAKE_STATIC_LIBRARY_SUFFIX}>")
+    endif()
     target_link_libraries(3rdparty_webrtc INTERFACE Open3D::3rdparty_threads ${CMAKE_DL_LIBS})
+    # libwebrtc.a and libturbojpeg.a both export jpeg_* symbols (WebRTC bundles libjpeg).
+    if(UNIX AND NOT APPLE)
+        target_link_options(3rdparty_webrtc INTERFACE "LINKER:--allow-multiple-definition")
+    endif()
     if (MSVC) # https://github.com/iimachines/webrtc-build/issues/2#issuecomment-503535704
         target_link_libraries(3rdparty_webrtc INTERFACE secur32 winmm dmoguids wmcodecdspuuid msdmo strmiids)
     endif()

3rdparty/webrtc/0001-build-enable-rtc_use_cxx11_abi-option.patch

@@ -1,29 +1,15 @@
-From c47a1b6c0faa2206395647cb83cb1a0542101847 Mon Sep 17 00:00:00 2001
-From: Yixing Lao <yixing.lao@gmail.com>
-Date: Wed, 7 Apr 2021 16:17:39 -0700
-Subject: [PATCH] build: enable rtc_use_cxx11_abi option
-
----
- config/BUILDCONFIG.gn | 6 ++++++
- 1 file changed, 6 insertions(+)
-
 diff --git a/config/BUILDCONFIG.gn b/config/BUILDCONFIG.gn
-index 0ef73ab2b..5ab677e27 100644
 --- a/config/BUILDCONFIG.gn
 +++ b/config/BUILDCONFIG.gn
-@@ -163,6 +163,12 @@ declare_args() {
-   is_component_build = is_debug && current_os != "ios"
+@@ -171,6 +171,11 @@ declare_args() {
+       is_debug && current_os != "ios" && current_os != "watchos"
  }
 
 +declare_args() {
-+  # Set to false to define "_GLIBCXX_USE_CXX11_ABI=0". If set to true, the
-+  # default will be used, which corresponds to the new CXX11 ABI.
++  # Open3D: GCC libstdc++ ABI selection on Linux.
 +  rtc_use_cxx11_abi = true
 +}
 +
  assert(!(is_debug && is_official_build), "Can't do official debug builds")
- 
- # ==============================================================================
--- 
-2.17.1
-
+ assert(!(current_os == "ios" && is_component_build),
+        "Can't use component build on iOS")

3rdparty/webrtc/0002-src-fix-nullptr_t-with-libstdcxx.patch

@@ -0,0 +1,12 @@
+diff --git a/rtc_base/ssl_stream_adapter.h b/rtc_base/ssl_stream_adapter.h
+--- a/rtc_base/ssl_stream_adapter.h
++++ b/rtc_base/ssl_stream_adapter.h
+@@ -129,7 +129,7 @@ class SSLStreamAdapter : public StreamInterface {
+   static std::unique_ptr<SSLStreamAdapter> Create(
+       std::unique_ptr<StreamInterface> stream,
+       absl::AnyInvocable<void(SSLHandshakeError)> handshake_error,
+-      nullptr_t /*field_trials*/) {
++      std::nullptr_t /*field_trials*/) {
+     return Create(std::move(stream), std::move(handshake_error));
+   }
+