In [ ]

# Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved.

渲染彩色點雲¶

本教學課程說明如何

設定渲染器
渲染點雲
調整渲染設定，例如合成與相機位置

導入模組¶

請確定已安裝 torch 和 torchvision。如果未安裝 pytorch3d，請使用下方的單元格進行安裝

In [ ]

import os
import sys
import torch
need_pytorch3d=False
try:
    import pytorch3d
except ModuleNotFoundError:
    need_pytorch3d=True
if need_pytorch3d:
    if torch.__version__.startswith("2.2.") and sys.platform.startswith("linux"):
        # We try to install PyTorch3D via a released wheel.
        pyt_version_str=torch.__version__.split("+")[0].replace(".", "")
        version_str="".join([
            f"py3{sys.version_info.minor}_cu",
            torch.version.cuda.replace(".",""),
            f"_pyt{pyt_version_str}"
        ])
        !pip install fvcore iopath
        !pip install --no-index --no-cache-dir pytorch3d -f https://dl.fbaipublicfiles.com/pytorch3d/packaging/wheels/{version_str}/download.html
    else:
        # We try to install PyTorch3D from source.
        !pip install 'git+https://github.com/facebookresearch/pytorch3d.git@stable'

In [ ]

import os
import torch
import torch.nn.functional as F
import matplotlib.pyplot as plt

# Util function for loading point clouds|
import numpy as np

# Data structures and functions for rendering
from pytorch3d.structures import Pointclouds
from pytorch3d.vis.plotly_vis import AxisArgs, plot_batch_individually, plot_scene
from pytorch3d.renderer import (
    look_at_view_transform,
    FoVOrthographicCameras, 
    PointsRasterizationSettings,
    PointsRenderer,
    PulsarPointsRenderer,
    PointsRasterizer,
    AlphaCompositor,
    NormWeightedCompositor
)

載入點雲及對應的顏色¶

載入並建立點雲物件。

點雲是 PyTorch3D 中提供的一個獨有資料結構，用於處理不同大小的成批點雲。

如果你使用 Google Colab 執行本筆記本，請執行下列單元格，取得點雲資料並儲存在路徑 data/PittsburghBridge 中：如果你在本地執行，則資料已存在正確的路徑中。

In [ ]

!mkdir -p data/PittsburghBridge
!wget -P data/PittsburghBridge https://dl.fbaipublicfiles.com/pytorch3d/data/PittsburghBridge/pointcloud.npz

In [ ]

# Setup
if torch.cuda.is_available():
    device = torch.device("cuda:0")
    torch.cuda.set_device(device)
else:
    device = torch.device("cpu")

# Set paths
DATA_DIR = "./data"
obj_filename = os.path.join(DATA_DIR, "PittsburghBridge/pointcloud.npz")

# Load point cloud
pointcloud = np.load(obj_filename)
verts = torch.Tensor(pointcloud['verts']).to(device)
        
rgb = torch.Tensor(pointcloud['rgb']).to(device)

point_cloud = Pointclouds(points=[verts], features=[rgb])

建立渲染器¶

PyTorch3D 中的渲染器由 光柵化器 及 著色器 組成，而每個渲染器各有許多子組件，例如相機（正交/透視）。在此，我們會初始化其中一些組件，並對其餘部分使用預設值。

在此範例中，我們會先建立一個 渲染器，該渲染器使用 正交相機 以及套用 alpha 合成。接著我們會了解如何透過模組化 API 調整不同的組件。

[1] SynSin：從單一影像進行端到端的檢視合成。 Olivia Wiles、Georgia Gkioxari、Richard Szeliski、Justin Johnson。CVPR 2020。

In [ ]

# Initialize a camera.
R, T = look_at_view_transform(20, 10, 0)
cameras = FoVOrthographicCameras(device=device, R=R, T=T, znear=0.01)

# Define the settings for rasterization and shading. Here we set the output image to be of size
# 512x512. As we are rendering images for visualization purposes only we will set faces_per_pixel=1
# and blur_radius=0.0. Refer to raster_points.py for explanations of these parameters. 
raster_settings = PointsRasterizationSettings(
    image_size=512, 
    radius = 0.003,
    points_per_pixel = 10
)


# Create a points renderer by compositing points using an alpha compositor (nearer points
# are weighted more heavily). See [1] for an explanation.
rasterizer = PointsRasterizer(cameras=cameras, raster_settings=raster_settings)
renderer = PointsRenderer(
    rasterizer=rasterizer,
    compositor=AlphaCompositor()
)

In [ ]

images = renderer(point_cloud)
plt.figure(figsize=(10, 10))
plt.imshow(images[0, ..., :3].cpu().numpy())
plt.axis("off");

現在，我們將修改 渲染器，使用 alpha 合成 與設定的背景顏色。

In [ ]

renderer = PointsRenderer(
    rasterizer=rasterizer,
    # Pass in background_color to the alpha compositor, setting the background color 
    # to the 3 item tuple, representing rgb on a scale of 0 -> 1, in this case blue
    compositor=AlphaCompositor(background_color=(0, 0, 1))
)
images = renderer(point_cloud)

plt.figure(figsize=(10, 10))
plt.imshow(images[0, ..., :3].cpu().numpy())
plt.axis("off");

在此範例中，我們會先建立一個 渲染器，該渲染器使用 正交相機 以及套用 加權合成。

In [ ]

# Initialize a camera.
R, T = look_at_view_transform(20, 10, 0)
cameras = FoVOrthographicCameras(device=device, R=R, T=T, znear=0.01)

# Define the settings for rasterization and shading. Here we set the output image to be of size
# 512x512. As we are rendering images for visualization purposes only we will set faces_per_pixel=1
# and blur_radius=0.0. Refer to rasterize_points.py for explanations of these parameters. 
raster_settings = PointsRasterizationSettings(
    image_size=512, 
    radius = 0.003,
    points_per_pixel = 10
)


# Create a points renderer by compositing points using an weighted compositor (3D points are
# weighted according to their distance to a pixel and accumulated using a weighted sum)
renderer = PointsRenderer(
    rasterizer=PointsRasterizer(cameras=cameras, raster_settings=raster_settings),
    compositor=NormWeightedCompositor()
)

In [ ]

images = renderer(point_cloud)
plt.figure(figsize=(10, 10))
plt.imshow(images[0, ..., :3].cpu().numpy())
plt.axis("off");

現在，我們將修改 渲染器，使用 加權合成 與設定的背景顏色。

In [ ]

renderer = PointsRenderer(
    rasterizer=PointsRasterizer(cameras=cameras, raster_settings=raster_settings),
    # Pass in background_color to the norm weighted compositor, setting the background color 
    # to the 3 item tuple, representing rgb on a scale of 0 -> 1, in this case red
    compositor=NormWeightedCompositor(background_color=(1,0,0))
)
images = renderer(point_cloud)
plt.figure(figsize=(10, 10))
plt.imshow(images[0, ..., :3].cpu().numpy())
plt.axis("off");

使用 pulsar 後端¶

切換到 pulsar 後端很簡單！pulsar 後端有一個內建的合成器，因此在建立時不需要 compositor 參數（即使你提供該參數，系統仍會顯示警告）。它會在渲染裝置上預先配置記憶體，因此在建立時需要 n_channels。

渲染器轉送函式的所有參數都是逐批的，背景顏色（在本例為 gamma）除外，而且你必須提供與批次中的範例數量相同的值。背景顏色是可選的，預設為全零。你可以在論文使用基於球形的表示進行神經渲染的快速可微分射線投射中找到有關 gamma 如何影響渲染函式的詳細說明。

你也可以將 native 後端用於 pulsar 後端，它已經提供存取點不透明性。native 後端可以從 pytorch3d.renderer.points.pulsar 匯入；你可以在 docs/examples 資料夾中找到相關範例。

In [ ]

renderer = PulsarPointsRenderer(
    rasterizer=PointsRasterizer(cameras=cameras, raster_settings=raster_settings),
    n_channels=4
).to(device)

images = renderer(point_cloud, gamma=(1e-4,),
                  bg_col=torch.tensor([0.0, 1.0, 0.0, 1.0], dtype=torch.float32, device=device))
plt.figure(figsize=(10, 10))
plt.imshow(images[0, ..., :3].cpu().numpy())
plt.axis("off");

在 Plotly 圖形中檢視點雲¶

在這裡，我們使用 pyTorch3D 函數 plot_scene 在 Plotly 圖形中呈現點雲。plot_scene 會傳回由輸入定義的軌跡和子繪製項的 Plotly 圖形。

In [ ]

plot_scene({
    "Pointcloud": {
        "person": point_cloud
    }
})

現在，我們將呈現一批點雲。第一個點雲和上面一樣，而第二個點雲全黑，且在所有維度中偏移 2，讓它們能呈現於同一張圖表上。

In [ ]

point_cloud_batch = Pointclouds(points=[verts, verts + 2], features=[rgb, torch.zeros_like(rgb)])
# render both in the same plot in different traces
fig = plot_scene({
    "Pointcloud": {
        "person": point_cloud_batch[0],
        "person2": point_cloud_batch[1]
    }
})
fig.show()

In [ ]

# render both in the same plot in one trace
fig = plot_scene({
    "Pointcloud": {
        "2 people": point_cloud_batch
    }
})
fig.show()

對於批次，我們也能使用 plot_batch_individually 來避免自行建構場景字典。

In [ ]

# render both in 1 row in different subplots
fig2 = plot_batch_individually(point_cloud_batch, ncols=2)
fig2.show()

In [ ]

# modify the plotly figure height and width
fig2.update_layout(height=500, width=500)
fig2.show()

我們也能修改這兩個函數的軸數引數和軸數背景，並將標題放置於 plot_batch_individually 中。

In [ ]

fig3 = plot_batch_individually(
    point_cloud_batch, 
    xaxis={"backgroundcolor":"rgb(200, 200, 230)"},
    yaxis={"backgroundcolor":"rgb(230, 200, 200)"},
    zaxis={"backgroundcolor":"rgb(200, 230, 200)"}, 
    subplot_titles=["Pointcloud1", "Pointcloud2"], # this should have a title for each subplot, titles can be ""
    axis_args=AxisArgs(showgrid=True))
fig3.show()