How to get the backbuffer of omniverse's current viewport?

I saw the capture_viewport_to_buffer interface in the omni.kit.viewport.utility library, but I couldn’t find the usage instructions for this interface. I need help.

Hi @Pitt_Z. I found a good example in the omni.kit.thumbnails.mdl extension. Here’s an excerpt from the module: omni.kit.thumbnails.mdl.viewport_thumbnail_generator. Please refer to the module for the full example.

async def _capture_viewport_async(self) -> bool:
        Setup the viewport in the desired state and capture the viewport
        Always capture to local and down sampling for better edge queality

        # we are gonna resets the view so we need to wait render_iterations
        for iteration in range(self._iterations):

        # Capture viewport
        viewport_api = get_active_viewport()
        capture = capture_viewport_to_buffer(viewport_api, self.on_viewport_captured)
        await capture.wait_for_result()


        return await self._is_url_exists_async(self.output_url)

    def on_viewport_captured(self, buffer, buffer_size, width, height, format) -> None:
        Function called when capturing viewport
            ctypes.pythonapi.PyCapsule_GetPointer.restype = ctypes.POINTER(ctypes.c_byte * buffer_size)
            ctypes.pythonapi.PyCapsule_GetPointer.argtypes = [ctypes.py_object, ctypes.c_char_p]
            content = ctypes.pythonapi.PyCapsule_GetPointer(buffer, None)
        except Exception as e:
            carb.log_error(f"[Thumbnail] Failed to get capture buffer: {e}")

        # Crop and down sample
        data = self._generate_thumbnail_data(content.contents, (width, height))

        # Save to output url
        result = omni.client.write_file(self.output_url, data)
        if result != omni.client.Result.OK:
            carb.log_error(f"[Thumbnail] Cannot write {self.output_url}, error code: {result}.")
            carb.log_info(f"[Thumbnail] Exported to {self.output_url}")

    def _generate_thumbnail_data(self, raw_data, size: Tuple[int, int]) -> bytes:
        Generate desired thumbnail data from raw data
        # Constrcut Image object
        im = Image.frombytes("RGBA", size, raw_data)

        # Crop the center of the image if necessary
        width, height = im.size  # Get dimensions
        if width != height:
            if width > height:
                left = (width - height) / 2
                right = width - left
                top = 0
                bottom = height
                left = 0
                right = width
                top = (height - width) / 2
                bottom = height - top
            im = im.crop((left, top, right, bottom))

        im.thumbnail((self._width, self._height), Image.ANTIALIAS)

        # Save to buffer
        buffer = io.BytesIO(), "png")
        return buffer.getvalue()

If you want to know how I found this example, I show you how here: Before You Create Metaverse Apps and Tools...Watch This! - YouTube

1 Like

That’s cool. Thank you very much for sharing the method with me

How to grab SemanticSegmentation image in this buffer contents.