Closed
Description
Hello all
I have a use case that requires passing a cpu generated background array to a pygfx image object to perform background subtraction in the shader. I have got things working but I noticed that if I change the background texture passed to the image object it does not update until I change a store variable in the object (such as apply_background_texture in my example). I think this is because changing this texture does not trigger an entry into the PropTracker to get the get_bindings function of the image object to be called, while changing the store variable does.
Below is a minimum example I was able to create. As you change the background subtracted value with the slider the image does not change, until you toggle the background subtraction on/off with the Apply Background Subtraction combo box.
Is there a way around this, such as a manual way to flag the image object to get the get_binding function to be called?
Thanks!
import imageio.v3 as iio
import numpy as np
from wgpu.gui.auto import WgpuCanvas, run
import pygfx as gfx
from pygfx.renderers.wgpu.shaders.imageshader import ImageShader
from wgpu.utils.imgui import ImguiRenderer
from imgui_bundle import imgui
from pygfx.renderers.wgpu.shaders.imageshader import vertex_and_fragment
from pygfx.renderers.wgpu import (
Binding,
GfxSampler,
GfxTextureView,
register_wgpu_render_function
)
# Get list of available standard images
standard_images = [
"astronaut",
]
def load_image(image_name):
if image_name.startswith("/"):
return iio.imread(image_name)
return iio.imread(f"imageio:{image_name}.png")
# Load initial image
current_image_name = standard_images[0]
im = load_image(current_image_name)
canvas_size = im.shape[0] + 100, im.shape[1] + 100
canvas = WgpuCanvas(size=canvas_size, max_fps=999)
renderer = gfx.renderers.WgpuRenderer(canvas, show_fps=True)
scene = gfx.Scene()
camera = gfx.OrthographicCamera(canvas_size[0], canvas_size[1])
controller = gfx.PanZoomController(camera, register_events=renderer)
camera.local.scale_y = -1
camera.local.scale_x = 1
image_texture = gfx.Texture(im,
dim=2,
)
background_texture = gfx.Texture(
np.full_like(im, 128, dtype=np.uint8),
dim=2,
)
class BackGroundRemovedImageMaterial(gfx.ImageBasicMaterial):
"""
An image that has the background removed.
"""
uniform_type = dict(
gfx.ImageBasicMaterial.uniform_type,
background_level="u4",
contrast="f4",
)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.apply_background_subtraction = True
@property
def apply_background_subtraction(self):
return self._store.apply_background_subtraction
@apply_background_subtraction.setter
def apply_background_subtraction(self, value):
self._store.apply_background_subtraction = value
class BackGroundRemovedImage(gfx.Image):
def __init__(self, background_texture, *args, **kwargs):
super().__init__(*args, **kwargs)
self.background_texture = background_texture
@register_wgpu_render_function(BackGroundRemovedImage, BackGroundRemovedImageMaterial)
class BackGroundRemovedImageShader(ImageShader):
def __init__(self, wobject):
super().__init__(wobject)
material = wobject.material
self['apply_background_subtraction'] = material.apply_background_subtraction
def get_bindings(self, wobject, shared):
geometry = wobject.geometry
material = wobject.material
bindings = [
Binding("u_stdinfo", "buffer/uniform", shared.uniform_buffer),
Binding("u_wobject", "buffer/uniform", wobject.uniform_buffer),
Binding("u_material", "buffer/uniform", material.uniform_buffer),
]
tex_view = GfxTextureView(geometry.grid)
sampler = GfxSampler(material.interpolation, "clamp")
bindings.append(Binding("s_img", "sampler/filtering", sampler, "FRAGMENT"))
bindings.append(Binding("t_img", "texture/auto", tex_view, vertex_and_fragment))
if self["three_grid_yuv"]:
u_tex_view = GfxTextureView(geometry.grid_u)
v_tex_view = GfxTextureView(geometry.grid_v)
bindings.append(
Binding("t_u_img", "texture/auto", u_tex_view, vertex_and_fragment)
)
bindings.append(
Binding("t_v_img", "texture/auto", v_tex_view, vertex_and_fragment)
)
# Background texture
background_texture = wobject.background_texture
bindings.append(Binding("s_background", "sampler/filtering",
GfxSampler("linear", "clamp"), "FRAGMENT"))
tex_view = GfxTextureView(background_texture)
bindings.append(Binding(f"t_background", "texture/auto",
tex_view, vertex_and_fragment))
# Background texture end
if self["use_colormap"]:
bindings.extend(self.define_img_colormap(material.map))
bindings = {i: b for i, b in enumerate(bindings)}
self.define_bindings(0, bindings)
return {
0: bindings,
}
def get_code(self):
code = super().get_code()
code = code.replace(
"""
let value = sample_im(varyings.texcoord.xy, sizef);
""",
"""
var value = sample_im(varyings.texcoord.xy, sizef);
$$ if apply_background_subtraction
value = value - textureSample(t_background, s_img, varyings.texcoord.xy);
$$ endif
"""
)
return code
image = BackGroundRemovedImage(
background_texture,
gfx.Geometry(grid=image_texture),
BackGroundRemovedImageMaterial(
clim=(0, 255),
interpolation="nearest"
)
)
image.local.x = -im.shape[1] / 2
image.local.y = -im.shape[0] / 2 + 50
scene.add(image)
current_apply_background_subtraction = 1
current_background_value = 128
def draw_imgui():
global current_apply_background_subtraction
global current_background_value
global im, image_texture
imgui.new_frame()
imgui.set_next_window_size((600, 0), imgui.Cond_.always)
imgui.set_next_window_pos((0, 0), imgui.Cond_.always)
is_expand, _ = imgui.begin("Controls", None)
if is_expand:
# Background level selection dropdown
changed, current_apply_background_subtraction = imgui.combo(
"Apply Background Subtraction", current_apply_background_subtraction, ['False', 'True'], 2
)
if changed:
image.material.apply_background_subtraction = bool(current_apply_background_subtraction)
# Background value slider
changed, current_background_value = imgui.slider_int(
"Background Value", current_background_value, 0, 255
)
if changed:
# Creating a new background texture with the selected value for this example.
# It is not necessary to create a new texture for this simple example,
# but in a real application you might want to update the texture dynamically,
# such as in cases where the shape of the background texture data changes.
background_texture = gfx.Texture(
np.full_like(im, current_background_value, dtype=np.uint8),
dim=2,
)
image.background_texture = background_texture
imgui.end()
imgui.end_frame()
imgui.render()
return imgui.get_draw_data()
# Create GUI renderer
gui_renderer = ImguiRenderer(renderer.device, canvas)
def animate():
renderer.render(scene, camera)
renderer.flush()
gui_renderer.render()
canvas.request_draw()
gui_renderer.set_gui(draw_imgui)
if __name__ == "__main__":
canvas.request_draw(animate)
run()Metadata
Metadata
Assignees
Labels
No labels