Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

SCUNet after FeMaSR -> RuntimeError: expected scalar type Half but found Float #4

Closed
Selur opened this issue Mar 17, 2023 · 2 comments
Closed

SCUNet after FeMaSR -> RuntimeError: expected scalar type Half but found Float #4

Selur opened this issue Mar 17, 2023 · 2 comments

Comments

@Selur
Copy link

Selur commented Mar 17, 2023 •
edited
Loading

when using SCUNet after FeMaSR like this:

# Imports
import vapoursynth as vs
# getting Vapoursynth core
core = vs.core
import site
import os
import sys
# Import scripts folder
scriptPath = 'i:/Hybrid/64bit/vsscripts'
sys.path.insert(0, os.path.abspath(scriptPath))
import ctypes
# Adding torch dependencies to PATH
path = site.getsitepackages()[0]+'/torch_dependencies/bin/'
ctypes.windll.kernel32.SetDllDirectoryW(path)
path = path.replace('\\', '/')
os.environ["PATH"] = path + os.pathsep + os.environ["PATH"]
os.environ["CUDA_MODULE_LOADING"] = "LAZY"
# Loading Plugins
core.std.LoadPlugin(path="i:/Hybrid/64bit/vsfilters/Support/fmtconv.dll")
core.std.LoadPlugin(path="i:/Hybrid/64bit/vsfilters/SourceFilter/LSmashSource/vslsmashsource.dll")
# source: 'G:\TestClips&Co\files\lossless\suzie_lossless..mp4'
# current color space: YUV444P8, bit depth: 8, resolution: 176x144, fps: 29.97, color matrix: 470bg, yuv luminance scale: limited, scanorder: progressive
# Loading G:\TestClips&Co\files\lossless\suzie_lossless..mp4 using LibavSMASHSource
clip = core.lsmas.LibavSMASHSource(source="G:/TestClips&Co/files/lossless/suzie_lossless..mp4")
# Setting detected color matrix (470bg).
clip = core.std.SetFrameProps(clip, _Matrix=5)
# Setting color transfer info (470bg), when it is not set
clip = clip if not core.text.FrameProps(clip,'_Transfer') else core.std.SetFrameProps(clip, _Transfer=5)
# Setting color primaries info (), when it is not set
clip = clip if not core.text.FrameProps(clip,'_Primaries') else core.std.SetFrameProps(clip, _Primaries=5)
# Setting color range to TV (limited) range.
clip = core.std.SetFrameProp(clip=clip, prop="_ColorRange", intval=1)
# making sure frame rate is set to 29.97
clip = core.std.AssumeFPS(clip=clip, fpsnum=30000, fpsden=1001)
clip = core.std.SetFrameProp(clip=clip, prop="_FieldBased", intval=0) # progressive
from vsfemasr import femasr as FeMaSR
# adjusting color space from YUV444P8 to RGBH for VsFeMaSR
clip = core.resize.Bicubic(clip=clip, format=vs.RGBH, matrix_in_s="470bg", range_s="limited")
# resizing using FeMaSR
clip = FeMaSR(clip=clip, model=1, device_index=0, trt=True, trt_cache_path=r"J:\tmp") # 704x576
# resizing 704x576 to 1280x1048
# adjusting resizing
clip = core.resize.Bicubic(clip=clip, format=vs.RGBS, range_s="limited")
clip = core.fmtc.resample(clip=clip, w=1280, h=1048, kernel="cubic", interlaced=False, interlacedd=False)
from vsscunet import scunet as SCUNet
# denoising using SCUNet
clip = SCUNet(clip=clip, model=4, cuda_graphs=True, nvfuser=True)
# adjusting output color from: RGBS to YUV420P10 for x265Model
clip = core.resize.Bicubic(clip=clip, format=vs.YUV420P10, matrix_s="470bg", range_s="limited", dither_type="error_diffusion")
# adjusted resizing to achieve PAR 1:1 for preview
# resizing for preview to square pixel to 1400x1048
clip = core.resize.Bicubic(clip=clip, width=1400, height=1048)
# set output frame rate to 29.97fps (progressive)
clip = core.std.AssumeFPS(clip=clip, fpsnum=30000, fpsden=1001)
# Output
clip.set_output()

fails with:

Failed to evaluate the script:
Python exception: expected scalar type Half but found Float

Traceback (most recent call last):
File "src\cython\vapoursynth.pyx", line 2866, in vapoursynth._vpy_evaluate
File "src\cython\vapoursynth.pyx", line 2867, in vapoursynth._vpy_evaluate
File "J:\tmp\tempPreviewVapoursynthFile10_46_44_212.vpy", line 47, in 
clip = SCUNet(clip=clip, model=4)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\autograd\grad_mode.py", line 27, in decorate_context
return func(*args, **kwargs)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\vsscunet\__init__.py", line 141, in scunet
module(static_input[i])
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\nn\modules\module.py", line 1190, in _call_impl
return forward_call(*input, **kwargs)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\functorch\_src\aot_autograd.py", line 705, in forward
return compiled_f(
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\functorch\_src\aot_autograd.py", line 656, in returned_function
compiled_fn = create_aot_dispatcher_function(
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\functorch\_src\aot_autograd.py", line 512, in create_aot_dispatcher_function
return aot_dispatch_base(flat_fn, fake_flat_tensor_args, aot_config)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\functorch\_src\aot_autograd.py", line 277, in aot_dispatch_base
fw_module = make_fx(flat_fn, aot_config.decompositions)(*flat_args)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\fx\experimental\proxy_tensor.py", line 652, in wrapped
t = dispatch_trace(wrap_key(func, args, fx_tracer), tracer=fx_tracer, concrete_args=tuple(phs))
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\fx\experimental\proxy_tensor.py", line 400, in dispatch_trace
graph = tracer.trace(root, concrete_args)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\fx\_symbolic_trace.py", line 739, in trace
(self.create_arg(fn(*args)),),
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\fx\experimental\proxy_tensor.py", line 414, in wrapped
out = f(*tensors)
File "", line 1, in 
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\functorch\_src\aot_autograd.py", line 636, in flat_fn
tree_out = fn(*args, **kwargs)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\functorch\_src\aot_autograd.py", line 695, in functional_call
return stateless.functional_call(mod, params_and_buffers, args, kwargs)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\nn\utils\stateless.py", line 156, in functional_call
out = module(*args, **kwargs)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\fx\_symbolic_trace.py", line 717, in module_call_wrapper
return self.call_module(mod, forward, args, kwargs)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\fx\experimental\proxy_tensor.py", line 366, in call_module
return forward(*args, **kwargs)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\fx\_symbolic_trace.py", line 710, in forward
return _orig_module_call(mod, *args, **kwargs)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\nn\modules\module.py", line 1190, in _call_impl
return forward_call(*input, **kwargs)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\vsscunet\network_scunet.py", line 226, in forward
x2 = self.m_down1(x1)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\fx\_symbolic_trace.py", line 717, in module_call_wrapper
return self.call_module(mod, forward, args, kwargs)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\fx\experimental\proxy_tensor.py", line 366, in call_module
return forward(*args, **kwargs)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\fx\_symbolic_trace.py", line 710, in forward
return _orig_module_call(mod, *args, **kwargs)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\nn\modules\module.py", line 1190, in _call_impl
return forward_call(*input, **kwargs)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\nn\modules\container.py", line 204, in forward
input = module(input)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\fx\_symbolic_trace.py", line 717, in module_call_wrapper
return self.call_module(mod, forward, args, kwargs)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\fx\experimental\proxy_tensor.py", line 366, in call_module
return forward(*args, **kwargs)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\fx\_symbolic_trace.py", line 710, in forward
return _orig_module_call(mod, *args, **kwargs)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\nn\modules\module.py", line 1190, in _call_impl
return forward_call(*input, **kwargs)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\vsscunet\network_scunet.py", line 156, in forward
trans_x = self.trans_block(trans_x)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\fx\_symbolic_trace.py", line 717, in module_call_wrapper
return self.call_module(mod, forward, args, kwargs)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\fx\experimental\proxy_tensor.py", line 366, in call_module
return forward(*args, **kwargs)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\fx\_symbolic_trace.py", line 710, in forward
return _orig_module_call(mod, *args, **kwargs)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\nn\modules\module.py", line 1190, in _call_impl
return forward_call(*input, **kwargs)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\vsscunet\network_scunet.py", line 120, in forward
x = x + self.drop_path(self.msa(self.ln1(x)))
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\fx\_symbolic_trace.py", line 717, in module_call_wrapper
return self.call_module(mod, forward, args, kwargs)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\fx\experimental\proxy_tensor.py", line 366, in call_module
return forward(*args, **kwargs)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\fx\_symbolic_trace.py", line 710, in forward
return _orig_module_call(mod, *args, **kwargs)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\nn\modules\module.py", line 1190, in _call_impl
return forward_call(*input, **kwargs)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\vsscunet\network_scunet.py", line 83, in forward
output = torch.einsum('hbwij,hbwjc->hbwic', probs, v)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\functional.py", line 378, in einsum
return _VF.einsum(equation, operands) # type: ignore[attr-defined]
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\utils\_python_dispatch.py", line 101, in __torch_dispatch__
return old.__torch_dispatch__(func, types, args, kwargs)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\fx\experimental\proxy_tensor.py", line 440, in __torch_dispatch__
return self.inner_torch_dispatch(func, types, args, kwargs)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\fx\experimental\proxy_tensor.py", line 465, in inner_torch_dispatch
out = proxy_call(self, func, args, kwargs)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\fx\experimental\proxy_tensor.py", line 218, in proxy_call
r = func.decompose(*args, **kwargs)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\_ops.py", line 279, in decompose
return self._op_dk(dk, *args, **kwargs)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\utils\_python_dispatch.py", line 101, in __torch_dispatch__
return old.__torch_dispatch__(func, types, args, kwargs)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\fx\experimental\proxy_tensor.py", line 440, in __torch_dispatch__
return self.inner_torch_dispatch(func, types, args, kwargs)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\fx\experimental\proxy_tensor.py", line 465, in inner_torch_dispatch
out = proxy_call(self, func, args, kwargs)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\fx\experimental\proxy_tensor.py", line 304, in proxy_call
out = func(*args, **kwargs)
File "I:\Hybrid\64bit\Vapoursynth\Lib\site-packages\torch\_ops.py", line 257, in __call__
return self._op(*args, **kwargs or {})
RuntimeError: expected scalar type Half but found Float

Using FeMaSR (32bit) and SCUNet:

# Imports
import vapoursynth as vs
# getting Vapoursynth core
core = vs.core
import site
import os
import sys
# Import scripts folder
scriptPath = 'i:/Hybrid/64bit/vsscripts'
sys.path.insert(0, os.path.abspath(scriptPath))
import ctypes
# Adding torch dependencies to PATH
path = site.getsitepackages()[0]+'/torch_dependencies/bin/'
ctypes.windll.kernel32.SetDllDirectoryW(path)
path = path.replace('\\', '/')
os.environ["PATH"] = path + os.pathsep + os.environ["PATH"]
os.environ["CUDA_MODULE_LOADING"] = "LAZY"
# Loading Plugins
core.std.LoadPlugin(path="i:/Hybrid/64bit/vsfilters/Support/fmtconv.dll")
core.std.LoadPlugin(path="i:/Hybrid/64bit/vsfilters/SourceFilter/LSmashSource/vslsmashsource.dll")
# source: 'G:\TestClips&Co\files\lossless\suzie_lossless..mp4'
# current color space: YUV444P8, bit depth: 8, resolution: 176x144, fps: 29.97, color matrix: 470bg, yuv luminance scale: limited, scanorder: progressive
# Loading G:\TestClips&Co\files\lossless\suzie_lossless..mp4 using LibavSMASHSource
clip = core.lsmas.LibavSMASHSource(source="G:/TestClips&Co/files/lossless/suzie_lossless..mp4")
# Setting detected color matrix (470bg).
clip = core.std.SetFrameProps(clip, _Matrix=5)
# Setting color transfer info (470bg), when it is not set
clip = clip if not core.text.FrameProps(clip,'_Transfer') else core.std.SetFrameProps(clip, _Transfer=5)
# Setting color primaries info (), when it is not set
clip = clip if not core.text.FrameProps(clip,'_Primaries') else core.std.SetFrameProps(clip, _Primaries=5)
# Setting color range to TV (limited) range.
clip = core.std.SetFrameProp(clip=clip, prop="_ColorRange", intval=1)
# making sure frame rate is set to 29.97
clip = core.std.AssumeFPS(clip=clip, fpsnum=30000, fpsden=1001)
clip = core.std.SetFrameProp(clip=clip, prop="_FieldBased", intval=0) # progressive
from vsfemasr import femasr as FeMaSR
# adjusting color space from YUV444P8 to RGBS for VsFeMaSR
clip = core.resize.Bicubic(clip=clip, format=vs.RGBS, matrix_in_s="470bg", range_s="limited")
# resizing using FeMaSR
clip = FeMaSR(clip=clip, model=1, device_index=0, trt=True, trt_cache_path=r"J:\tmp") # 704x576
# resizing 704x576 to 1280x1048
# adjusting resizing
clip = core.fmtc.resample(clip=clip, w=1280, h=1048, kernel="cubic", interlaced=False, interlacedd=False)
from vsscunet import scunet as SCUNet
# denoising using SCUNet
clip = SCUNet(clip=clip, model=4, cuda_graphs=True, nvfuser=True)
# adjusting output color from: RGBS to YUV420P10 for x265Model
clip = core.resize.Bicubic(clip=clip, format=vs.YUV420P10, matrix_s="470bg", range_s="limited", dither_type="error_diffusion")
# set output frame rate to 29.97fps (progressive)
clip = core.std.AssumeFPS(clip=clip, fpsnum=30000, fpsden=1001)
# Output
clip.set_output()

works.

Is it a bug or am I doing something wrong when using 16bit FeMaSR followed by SCUNet?
@HolyWu
Copy link
Owner

HolyWu commented Mar 18, 2023

Update FeMaSR to 1.2.0 should fix your issue.

@Selur
Copy link
Author

Selur commented Mar 18, 2023

Thanks, that fixed the issue. :) 👍

@Selur Selur closed this as completed Mar 18, 2023
@Selur Selur mentioned this issue Mar 28, 2023
Closed
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees
No one assigned
Labels
None yet
Projects
None yet
Milestone
No milestone
Development

No branches or pull requests
2 participants
@Selur @HolyWu