sd_hijack_unet.py 4.5 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485
  1. import torch
  2. from packaging import version
  3. from modules import devices
  4. from modules.sd_hijack_utils import CondFunc
  5. class TorchHijackForUnet:
  6. """
  7. This is torch, but with cat that resizes tensors to appropriate dimensions if they do not match;
  8. this makes it possible to create pictures with dimensions that are multiples of 8 rather than 64
  9. """
  10. def __getattr__(self, item):
  11. if item == 'cat':
  12. return self.cat
  13. if hasattr(torch, item):
  14. return getattr(torch, item)
  15. raise AttributeError(f"'{type(self).__name__}' object has no attribute '{item}'")
  16. def cat(self, tensors, *args, **kwargs):
  17. if len(tensors) == 2:
  18. a, b = tensors
  19. if a.shape[-2:] != b.shape[-2:]:
  20. a = torch.nn.functional.interpolate(a, b.shape[-2:], mode="nearest")
  21. tensors = (a, b)
  22. return torch.cat(tensors, *args, **kwargs)
  23. th = TorchHijackForUnet()
  24. # Below are monkey patches to enable upcasting a float16 UNet for float32 sampling
  25. def apply_model(orig_func, self, x_noisy, t, cond, **kwargs):
  26. if isinstance(cond, dict):
  27. for y in cond.keys():
  28. if isinstance(cond[y], list):
  29. cond[y] = [x.to(devices.dtype_unet) if isinstance(x, torch.Tensor) else x for x in cond[y]]
  30. else:
  31. cond[y] = cond[y].to(devices.dtype_unet) if isinstance(cond[y], torch.Tensor) else cond[y]
  32. with devices.autocast():
  33. return orig_func(self, x_noisy.to(devices.dtype_unet), t.to(devices.dtype_unet), cond, **kwargs).float()
  34. class GELUHijack(torch.nn.GELU, torch.nn.Module):
  35. def __init__(self, *args, **kwargs):
  36. torch.nn.GELU.__init__(self, *args, **kwargs)
  37. def forward(self, x):
  38. if devices.unet_needs_upcast:
  39. return torch.nn.GELU.forward(self.float(), x.float()).to(devices.dtype_unet)
  40. else:
  41. return torch.nn.GELU.forward(self, x)
  42. ddpm_edit_hijack = None
  43. def hijack_ddpm_edit():
  44. global ddpm_edit_hijack
  45. if not ddpm_edit_hijack:
  46. CondFunc('modules.models.diffusion.ddpm_edit.LatentDiffusion.decode_first_stage', first_stage_sub, first_stage_cond)
  47. CondFunc('modules.models.diffusion.ddpm_edit.LatentDiffusion.encode_first_stage', first_stage_sub, first_stage_cond)
  48. ddpm_edit_hijack = CondFunc('modules.models.diffusion.ddpm_edit.LatentDiffusion.apply_model', apply_model, unet_needs_upcast)
  49. unet_needs_upcast = lambda *args, **kwargs: devices.unet_needs_upcast
  50. CondFunc('ldm.models.diffusion.ddpm.LatentDiffusion.apply_model', apply_model, unet_needs_upcast)
  51. CondFunc('ldm.modules.diffusionmodules.openaimodel.timestep_embedding', lambda orig_func, timesteps, *args, **kwargs: orig_func(timesteps, *args, **kwargs).to(torch.float32 if timesteps.dtype == torch.int64 else devices.dtype_unet), unet_needs_upcast)
  52. if version.parse(torch.__version__) <= version.parse("1.13.2") or torch.cuda.is_available():
  53. CondFunc('ldm.modules.diffusionmodules.util.GroupNorm32.forward', lambda orig_func, self, *args, **kwargs: orig_func(self.float(), *args, **kwargs), unet_needs_upcast)
  54. CondFunc('ldm.modules.attention.GEGLU.forward', lambda orig_func, self, x: orig_func(self.float(), x.float()).to(devices.dtype_unet), unet_needs_upcast)
  55. CondFunc('open_clip.transformer.ResidualAttentionBlock.__init__', lambda orig_func, *args, **kwargs: kwargs.update({'act_layer': GELUHijack}) and False or orig_func(*args, **kwargs), lambda _, *args, **kwargs: kwargs.get('act_layer') is None or kwargs['act_layer'] == torch.nn.GELU)
  56. first_stage_cond = lambda _, self, *args, **kwargs: devices.unet_needs_upcast and self.model.diffusion_model.dtype == torch.float16
  57. first_stage_sub = lambda orig_func, self, x, **kwargs: orig_func(self, x.to(devices.dtype_vae), **kwargs)
  58. CondFunc('ldm.models.diffusion.ddpm.LatentDiffusion.decode_first_stage', first_stage_sub, first_stage_cond)
  59. CondFunc('ldm.models.diffusion.ddpm.LatentDiffusion.encode_first_stage', first_stage_sub, first_stage_cond)
  60. CondFunc('ldm.models.diffusion.ddpm.LatentDiffusion.get_first_stage_encoding', lambda orig_func, *args, **kwargs: orig_func(*args, **kwargs).float(), first_stage_cond)
  61. CondFunc('sgm.modules.diffusionmodules.wrappers.OpenAIWrapper.forward', apply_model, unet_needs_upcast)
  62. CondFunc('sgm.modules.diffusionmodules.openaimodel.timestep_embedding', lambda orig_func, timesteps, *args, **kwargs: orig_func(timesteps, *args, **kwargs).to(torch.float32 if timesteps.dtype == torch.int64 else devices.dtype_unet), unet_needs_upcast)