From 53fc6f1b312d7cbad9b90b03c0dd40a4e962ec77 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Sat, 10 Dec 2022 15:01:24 +0000 Subject: [PATCH 1/6] Deprecate init image correctl --- .../pipelines/alt_diffusion/pipeline_alt_diffusion_img2img.py | 2 +- .../pipeline_latent_diffusion_superresolution.py | 2 +- .../pipelines/stable_diffusion/pipeline_cycle_diffusion.py | 2 +- .../pipeline_onnx_stable_diffusion_img2img.py | 2 +- .../pipeline_onnx_stable_diffusion_inpaint_legacy.py | 4 ++-- .../stable_diffusion/pipeline_stable_diffusion_img2img.py | 2 +- .../pipeline_stable_diffusion_inpaint_legacy.py | 4 ++-- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/diffusers/pipelines/alt_diffusion/pipeline_alt_diffusion_img2img.py b/src/diffusers/pipelines/alt_diffusion/pipeline_alt_diffusion_img2img.py index 28aea4acc2b0..1d4e5fdef9f6 100644 --- a/src/diffusers/pipelines/alt_diffusion/pipeline_alt_diffusion_img2img.py +++ b/src/diffusers/pipelines/alt_diffusion/pipeline_alt_diffusion_img2img.py @@ -422,7 +422,7 @@ def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dt def __call__( self, prompt: Union[str, List[str]], - image: Union[torch.FloatTensor, PIL.Image.Image], + image: Union[torch.FloatTensor, PIL.Image.Image] = None, strength: float = 0.8, num_inference_steps: Optional[int] = 50, guidance_scale: Optional[float] = 7.5, diff --git a/src/diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion_superresolution.py b/src/diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion_superresolution.py index 09bdca54accf..e417424625f1 100644 --- a/src/diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion_superresolution.py +++ b/src/diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion_superresolution.py @@ -66,7 +66,7 @@ def __init__( @torch.no_grad() def __call__( self, - image: Union[torch.Tensor, PIL.Image.Image], + image: Union[torch.Tensor, PIL.Image.Image] = None, batch_size: Optional[int] = 1, num_inference_steps: Optional[int] = 100, eta: Optional[float] = 0.0, diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_cycle_diffusion.py b/src/diffusers/pipelines/stable_diffusion/pipeline_cycle_diffusion.py index 1639b723af99..0190b1bf373d 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_cycle_diffusion.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_cycle_diffusion.py @@ -462,7 +462,7 @@ def __call__( self, prompt: Union[str, List[str]], source_prompt: Union[str, List[str]], - image: Union[torch.FloatTensor, PIL.Image.Image], + image: Union[torch.FloatTensor, PIL.Image.Image] = None, strength: float = 0.8, num_inference_steps: Optional[int] = 50, guidance_scale: Optional[float] = 7.5, diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_img2img.py b/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_img2img.py index 2242d21b1d91..41128f29949e 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_img2img.py @@ -229,7 +229,7 @@ def _encode_prompt(self, prompt, num_images_per_prompt, do_classifier_free_guida def __call__( self, prompt: Union[str, List[str]], - image: Union[np.ndarray, PIL.Image.Image], + image: Union[np.ndarray, PIL.Image.Image] = None, strength: float = 0.8, num_inference_steps: Optional[int] = 50, guidance_scale: Optional[float] = 7.5, diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_inpaint_legacy.py b/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_inpaint_legacy.py index 84e85e51cca2..1b25f04d67e3 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_inpaint_legacy.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_inpaint_legacy.py @@ -228,8 +228,8 @@ def _encode_prompt(self, prompt, num_images_per_prompt, do_classifier_free_guida def __call__( self, prompt: Union[str, List[str]], - image: Union[np.ndarray, PIL.Image.Image], - mask_image: Union[np.ndarray, PIL.Image.Image], + image: Union[np.ndarray, PIL.Image.Image] = None, + mask_image: Union[np.ndarray, PIL.Image.Image] = None, strength: float = 0.8, num_inference_steps: Optional[int] = 50, guidance_scale: Optional[float] = 7.5, diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py index 2879fd275543..4e2f4de46923 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py @@ -427,7 +427,7 @@ def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dt def __call__( self, prompt: Union[str, List[str]], - image: Union[torch.FloatTensor, PIL.Image.Image], + image: Union[torch.FloatTensor, PIL.Image.Image] = None, strength: float = 0.8, num_inference_steps: Optional[int] = 50, guidance_scale: Optional[float] = 7.5, diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint_legacy.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint_legacy.py index a5416b30a98a..e3cbcc81ec0b 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint_legacy.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint_legacy.py @@ -425,8 +425,8 @@ def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dt def __call__( self, prompt: Union[str, List[str]], - image: Union[torch.FloatTensor, PIL.Image.Image], - mask_image: Union[torch.FloatTensor, PIL.Image.Image], + image: Union[torch.FloatTensor, PIL.Image.Image] = None, + mask_image: Union[torch.FloatTensor, PIL.Image.Image] = None, strength: float = 0.8, num_inference_steps: Optional[int] = 50, guidance_scale: Optional[float] = 7.5, From 626f86f48ba93ba72aed79b32ac2b5a9f3cb25f4 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Sat, 10 Dec 2022 17:03:03 +0000 Subject: [PATCH 2/6] fix some stuff --- .../pipeline_alt_diffusion_img2img.py | 25 ++++++++++++------- .../pipeline_cycle_diffusion.py | 13 ++++++---- .../pipeline_stable_diffusion_depth2img.py | 17 +++++++------ .../pipeline_stable_diffusion_img2img.py | 22 +++++++++------- ...ipeline_stable_diffusion_inpaint_legacy.py | 13 ++++++---- 5 files changed, 55 insertions(+), 35 deletions(-) diff --git a/src/diffusers/pipelines/alt_diffusion/pipeline_alt_diffusion_img2img.py b/src/diffusers/pipelines/alt_diffusion/pipeline_alt_diffusion_img2img.py index 1d4e5fdef9f6..29642aa87707 100644 --- a/src/diffusers/pipelines/alt_diffusion/pipeline_alt_diffusion_img2img.py +++ b/src/diffusers/pipelines/alt_diffusion/pipeline_alt_diffusion_img2img.py @@ -376,14 +376,20 @@ def check_inputs(self, prompt, strength, callback_steps): def get_timesteps(self, num_inference_steps, strength, device): # get the original timestep using init_timestep - offset = self.scheduler.config.get("steps_offset", 0) - init_timestep = int(num_inference_steps * strength) + offset - init_timestep = min(init_timestep, num_inference_steps) + if not strength < 1.0: + raise ValueError( + f"strength={strength} is too high for the original image to be taken into account. Make sure that" + " strength < 1.0." + ) + + init_timestep = int(num_inference_steps * strength) + + t_start = num_inference_steps - init_timestep - t_start = max(num_inference_steps - init_timestep + offset, 0) timesteps = self.scheduler.timesteps[t_start:] + latent_timestep = self.scheduler.timesteps[t_start - 1] - return timesteps, num_inference_steps - t_start + return timesteps, latent_timestep, num_inference_steps - t_start def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dtype, device, generator=None): image = image.to(device=device, dtype=dtype) @@ -410,11 +416,11 @@ def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dt init_latents = torch.cat([init_latents] * num_images_per_prompt, dim=0) # add noise to latents using the timesteps + torch.manual_seed(0) noise = torch.randn(init_latents.shape, generator=generator, device=device, dtype=dtype) # get latents - init_latents = self.scheduler.add_noise(init_latents, noise, timestep) - latents = init_latents + latents = self.scheduler.add_noise(init_latents, noise, timestep) return latents @@ -517,10 +523,11 @@ def __call__( # 5. set timesteps self.scheduler.set_timesteps(num_inference_steps, device=device) - timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device) - latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt) + timesteps, latent_timestep, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device) + latent_timestep = latent_timestep.repeat(batch_size * num_images_per_prompt) # 6. Prepare latent variables + torch.manual_seed(0) latents = self.prepare_latents( image, latent_timestep, batch_size, num_images_per_prompt, text_embeddings.dtype, device, generator ) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_cycle_diffusion.py b/src/diffusers/pipelines/stable_diffusion/pipeline_cycle_diffusion.py index 0190b1bf373d..09128377e385 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_cycle_diffusion.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_cycle_diffusion.py @@ -414,14 +414,17 @@ def decode_latents(self, latents): # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.StableDiffusionImg2ImgPipeline.get_timesteps def get_timesteps(self, num_inference_steps, strength, device): # get the original timestep using init_timestep - offset = self.scheduler.config.get("steps_offset", 0) - init_timestep = int(num_inference_steps * strength) + offset - init_timestep = min(init_timestep, num_inference_steps) + if not strength < 1.0: + raise ValueError(f"strength={strength} is too high for the original image to be taken into account. Make sure that strength < 1.0.") + + init_timestep = int(num_inference_steps * strength) + + t_start = num_inference_steps - init_timestep - t_start = max(num_inference_steps - init_timestep + offset, 0) timesteps = self.scheduler.timesteps[t_start:] + latent_timestep = self.scheduler.timesteps[t_start - 1] - return timesteps, num_inference_steps - t_start + return timesteps, latent_timestep, num_inference_steps - t_start def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dtype, device, generator=None): image = image.to(device=device, dtype=dtype) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_depth2img.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_depth2img.py index f8751410e933..6564bd64f744 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_depth2img.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_depth2img.py @@ -323,14 +323,17 @@ def check_inputs(self, prompt, strength, callback_steps): # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.StableDiffusionImg2ImgPipeline.get_timesteps def get_timesteps(self, num_inference_steps, strength, device): # get the original timestep using init_timestep - offset = self.scheduler.config.get("steps_offset", 0) - init_timestep = int(num_inference_steps * strength) + offset - init_timestep = min(init_timestep, num_inference_steps) + if not strength < 1.0: + raise ValueError(f"strength={strength} is too high for the original image to be taken into account. Make sure that strength < 1.0.") + + init_timestep = int(num_inference_steps * strength) + + t_start = num_inference_steps - init_timestep - t_start = max(num_inference_steps - init_timestep + offset, 0) timesteps = self.scheduler.timesteps[t_start:] + latent_timestep = self.scheduler.timesteps[t_start - 1] - return timesteps, num_inference_steps - t_start + return timesteps, latent_timestep, num_inference_steps - t_start # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.StableDiffusionImg2ImgPipeline.prepare_latents def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dtype, device, generator=None): @@ -358,11 +361,11 @@ def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dt init_latents = torch.cat([init_latents] * num_images_per_prompt, dim=0) # add noise to latents using the timesteps + torch.manual_seed(0) noise = torch.randn(init_latents.shape, generator=generator, device=device, dtype=dtype) # get latents - init_latents = self.scheduler.add_noise(init_latents, noise, timestep) - latents = init_latents + latents = self.scheduler.add_noise(init_latents, noise, timestep) return latents diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py index 4e2f4de46923..a241e7eb1f6b 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py @@ -381,14 +381,17 @@ def check_inputs(self, prompt, strength, callback_steps): def get_timesteps(self, num_inference_steps, strength, device): # get the original timestep using init_timestep - offset = self.scheduler.config.get("steps_offset", 0) - init_timestep = int(num_inference_steps * strength) + offset - init_timestep = min(init_timestep, num_inference_steps) + if not strength < 1.0: + raise ValueError(f"strength={strength} is too high for the original image to be taken into account. Make sure that strength < 1.0.") + + init_timestep = int(num_inference_steps * strength) + + t_start = num_inference_steps - init_timestep - t_start = max(num_inference_steps - init_timestep + offset, 0) timesteps = self.scheduler.timesteps[t_start:] + latent_timestep = self.scheduler.timesteps[t_start - 1] - return timesteps, num_inference_steps - t_start + return timesteps, latent_timestep, num_inference_steps - t_start def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dtype, device, generator=None): image = image.to(device=device, dtype=dtype) @@ -415,11 +418,11 @@ def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dt init_latents = torch.cat([init_latents] * num_images_per_prompt, dim=0) # add noise to latents using the timesteps + torch.manual_seed(0) noise = torch.randn(init_latents.shape, generator=generator, device=device, dtype=dtype) # get latents - init_latents = self.scheduler.add_noise(init_latents, noise, timestep) - latents = init_latents + latents = self.scheduler.add_noise(init_latents, noise, timestep) return latents @@ -522,10 +525,11 @@ def __call__( # 5. set timesteps self.scheduler.set_timesteps(num_inference_steps, device=device) - timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device) - latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt) + timesteps, latent_timestep, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device) + latent_timestep = latent_timestep.repeat(batch_size * num_images_per_prompt) # 6. Prepare latent variables + torch.manual_seed(0) latents = self.prepare_latents( image, latent_timestep, batch_size, num_images_per_prompt, text_embeddings.dtype, device, generator ) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint_legacy.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint_legacy.py index e3cbcc81ec0b..93b6b068f8e3 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint_legacy.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint_legacy.py @@ -396,14 +396,17 @@ def check_inputs(self, prompt, strength, callback_steps): # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.StableDiffusionImg2ImgPipeline.get_timesteps def get_timesteps(self, num_inference_steps, strength, device): # get the original timestep using init_timestep - offset = self.scheduler.config.get("steps_offset", 0) - init_timestep = int(num_inference_steps * strength) + offset - init_timestep = min(init_timestep, num_inference_steps) + if not strength < 1.0: + raise ValueError(f"strength={strength} is too high for the original image to be taken into account. Make sure that strength < 1.0.") + + init_timestep = int(num_inference_steps * strength) + + t_start = num_inference_steps - init_timestep - t_start = max(num_inference_steps - init_timestep + offset, 0) timesteps = self.scheduler.timesteps[t_start:] + latent_timestep = self.scheduler.timesteps[t_start - 1] - return timesteps, num_inference_steps - t_start + return timesteps, latent_timestep, num_inference_steps - t_start def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dtype, device, generator): image = image.to(device=self.device, dtype=dtype) From 1abcd21d905e59b8ec2380c75dcf1d5210135c8f Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Sat, 10 Dec 2022 18:05:20 +0100 Subject: [PATCH 3/6] Apply suggestions from code review --- .../pipelines/alt_diffusion/pipeline_alt_diffusion_img2img.py | 2 +- .../pipeline_latent_diffusion_superresolution.py | 2 +- .../pipelines/stable_diffusion/pipeline_cycle_diffusion.py | 2 +- .../pipeline_onnx_stable_diffusion_img2img.py | 2 +- .../pipeline_onnx_stable_diffusion_inpaint_legacy.py | 4 ++-- .../stable_diffusion/pipeline_stable_diffusion_img2img.py | 2 +- .../pipeline_stable_diffusion_inpaint_legacy.py | 4 ++-- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/diffusers/pipelines/alt_diffusion/pipeline_alt_diffusion_img2img.py b/src/diffusers/pipelines/alt_diffusion/pipeline_alt_diffusion_img2img.py index 29642aa87707..39ba0d54a500 100644 --- a/src/diffusers/pipelines/alt_diffusion/pipeline_alt_diffusion_img2img.py +++ b/src/diffusers/pipelines/alt_diffusion/pipeline_alt_diffusion_img2img.py @@ -428,7 +428,7 @@ def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dt def __call__( self, prompt: Union[str, List[str]], - image: Union[torch.FloatTensor, PIL.Image.Image] = None, + image: Union[torch.FloatTensor, PIL.Image.Image], strength: float = 0.8, num_inference_steps: Optional[int] = 50, guidance_scale: Optional[float] = 7.5, diff --git a/src/diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion_superresolution.py b/src/diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion_superresolution.py index e417424625f1..09bdca54accf 100644 --- a/src/diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion_superresolution.py +++ b/src/diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion_superresolution.py @@ -66,7 +66,7 @@ def __init__( @torch.no_grad() def __call__( self, - image: Union[torch.Tensor, PIL.Image.Image] = None, + image: Union[torch.Tensor, PIL.Image.Image], batch_size: Optional[int] = 1, num_inference_steps: Optional[int] = 100, eta: Optional[float] = 0.0, diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_cycle_diffusion.py b/src/diffusers/pipelines/stable_diffusion/pipeline_cycle_diffusion.py index 09128377e385..bcb09aa5fc4e 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_cycle_diffusion.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_cycle_diffusion.py @@ -465,7 +465,7 @@ def __call__( self, prompt: Union[str, List[str]], source_prompt: Union[str, List[str]], - image: Union[torch.FloatTensor, PIL.Image.Image] = None, + image: Union[torch.FloatTensor, PIL.Image.Image], strength: float = 0.8, num_inference_steps: Optional[int] = 50, guidance_scale: Optional[float] = 7.5, diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_img2img.py b/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_img2img.py index 41128f29949e..2242d21b1d91 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_img2img.py @@ -229,7 +229,7 @@ def _encode_prompt(self, prompt, num_images_per_prompt, do_classifier_free_guida def __call__( self, prompt: Union[str, List[str]], - image: Union[np.ndarray, PIL.Image.Image] = None, + image: Union[np.ndarray, PIL.Image.Image], strength: float = 0.8, num_inference_steps: Optional[int] = 50, guidance_scale: Optional[float] = 7.5, diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_inpaint_legacy.py b/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_inpaint_legacy.py index 1b25f04d67e3..84e85e51cca2 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_inpaint_legacy.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_inpaint_legacy.py @@ -228,8 +228,8 @@ def _encode_prompt(self, prompt, num_images_per_prompt, do_classifier_free_guida def __call__( self, prompt: Union[str, List[str]], - image: Union[np.ndarray, PIL.Image.Image] = None, - mask_image: Union[np.ndarray, PIL.Image.Image] = None, + image: Union[np.ndarray, PIL.Image.Image], + mask_image: Union[np.ndarray, PIL.Image.Image], strength: float = 0.8, num_inference_steps: Optional[int] = 50, guidance_scale: Optional[float] = 7.5, diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py index a241e7eb1f6b..1d87763fe1ea 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py @@ -430,7 +430,7 @@ def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dt def __call__( self, prompt: Union[str, List[str]], - image: Union[torch.FloatTensor, PIL.Image.Image] = None, + image: Union[torch.FloatTensor, PIL.Image.Image], strength: float = 0.8, num_inference_steps: Optional[int] = 50, guidance_scale: Optional[float] = 7.5, diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint_legacy.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint_legacy.py index 93b6b068f8e3..e922549082eb 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint_legacy.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint_legacy.py @@ -428,8 +428,8 @@ def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dt def __call__( self, prompt: Union[str, List[str]], - image: Union[torch.FloatTensor, PIL.Image.Image] = None, - mask_image: Union[torch.FloatTensor, PIL.Image.Image] = None, + image: Union[torch.FloatTensor, PIL.Image.Image], + mask_image: Union[torch.FloatTensor, PIL.Image.Image], strength: float = 0.8, num_inference_steps: Optional[int] = 50, guidance_scale: Optional[float] = 7.5, From c6598d23d9a744f875d4e8006ebb75235fbd291f Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Sat, 10 Dec 2022 17:08:53 +0000 Subject: [PATCH 4/6] upload --- .../stable_diffusion/pipeline_cycle_diffusion.py | 9 ++++++--- .../pipeline_stable_diffusion_depth2img.py | 9 ++++++--- .../pipeline_stable_diffusion_img2img.py | 5 ++++- .../pipeline_stable_diffusion_inpaint_legacy.py | 9 ++++++--- 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_cycle_diffusion.py b/src/diffusers/pipelines/stable_diffusion/pipeline_cycle_diffusion.py index 09128377e385..3d485aebc60b 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_cycle_diffusion.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_cycle_diffusion.py @@ -415,7 +415,10 @@ def decode_latents(self, latents): def get_timesteps(self, num_inference_steps, strength, device): # get the original timestep using init_timestep if not strength < 1.0: - raise ValueError(f"strength={strength} is too high for the original image to be taken into account. Make sure that strength < 1.0.") + raise ValueError( + f"strength={strength} is too high for the original image to be taken into account. Make sure that" + " strength < 1.0." + ) init_timestep = int(num_inference_steps * strength) @@ -561,8 +564,8 @@ def __call__( # 5. Prepare timesteps self.scheduler.set_timesteps(num_inference_steps, device=device) - timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device) - latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt) + timesteps, num_inference_steps, latent_timestep = self.get_timesteps(num_inference_steps, strength, device) + latent_timestep = latent_timestep.repeat(batch_size * num_images_per_prompt) # 6. Prepare latent variables latents, clean_latents = self.prepare_latents( diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_depth2img.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_depth2img.py index 6564bd64f744..66432f166816 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_depth2img.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_depth2img.py @@ -324,7 +324,10 @@ def check_inputs(self, prompt, strength, callback_steps): def get_timesteps(self, num_inference_steps, strength, device): # get the original timestep using init_timestep if not strength < 1.0: - raise ValueError(f"strength={strength} is too high for the original image to be taken into account. Make sure that strength < 1.0.") + raise ValueError( + f"strength={strength} is too high for the original image to be taken into account. Make sure that" + " strength < 1.0." + ) init_timestep = int(num_inference_steps * strength) @@ -517,8 +520,8 @@ def __call__( # 6. set timesteps self.scheduler.set_timesteps(num_inference_steps, device=device) - timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device) - latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt) + timesteps, num_inference_steps, latent_timestep = self.get_timesteps(num_inference_steps, strength, device) + latent_timestep = latent_timestep.repeat(batch_size * num_images_per_prompt) # 7. Prepare latent variables latents = self.prepare_latents( diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py index a241e7eb1f6b..6edf22c64d7c 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py @@ -382,7 +382,10 @@ def check_inputs(self, prompt, strength, callback_steps): def get_timesteps(self, num_inference_steps, strength, device): # get the original timestep using init_timestep if not strength < 1.0: - raise ValueError(f"strength={strength} is too high for the original image to be taken into account. Make sure that strength < 1.0.") + raise ValueError( + f"strength={strength} is too high for the original image to be taken into account. Make sure that" + " strength < 1.0." + ) init_timestep = int(num_inference_steps * strength) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint_legacy.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint_legacy.py index 93b6b068f8e3..72ed2ffb79ec 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint_legacy.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint_legacy.py @@ -397,7 +397,10 @@ def check_inputs(self, prompt, strength, callback_steps): def get_timesteps(self, num_inference_steps, strength, device): # get the original timestep using init_timestep if not strength < 1.0: - raise ValueError(f"strength={strength} is too high for the original image to be taken into account. Make sure that strength < 1.0.") + raise ValueError( + f"strength={strength} is too high for the original image to be taken into account. Make sure that" + " strength < 1.0." + ) init_timestep = int(num_inference_steps * strength) @@ -531,8 +534,8 @@ def __call__( # 5. set timesteps self.scheduler.set_timesteps(num_inference_steps, device=device) - timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device) - latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt) + timesteps, num_inference_steps, latent_timestep = self.get_timesteps(num_inference_steps, strength, device) + latent_timestep = latent_timestep.repeat(batch_size * num_images_per_prompt) # 6. Prepare latent variables # encode the init image into latents and scale the latents From 1c0b63c0f62e2059c605f4266afd85066478c62b Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Sat, 10 Dec 2022 17:10:45 +0000 Subject: [PATCH 5/6] up --- .../pipelines/alt_diffusion/pipeline_alt_diffusion_img2img.py | 1 - .../stable_diffusion/pipeline_stable_diffusion_img2img.py | 1 - 2 files changed, 2 deletions(-) diff --git a/src/diffusers/pipelines/alt_diffusion/pipeline_alt_diffusion_img2img.py b/src/diffusers/pipelines/alt_diffusion/pipeline_alt_diffusion_img2img.py index 39ba0d54a500..d11864916091 100644 --- a/src/diffusers/pipelines/alt_diffusion/pipeline_alt_diffusion_img2img.py +++ b/src/diffusers/pipelines/alt_diffusion/pipeline_alt_diffusion_img2img.py @@ -527,7 +527,6 @@ def __call__( latent_timestep = latent_timestep.repeat(batch_size * num_images_per_prompt) # 6. Prepare latent variables - torch.manual_seed(0) latents = self.prepare_latents( image, latent_timestep, batch_size, num_images_per_prompt, text_embeddings.dtype, device, generator ) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py index aa00c73c889a..62347caee5d0 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py @@ -532,7 +532,6 @@ def __call__( latent_timestep = latent_timestep.repeat(batch_size * num_images_per_prompt) # 6. Prepare latent variables - torch.manual_seed(0) latents = self.prepare_latents( image, latent_timestep, batch_size, num_images_per_prompt, text_embeddings.dtype, device, generator ) From 2b76136885ab1538fcbb47e7100313cc2abd0179 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Sat, 10 Dec 2022 17:28:02 +0000 Subject: [PATCH 6/6] up --- .../pipelines/stable_diffusion/pipeline_cycle_diffusion.py | 2 +- .../stable_diffusion/pipeline_stable_diffusion_depth2img.py | 2 +- .../pipeline_stable_diffusion_inpaint_legacy.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_cycle_diffusion.py b/src/diffusers/pipelines/stable_diffusion/pipeline_cycle_diffusion.py index 0ceb2eb3d813..29b148a91805 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_cycle_diffusion.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_cycle_diffusion.py @@ -564,7 +564,7 @@ def __call__( # 5. Prepare timesteps self.scheduler.set_timesteps(num_inference_steps, device=device) - timesteps, num_inference_steps, latent_timestep = self.get_timesteps(num_inference_steps, strength, device) + timesteps, latent_timestep, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device) latent_timestep = latent_timestep.repeat(batch_size * num_images_per_prompt) # 6. Prepare latent variables diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_depth2img.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_depth2img.py index 66432f166816..78040b1bae6a 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_depth2img.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_depth2img.py @@ -520,7 +520,7 @@ def __call__( # 6. set timesteps self.scheduler.set_timesteps(num_inference_steps, device=device) - timesteps, num_inference_steps, latent_timestep = self.get_timesteps(num_inference_steps, strength, device) + timesteps, latent_timestep, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device) latent_timestep = latent_timestep.repeat(batch_size * num_images_per_prompt) # 7. Prepare latent variables diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint_legacy.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint_legacy.py index ec360a3cca8b..f08c15227e66 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint_legacy.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint_legacy.py @@ -534,7 +534,7 @@ def __call__( # 5. set timesteps self.scheduler.set_timesteps(num_inference_steps, device=device) - timesteps, num_inference_steps, latent_timestep = self.get_timesteps(num_inference_steps, strength, device) + timesteps, latent_timestep, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device) latent_timestep = latent_timestep.repeat(batch_size * num_images_per_prompt) # 6. Prepare latent variables