diff --git a/src/diffusers/pipelines/alt_diffusion/pipeline_alt_diffusion_img2img.py b/src/diffusers/pipelines/alt_diffusion/pipeline_alt_diffusion_img2img.py index 28aea4acc2b0..d11864916091 100644 --- a/src/diffusers/pipelines/alt_diffusion/pipeline_alt_diffusion_img2img.py +++ b/src/diffusers/pipelines/alt_diffusion/pipeline_alt_diffusion_img2img.py @@ -376,14 +376,20 @@ def check_inputs(self, prompt, strength, callback_steps): def get_timesteps(self, num_inference_steps, strength, device): # get the original timestep using init_timestep - offset = self.scheduler.config.get("steps_offset", 0) - init_timestep = int(num_inference_steps * strength) + offset - init_timestep = min(init_timestep, num_inference_steps) + if not strength < 1.0: + raise ValueError( + f"strength={strength} is too high for the original image to be taken into account. Make sure that" + " strength < 1.0." + ) + + init_timestep = int(num_inference_steps * strength) + + t_start = num_inference_steps - init_timestep - t_start = max(num_inference_steps - init_timestep + offset, 0) timesteps = self.scheduler.timesteps[t_start:] + latent_timestep = self.scheduler.timesteps[t_start - 1] - return timesteps, num_inference_steps - t_start + return timesteps, latent_timestep, num_inference_steps - t_start def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dtype, device, generator=None): image = image.to(device=device, dtype=dtype) @@ -410,11 +416,11 @@ def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dt init_latents = torch.cat([init_latents] * num_images_per_prompt, dim=0) # add noise to latents using the timesteps + torch.manual_seed(0) noise = torch.randn(init_latents.shape, generator=generator, device=device, dtype=dtype) # get latents - init_latents = self.scheduler.add_noise(init_latents, noise, timestep) - latents = init_latents + latents = self.scheduler.add_noise(init_latents, noise, timestep) return latents @@ -517,8 +523,8 @@ def __call__( # 5. set timesteps self.scheduler.set_timesteps(num_inference_steps, device=device) - timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device) - latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt) + timesteps, latent_timestep, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device) + latent_timestep = latent_timestep.repeat(batch_size * num_images_per_prompt) # 6. Prepare latent variables latents = self.prepare_latents( diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_cycle_diffusion.py b/src/diffusers/pipelines/stable_diffusion/pipeline_cycle_diffusion.py index 1639b723af99..29b148a91805 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_cycle_diffusion.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_cycle_diffusion.py @@ -414,14 +414,20 @@ def decode_latents(self, latents): # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.StableDiffusionImg2ImgPipeline.get_timesteps def get_timesteps(self, num_inference_steps, strength, device): # get the original timestep using init_timestep - offset = self.scheduler.config.get("steps_offset", 0) - init_timestep = int(num_inference_steps * strength) + offset - init_timestep = min(init_timestep, num_inference_steps) + if not strength < 1.0: + raise ValueError( + f"strength={strength} is too high for the original image to be taken into account. Make sure that" + " strength < 1.0." + ) + + init_timestep = int(num_inference_steps * strength) + + t_start = num_inference_steps - init_timestep - t_start = max(num_inference_steps - init_timestep + offset, 0) timesteps = self.scheduler.timesteps[t_start:] + latent_timestep = self.scheduler.timesteps[t_start - 1] - return timesteps, num_inference_steps - t_start + return timesteps, latent_timestep, num_inference_steps - t_start def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dtype, device, generator=None): image = image.to(device=device, dtype=dtype) @@ -558,8 +564,8 @@ def __call__( # 5. Prepare timesteps self.scheduler.set_timesteps(num_inference_steps, device=device) - timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device) - latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt) + timesteps, latent_timestep, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device) + latent_timestep = latent_timestep.repeat(batch_size * num_images_per_prompt) # 6. Prepare latent variables latents, clean_latents = self.prepare_latents( diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_depth2img.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_depth2img.py index f8751410e933..78040b1bae6a 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_depth2img.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_depth2img.py @@ -323,14 +323,20 @@ def check_inputs(self, prompt, strength, callback_steps): # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.StableDiffusionImg2ImgPipeline.get_timesteps def get_timesteps(self, num_inference_steps, strength, device): # get the original timestep using init_timestep - offset = self.scheduler.config.get("steps_offset", 0) - init_timestep = int(num_inference_steps * strength) + offset - init_timestep = min(init_timestep, num_inference_steps) + if not strength < 1.0: + raise ValueError( + f"strength={strength} is too high for the original image to be taken into account. Make sure that" + " strength < 1.0." + ) + + init_timestep = int(num_inference_steps * strength) + + t_start = num_inference_steps - init_timestep - t_start = max(num_inference_steps - init_timestep + offset, 0) timesteps = self.scheduler.timesteps[t_start:] + latent_timestep = self.scheduler.timesteps[t_start - 1] - return timesteps, num_inference_steps - t_start + return timesteps, latent_timestep, num_inference_steps - t_start # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.StableDiffusionImg2ImgPipeline.prepare_latents def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dtype, device, generator=None): @@ -358,11 +364,11 @@ def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dt init_latents = torch.cat([init_latents] * num_images_per_prompt, dim=0) # add noise to latents using the timesteps + torch.manual_seed(0) noise = torch.randn(init_latents.shape, generator=generator, device=device, dtype=dtype) # get latents - init_latents = self.scheduler.add_noise(init_latents, noise, timestep) - latents = init_latents + latents = self.scheduler.add_noise(init_latents, noise, timestep) return latents @@ -514,8 +520,8 @@ def __call__( # 6. set timesteps self.scheduler.set_timesteps(num_inference_steps, device=device) - timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device) - latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt) + timesteps, latent_timestep, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device) + latent_timestep = latent_timestep.repeat(batch_size * num_images_per_prompt) # 7. Prepare latent variables latents = self.prepare_latents( diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py index 2879fd275543..62347caee5d0 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py @@ -381,14 +381,20 @@ def check_inputs(self, prompt, strength, callback_steps): def get_timesteps(self, num_inference_steps, strength, device): # get the original timestep using init_timestep - offset = self.scheduler.config.get("steps_offset", 0) - init_timestep = int(num_inference_steps * strength) + offset - init_timestep = min(init_timestep, num_inference_steps) + if not strength < 1.0: + raise ValueError( + f"strength={strength} is too high for the original image to be taken into account. Make sure that" + " strength < 1.0." + ) + + init_timestep = int(num_inference_steps * strength) + + t_start = num_inference_steps - init_timestep - t_start = max(num_inference_steps - init_timestep + offset, 0) timesteps = self.scheduler.timesteps[t_start:] + latent_timestep = self.scheduler.timesteps[t_start - 1] - return timesteps, num_inference_steps - t_start + return timesteps, latent_timestep, num_inference_steps - t_start def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dtype, device, generator=None): image = image.to(device=device, dtype=dtype) @@ -415,11 +421,11 @@ def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dt init_latents = torch.cat([init_latents] * num_images_per_prompt, dim=0) # add noise to latents using the timesteps + torch.manual_seed(0) noise = torch.randn(init_latents.shape, generator=generator, device=device, dtype=dtype) # get latents - init_latents = self.scheduler.add_noise(init_latents, noise, timestep) - latents = init_latents + latents = self.scheduler.add_noise(init_latents, noise, timestep) return latents @@ -522,8 +528,8 @@ def __call__( # 5. set timesteps self.scheduler.set_timesteps(num_inference_steps, device=device) - timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device) - latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt) + timesteps, latent_timestep, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device) + latent_timestep = latent_timestep.repeat(batch_size * num_images_per_prompt) # 6. Prepare latent variables latents = self.prepare_latents( diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint_legacy.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint_legacy.py index a5416b30a98a..f08c15227e66 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint_legacy.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint_legacy.py @@ -396,14 +396,20 @@ def check_inputs(self, prompt, strength, callback_steps): # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.StableDiffusionImg2ImgPipeline.get_timesteps def get_timesteps(self, num_inference_steps, strength, device): # get the original timestep using init_timestep - offset = self.scheduler.config.get("steps_offset", 0) - init_timestep = int(num_inference_steps * strength) + offset - init_timestep = min(init_timestep, num_inference_steps) + if not strength < 1.0: + raise ValueError( + f"strength={strength} is too high for the original image to be taken into account. Make sure that" + " strength < 1.0." + ) + + init_timestep = int(num_inference_steps * strength) + + t_start = num_inference_steps - init_timestep - t_start = max(num_inference_steps - init_timestep + offset, 0) timesteps = self.scheduler.timesteps[t_start:] + latent_timestep = self.scheduler.timesteps[t_start - 1] - return timesteps, num_inference_steps - t_start + return timesteps, latent_timestep, num_inference_steps - t_start def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dtype, device, generator): image = image.to(device=self.device, dtype=dtype) @@ -528,8 +534,8 @@ def __call__( # 5. set timesteps self.scheduler.set_timesteps(num_inference_steps, device=device) - timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device) - latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt) + timesteps, latent_timestep, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device) + latent_timestep = latent_timestep.repeat(batch_size * num_images_per_prompt) # 6. Prepare latent variables # encode the init image into latents and scale the latents