Skip to content

🚨🚨 Difference between CompVis/img2img and diffusers 🚨🚨 #1652

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
patrickvonplaten wants to merge 7 commits into from
Closed

🚨🚨 Difference between CompVis/img2img and diffusers 🚨🚨 #1652

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
53fc6f1
Deprecate init image correctl
patrickvonplaten Dec 10, 2022
626f86f
fix some stuff
patrickvonplaten Dec 10, 2022
1abcd21
Apply suggestions from code review
patrickvonplaten Dec 10, 2022
c6598d2
upload
patrickvonplaten Dec 10, 2022
bae06d8
Merge branch 'correct_img2img' of https://github.com/huggingface/diff…
patrickvonplaten Dec 10, 2022
1c0b63c
up
patrickvonplaten Dec 10, 2022
2b76136
up
patrickvonplaten Dec 10, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 15 additions & 9 deletions src/diffusers/pipelines/alt_diffusion/pipeline_alt_diffusion_img2img.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -376,14 +376,20 @@ def check_inputs(self, prompt, strength, callback_steps):

def get_timesteps(self, num_inference_steps, strength, device):
# get the original timestep using init_timestep
offset = self.scheduler.config.get("steps_offset", 0)
init_timestep = int(num_inference_steps * strength) + offset
init_timestep = min(init_timestep, num_inference_steps)
if not strength < 1.0:
raise ValueError(
f"strength={strength} is too high for the original image to be taken into account. Make sure that"
" strength < 1.0."
)

init_timestep = int(num_inference_steps * strength)

t_start = num_inference_steps - init_timestep

t_start = max(num_inference_steps - init_timestep + offset, 0)
timesteps = self.scheduler.timesteps[t_start:]
latent_timestep = self.scheduler.timesteps[t_start - 1]

return timesteps, num_inference_steps - t_start
return timesteps, latent_timestep, num_inference_steps - t_start

def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dtype, device, generator=None):
image = image.to(device=device, dtype=dtype)
Expand All @@ -410,11 +416,11 @@ def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dt
init_latents = torch.cat([init_latents] * num_images_per_prompt, dim=0)

# add noise to latents using the timesteps
torch.manual_seed(0)
noise = torch.randn(init_latents.shape, generator=generator, device=device, dtype=dtype)

# get latents
init_latents = self.scheduler.add_noise(init_latents, noise, timestep)
latents = init_latents
latents = self.scheduler.add_noise(init_latents, noise, timestep)

return latents

Expand Down Expand Up @@ -517,8 +523,8 @@ def __call__(

# 5. set timesteps
self.scheduler.set_timesteps(num_inference_steps, device=device)
timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device)
latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt)
timesteps, latent_timestep, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device)
latent_timestep = latent_timestep.repeat(batch_size * num_images_per_prompt)

# 6. Prepare latent variables
latents = self.prepare_latents(
Expand Down
20 changes: 13 additions & 7 deletions src/diffusers/pipelines/stable_diffusion/pipeline_cycle_diffusion.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -414,14 +414,20 @@ def decode_latents(self, latents):
# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.StableDiffusionImg2ImgPipeline.get_timesteps
def get_timesteps(self, num_inference_steps, strength, device):
# get the original timestep using init_timestep
offset = self.scheduler.config.get("steps_offset", 0)
init_timestep = int(num_inference_steps * strength) + offset
init_timestep = min(init_timestep, num_inference_steps)
if not strength < 1.0:
raise ValueError(
f"strength={strength} is too high for the original image to be taken into account. Make sure that"
" strength < 1.0."
)

init_timestep = int(num_inference_steps * strength)

t_start = num_inference_steps - init_timestep

t_start = max(num_inference_steps - init_timestep + offset, 0)
timesteps = self.scheduler.timesteps[t_start:]
latent_timestep = self.scheduler.timesteps[t_start - 1]

return timesteps, num_inference_steps - t_start
return timesteps, latent_timestep, num_inference_steps - t_start

def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dtype, device, generator=None):
image = image.to(device=device, dtype=dtype)
Expand Down Expand Up @@ -558,8 +564,8 @@ def __call__(

# 5. Prepare timesteps
self.scheduler.set_timesteps(num_inference_steps, device=device)
timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device)
latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt)
timesteps, latent_timestep, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device)
latent_timestep = latent_timestep.repeat(batch_size * num_images_per_prompt)

# 6. Prepare latent variables
latents, clean_latents = self.prepare_latents(
Expand Down
24 changes: 15 additions & 9 deletions src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_depth2img.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -323,14 +323,20 @@ def check_inputs(self, prompt, strength, callback_steps):
# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.StableDiffusionImg2ImgPipeline.get_timesteps
def get_timesteps(self, num_inference_steps, strength, device):
# get the original timestep using init_timestep
offset = self.scheduler.config.get("steps_offset", 0)
init_timestep = int(num_inference_steps * strength) + offset
init_timestep = min(init_timestep, num_inference_steps)
if not strength < 1.0:
raise ValueError(
f"strength={strength} is too high for the original image to be taken into account. Make sure that"
" strength < 1.0."
)

init_timestep = int(num_inference_steps * strength)

t_start = num_inference_steps - init_timestep

t_start = max(num_inference_steps - init_timestep + offset, 0)
timesteps = self.scheduler.timesteps[t_start:]
latent_timestep = self.scheduler.timesteps[t_start - 1]

return timesteps, num_inference_steps - t_start
return timesteps, latent_timestep, num_inference_steps - t_start

# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.StableDiffusionImg2ImgPipeline.prepare_latents
def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dtype, device, generator=None):
Expand Down Expand Up @@ -358,11 +364,11 @@ def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dt
init_latents = torch.cat([init_latents] * num_images_per_prompt, dim=0)

# add noise to latents using the timesteps
torch.manual_seed(0)
noise = torch.randn(init_latents.shape, generator=generator, device=device, dtype=dtype)

# get latents
init_latents = self.scheduler.add_noise(init_latents, noise, timestep)
latents = init_latents
latents = self.scheduler.add_noise(init_latents, noise, timestep)

return latents

Expand Down Expand Up @@ -514,8 +520,8 @@ def __call__(

# 6. set timesteps
self.scheduler.set_timesteps(num_inference_steps, device=device)
timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device)
latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt)
timesteps, latent_timestep, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device)
latent_timestep = latent_timestep.repeat(batch_size * num_images_per_prompt)

# 7. Prepare latent variables
latents = self.prepare_latents(
Expand Down
24 changes: 15 additions & 9 deletions src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -381,14 +381,20 @@ def check_inputs(self, prompt, strength, callback_steps):

def get_timesteps(self, num_inference_steps, strength, device):
# get the original timestep using init_timestep
offset = self.scheduler.config.get("steps_offset", 0)
init_timestep = int(num_inference_steps * strength) + offset
init_timestep = min(init_timestep, num_inference_steps)
if not strength < 1.0:
raise ValueError(
f"strength={strength} is too high for the original image to be taken into account. Make sure that"
" strength < 1.0."
)

init_timestep = int(num_inference_steps * strength)

t_start = num_inference_steps - init_timestep

t_start = max(num_inference_steps - init_timestep + offset, 0)
timesteps = self.scheduler.timesteps[t_start:]
latent_timestep = self.scheduler.timesteps[t_start - 1]

return timesteps, num_inference_steps - t_start
return timesteps, latent_timestep, num_inference_steps - t_start

def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dtype, device, generator=None):
image = image.to(device=device, dtype=dtype)
Expand All @@ -415,11 +421,11 @@ def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dt
init_latents = torch.cat([init_latents] * num_images_per_prompt, dim=0)

# add noise to latents using the timesteps
torch.manual_seed(0)
noise = torch.randn(init_latents.shape, generator=generator, device=device, dtype=dtype)

# get latents
init_latents = self.scheduler.add_noise(init_latents, noise, timestep)
latents = init_latents
latents = self.scheduler.add_noise(init_latents, noise, timestep)

return latents

Expand Down Expand Up @@ -522,8 +528,8 @@ def __call__(

# 5. set timesteps
self.scheduler.set_timesteps(num_inference_steps, device=device)
timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device)
latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt)
timesteps, latent_timestep, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device)
latent_timestep = latent_timestep.repeat(batch_size * num_images_per_prompt)

# 6. Prepare latent variables
latents = self.prepare_latents(
Expand Down
20 changes: 13 additions & 7 deletions src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint_legacy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -396,14 +396,20 @@ def check_inputs(self, prompt, strength, callback_steps):
# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.StableDiffusionImg2ImgPipeline.get_timesteps
def get_timesteps(self, num_inference_steps, strength, device):
# get the original timestep using init_timestep
offset = self.scheduler.config.get("steps_offset", 0)
init_timestep = int(num_inference_steps * strength) + offset
init_timestep = min(init_timestep, num_inference_steps)
if not strength < 1.0:
raise ValueError(
f"strength={strength} is too high for the original image to be taken into account. Make sure that"
" strength < 1.0."
)

init_timestep = int(num_inference_steps * strength)

t_start = num_inference_steps - init_timestep

t_start = max(num_inference_steps - init_timestep + offset, 0)
timesteps = self.scheduler.timesteps[t_start:]
latent_timestep = self.scheduler.timesteps[t_start - 1]

return timesteps, num_inference_steps - t_start
return timesteps, latent_timestep, num_inference_steps - t_start

def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dtype, device, generator):
image = image.to(device=self.device, dtype=dtype)
Expand Down Expand Up @@ -528,8 +534,8 @@ def __call__(

# 5. set timesteps
self.scheduler.set_timesteps(num_inference_steps, device=device)
timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device)
latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt)
timesteps, latent_timestep, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device)
latent_timestep = latent_timestep.repeat(batch_size * num_images_per_prompt)

# 6. Prepare latent variables
# encode the init image into latents and scale the latents
Expand Down