Fix: use s_scale=0 when IP-Adapter loaded but no face requested
When IP-Adapter FaceID is initialized, it modifies the pipeline's UNet cross-attention layers. Calling raw pipeline() without face embeddings leaves these layers in a broken state, causing corrupted output. Solution: When IP-Adapter is loaded but no face_image provided, call ip_model.generate() with s_scale=0.0 and zero embeddings to properly disable face conditioning while satisfying the modified layers.
This commit is contained in:
@@ -289,30 +289,52 @@ class SDXLGenerator:
|
||||
)[0]
|
||||
)
|
||||
else:
|
||||
# Progress callback wrapper (only for standard pipeline)
|
||||
def callback_wrapper(step: int, timestep: int, latents: torch.FloatTensor):
|
||||
if progress_callback:
|
||||
progress = int((step / num_inference_steps) * 100)
|
||||
try:
|
||||
asyncio.create_task(progress_callback(progress))
|
||||
except:
|
||||
pass
|
||||
# Check if IP-Adapter is loaded - if so, we must use it with s_scale=0
|
||||
# to avoid corrupted output from dangling adapter layers
|
||||
if self.ip_adapter_loaded:
|
||||
logger.info("IP-Adapter loaded but no face requested, using s_scale=0")
|
||||
# Create zero embedding (512-dim for FaceID)
|
||||
zero_embed = torch.zeros((1, 512), device=self.device, dtype=torch.float16)
|
||||
|
||||
image = await loop.run_in_executor(
|
||||
None,
|
||||
lambda: self.ip_model.generate(
|
||||
prompt=prompt,
|
||||
negative_prompt=negative_prompt,
|
||||
faceid_embeds=zero_embed,
|
||||
width=width,
|
||||
height=height,
|
||||
num_inference_steps=num_inference_steps,
|
||||
guidance_scale=guidance_scale,
|
||||
num_samples=1,
|
||||
seed=seed,
|
||||
s_scale=0.0, # Disable face conditioning
|
||||
)[0]
|
||||
)
|
||||
else:
|
||||
# Standard generation - IP-Adapter not loaded
|
||||
def callback_wrapper(step: int, timestep: int, latents: torch.FloatTensor):
|
||||
if progress_callback:
|
||||
progress = int((step / num_inference_steps) * 100)
|
||||
try:
|
||||
asyncio.create_task(progress_callback(progress))
|
||||
except:
|
||||
pass
|
||||
|
||||
# Standard generation without face lock
|
||||
image = await loop.run_in_executor(
|
||||
None,
|
||||
lambda: self.pipeline(
|
||||
prompt=prompt,
|
||||
negative_prompt=negative_prompt,
|
||||
width=width,
|
||||
height=height,
|
||||
num_inference_steps=num_inference_steps,
|
||||
guidance_scale=guidance_scale,
|
||||
generator=generator,
|
||||
callback=callback_wrapper,
|
||||
callback_steps=1
|
||||
).images[0]
|
||||
)
|
||||
image = await loop.run_in_executor(
|
||||
None,
|
||||
lambda: self.pipeline(
|
||||
prompt=prompt,
|
||||
negative_prompt=negative_prompt,
|
||||
width=width,
|
||||
height=height,
|
||||
num_inference_steps=num_inference_steps,
|
||||
guidance_scale=guidance_scale,
|
||||
generator=generator,
|
||||
callback=callback_wrapper,
|
||||
callback_steps=1
|
||||
).images[0]
|
||||
)
|
||||
|
||||
logger.info("Image generated successfully")
|
||||
return image
|
||||
|
||||
Reference in New Issue
Block a user