vixy/dreamtail
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix: use s_scale=0 when IP-Adapter loaded but no face requested

Browse Source 
When IP-Adapter FaceID is initialized, it modifies the pipeline's UNet
cross-attention layers. Calling raw pipeline() without face embeddings
leaves these layers in a broken state, causing corrupted output.

Solution: When IP-Adapter is loaded but no face_image provided, call
ip_model.generate() with s_scale=0.0 and zero embeddings to properly
disable face conditioning while satisfying the modified layers.
This commit is contained in:
Vixy
2026-01-01 22:01:18 -06:00
parent 28e00e88e6
commit f5e37bc5b2
2 changed files with 46 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
requirements.txt
View File

@@ -28,6 +28,7 @@ insightface==0.7.3
# IP-Adapter from GitHub # IP-Adapter from GitHub
git+https://github.com/tencent-ailab/IP-Adapter.git git+https://github.com/tencent-ailab/IP-Adapter.git
einops # Required by IP-Adapter
# Utilities # Utilities
pydantic==2.6.0 pydantic==2.6.0

68
worker/generator.py
View File

@@ -289,30 +289,52 @@ class SDXLGenerator:
)[0] )[0]
) )
else: else:
# Progress callback wrapper (only for standard pipeline) # Check if IP-Adapter is loaded - if so, we must use it with s_scale=0
def callback_wrapper(step: int, timestep: int, latents: torch.FloatTensor): # to avoid corrupted output from dangling adapter layers
if progress_callback: if self.ip_adapter_loaded:
progress = int((step / num_inference_steps) * 100) logger.info("IP-Adapter loaded but no face requested, using s_scale=0")
try: # Create zero embedding (512-dim for FaceID)
asyncio.create_task(progress_callback(progress)) zero_embed = torch.zeros((1, 512), device=self.device, dtype=torch.float16)
except:
pass image = await loop.run_in_executor(
None,
lambda: self.ip_model.generate(
prompt=prompt,
negative_prompt=negative_prompt,
faceid_embeds=zero_embed,
width=width,
height=height,
num_inference_steps=num_inference_steps,
guidance_scale=guidance_scale,
num_samples=1,
seed=seed,
s_scale=0.0, # Disable face conditioning
)[0]
)
else:
# Standard generation - IP-Adapter not loaded
def callback_wrapper(step: int, timestep: int, latents: torch.FloatTensor):
if progress_callback:
progress = int((step / num_inference_steps) * 100)
try:
asyncio.create_task(progress_callback(progress))
except:
pass
# Standard generation without face lock image = await loop.run_in_executor(
image = await loop.run_in_executor( None,
None, lambda: self.pipeline(
lambda: self.pipeline( prompt=prompt,
prompt=prompt, negative_prompt=negative_prompt,
negative_prompt=negative_prompt, width=width,
width=width, height=height,
height=height, num_inference_steps=num_inference_steps,
num_inference_steps=num_inference_steps, guidance_scale=guidance_scale,
guidance_scale=guidance_scale, generator=generator,
generator=generator, callback=callback_wrapper,
callback=callback_wrapper, callback_steps=1
callback_steps=1 ).images[0]
).images[0] )
)
logger.info("Image generated successfully") logger.info("Image generated successfully")
return image return image