|
|
--- |
|
|
license: |
|
|
- apache-2.0 |
|
|
- other |
|
|
license_name: flux-1-dev-non-commercial-license |
|
|
license_link: https://huggingface.co/black-forest-labs/FLUX.1-dev/blob/main/LICENSE.md |
|
|
library_name: diffusers |
|
|
pipeline_tag: text-to-image |
|
|
datasets: |
|
|
- SA1B |
|
|
- opendiffusionai/laion2b-squareish-1024px |
|
|
base_model: |
|
|
- jimmycarter/LibreFLUX |
|
|
--- |
|
|
# LibreFLUX-IP-Adapter-ControlNet |
|
|
 |
|
|
|
|
|
This model/pipeline combines my [LibreFlux-IP-Adapter](https://huggingface.co/neuralvfx/LibreFlux-IP-Adapter) and [LibreFlux ControlNet](https://huggingface.co/neuralvfx/LibreFlux-ControlNet), into a single pipeline. [LibreFLUX](https://huggingface.co/jimmycarter/LibreFLUX) is used as the underlying Transformer model. |
|
|
|
|
|
# How does this relate to LibreFLUX? |
|
|
- Base model is [LibreFLUX](https://huggingface.co/jimmycarter/LibreFLUX) |
|
|
- Trained in same non-distilled fashion |
|
|
- Uses Attention Masking |
|
|
- Uses CFG during Inference |
|
|
|
|
|
# Compatibility |
|
|
```py |
|
|
pip install -U diffusers==0.35.2 |
|
|
pip install -U transformers==4.57.1 |
|
|
``` |
|
|
|
|
|
Low VRAM: |
|
|
```py |
|
|
pip install optimum.quanto |
|
|
``` |
|
|
|
|
|
|
|
|
# Load Pipeline |
|
|
```py |
|
|
import torch |
|
|
from diffusers import DiffusionPipeline |
|
|
from huggingface_hub import hf_hub_download |
|
|
|
|
|
model_id = "neuralvfx/LibreFlux-IP-Adapter-ControlNet" |
|
|
|
|
|
device = "cuda" if torch.cuda.is_available() else "cpu" |
|
|
dtype = torch.bfloat16 if device == "cuda" else torch.float32 |
|
|
|
|
|
pipe = DiffusionPipeline.from_pretrained( |
|
|
model_id, |
|
|
custom_pipeline=model_id, |
|
|
trust_remote_code=True, |
|
|
torch_dtype=dtype, |
|
|
safety_checker=None |
|
|
) |
|
|
|
|
|
# Optional way to download the weights |
|
|
hf_hub_download(repo_id="neuralvfx/LibreFlux-IP-Adapter-ControlNet", |
|
|
filename="ip_adapter.pt", |
|
|
local_dir=".", |
|
|
local_dir_use_symlinks=False) |
|
|
|
|
|
pipe.load_ip_adapter('ip_adapter.pt') |
|
|
|
|
|
pipe.to(device) |
|
|
``` |
|
|
|
|
|
# Inference |
|
|
```py |
|
|
from PIL import Image |
|
|
from torchvision.transforms import ToTensor |
|
|
|
|
|
|
|
|
# Optional way to download test Control Net Image |
|
|
hf_hub_download(repo_id="neuralvfx/LibreFlux-IP-Adapter-ControlNet", |
|
|
filename="examples/libre_flux_control_image.png", |
|
|
local_dir=".", |
|
|
local_dir_use_symlinks=False) |
|
|
|
|
|
# Load Control Image |
|
|
cond = Image.open("examples/libre_flux_control_image.png").convert("RGB") |
|
|
cond = cond.resize((1024, 1024)) |
|
|
|
|
|
# Optional way to download test IP Adapter Image |
|
|
hf_hub_download(repo_id="neuralvfx/LibreFlux-IP-Adapter-ControlNet", |
|
|
filename="examples/merc.jpeg", |
|
|
local_dir=".", |
|
|
local_dir_use_symlinks=False) |
|
|
|
|
|
# Load IP Adapter Image |
|
|
ip_image = Image.open("examples/merc.jpeg").convert("RGB") |
|
|
ip_image = ip_image.resize((512, 512)) |
|
|
|
|
|
out = pipe( |
|
|
prompt="the words libre flux", |
|
|
negative_prompt="blurry", |
|
|
control_image=cond, # Use the tensor here |
|
|
num_inference_steps=75, |
|
|
guidance_scale=4.0, |
|
|
controlnet_conditioning_scale=1.0, |
|
|
ip_adapter_image=ip_image, |
|
|
ip_adapter_scale=1.0, |
|
|
num_images_per_prompt=1, |
|
|
generator= torch.Generator().manual_seed(74), |
|
|
return_dict=True, |
|
|
) |
|
|
out.images[0] |
|
|
``` |
|
|
|
|
|
# Load Pipeline ( Low VRAM ) |
|
|
```py |
|
|
import torch |
|
|
from huggingface_hub import hf_hub_download |
|
|
from diffusers import DiffusionPipeline |
|
|
from optimum.quanto import freeze, quantize, qint8 |
|
|
|
|
|
model_id = "neuralvfx/LibreFlux-IP-Adapter-ControlNet" |
|
|
|
|
|
device = "cuda" if torch.cuda.is_available() else "cpu" |
|
|
dtype = torch.bfloat16 if device == "cuda" else torch.float32 |
|
|
|
|
|
pipe = DiffusionPipeline.from_pretrained( |
|
|
model_id, |
|
|
custom_pipeline=model_id, |
|
|
trust_remote_code=True, |
|
|
torch_dtype=dtype, |
|
|
safety_checker=None |
|
|
) |
|
|
|
|
|
# Optional way to download the weights |
|
|
hf_hub_download(repo_id="neuralvfx/LibreFlux-IP-Adapter-ControlNet", |
|
|
filename="ip_adapter.pt", |
|
|
local_dir=".", |
|
|
local_dir_use_symlinks=False) |
|
|
|
|
|
# Load the IP Adapter First |
|
|
pipe.load_ip_adapter('ip_adapter.pt') |
|
|
|
|
|
# Quantize and Freeze |
|
|
quantize( |
|
|
pipe.transformer, |
|
|
weights=qint8, |
|
|
exclude=[ |
|
|
"*.norm", "*.norm1", "*.norm2", "*.norm2_context", |
|
|
"proj_out", "x_embedder", "norm_out", "context_embedder", |
|
|
], |
|
|
) |
|
|
|
|
|
quantize( |
|
|
pipe.ip_adapter, |
|
|
weights=qint8, |
|
|
exclude=[ |
|
|
"*.norm", "*.norm1", "*.norm2", "*.norm2_context", |
|
|
"proj_out", "x_embedder", "norm_out", "context_embedder", |
|
|
], |
|
|
) |
|
|
|
|
|
quantize( |
|
|
pipe.controlnet, |
|
|
weights=qint8, |
|
|
exclude=[ |
|
|
"*.norm", "*.norm1", "*.norm2", "*.norm2_context", |
|
|
"proj_out", "x_embedder", "norm_out", "context_embedder", |
|
|
], |
|
|
) |
|
|
|
|
|
freeze(pipe.transformer) |
|
|
freeze(pipe.ip_adapter) |
|
|
freeze(pipe.controlnet) |
|
|
|
|
|
# Enable Model Offloading |
|
|
pipe.enable_model_cpu_offload() |
|
|
``` |
|
|
|
