Commit
·
a40eaa8
1
Parent(s):
0e5e0f5
commit
Browse files
cogvideo_EF_Net.py → EF_Net.py
RENAMED
|
@@ -16,7 +16,7 @@ from diffusers.models.normalization import AdaLayerNorm, CogVideoXLayerNormZero,
|
|
| 16 |
from diffusers.configuration_utils import ConfigMixin, register_to_config
|
| 17 |
|
| 18 |
|
| 19 |
-
class
|
| 20 |
_supports_gradient_checkpointing = True
|
| 21 |
|
| 22 |
@register_to_config
|
|
|
|
| 16 |
from diffusers.configuration_utils import ConfigMixin, register_to_config
|
| 17 |
|
| 18 |
|
| 19 |
+
class EF_Net(ModelMixin, ConfigMixin, PeftAdapterMixin):
|
| 20 |
_supports_gradient_checkpointing = True
|
| 21 |
|
| 22 |
@register_to_config
|
Sci_Fi_frame_inbetweening.py
CHANGED
|
@@ -31,9 +31,9 @@ from diffusers import (
|
|
| 31 |
AutoencoderKLCogVideoX
|
| 32 |
)
|
| 33 |
from diffusers.utils import export_to_video, load_image
|
| 34 |
-
from
|
| 35 |
from cogvideo_transformer import CustomCogVideoXTransformer3DModel
|
| 36 |
-
from
|
| 37 |
import cv2
|
| 38 |
import os
|
| 39 |
import sys
|
|
@@ -79,7 +79,7 @@ def generate_video(
|
|
| 79 |
scheduler = CogVideoXDDIMScheduler.from_pretrained(pretrained_model_name_or_path, subfolder="scheduler")
|
| 80 |
|
| 81 |
# 2. Load the pre-trained EF_Net
|
| 82 |
-
EF_Net =
|
| 83 |
ckpt = torch.load(EF_Net_model_path, map_location='cpu', weights_only=False)
|
| 84 |
EF_Net_state_dict = {}
|
| 85 |
for name, params in ckpt['state_dict'].items():
|
|
|
|
| 31 |
AutoencoderKLCogVideoX
|
| 32 |
)
|
| 33 |
from diffusers.utils import export_to_video, load_image
|
| 34 |
+
from Sci_Fi_inbetweening_pipeline import CogVideoXEFNetInbetweeningPipeline
|
| 35 |
from cogvideo_transformer import CustomCogVideoXTransformer3DModel
|
| 36 |
+
from EF_Net import EF_Net
|
| 37 |
import cv2
|
| 38 |
import os
|
| 39 |
import sys
|
|
|
|
| 79 |
scheduler = CogVideoXDDIMScheduler.from_pretrained(pretrained_model_name_or_path, subfolder="scheduler")
|
| 80 |
|
| 81 |
# 2. Load the pre-trained EF_Net
|
| 82 |
+
EF_Net = EF_Net(num_layers=4, downscale_coef=8, in_channels=2, num_attention_heads=48,).requires_grad_(False).eval()
|
| 83 |
ckpt = torch.load(EF_Net_model_path, map_location='cpu', weights_only=False)
|
| 84 |
EF_Net_state_dict = {}
|
| 85 |
for name, params in ckpt['state_dict'].items():
|
cogvideo_Sci_Fi_inbetweening_pipeline.py → Sci_Fi_inbetweening_pipeline.py
RENAMED
|
@@ -18,7 +18,7 @@ from diffusers import CogVideoXDDIMScheduler, CogVideoXDPMScheduler, CogVideoXIm
|
|
| 18 |
from diffusers.callbacks import MultiPipelineCallbacks, PipelineCallback
|
| 19 |
from diffusers.pipelines.cogvideo.pipeline_cogvideox import CogVideoXPipelineOutput, CogVideoXLoraLoaderMixin
|
| 20 |
|
| 21 |
-
from
|
| 22 |
import torch
|
| 23 |
|
| 24 |
def resize_for_crop(image, crop_h, crop_w):
|
|
@@ -177,7 +177,7 @@ class CogVideoXEFNetInbetweeningPipeline(DiffusionPipeline, CogVideoXLoraLoaderM
|
|
| 177 |
text_encoder: T5EncoderModel,
|
| 178 |
vae: AutoencoderKLCogVideoX,
|
| 179 |
transformer: CogVideoXTransformer3DModel,
|
| 180 |
-
EF_Net:
|
| 181 |
scheduler: CogVideoXDDIMScheduler,
|
| 182 |
):
|
| 183 |
super().__init__()
|
|
|
|
| 18 |
from diffusers.callbacks import MultiPipelineCallbacks, PipelineCallback
|
| 19 |
from diffusers.pipelines.cogvideo.pipeline_cogvideox import CogVideoXPipelineOutput, CogVideoXLoraLoaderMixin
|
| 20 |
|
| 21 |
+
from EF_Net import EF_Net
|
| 22 |
import torch
|
| 23 |
|
| 24 |
def resize_for_crop(image, crop_h, crop_w):
|
|
|
|
| 177 |
text_encoder: T5EncoderModel,
|
| 178 |
vae: AutoencoderKLCogVideoX,
|
| 179 |
transformer: CogVideoXTransformer3DModel,
|
| 180 |
+
EF_Net: EF_Net,
|
| 181 |
scheduler: CogVideoXDDIMScheduler,
|
| 182 |
):
|
| 183 |
super().__init__()
|