commit

Sci_Fi_frame_inbetweening.py

@@ -79,12 +79,12 @@ def generate_video(
     scheduler = CogVideoXDDIMScheduler.from_pretrained(pretrained_model_name_or_path, subfolder="scheduler")
     
     # 2. Load the pre-trained EF_Net
-    EF_Net = EF_Net(num_layers=4, downscale_coef=8, in_channels=2, num_attention_heads=48,).requires_grad_(False).eval()
+    EF_Net_model = EF_Net(num_layers=4, downscale_coef=8, in_channels=2, num_attention_heads=48,).requires_grad_(False).eval()
     ckpt = torch.load(EF_Net_model_path, map_location='cpu', weights_only=False)
     EF_Net_state_dict = {}
     for name, params in ckpt['state_dict'].items():
         EF_Net_state_dict[name] = params
-    m, u = EF_Net.load_state_dict(EF_Net_state_dict, strict=False)
+    m, u = EF_Net_model.load_state_dict(EF_Net_state_dict, strict=False)
     print(f'[ Weights from pretrained EF-Net was loaded into EF-Net ] [M: {len(m)} | U: {len(u)}]')
     
     #3. Load the prompt (Can be modified independently according to specific needs.)
@@ -98,7 +98,7 @@ def generate_video(
         text_encoder=text_encoder,
         transformer=transformer,
         vae=vae,
-        EF_Net=EF_Net,
+        EF_Net_model=EF_Net_model,
         scheduler=scheduler,
     )
     pipe.scheduler = CogVideoXDDIMScheduler.from_config(pipe.scheduler.config, timestep_spacing="trailing")

Sci_Fi_frame_inbetweening.sh

@@ -1,12 +1,13 @@
-export CUDA_VISIBLE_DEVICES=6
+export CUDA_VISIBLE_DEVICES=7
 EVAL_DIR=/home/lhchen/Sci-Fi/example_input_pairs
 MODEL_NAME=CogVideoX-5b-I2V
 OUT_DIR=outputs
+basename=eval_videos_dir
 
 mkdir -p $OUT_DIR
 for example_dir in $(ls -d $EVAL_DIR/*)
 do
-    example_name=$(EVAL_DIR $example_dir)
+    example_name=$(basename $example_dir)
     echo $example_name
 
     out_fn=$OUT_DIR/$example_name'.mp4'

Sci_Fi_inbetweening_pipeline.py

@@ -177,7 +177,7 @@ class CogVideoXEFNetInbetweeningPipeline(DiffusionPipeline, CogVideoXLoraLoaderM
         text_encoder: T5EncoderModel,
         vae: AutoencoderKLCogVideoX,
         transformer: CogVideoXTransformer3DModel,
-        EF_Net: EF_Net,
+        EF_Net_model: EF_Net,
         scheduler: CogVideoXDDIMScheduler,
     ):
         super().__init__()
@@ -187,7 +187,7 @@ class CogVideoXEFNetInbetweeningPipeline(DiffusionPipeline, CogVideoXLoraLoaderM
             text_encoder=text_encoder,
             vae=vae,
             transformer=transformer,
-            EF_Net=EF_Net,
+            EF_Net_model=EF_Net_model,
             scheduler=scheduler,
         )
         self.vae_scale_factor_spatial = (
@@ -742,7 +742,7 @@ class CogVideoXEFNetInbetweeningPipeline(DiffusionPipeline, CogVideoXLoraLoaderM
                 EF_Net_states = []
                 if (EF_Net_guidance_start <= current_sampling_percent < EF_Net_guidance_end):
                     # extract EF_Net hidden state
-                    EF_Net_states = self.EF_Net(
+                    EF_Net_states = self.EF_Net_model(
                         hidden_states=latent_image_input[:,:,0:16,:,:],
                         encoder_hidden_states=prompt_embeds,
                         image_rotary_emb=None,