From 85f929d12634883b48c96ccfd055b23c526c9e2d Mon Sep 17 00:00:00 2001 From: Shen-Chenhui Date: Mon, 1 Apr 2024 15:26:16 +0800 Subject: [PATCH] debug --- opensora/models/vae/model_utils.py | 136 +++++++++++++++++++++++++++-- 1 file changed, 131 insertions(+), 5 deletions(-) diff --git a/opensora/models/vae/model_utils.py b/opensora/models/vae/model_utils.py index 2b08cdc..870a085 100644 --- a/opensora/models/vae/model_utils.py +++ b/opensora/models/vae/model_utils.py @@ -5,9 +5,132 @@ from typing import Any, Optional, Sequence, Type import torch.nn as nn import numpy as np import torch -from taming.modules.losses.lpips import LPIPS # need to pip install https://github.com/CompVis/taming-transformers +# from taming.modules.losses.lpips import LPIPS # need to pip install https://github.com/CompVis/taming-transformers from taming.modules.discriminator.model import NLayerDiscriminator, weights_init + +"""Stripped version of https://github.com/richzhang/PerceptualSimilarity/tree/master/models""" + +import torch +from torchvision import models +from collections import namedtuple +from taming.util import get_ckpt_path + + +class LPIPS(nn.Module): + # Learned perceptual metric + def __init__(self, use_dropout=True): + super().__init__() + self.scaling_layer = ScalingLayer() + self.chns = [64, 128, 256, 512, 512] # vg16 features + self.net = vgg16(pretrained=True, requires_grad=False) + self.lin0 = NetLinLayer(self.chns[0], use_dropout=use_dropout) + self.lin1 = NetLinLayer(self.chns[1], use_dropout=use_dropout) + self.lin2 = NetLinLayer(self.chns[2], use_dropout=use_dropout) + self.lin3 = NetLinLayer(self.chns[3], use_dropout=use_dropout) + self.lin4 = NetLinLayer(self.chns[4], use_dropout=use_dropout) + self.load_from_pretrained() + for param in self.parameters(): + param.requires_grad = False + + def load_from_pretrained(self, name="vgg_lpips"): + ckpt = get_ckpt_path(name, "taming/modules/autoencoder/lpips") + self.load_state_dict(torch.load(ckpt, map_location=torch.device("cpu")), strict=False) + print("loaded pretrained LPIPS loss from {}".format(ckpt)) + + @classmethod + def from_pretrained(cls, name="vgg_lpips"): + if name != "vgg_lpips": + raise NotImplementedError + model = cls() + ckpt = get_ckpt_path(name) + model.load_state_dict(torch.load(ckpt, map_location=torch.device("cpu")), strict=False) + return model + + def forward(self, input, target): + in0_input, in1_input = (self.scaling_layer(input), self.scaling_layer(target)) + outs0, outs1 = self.net(in0_input), self.net(in1_input) + feats0, feats1, diffs = {}, {}, {} + lins = [self.lin0, self.lin1, self.lin2, self.lin3, self.lin4] + for kk in range(len(self.chns)): + feats0[kk], feats1[kk] = normalize_tensor(outs0[kk]), normalize_tensor(outs1[kk]) + diffs[kk] = (feats0[kk] - feats1[kk]) ** 2 + + res = [spatial_average(lins[kk].model(diffs[kk]), keepdim=True) for kk in range(len(self.chns))] + val = res[0] + for l in range(1, len(self.chns)): + val += res[l] + return val + + +# SCH: TODO: this channel shift & scale may need to be changed +class ScalingLayer(nn.Module): + def __init__(self): + super(ScalingLayer, self).__init__() + self.register_buffer('shift', torch.Tensor([-.030, -.088, -.188])[None, :, None, None, None]) + self.register_buffer('scale', torch.Tensor([.458, .448, .450])[None, :, None, None, None]) + + def forward(self, inp): + return (inp - self.shift) / self.scale + + +class NetLinLayer(nn.Module): + """ A single linear layer which does a 1x1 conv """ + def __init__(self, chn_in, chn_out=1, use_dropout=False): + super(NetLinLayer, self).__init__() + layers = [nn.Dropout(), ] if (use_dropout) else [] + layers += [nn.Conv2d(chn_in, chn_out, 1, stride=1, padding=0, bias=False), ] + self.model = nn.Sequential(*layers) + + +class vgg16(torch.nn.Module): + def __init__(self, requires_grad=False, pretrained=True): + super(vgg16, self).__init__() + vgg_pretrained_features = models.vgg16(pretrained=pretrained).features + self.slice1 = torch.nn.Sequential() + self.slice2 = torch.nn.Sequential() + self.slice3 = torch.nn.Sequential() + self.slice4 = torch.nn.Sequential() + self.slice5 = torch.nn.Sequential() + self.N_slices = 5 + for x in range(4): + self.slice1.add_module(str(x), vgg_pretrained_features[x]) + for x in range(4, 9): + self.slice2.add_module(str(x), vgg_pretrained_features[x]) + for x in range(9, 16): + self.slice3.add_module(str(x), vgg_pretrained_features[x]) + for x in range(16, 23): + self.slice4.add_module(str(x), vgg_pretrained_features[x]) + for x in range(23, 30): + self.slice5.add_module(str(x), vgg_pretrained_features[x]) + if not requires_grad: + for param in self.parameters(): + param.requires_grad = False + + def forward(self, X): + h = self.slice1(X) + h_relu1_2 = h + h = self.slice2(h) + h_relu2_2 = h + h = self.slice3(h) + h_relu3_3 = h + h = self.slice4(h) + h_relu4_3 = h + h = self.slice5(h) + h_relu5_3 = h + vgg_outputs = namedtuple("VggOutputs", ['relu1_2', 'relu2_2', 'relu3_3', 'relu4_3', 'relu5_3']) + out = vgg_outputs(h_relu1_2, h_relu2_2, h_relu3_3, h_relu4_3, h_relu5_3) + return out + + +def normalize_tensor(x,eps=1e-10): + norm_factor = torch.sqrt(torch.sum(x**2,dim=1,keepdim=True)) + return x/(norm_factor+eps) + + +def spatial_average(x, keepdim=True): + return x.mean([2,3],keepdim=keepdim) + ## NOTE: not used since we only have 'GN' # def get_norm_layer(norm_type, dtype): # if norm_type == 'LN': @@ -171,11 +294,13 @@ class VEA3DLossWithPerceptualLoss(nn.Module): ): rec_loss = torch.abs(inputs.contiguous() - reconstructions.contiguous()) if self.perceptual_weight > 0: - # SCH: transform to [B, (C,T), H, W] shape for percetual loss + # SCH: transform to [(B,T), C, H, W] shape for percetual loss over each frame + permutated_input = torch.permute(inputs, (0, 2, 1, 3, 4)) # [B, C, T, H, W] --> [B, T, C, H, W] + permutated_rec = torch.permute(reconstructions, (0, 2, 1, 3, 4)) data_shape = inputs.size() p_loss = self.perceptual_loss( - inputs.view(data_shape[0], -1, data_shape[-2],data_shape[-1]).contiguous(), - reconstructions.view(data_shape[0], -1, data_shape[-2],data_shape[-1]).contiguous() + permutated_input.view(-1, data_shape[-3], data_shape[-2],data_shape[-1]).contiguous(), + permutated_rec.view(-1, data_shape[-3], data_shape[-2],data_shape[-1]).contiguous() ) rec_loss = rec_loss + self.perceptual_weight * p_loss @@ -200,4 +325,5 @@ class VEA3DLossWithPerceptualLoss(nn.Module): # # "{}/g_loss".format(split): g_loss.detach().mean(), # } - return loss \ No newline at end of file + return loss +