edge,dgcnz

edge's People

Contributors

Watchers

edge's Issues

Implement YOLOv8 Lightning Module

Adapt dinov2 as a detectron2's Backbone

Relevant DinoV2 snippet

class DinoVisionTransformer(nn.Module):
    ....
    def forward_features(self, x, masks=None):
        if isinstance(x, list):
            return self.forward_features_list(x, masks)

        x = self.prepare_tokens_with_masks(x, masks)

        for blk in self.blocks:
            x = blk(x)

        x_norm = self.norm(x)
        return {
            "x_norm_clstoken": x_norm[:, 0],
            "x_norm_regtokens": x_norm[:, 1 : self.num_register_tokens + 1],
            "x_norm_patchtokens": x_norm[:, self.num_register_tokens + 1 :],
            "x_prenorm": x,
            "masks": masks,
        }

Notes:

x_prenorm is of shape [1, 1370, 768] or [B, P*P + 1, D]

Relevant detectron2 snippet

class ViT(Backbone):
    ...
    def forward(self, x):
        x = self.patch_embed(x)
        if self.pos_embed is not None:
            x = x + get_abs_pos(
                self.pos_embed, self.pretrain_use_cls_token, (x.shape[1], x.shape[2])
            )

        for blk in self.blocks:
            x = blk(x)

        outputs = {self._out_features[0]: x.permute(0, 3, 1, 2)}
        return outputs