Upload folder using huggingface_hub

config.json

@@ -0,0 +1,178 @@
+{
+  "architectures": [
+    "EncoderForSequenceClassification"
+  ],
+  "auto_map": {
+    "AutoModel": "modeling_lfm2_bidirectional.Lfm2BidirectionalModel",
+    "AutoModelForMaskedLM": "modeling_lfm2_bidirectional.Lfm2BidirectionalForMaskedLM",
+    "AutoModelForSequenceClassification": "modeling_encoder_seq_cls.EncoderForSequenceClassification"
+  },
+  "block_auto_adjust_ff_dim": true,
+  "block_dim": 1024,
+  "block_ffn_dim_multiplier": 1.0,
+  "block_mlp_init_scale": 1.0,
+  "block_multiple_of": 256,
+  "block_norm_eps": 1e-05,
+  "block_out_init_scale": 1.0,
+  "block_use_swiglu": true,
+  "block_use_xavier_init": true,
+  "bos_token_id": 1,
+  "classifier_thresholds": [
+    0.35,
+    0.15,
+    0.2,
+    0.15,
+    0.15,
+    0.4,
+    0.1,
+    0.15,
+    0.2,
+    0.1,
+    0.2,
+    0.3,
+    0.2,
+    0.35,
+    0.3,
+    0.25,
+    0.55,
+    0.5,
+    0.3,
+    0.05,
+    0.4,
+    0.25,
+    0.4,
+    0.3,
+    0.2,
+    0.35,
+    0.25,
+    0.45,
+    0.3,
+    0.25,
+    0.25,
+    0.1,
+    0.1,
+    0.2,
+    0.15
+  ],
+  "conv_L_cache": 3,
+  "conv_bias": false,
+  "conv_dim": 1024,
+  "conv_dim_out": 1024,
+  "conv_use_xavier_init": true,
+  "dtype": "float32",
+  "eos_token_id": 7,
+  "full_attn_idxs": null,
+  "hidden_size": 1024,
+  "id2label": {
+    "0": "audio-classification",
+    "1": "audio-to-audio",
+    "10": "image-to-image",
+    "11": "image-to-text",
+    "12": "multiple-choice",
+    "13": "object-detection",
+    "14": "question-answering",
+    "15": "reinforcement-learning",
+    "16": "robotics",
+    "17": "sentence-similarity",
+    "18": "summarization",
+    "19": "table-question-answering",
+    "2": "automatic-speech-recognition",
+    "20": "tabular-classification",
+    "21": "tabular-regression",
+    "22": "text-classification",
+    "23": "text-generation",
+    "24": "text-retrieval",
+    "25": "text-to-image",
+    "26": "text-to-speech",
+    "27": "text-to-video",
+    "28": "time-series-forecasting",
+    "29": "token-classification",
+    "3": "feature-extraction",
+    "30": "translation",
+    "31": "video-classification",
+    "32": "video-text-to-text",
+    "33": "visual-question-answering",
+    "34": "zero-shot-classification",
+    "4": "fill-mask",
+    "5": "image-classification",
+    "6": "image-feature-extraction",
+    "7": "image-segmentation",
+    "8": "image-text-to-text",
+    "9": "image-to-3d"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 6656,
+  "label2id": {
+    "audio-classification": 0,
+    "audio-to-audio": 1,
+    "automatic-speech-recognition": 2,
+    "feature-extraction": 3,
+    "fill-mask": 4,
+    "image-classification": 5,
+    "image-feature-extraction": 6,
+    "image-segmentation": 7,
+    "image-text-to-text": 8,
+    "image-to-3d": 9,
+    "image-to-image": 10,
+    "image-to-text": 11,
+    "multiple-choice": 12,
+    "object-detection": 13,
+    "question-answering": 14,
+    "reinforcement-learning": 15,
+    "robotics": 16,
+    "sentence-similarity": 17,
+    "summarization": 18,
+    "table-question-answering": 19,
+    "tabular-classification": 20,
+    "tabular-regression": 21,
+    "text-classification": 22,
+    "text-generation": 23,
+    "text-retrieval": 24,
+    "text-to-image": 25,
+    "text-to-speech": 26,
+    "text-to-video": 27,
+    "time-series-forecasting": 28,
+    "token-classification": 29,
+    "translation": 30,
+    "video-classification": 31,
+    "video-text-to-text": 32,
+    "visual-question-answering": 33,
+    "zero-shot-classification": 34
+  },
+  "layer_types": [
+    "conv",
+    "conv",
+    "full_attention",
+    "conv",
+    "conv",
+    "full_attention",
+    "conv",
+    "conv",
+    "full_attention",
+    "conv",
+    "full_attention",
+    "conv",
+    "full_attention",
+    "conv",
+    "full_attention",
+    "conv"
+  ],
+  "max_position_embeddings": 128000,
+  "model_type": "lfm2",
+  "norm_eps": 1e-05,
+  "num_attention_heads": 16,
+  "num_heads": 16,
+  "num_hidden_layers": 16,
+  "num_key_value_heads": 8,
+  "pad_token_id": 0,
+  "problem_type": "multi_label_classification",
+  "rope_parameters": {
+    "rope_theta": 1000000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.14.1",
+  "use_cache": false,
+  "use_pos_enc": true,
+  "vocab_size": 65536
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3bf5c212f49c333d3e0e522e4cd61672774559c8292146670d7fbc2269f214a0
+size 1418095980

modeling_encoder_seq_cls.py

@@ -0,0 +1,53 @@
+"""Generic sequence classification head: AutoModel backbone + mean pooling + linear.
+
+Auto-generated by the uv-scripts `train-classifier.py` recipe. Loaded via
+`AutoModelForSequenceClassification.from_pretrained(repo, trust_remote_code=True)`;
+the backbone class is resolved from this repo's own `auto_map`/code files.
+"""
+
+import torch
+from torch import nn
+from transformers import AutoModel, PreTrainedModel
+from transformers.modeling_outputs import SequenceClassifierOutput
+
+
+class EncoderForSequenceClassification(PreTrainedModel):
+    base_model_prefix = "model"
+    supports_gradient_checkpointing = True
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.num_labels = config.num_labels
+        self.model = AutoModel.from_config(config, trust_remote_code=True)
+        dropout = getattr(config, "classifier_dropout", None)
+        self.dropout = nn.Dropout(0.1 if dropout is None else dropout)
+        self.classifier = nn.Linear(config.hidden_size, config.num_labels)
+        self.post_init()
+
+    def forward(self, input_ids=None, attention_mask=None, labels=None, **kwargs):
+        outputs = self.model(input_ids=input_ids, attention_mask=attention_mask)
+        hidden = outputs.last_hidden_state
+        if attention_mask is None:
+            pooled = hidden.mean(dim=1)
+        else:
+            mask = attention_mask.unsqueeze(-1).to(hidden.dtype)
+            pooled = (hidden * mask).sum(dim=1) / mask.sum(dim=1).clamp(min=1e-9)
+        logits = self.classifier(self.dropout(pooled))
+        loss = None
+        if labels is not None:
+            if self.config.problem_type == "multi_label_classification":
+                loss = nn.functional.binary_cross_entropy_with_logits(
+                    logits, labels.to(logits.dtype)
+                )
+            else:
+                loss = nn.functional.cross_entropy(logits, labels.view(-1))
+        return SequenceClassifierOutput(loss=loss, logits=logits)
+
+
+# AutoModelForSequenceClassification.from_pretrained registers this class against the
+# config class, and that requires config_class to be set (transformers v5 crashes on None).
+try:
+    from transformers import Lfm2Config
+    EncoderForSequenceClassification.config_class = Lfm2Config
+except ImportError:  # flat import during training; the trainer sets config_class itself
+    pass

modeling_lfm2_bidirectional.py

@@ -0,0 +1,211 @@
+"""LFM2 backbone with bidirectional attention + non-causal short-conv.
+
+Wired into the HF repo via `auto_map` in config.json so that
+
+AutoModel.from_pretrained(repo, trust_remote_code=True)
+AutoModelForMaskedLM.from_pretrained(repo, trust_remote_code=True)
+
+both return a model with the encoder-style patches already applied.
+
+Supports `attn_implementation` in {"eager", "sdpa", "flash_attention_2"}:
+
+eager/sdpa consume a 4D additive pad-only mask and reproduce the exact
+training-time behavior; flash_attention_2 receives the 2D padding mask (or
+None) and runs the kernel non-causally via `Lfm2Attention.is_causal = False`,
+yielding outputs equivalent to the unpadded forward.
+"""
+
+from typing import Optional
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from transformers.configuration_utils import PretrainedConfig
+from transformers.modeling_outputs import BaseModelOutput, MaskedLMOutput
+from transformers.modeling_utils import PreTrainedModel
+from transformers.models.lfm2 import modeling_lfm2 as _lfm2_mod
+from transformers.models.lfm2.configuration_lfm2 import Lfm2Config
+from transformers.models.lfm2.modeling_lfm2 import (
+    Lfm2Attention,
+    Lfm2Model,
+    Lfm2PreTrainedModel,
+    Lfm2ShortConv,
+    apply_mask_to_padding_states,
+)
+
+
+def _bidirectional_mask(
+    config,
+    input_embeds: torch.Tensor = None,
+    attention_mask: Optional[torch.Tensor] = None,
+    cache_position: Optional[torch.LongTensor] = None,
+    past_key_values=None,
+    position_ids: Optional[torch.LongTensor] = None,
+    **kwargs,
+) -> Optional[torch.Tensor]:
+    # transformers has renamed the embeds kwarg across versions
+    # (input_embeds <-> inputs_embeds); accept either to stay forward-compatible.
+    if input_embeds is None:
+        input_embeds = kwargs.get("inputs_embeds")
+
+    if config._attn_implementation == "flash_attention_2":
+        # FA2 only uses the 2D padding mask to unpad sequences; causality is
+        # controlled by `Lfm2Attention.is_causal` (set to False below).
+        if attention_mask is not None and not attention_mask.all():
+            return attention_mask
+        return None
+
+    device = input_embeds.device
+    dtype = input_embeds.dtype
+    bsz, q_len = input_embeds.shape[:2]
+    past = past_key_values.get_seq_length() if past_key_values is not None else 0
+    kv_len = past + q_len
+
+    mask = torch.zeros((bsz, 1, q_len, kv_len), device=device, dtype=dtype)
+    if attention_mask is not None:
+        cur_len = attention_mask.size(-1)
+        key_pad_flags = (attention_mask == 0).to(device=device, dtype=torch.float32)
+        pad_vec = torch.zeros((bsz, kv_len), device=device, dtype=torch.float32)
+        if cur_len > 0:
+            pad_vec[:, past:past + cur_len] = key_pad_flags * -1e9
+        mask = mask + pad_vec.to(dtype)[:, None, None, :]
+    return mask
+
+
+def _noncausal_shortconv_forward(
+    self,
+    hidden_states: torch.Tensor,
+    past_key_values=None,
+    cache_position=None,
+    attention_mask: Optional[torch.Tensor] = None,
+    **kwargs,
+) -> torch.Tensor:
+    x = apply_mask_to_padding_states(hidden_states, attention_mask)
+
+    BCx = self.in_proj(x).transpose(-1, -2)
+    B, C, x = BCx.chunk(3, dim=-2)
+    Bx = B * x
+
+    k = self.conv.weight.shape[-1]
+    pad = k // 2
+    conv_out = F.conv1d(
+        Bx, weight=self.conv.weight, bias=self.conv.bias,
+        stride=1, padding=pad, dilation=1, groups=Bx.shape[1],
+    )
+    if conv_out.shape[-1] > Bx.shape[-1]:
+        conv_out = conv_out[..., :Bx.shape[-1]]
+    elif conv_out.shape[-1] < Bx.shape[-1]:
+        conv_out = F.pad(conv_out, (0, Bx.shape[-1] - conv_out.shape[-1]))
+
+    y = C * conv_out
+    y = y.transpose(-1, -2).contiguous()
+    return self.out_proj(y)
+
+
+def _shortconv_forward(self, *args, **kwargs):
+    return self.slow_forward(*args, **kwargs)
+
+
+_PATCHED = False
+
+
+def _install_patches() -> None:
+    global _PATCHED
+    if _PATCHED:
+        return
+    _lfm2_mod.create_causal_mask = _bidirectional_mask
+    Lfm2ShortConv.slow_forward = _noncausal_shortconv_forward
+    Lfm2ShortConv.forward = _shortconv_forward
+    _PATCHED = True
+
+
+_install_patches()
+
+
+def _set_attention_noncausal(model) -> None:
+    for module in model.modules():
+        if isinstance(module, Lfm2Attention):
+            module.is_causal = False
+
+
+class Lfm2BidirectionalModel(Lfm2Model):
+    """LFM2 patched for encoder-style use: 
+    full bidirectional attention + non-causal short-conv."""
+
+    def __init__(self, config):
+        _install_patches()
+        super().__init__(config)
+        _set_attention_noncausal(self)
+
+
+class Lfm2BidirectionalForMaskedLM(Lfm2PreTrainedModel):
+    """LFM2 bidirectional encoder with a tied masked-LM head."""
+
+    config_class = Lfm2Config
+    base_model_prefix = "lfm2"
+    _tied_weights_keys = {"lm_head.weight": "lfm2.embed_tokens.weight"}
+
+    def __init__(self, config: Lfm2Config):
+        _install_patches()
+        config = type(config).from_dict({**config.to_dict(), "use_cache": False})
+        super().__init__(config)
+        self.lfm2 = Lfm2BidirectionalModel(config)
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+        self.post_init()
+        self.lm_head.weight = self.lfm2.embed_tokens.weight
+
+    def get_input_embeddings(self):
+        return self.lfm2.embed_tokens
+
+    def set_input_embeddings(self, value):
+        self.lfm2.embed_tokens = value
+
+    def get_output_embeddings(self):
+        return self.lm_head
+
+    def set_output_embeddings(self, new_embeddings):
+        self.lm_head = new_embeddings
+
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        output_hidden_states: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+        **kwargs,
+    ) -> MaskedLMOutput:
+        return_dict = True if return_dict is None else return_dict
+        outputs = self.lfm2(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            inputs_embeds=inputs_embeds,
+            use_cache=False,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=True,
+        )
+        hidden = outputs.last_hidden_state
+        logits = self.lm_head(hidden)
+
+        loss = None
+        if labels is not None:
+            loss = F.cross_entropy(
+                logits.view(-1, self.config.vocab_size),
+                labels.view(-1),
+                ignore_index=-100,
+            )
+
+        if not return_dict:
+            out = (logits,) + outputs[1:]
+            return ((loss,) + out) if loss is not None else out
+        return MaskedLMOutput(
+            loss=loss,
+            logits=logits,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )

tokenizer_config.json

@@ -0,0 +1,12 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|startoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "is_local": false,
+  "local_files_only": false,
+  "mask_token": "<|mask|>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|pad|>",
+  "tokenizer_class": "TokenizersBackend"
+}