virtuous7373/Lambent-Mira-Aphrodite-v1zf

All credits go to:

Lambent for the wonderful model series.

ϻira Λphrodite-v1zf-27B

(I ran out of letters :P) Maybe have finally earned the namesake, we'll see.

📖 Overview

This is a merge of pre-trained language models created using mergekit. It utilizes custom merge methods.

The Recipe

ϻira Λphrodite-v1zf | Methods: [a lot]

View Configuration

# https://huggingface.co/Lambent
name: Mira M2

merge_method: cabs

base_model: ./Ingredients/Lambent_Mira-v1.8.1a-27B

models:
  - model: ./Ingredients/Lambent_Mira-v1.8.1a-27B
  - model: ./Ingredients/Lambent_Mira-v1.17-27B-Custom-Heretic
    parameters:
      weight: 0.4
      n_val: 16
      m_val: 32
  - model: ./Ingredients/mira-mix3
    parameters:
      weight: 0.2
      n_val: 12
      m_val: 32

default_n_val: 8
default_m_val: 32

pruning_order:
  - ./Ingredients/Lambent_Mira-v1.17-27B-Custom-Heretic
  - ./Ingredients/mira-mix3


dtype: float32
out_dtype: bfloat16

tokenizer:
  source: union
---
name: Mira-Aphrodite-v1d

merge_method: variance_crossover

base_model: ./Ingredients/Lambent_Mira-v1.16-Ties-27B

models:
  - model: ./Ingredients/Lambent_Mira-v1.16-Ties-27B
  - model: ./Ingredients/Lambent_Mira-v1.14-27B

tokenizer:
  source: union

dtype: bfloat16
out_dtype: bfloat16

---
name: Mira-Aphrodite-v1j

merge_method: variance_crossover

base_model: ./Ingredients/Mira-M2

models:
  - model: ./Ingredients/Mira-M2
  - model: ./Ingredients/Lambent_Mira-v1.8.1a-27B

parameters:
  select_topk: 0.51
  epsilon: 1e-6

tokenizer:
  source: union

dtype: float32
out_dtype: bfloat16

---
#name: Mira-Aphrodite-v1I

merge_method: ties_sb

base_model: ./Ingredients/Lambent_Mira-v1.16-Ties-27B
name: m16

models:
  - model: ./Ingredients/Lambent_Mira-v1.16-Ties-27B
  - model: ./Ingredients/Lambent_Mira-v1.11-Ties-27B
    name: m11
    parameters:
      weight: 0.5
      density: 1.0
  - model: ./Ingredients/Lambent_Mira-v1.14-27B
    name: m14
    parameters:
      weight: 0.5
      density: 1.0

parameters:
  normalize: true
  int8_mask: false
  svd_thresh: 0.62
  cumsum: true
  rescale: true
  prescale: true

dtype: bfloat16

tokenizer:
  source: union

layer_parameters:
  - filter: ".*(norm).*"
    models: { m16: 1.0 }

  - filter: ".*(embed).*"
    models: { m16: 1.0 }

  - filter: ".*(self_attn.q_proj).*"
    models: { m11: 0.65, m14: 0.35 }

  - filter: ".*(self_attn.k_proj).*"
    models: { m11: 0.35, m14: 0.65 }

  - filter: ".*(self_attn.v_proj).*"
    models: { m11: 0.55, m14: 0.45 }

  - filter: ".*(self_attn.o_proj).*"
    models: { m11: 0.50, m14: 0.50 }

  - filter: ".*(mlp|ffn).*"
    models: { m11: 0.40, m14: 0.60 }

---
name: Mira-Aphrodite-v1q

merge_method: variance_crossover

base_model: ./Ingredients/Mira-M2

models:
  - model: ./Ingredients/Lambent_Mira-v1.16-Ties-27B
  - model: ./Ingredients/Lambent_Mira-v1.14-27B
  - model: ./Ingredients/Mira-Aphrodite-v1d
  - model: ./Ingredients/Mira-M2

parameters:
  select_topk: 0.18
  epsilon: 1e-6

tokenizer:
  source: base

dtype: float32
out_dtype: bfloat16

---
name: Mira-Aphrodite-v1zk

merge_method: ties_sb

base_model: ./Ingredients/Mira-M2

models:
  - model: ./Ingredients/Mira-M2
  - model: ./Ingredients/Mira-Aprodite-v1j
    name: v1j
    parameters:
      weight: 0.5
      density: 1.0
  - model: ./Ingredients/Mira-Aprodite-v1I
    name: v1I
    parameters:
      weight: 0.25
      density: 1.0
  - model: ./Ingredients/Mira-Aprodite-v1g
    name: v1g
    parameters:
      weight: 0.25
      density: 1.0

parameters:
  normalize: true
  int8_mask: false
  svd_thresh: 0.62
  cumsum: true
  rescale: true
  prescale: true

dtype: bfloat16

tokenizer:
  source: union

layer_parameters:
  - filter: ".*(norm).*"
    models: { m2: 1.0 }

  - filter: ".*(embed).*"
    models: { m2: 1.0 }

  - filter: ".*(self_attn.q_proj).*"
    models: { v1j: 0.25, v1I: 0.20, v1g: 0.55 }

  - filter: ".*(self_attn.k_proj).*"
    models: { v1j: 0.45, v1I: 0.15, v1g: 0.40 }

  - filter: ".*(self_attn.v_proj).*"
    models: { v1j: 0.35, v1I: 0.35, v1g: 0.30 }

  - filter: ".*(self_attn.o_proj).*"
    models: { v1j: 0.50, v1I: 0.25, v1g: 0.25 }

  - filter: ".*(mlp|ffn).*"
    models: { v1j: 0.60, v1I: 0.20, v1g: 0.20 }

---
name: Mira-Aphrodite-v1q

merge_method: cabs

base_model: ./Ingredients/Lambent_Mira-v1-18-27B

models:
  - model: ./Ingredients/Lambent_Mira-v1-18-27B
    parameters:
      weight: 0.6
      n_val: 16
      m_val: 32
  - model: ./Ingredients/Lambent_Mira-v1.5-27B
    parameters:
      weight: 0.4
      n_val: 11
      m_val: 33

default_n_val: 8
default_m_val: 32

pruning_order:
  - ./Ingredients/Lambent_Mira-v1-18-27B
  - ./Ingredients/Lambent_Mira-v1.5-27B

# base_model: base

dtype: bfloat16
out_dtype: bfloat16

tokenizer:
  source: base

---
name: Mira-Aphrodite-v1za

merge_method: variance_crossover

base_model: ./Ingredients/Mira-M2

models:
  - model: ./Ingredients/Lambent_Mira-v1.12-Ties-27B
    parameters:
      weight: 0.25
  - model: ./Ingredients/Lambent_Mira-v1.5-27B
    parameters:
      weight: 0.25
  - model: ./Ingredients/Lambent_Mira-v1.18-27B-heretic-0.019/
    parameters:
      weight: 0.25
  - model: ./Ingredients/Lambent_Mira-v1.19-27B-dpo
    parameters:
      weight: 0.25

parameters:
  select_topk: 0.69 #nice
  epsilon: 1e-6
  rescale: true
  prescale: true

tokenizer:
  source: base

dtype: bfloat16
out_dtype: bfloat16

---
name: Mira Aphrodite v1zf

merge_method: cabs

models:
  - model: ./Ingredients/Lambent_Mira-v1.8.1a-27B

  - model: ./Ingredients/Mira-Aphrodite-v1q
    parameters:
      weight: 0.6
      n_val: 16
      m_val: 32
  - model: ./Ingredients/Mira-Aphrodite-v1za
    parameters:
      weight: 0.4
      n_val: 11
      m_val: 33

default_n_val: 8
default_m_val: 32

pruning_order:
  - ./Ingredients/Mira-Aphrodite-v1q
  - ./Ingredients/Mira-Aphrodite-v1zk

base_model: ./Ingredients/Lambent_Mira-v1.8.1a-27B

dtype: bfloat16
out_dtype: bfloat16

tokenizer:
  source: union
```
```