first commit

.gitattributes

@@ -0,0 +1,40 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pdf filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.jpg filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,5 @@
+node_modules/
+*.log
+*.env
+*.cache
+python/**/__pycache__

README.md

@@ -0,0 +1,48 @@
+---
+title: The Distill Template
+emoji: 🌌
+colorFrom: yellow
+colorTo: purple
+sdk: static
+pinned: true
+license: apache-2.0
+header: mini
+app_file: dist/index.html
+thumbnail: https://huggingface.co/spaces/nanotron/distill-blog-template/resolve/main/thumbnail.png
+short_description: Craft Beautiful Blogs
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+
+
+Instruction to install and run locally
+
+```bash
+npm install
+npm run build
+npm run dev
+
+// If you want to change something change it in src/....
+
+// Once you are finished
+npm run build
+// And commit the dist folder
+```
+
+## Loading HTML fragments:
+There are two way to load HTML fragments:
+1. Compile them into html during build time
+2. Fetch them and insert them during run-time
+
+## When to use what
+- Use compile time fragments only on parts which you want to ensure are seen by every user right after page load (e.g logo)
+- Use run-time fragments for everything else so that the final HTML is of reasonable size (<1MB idealy)
+
+## How to add a new fragment
+- Add it to the `src/fragments` folder (e.g. `src/fragments/banner.html`)
+- For run-time fragments, add {{{fragment-name}}} to appropriate place in `src/index.html` (e.g. {{{fragment-banner}}})
+- For compile-time fragments, add <div id="fragment-name"></div> to `src/index.html` where you want to insert the fragment (e.g. <div id="fragment-banner"></div>)
+
+
+## How to know which fragments are available
+- Run `npm run dev` and look at the console for available fragments

package.json

@@ -0,0 +1,41 @@
+{
+  "dependencies": {
+    "d3": "^7.9.0",
+    "katex": "^0.16.11",
+    "lodash": "^4.17.21",
+    "papaparse": "^5.4.1",
+    "plotly.js-basic-dist-min": "^2.33.0"
+  },
+  "name": "blogpost",
+  "version": "1.0.0",
+  "description": "--- title: \"The Nanotron Gigablogpost\" emoji: 🍷 colorFrom: pink colorTo: red sdk: static pinned: false header: mini ---",
+  "main": "index.js",
+  "scripts": {
+    "dev": "webpack serve --open",
+    "build": "NODE_ENV=production webpack"
+  },
+  "author": "",
+  "license": "ISC",
+  "devDependencies": {
+    "@babel/preset-env": "^7.24.6",
+    "@swc/html": "^1.10.17",
+    "babel-loader": "^9.1.3",
+    "clean-webpack-plugin": "^4.0.0",
+    "compression-webpack-plugin": "^11.1.0",
+    "copy-webpack-plugin": "^12.0.2",
+    "css-loader": "^7.1.2",
+    "fs": "^0.0.1-security",
+    "handlebars": "^4.7.8",
+    "html-minimizer-webpack-plugin": "^5.0.0",
+    "html-webpack-change-assets-extension-plugin": "^1.3.1",
+    "html-webpack-plugin": "^5.6.0",
+    "image-minimizer-webpack-plugin": "^4.1.3",
+    "sharp": "^0.33.5",
+    "style-loader": "^4.0.0",
+    "svgo": "^3.3.2",
+    "webpack": "^5.91.0",
+    "webpack-bundle-analyzer": "^4.10.2",
+    "webpack-cli": "^5.1.4",
+    "webpack-dev-server": "^5.0.4"
+  }
+}

python/memory/formulas/explorations.ipynb

@@ -0,0 +1,191 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from utils import activation_memory, param_grads_opt"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 48,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def activation_memory(\n",
+    "    a, # attention heads\n",
+    "    b, # micro batch size\n",
+    "    h, # hidden dimension size\n",
+    "    h_ff, # feedforward dimension size (often h_ff = 4h)\n",
+    "    L, # number of layers\n",
+    "    s, # sequence length\n",
+    "    mixed=True,\n",
+    "    recomputation=\"none\",\n",
+    "    ff_activation=\"relu\"\n",
+    "    ):\n",
+    "    \n",
+    "    # https://arxiv.org/pdf/2205.05198\n",
+    "    if mixed:\n",
+    "        bytes_per_value = 2 \n",
+    "    else:\n",
+    "        bytes_per_value = 4\n",
+    "\n",
+    "    one_layer_attention = s * b * h * (bytes_per_value * 5 + 1) + ((2 * bytes_per_value + 1) * a * s * s * b) # eq (2)\n",
+    "\n",
+    "    if ff_activation == \"relu\":\n",
+    "        one_layer_feedforward = (s * b * h * bytes_per_value + (s * b * h_ff * bytes_per_value)   # inputs of 1st/2nd linear layers\n",
+    "                + s * b * h)  # dropout\n",
+    "    elif ff_activation == \"gelu\":\n",
+    "        one_layer_feedforward = (s * b * h * bytes_per_value + (s * b * h_ff * bytes_per_value)   # inputs of 1st/2nd linear layers\n",
+    "                + s * b * h_ff * bytes_per_value # inputs of activation function (not really necessary for Relu)\n",
+    "                + s * b * h)  # dropout\n",
+    "    elif ff_activation == \"swiglu\":\n",
+    "        one_layer_feedforward = (s * b * h * bytes_per_value + (s * b * h_ff * bytes_per_value)   # inputs of input/output linear layers\n",
+    "         + s * b * h_ff * bytes_per_value * 3 # inputs of activation function\n",
+    "            + s * b * h)  # dropout (note that dropout is lower-precision - boolean)\n",
+    "\n",
+    "\n",
+    "    layer_norm = s * b * h * bytes_per_value\n",
+    "\n",
+    "    if recomputation == \"none\":\n",
+    "        one_layer =  one_layer_attention + one_layer_feedforward + 2 * layer_norm # eq (2)\n",
+    "    elif recomputation ==\"selective\":\n",
+    "        one_layer = s * b * h * 34 # eq (6)\n",
+    "    elif recomputation ==\"full\":\n",
+    "        one_layer = s * b * h * 2\n",
+    "    else:\n",
+    "        raise ValueError()\n",
+    "    \n",
+    "    input_dropout = s * b * h # section 4.3\n",
+    "\n",
+    "    total = L * one_layer + input_dropout\n",
+    "        \n",
+    "    return total\n",
+    "\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 51,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "a = 16\n",
+    "b = 3\n",
+    "h = 1024\n",
+    "h_ff = 4 * h\n",
+    "L = 1\n",
+    "s = 7  # 128000\n",
+    "recomputation = \"none\"\n",
+    "mixed = True\n",
+    "ff_activation = \"swiglu\"\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 52,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "1086960"
+      ]
+     },
+     "execution_count": 52,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "activation_memory(a=a, b=b, h=h, h_ff=h_ff, L=L, s=s, recomputation=recomputation, mixed=mixed, ff_activation=ff_activation)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from math import log\n",
+    "\n",
+    "def format_bytes(bytes):\n",
+    "    sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB']\n",
+    "    if bytes == 0:\n",
+    "        return '0 Bytes'\n",
+    "    i = int(log(bytes, 1024))\n",
+    "    print(i)\n",
+    "    p = 1024 ** i\n",
+    "    s = round(bytes / p, 2)\n",
+    "    return f\"{s} {sizes[i]}\"\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "4\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "'22.13 TB'"
+      ]
+     },
+     "execution_count": 19,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "format_bytes(activation_memory(a, b, h, L, s, recomputation))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "jupyter",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.14"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

python/memory/formulas/utils.py

@@ -0,0 +1,69 @@
+def activation_memory(
+    a, # attention heads
+    b, # micro batch size
+    h, # hidden dimension size
+    h_ff, # feedforward dimension size (often h_ff = 4h)
+    L, # number of layers
+    s, # sequence length
+    mixed=True,
+    recomputation="none"
+    ):
+    
+    # https://arxiv.org/pdf/2205.05198
+    if mixed:
+        bytes_per_value = 2 
+    else:
+        bytes_per_value = 4
+
+    one_layer_attention = s * b * h * (bytes_per_value * 5 + 1) + ((2 * bytes_per_value + 1) * a * s * s * b) # eq (2)
+    one_layer_feedforward_mlp = (s * b * h * bytes_per_value + (s * b * h_ff * bytes_per_value)   # inputs of 1st/2nd linear layers
+         + s * b * h_ff * bytes_per_value # inputs of activation function (not really necessary for Relu though)
+            + s * b * h)  # dropout
+    one_layer_feedforward_swiglu = (s * b * h * bytes_per_value + (s * b * h_ff * bytes_per_value)   # inputs of input/output linear layers
+         + s * b * h_ff * bytes_per_value * 3 # inputs of activation function
+            + s * b * h)  # dropout (note that dropout is lower-precision - boolean)
+
+
+    if recomputation == "none":
+        one_layer = one_layer_attention # eq (2)
+    elif recomputation =="selective":
+        one_layer = s * b * h * 34 # eq (6)
+    elif recomputation =="full":
+        one_layer = s * b * h * 2
+    else:
+        raise ValueError()
+    
+    input_dropout = 0  # s * b * h # section 4.3
+
+    total = L * one_layer + input_dropout
+        
+    return total
+
+
+def param_grads_opt(
+    h, # hidden dimension size
+    L, # number of layers
+    s, # sequence length
+    v, # vocab size
+    k=8, # parameters for optimizer (Adam: 8 = 4 bytes moments + 4 bytes variance)
+    mixed=True # mixed precision training
+    ):
+    
+    # https://michaelwornow.net/2024/01/18/counting-params-in-transformer
+    # note: this is without GQA or MQA
+    
+    emb = h*(v+s)
+    one_layer = 12 * h**2 + 13*h
+    other = 2*h
+
+    n = emb + L * one_layer + other
+    
+    # 3.1 https://arxiv.org/pdf/1910.02054
+    
+    if mixed:
+        k += 4 # additional full precision weights
+        bytes_per_paramter = 2
+    else:
+        bytes_per_paramter = 4
+    
+    return bytes_per_paramter*n, bytes_per_paramter*n, k*n

python/memory/measurments/act-mem-2/README.md

@@ -0,0 +1,16 @@
+# Activation Memory: Part 2
+
+Code accompanying the deep-dive [blog post on activation memory](https://determined.ai/blog/act-mem-2).
+
+- The main utility code is in `act_mem.py`. 
+- Basic transformer layers are implemented in `layers.py`.
+- The scripts `{block,mlp}_script.py` demonstrate how replacing `GELU` with `ReLU` affects activation
+memory. 
+- `attn_script.py` shows the cost of activation memory in the attention layer. 
+- Tests of the code are in `test.py`. 
+- See `requirements.txt` for versions the code was built against.
+
+
+## Contributors
+
+- [Garrett Goon](https://github.com/garrett361)

python/memory/measurments/act-mem-2/act_mem.py

@@ -0,0 +1,132 @@
+from typing import Any, Iterable, Optional, Union
+
+import torch
+
+
+def B_to_GiB(bytes: Union[int, float]) -> float:
+    return bytes / 2**30
+
+
+def get_tensor_bytes(tensor: torch.Tensor) -> int:
+    """
+    Returns the bytes of storage a given tensor takes up. If `tensor` is a view of a larger tensor,
+    this function only returns the bytes associated with the view.
+    """
+    tensor_bytes = tensor.numel() * tensor.element_size()
+    return tensor_bytes
+
+
+class AllocatedMemContext:
+    """
+    Context manager which captures the allocated GPU memory at context exit and the change between
+    enter and exit.
+
+    Only includes `allocated_bytes.all.`-prefixed keys in `memory_stats` with all readings converted
+    to GiB.
+
+    Example:
+
+    ```python
+
+    ```
+    """
+
+    def __init__(self) -> None:
+        # Ensure CUDA libraries are loaded:
+        torch.cuda.current_blas_handle()
+
+        self.before: dict[str, int] = {}
+        self.after: dict[str, int] = {}
+        self.delta: dict[str, int] = {}
+
+        self._mem_key_prefix = "allocated_bytes.all."
+
+    def _get_mem_dict(self) -> dict[str, int]:
+        return {
+            k.replace(self._mem_key_prefix, ""): v
+            for k, v in torch.cuda.memory_stats().items()
+            if self._mem_key_prefix in k
+        }
+
+    def __enter__(self) -> "AllocatedMemContext":
+        self.before = self._get_mem_dict()
+        return self
+
+    def __exit__(self, *args: Any, **kwargs: Any) -> None:
+        self.after = self._get_mem_dict()
+        self.delta = {k: v - self.before[k] for k, v in self.after.items()}
+
+
+class SavedTensorContext:
+    """
+    Context manager which captures all tensors which are registered as being saved for backwards
+    within the context window. Does not work with `meta`-device tensors.
+
+    All saved tensors are stored in the `saved_tensor_dict` attr, which is an instance of torch's
+    WeakTensorKeyDictionary with tensor/data_ptr key/value pairs. Some of these tensors may be
+    views of the same underlying storage. The total memory of all saved tensors in bytes, accounting
+    for redundant views, can be accessed through `saved_tensor_mem`.
+
+    Use:
+    ```
+    model = ...
+    with SavedTensorContext(ignored_tensors=model.parameters()) as saved:
+        # Do some computation with `model` and capture saved tensors which are not model weights
+
+    ```
+    saved.saved_tensor_dict # WeakTensorKeyDictionary of all saved tensors.
+    saved.saved_tensor_mem # bytes from all saved tensors (activation memory).
+    """
+
+    def __init__(
+        self,
+        ignored_tensors: Optional[Iterable[torch.Tensor]] = None,
+    ) -> None:
+        # Track ignored tensors by their storage's data_ptr. Important to use storage's data_ptr,
+        # not just the data_ptr of the tensor itself.
+        self._ignored_data_ptrs = (
+            set()
+            if ignored_tensors is None
+            else {t.untyped_storage().data_ptr() for t in ignored_tensors}
+        )
+
+        # Use WeakTensorKeyDictionary instances to save non-trivial tensor references, since these
+        # won't keep the tensor alive if the only references to the tensor are within this object.
+        self.saved_tensor_dict = torch.utils.weak.WeakTensorKeyDictionary()
+
+        def pack_hook(saved_tensor: torch.Tensor) -> torch.Tensor:
+            data_ptr = saved_tensor.untyped_storage().data_ptr()
+            if data_ptr not in self._ignored_data_ptrs:
+                self.saved_tensor_dict[saved_tensor] = data_ptr
+            return saved_tensor
+
+        def unpack_hook(saved_tensor: torch.Tensor) -> torch.Tensor:
+            return saved_tensor
+
+        self._saved_tensors_hook = torch.autograd.graph.saved_tensors_hooks(pack_hook, unpack_hook)
+
+    def __enter__(self) -> "SavedTensorContext":
+        self._saved_tensors_hook.__enter__()
+        return self
+
+    def __exit__(self, *args: Any, **kwargs: Any) -> None:
+        self._saved_tensors_hook.__exit__(*args, **kwargs)
+
+    @property
+    def saved_tensor_mem(self) -> int:
+        """
+        The memory in bytes of all saved tensors, accounting for views into the same storage.
+        """
+        accounted_for = self._ignored_data_ptrs.copy()
+        total_bytes = 0
+        for t in self.saved_tensor_dict:
+            data_ptr = t.untyped_storage().data_ptr()
+            if data_ptr not in accounted_for:
+                print(f"Tensor ptr: {t.untyped_storage().data_ptr()}, "
+                    f"shape: {t.shape}, "
+                    f"dtype: {t.dtype}, "
+                    f"device: {t.device}"
+                    )
+                total_bytes += t.untyped_storage().nbytes()
+                accounted_for.add(data_ptr)
+        return total_bytes

python/memory/measurments/act-mem-2/attn_script.py

@@ -0,0 +1,32 @@
+import torch
+
+import act_mem
+import layers
+
+if __name__ == "__main__":
+    batch_size, seq_len, d_model, n_heads = 1, 128, 1024, 32
+    print(f"Batch size: {batch_size}, sequence length: {seq_len}, d_model: {d_model}, n_heads: {n_heads}")
+    dtype = torch.bfloat16
+    inputs = torch.randn(
+        batch_size,
+        seq_len,
+        d_model,
+        device="cuda",
+        requires_grad=True,
+        dtype=dtype,
+    )
+
+    attn = layers.Attention(
+        d_model=d_model,
+        n_heads=n_heads,
+        device="cuda",
+        dtype=dtype,
+    )
+    with act_mem.AllocatedMemContext() as mem, act_mem.SavedTensorContext(
+        ignored_tensors=attn.parameters()
+    ) as saved:
+        out = attn(inputs)
+    stm = saved.saved_tensor_mem
+    print(f'{mem.delta["current"]=}')
+    print(f"{stm=}")
+    print(f"{stm/out.numel()=}")

python/memory/measurments/act-mem-2/block_script.py

@@ -0,0 +1,44 @@
+"""
+Prints out the ratio of activation memory for the a transformer Block when using ReLU vs GELU.
+"""
+
+import torch
+import torch.nn as nn
+
+import act_mem
+import layers
+
+if __name__ == "__main__":
+    batch_size, seq_len, d_model, n_heads = 2, 4096, 1024, 2
+    dtype = torch.bfloat16
+    inputs = torch.randn(
+        batch_size,
+        seq_len,
+        d_model,
+        device="cuda",
+        requires_grad=True,
+        dtype=dtype,
+    )
+
+    act_fn_dict = {"ReLU": nn.ReLU(), "GELU": nn.GELU()}
+    # Append outputs to a list to keep tensors alive
+    outputs = []
+    mem_bytes = []
+
+    for name, act_fn in act_fn_dict.items():
+        block = layers.Block(
+            d_model=d_model,
+            act_fn=act_fn,
+            n_heads=n_heads,
+            device="cuda",
+            dtype=dtype,
+        )
+        with act_mem.AllocatedMemContext() as mem, act_mem.SavedTensorContext(
+            ignored_tensors=block.parameters()
+        ) as saved:
+            out = block(inputs)
+            outputs.append(out)
+        print(f"{name} block bytes: {saved.saved_tensor_mem}")
+        mem_bytes.append(saved.saved_tensor_mem)
+
+    print(f"ReLU/GeLU block act mem ratio: {mem_bytes[0]/mem_bytes[1]}")

python/memory/measurments/act-mem-2/layers.py

@@ -0,0 +1,190 @@
+from typing import Optional, Union
+
+import einops
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+class Attention(nn.Module):
+    """
+    Minimal multi-head attention layer.
+    """
+
+    def __init__(
+        self,
+        d_model: int,
+        n_heads: int,
+        device: Optional[Union[str, torch.device]] = None,
+        dtype: Optional[torch.dtype] = None,
+    ):
+        super().__init__()
+        self.d_model = d_model
+        self.n_heads = n_heads
+        factory_kwargs = {"device": device, "dtype": dtype}
+
+        self.d_head, remainder = divmod(self.d_model, self.n_heads)
+        assert not remainder, f"{n_heads=} must divide {d_model=} evenly"
+
+        self.lin_qkv = nn.Linear(
+            self.d_model,
+            3 * self.d_model,
+            **factory_kwargs,
+        )
+
+        self.lin_out = nn.Linear(self.d_model, self.d_model, **factory_kwargs)
+
+    def forward(
+        self,
+        inputs: torch.Tensor,
+    ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+        bsz, seq_len, _ = inputs.size()
+
+        # Create the queries, keys, values
+        qkv = einops.rearrange(
+            self.lin_qkv(inputs),
+            "b s (three n_h d_h) -> three b s n_h d_h",
+            b=bsz,
+            s=seq_len,
+            three=3,
+            n_h=self.n_heads,
+            d_h=self.d_head,
+        )
+        q, k, v = qkv
+
+        bsz, seq_len, n_heads, d_head = q.shape
+
+        shape_kwargs = dict(b=bsz, n_h=n_heads, s=seq_len, d_h=d_head)
+        q = einops.rearrange(q, "b s n_h d_h -> b n_h s d_h", **shape_kwargs)
+        k = einops.rearrange(k, "b s n_h d_h -> b n_h s d_h", **shape_kwargs)
+        v = einops.rearrange(v, "b s n_h d_h -> b n_h s d_h", **shape_kwargs)
+
+        # Multi-head self-attention
+        attn_output = F.scaled_dot_product_attention(q, k, v, is_causal=True)
+        attn_output = einops.rearrange(
+            attn_output,
+            "b n_h s d_h -> b s (n_h d_h)",
+            b=bsz,
+            n_h=n_heads,
+            s=seq_len,
+            d_h=d_head,
+        )
+
+        # Final projection
+        out = self.lin_out(attn_output)
+
+        return out
+
+
+class MLP(nn.Module):
+    """
+    Basic MLP layer with optional Dropout.
+    """
+
+    def __init__(
+        self,
+        d_model: int,
+        act_fn: nn.Module,
+        dropout_prob: Optional[float] = None,
+        device: Optional[Union[str, torch.device]] = None,
+        dtype: Optional[torch.dtype] = None,
+    ) -> None:
+        super().__init__()
+        print(f"Shapes: d_model: {d_model}, act_fn: {act_fn}, dropout_prob: {dropout_prob}, device: {device}, dtype: {dtype}")
+        self.d_model = d_model
+        self.act_fn = act_fn
+        self.dropout_prob = dropout_prob
+        factory_kwargs = {"device": device, "dtype": dtype}
+
+        self.lin_0 = nn.Linear(self.d_model, 4 * self.d_model, **factory_kwargs)
+        self.lin_1 = nn.Linear(4 * self.d_model, self.d_model, **factory_kwargs)
+        self.dropout = nn.Dropout(self.dropout_prob) if self.dropout_prob else None
+
+    def forward(self, inputs: torch.Tensor) -> torch.Tensor:
+        x = self.lin_0(inputs)
+        x = self.act_fn(x)
+        x = self.lin_1(x)
+        if self.dropout is not None:
+            x = self.dropout(x)
+        return x
+
+
+class SwiGLUMLP(nn.Module):
+    """
+    Llama 3 SwiGLU MLP layer with optional Dropout.
+    """
+
+    def __init__(
+        self,
+        d_model: int,
+        intermediate_size: int,
+        act_fn: nn.Module,
+        dropout_prob: Optional[float] = None,
+        device: Optional[Union[str, torch.device]] = None,
+        dtype: Optional[torch.dtype] = None,
+    ) -> None:
+        super().__init__()
+        print(f"Shapes: d_model: {d_model}, intermediate_size: {intermediate_size}, act_fn: {act_fn}, dropout_prob: {dropout_prob}, device: {device}, dtype: {dtype}")
+        self.d_model = d_model
+        self.intermediate_size = intermediate_size
+        self.act_fn = act_fn
+        self.dropout_prob = dropout_prob
+        factory_kwargs = {"device": device, "dtype": dtype}
+
+        self.gate_proj = nn.Linear(self.d_model, self.intermediate_size, **factory_kwargs)
+        self.up_proj = nn.Linear(self.d_model, self.intermediate_size, **factory_kwargs)
+        self.down_proj = nn.Linear(self.intermediate_size, self.d_model, **factory_kwargs)
+        self.dropout = nn.Dropout(self.dropout_prob) if self.dropout_prob else None
+
+    def forward(self, inputs: torch.Tensor) -> torch.Tensor:
+        x = self.down_proj(self.act_fn(self.gate_proj(inputs)) * self.up_proj(inputs))
+        if self.dropout is not None:
+            x = self.dropout(x)
+        return x
+
+
+class Block(nn.Module):
+    """
+    Basic transformer block.
+
+    Schematic:
+    ┌──────┐
+    │inputs│
+    └┬─┬───┘
+     │┌▽───────────┐
+     ││norm_0, attn│
+     │└┬───────────┘
+    ┌▽─▽──┐
+    │ add │
+    └┬─┬──┘
+     │┌▽──────────┐
+     ││norm_1, mlp│
+     │└┬──────────┘
+    ┌▽─▽──┐
+    │ add │
+    └┬────┘
+    ┌▽──────┐
+    │outputs│
+    └───────┘
+    """
+
+    def __init__(
+        self,
+        d_model: int,
+        n_heads: int,
+        act_fn: nn.Module,
+        dropout_prob: Optional[float] = None,
+        dtype: Optional[torch.dtype] = None,
+        device: Optional[Union[str, torch.device]] = None,
+    ):
+        super().__init__()
+        factory_kwargs = {"device": device, "dtype": dtype}
+        self.attn = Attention(d_model=d_model, n_heads=n_heads, **factory_kwargs)
+        self.mlp = MLP(d_model=d_model, act_fn=act_fn, dropout_prob=dropout_prob, **factory_kwargs)
+        self.norm_0 = nn.LayerNorm(d_model, **factory_kwargs)
+        self.norm_1 = nn.LayerNorm(d_model, **factory_kwargs)
+
+    def forward(self, inputs: torch.Tensor) -> torch.Tensor:
+        outputs = self.attn(self.norm_0(inputs)) + inputs
+        outputs = self.mlp(self.norm_1(outputs)) + outputs
+        return outputs

python/memory/measurments/act-mem-2/mlp_script.py

@@ -0,0 +1,57 @@
+"""
+Prints out the ratio of activation memory for the MLP layer when using ReLU vs GELU.
+"""
+
+import torch
+import torch.nn as nn
+
+import act_mem
+import layers
+
+if __name__ == "__main__":
+    batch_size, seq_len, d_model, dropout_prob = 1, 128, 1024, 0.1
+    print(f"Batch size: {batch_size}, sequence length: {seq_len}, d_model: {d_model}, dropout_prob: {dropout_prob}  ")
+    dtype = torch.bfloat16
+    inputs = torch.randn(
+        batch_size,
+        seq_len,
+        d_model,
+        device="cuda",
+        requires_grad=True,
+        dtype=dtype,
+    )
+
+    act_fn_dict = {"ReLU": nn.ReLU() , "GELU": nn.GELU(), "silu": nn.SiLU()}
+    # Append outputs to a list to keep tensors alive
+    outputs = []
+    mem_bytes = []
+
+    for name, act_fn in act_fn_dict.items():
+        if name == "silu":
+            mlp = layers.SwiGLUMLP(
+                d_model=d_model,
+                intermediate_size=4 * d_model,
+                act_fn=act_fn,
+                dropout_prob=dropout_prob,
+                device="cuda",
+                dtype=dtype,
+            )
+        else:
+            mlp = layers.MLP(
+                d_model=d_model,
+                act_fn=act_fn,
+                dropout_prob=dropout_prob,
+                device="cuda",
+                dtype=dtype,
+        )
+        with act_mem.AllocatedMemContext() as mem, act_mem.SavedTensorContext(
+            ignored_tensors=mlp.parameters()
+        ) as saved:
+            out = mlp(inputs)
+            outputs.append(out)
+        stm = saved.saved_tensor_mem
+        assert mem.delta["current"] == stm
+        print(f"{name} bytes: {act_mem.B_to_GiB(stm)}")
+        mem_bytes.append(stm)
+
+    print(f"ReLU/GELU act mem ratio: {mem_bytes[0]/mem_bytes[1]}")

python/memory/measurments/act-mem-2/requirements.txt

@@ -0,0 +1,14 @@
+einops==0.8.0
+filelock==3.14.0
+fsspec==2024.5.0
+iniconfig==2.0.0
+Jinja2==3.1.4
+MarkupSafe==2.1.5
+mpmath==1.3.0
+networkx==3.3
+packaging==24.0
+pluggy==1.5.0
+pytest==8.2.1
+sympy==1.12.1
+torch==2.3.0
+typing_extensions==4.12.0

python/memory/measurments/act-mem-2/test.py

@@ -0,0 +1,203 @@
+from typing import Optional
+
+import pytest
+import torch
+import torch.nn as nn
+
+import act_mem
+import layers
+
+BATCH_SIZES = (1, 2)
+D_MODELS = (128, 256)
+SEQ_LENS = (64, 128)
+N_HEADS = (2, 4)
+
+
+DEVICES = ["cpu"]
+if torch.cuda.is_available():
+    DEVICES.append("cuda")
+
+
+ZERO_MEM_ACT_FNS = [
+    nn.ReLU(),
+    nn.Sigmoid(),
+    nn.Tanh(),
+    nn.LeakyReLU(inplace=True),
+    nn.Sigmoid(),
+]
+ALL_ACT_FNS = ZERO_MEM_ACT_FNS + [
+    nn.ELU(),
+    nn.GELU(),
+    nn.Hardshrink(),
+    nn.Hardsigmoid(),
+    nn.Hardswish(),
+    nn.Hardtanh(),
+    nn.LeakyReLU(),
+    nn.SELU(),
+    nn.SiLU(),
+]
+
+
+class TestSavedTensorContext:
+    @pytest.mark.parametrize("device", DEVICES)
+    @pytest.mark.parametrize("d_model", D_MODELS)
+    @pytest.mark.parametrize("batch_size", BATCH_SIZES)
+    def test_linear(self, device: str, d_model: int, batch_size: int) -> None:
+        """
+        Test a simple linear layer. The inputs should be saved for backwards
+        """
+        inputs = torch.randn(batch_size, d_model, requires_grad=True, device=device)
+        lin = nn.Linear(d_model, d_model, device=device)
+        with act_mem.SavedTensorContext(ignored_tensors=lin.parameters()) as saved:
+            _ = lin(inputs)
+        assert saved.saved_tensor_mem == inputs.numel() * inputs.element_size()
+
+    @pytest.mark.parametrize("device", DEVICES)
+    @pytest.mark.parametrize("d_model", D_MODELS)
+    @pytest.mark.parametrize("batch_size", BATCH_SIZES)
+    def test_linear_amp(self, device: str, d_model: int, batch_size: int) -> None:
+        """
+        Test a linear layer with AMP. The saved tensors should now be a low-precision version of the
+        inputs and the low-precision version of the weights version of the weights
+        """
+        inputs = torch.randn(batch_size, d_model, requires_grad=True, device=device)
+        lin = nn.Linear(d_model, d_model, device=device)
+        dtype = torch.bfloat16
+        with torch.autocast(device_type=device, dtype=dtype):
+            with act_mem.SavedTensorContext(ignored_tensors=lin.parameters()) as saved:
+                out = lin(inputs)
+        assert (
+            saved.saved_tensor_mem
+            == out.numel() * out.element_size() + lin.weight.numel() * dtype.itemsize
+        )
+
+    @pytest.mark.parametrize("act_fn", ALL_ACT_FNS)
+    @pytest.mark.parametrize("dropout_prob", (None, 0.5))
+    @pytest.mark.parametrize("device", DEVICES)
+    @pytest.mark.parametrize("d_model", D_MODELS)
+    @pytest.mark.parametrize("batch_size", BATCH_SIZES)
+    @pytest.mark.parametrize("seq_len", SEQ_LENS)
+    def test_mlp(
+        self,
+        act_fn: nn.Module,
+        dropout_prob: Optional[float],
+        device: str,
+        d_model: int,
+        batch_size: int,
+        seq_len: int,
+    ) -> None:
+        """
+        For the transformer MLP layer with a ReLU non-linearity, the initial inputs and the inputs
+        to the final linear layer (which are four times as large) must always be saved. If the
+        derivative of the activation function cannot be expressed in terms of the activation
+        function's *outputs*, then the activation inputs must also be saved (which are again four
+        times as large as the MLP's inputs). The MLP activation memory can be nearly halved by a
+        choice of activation function.
+        """
+        inputs = torch.randn(
+            batch_size, seq_len, d_model, requires_grad=True, device=device
+        )
+        expansion_factor = 4
+        mlp = layers.MLP(
+            d_model=d_model, act_fn=act_fn, dropout_prob=dropout_prob, device=device
+        )
+        with act_mem.SavedTensorContext(ignored_tensors=mlp.parameters()) as saved:
+            _ = mlp(inputs)
+
+        # Compare measured memory against expected
+        first_lin_input_mem = act_mem.get_tensor_bytes(inputs)
+        second_lin_input_mem = expansion_factor * first_lin_input_mem
+        # Only some activations require additional activation memory
+        activation_input_mem = 0 if act_fn in ZERO_MEM_ACT_FNS else second_lin_input_mem
+        dropout_act_mem = (
+            0 if not dropout_prob else inputs.numel() * (4 if device == "cpu" else 1)
+        )
+
+        expected_mem = (
+            first_lin_input_mem
+            + second_lin_input_mem
+            + activation_input_mem
+            + dropout_act_mem
+        )
+        assert saved.saved_tensor_mem == expected_mem
+
+    @pytest.mark.parametrize("act_fn", ALL_ACT_FNS)
+    @pytest.mark.parametrize("dropout_prob", (None, 0.5))
+    @pytest.mark.parametrize("device", DEVICES)
+    @pytest.mark.parametrize("d_model", D_MODELS)
+    @pytest.mark.parametrize("batch_size", BATCH_SIZES)
+    @pytest.mark.parametrize("seq_len", SEQ_LENS)
+    def test_mlp_amp(
+        self,
+        act_fn: nn.Module,
+        dropout_prob: Optional[float],
+        device: str,
+        d_model: int,
+        batch_size: int,
+        seq_len: int,
+    ) -> None:
+        """
+        Similar story with AMP. The only changes come from the modified dtypes and needing to also
+        save references to the low-precision weights in the Linear layers.
+        """
+        inputs = torch.randn(
+            batch_size, seq_len, d_model, requires_grad=True, device=device
+        )
+        expansion_factor = 4
+        mlp = layers.MLP(
+            d_model=d_model, act_fn=act_fn, dropout_prob=dropout_prob, device=device
+        )
+        dtype = torch.bfloat16
+        with torch.autocast(device_type=device, dtype=dtype):
+            with act_mem.SavedTensorContext(ignored_tensors=mlp.parameters()) as saved:
+                _ = mlp(inputs)
+
+        # Compare measured memory against expected
+        amp_weight_mem = 2 * expansion_factor * d_model**2 * dtype.itemsize
+        first_lin_input_mem = inputs.numel() * dtype.itemsize
+        second_lin_input_mem = expansion_factor * inputs.numel() * dtype.itemsize
+        # Only some activations require additional activation memory
+        activation_input_mem = 0 if act_fn in ZERO_MEM_ACT_FNS else second_lin_input_mem
+        dropout_act_mem = (
+            0
+            if not dropout_prob
+            else inputs.numel() * (dtype.itemsize if device == "cpu" else 1)
+        )
+
+        expected_mem = (
+            amp_weight_mem
+            + first_lin_input_mem
+            + second_lin_input_mem
+            + activation_input_mem
+            + dropout_act_mem
+        )
+        assert (
+            saved.saved_tensor_mem == expected_mem
+        ), f"Failed on {act_fn=}, {dropout_prob=}"
+
+
+@pytest.mark.skipif(not torch.cuda.is_available(), reason="cuda not available")
+class TestCUDAMemReadings:
+    @pytest.mark.parametrize("d_model", D_MODELS)
+    @pytest.mark.parametrize("batch_size", BATCH_SIZES)
+    @pytest.mark.parametrize("seq_len", SEQ_LENS)
+    @pytest.mark.parametrize("act_fn", ALL_ACT_FNS)
+    def test_mlp(
+        self, d_model: int, batch_size: int, seq_len: int, act_fn: nn.Module
+    ) -> None:
+        """
+        Track saved tensors and allocated memory and verify they agree.
+        """
+
+        inputs = torch.randn(batch_size, seq_len, d_model, device="cuda")
+        mlp = layers.MLP(d_model=d_model, act_fn=act_fn, device="cuda")
+
+        with act_mem.AllocatedMemContext() as mem, act_mem.SavedTensorContext(
+            ignored_tensors=mlp.parameters()
+        ) as saved:
+            outputs = mlp(inputs)
+
+        # AllocatedMemContext captures the outputs, but not inputs, while SavedTensorContext
+        # captures inputs and not outputs. Nevertheless, the readings agree because inputs and
+        # outputs are tensors of the same size and `dtype`.
+        assert mem.delta["current"] == saved.saved_tensor_mem

src/bibliography.bib

@@ -0,0 +1,24 @@
+@article{example2023,
+  title={Example Paper Title},
+  author={Example, Author and Another, Author},
+  journal={Journal of Examples},
+  volume={1},
+  number={1},
+  pages={1--10},
+  year={2023},
+  publisher={Example Publisher}
+}
+
+@online{distill,
+  title={Distill},
+  url={https://distill.pub},
+  year={2016},
+  publisher={Distill Working Group}
+}
+
+@book{textbook2022,
+  title={An Example Textbook},
+  author={Author, Example},
+  year={2022},
+  publisher={Example Academic Press}
+}

src/fragmentLoader.js

@@ -0,0 +1,114 @@
+async function loadFragments() {
+    // Find all elements with ids starting with 'fragment-'
+    const fragmentElements = Array.from(document.querySelectorAll('[id^="fragment-"]'));
+    
+    class FetchQueue {
+        constructor(maxConcurrent = 3) {
+            this.queue = [];
+            this.maxConcurrent = maxConcurrent;
+            this.activeFetches = 0;
+            this.maxRetries = 3; // Maximum number of retry attempts
+            this.baseDelay = 1000; // Base delay in milliseconds (1 second)
+        }
+
+        async sleep(ms) {
+            return new Promise(resolve => setTimeout(resolve, ms));
+        }
+
+        async fetchWithRetry(fragmentPath, retryCount = 0) {
+            try {
+                const response = await fetch(fragmentPath);
+                if (!response.ok) {
+                    throw new Error(`HTTP error! status: ${response.status}`);
+                }
+                return await response.text();
+            } catch (error) {
+                if (retryCount < this.maxRetries) {
+                    // Exponential backoff: 1s, 2s, 4s
+                    const delay = this.baseDelay * Math.pow(2, retryCount);
+                    console.warn(`Retry ${retryCount + 1}/${this.maxRetries} for ${fragmentPath} after ${delay}ms`);
+                    await this.sleep(delay);
+                    return this.fetchWithRetry(fragmentPath, retryCount + 1);
+                }
+                throw error;
+            }
+        }
+
+        async addFetch(element) {
+            const fragmentName = element.id.replace('fragment-', '');
+            const fragmentPath = `fragments/${fragmentName}.html`;
+            
+            return new Promise(async (resolve, reject) => {
+                try {
+                    const fetchPromise = (async () => {
+                        try {
+                            const html = await this.fetchWithRetry(fragmentPath);
+                            
+                            // Process the fragment
+                            const temp = document.createElement('div');
+                            temp.innerHTML = html;
+                            element.innerHTML = temp.innerHTML;
+                            
+                            // Handle scripts
+                            const scripts = temp.getElementsByTagName('script');
+                            Array.from(scripts).forEach(oldScript => {
+                                const newScript = document.createElement('script');
+                                Array.from(oldScript.attributes).forEach(attr => {
+                                    newScript.setAttribute(attr.name, attr.value);
+                                });
+                                newScript.textContent = oldScript.textContent;
+                                oldScript.parentNode.removeChild(oldScript);
+                                document.body.appendChild(newScript);
+                            });
+                            
+                            this.activeFetches--;
+                            resolve();
+                        } catch (error) {
+                            console.error(`Failed to load fragment ${fragmentPath} after ${this.maxRetries} retries:`, error);
+                            this.activeFetches--;
+                            reject(error);
+                        }
+                    })();
+
+                    this.queue.push(fetchPromise);
+                    this.activeFetches++;
+                } catch (error) {
+                    reject(error);
+                }
+            });
+        }
+
+        async processNext(element) {
+            if (this.activeFetches < this.maxConcurrent && element) {
+                await this.addFetch(element);
+            }
+        }
+    }
+
+    // Initialize queue
+    const fetchQueue = new FetchQueue(3);
+    let currentIndex = 0;
+    const elements = fragmentElements; // Assuming this is defined elsewhere
+
+    // Initial loading of first 3 elements
+    while (currentIndex < elements.length && currentIndex < 3) {
+        await fetchQueue.processNext(elements[currentIndex]);
+        currentIndex++;
+    }
+
+    // Process remaining elements as fetches complete
+    while (currentIndex < elements.length) {
+        // Wait for any fetch to complete
+        await Promise.race(fetchQueue.queue);
+        // Remove completed fetch from queue
+        fetchQueue.queue = fetchQueue.queue.filter(p => p.status === 'pending');
+        // Add next element to queue
+        await fetchQueue.processNext(elements[currentIndex]);
+        currentIndex++;
+    }
+
+    // Wait for remaining fetches to complete
+    await Promise.all(fetchQueue.queue);
+}
+
+export { loadFragments }

src/fragments/banner.html

@@ -0,0 +1 @@
+<div>                            <div id="88bd2bde-2a9d-42b6-bea3-2a6d06b034ea" class="plotly-graph-div" style="height:400px; width:1200px;"></div>            <script type="text/javascript">                window.PLOTLYENV=window.PLOTLYENV || {};                                if (document.getElementById("88bd2bde-2a9d-42b6-bea3-2a6d06b034ea")) {                    Plotly.newPlot(                        "88bd2bde-2a9d-42b6-bea3-2a6d06b034ea",                        [{"customdata":[["biiig dot"],["ok-ish dot"],["a dot"],["a dot"],["biiig dot"],["a dot"],["ok-ish dot"],["smol dot"],["a dot"],["a dot"],["ok-ish dot"],["smol dot"],["a dot"],["biiig dot"],["smol dot"],["ok-ish dot"],["ok-ish dot"],["smol dot"],["smol dot"],["a dot"],["smol dot"],["smol dot"],["biiig dot"],["a dot"],["smol dot"],["biiig dot"],["a dot"],["smol dot"],["a dot"],["biiig dot"],["ok-ish dot"],["biiig dot"],["biiig dot"],["a dot"],["ok-ish dot"],["ok-ish dot"],["a dot"],["biiig dot"],["ok-ish dot"],["ok-ish dot"],["smol dot"],["smol dot"],["smol dot"],["smol dot"],["smol dot"],["smol dot"],["ok-ish dot"],["a dot"],["a dot"],["biiig dot"],["ok-ish dot"],["smol dot"],["ok-ish dot"],["a dot"],["ok-ish dot"],["ok-ish dot"],["smol dot"],["ok-ish dot"],["smol dot"],["smol dot"],["a dot"],["a dot"],["a dot"],["smol dot"],["ok-ish dot"],["a dot"],["ok-ish dot"],["biiig dot"],["a dot"],["smol dot"],["ok-ish dot"],["a dot"],["ok-ish dot"],["a dot"],["ok-ish dot"],["a dot"],["ok-ish dot"],["smol dot"],["smol dot"],["a dot"],["ok-ish dot"],["biiig dot"],["biiig dot"],["a dot"],["a dot"],["biiig dot"],["smol dot"],["ok-ish dot"],["ok-ish dot"],["smol dot"],["ok-ish dot"],["a dot"],["ok-ish dot"],["biiig dot"],["biiig dot"],["biiig dot"],["a dot"],["ok-ish dot"],["biiig dot"],["ok-ish dot"],["smol dot"],["biiig dot"],["ok-ish dot"],["a dot"],["ok-ish dot"],["a dot"],["a dot"],["ok-ish dot"],["biiig dot"],["ok-ish dot"],["a dot"],["biiig dot"],["a dot"],["biiig dot"],["smol dot"],["a dot"],["biiig dot"],["smol dot"],["a dot"],["a dot"],["a dot"],["biiig dot"],["a dot"],["biiig dot"],["ok-ish dot"],["a dot"],["a dot"],["a dot"],["smol dot"],["a dot"],["smol dot"],["biiig dot"],["ok-ish dot"],["ok-ish dot"],["smol dot"],["a dot"],["smol dot"],["a dot"],["smol dot"],["ok-ish dot"],["smol dot"],["ok-ish dot"],["smol dot"],["ok-ish dot"],["smol dot"],["ok-ish dot"],["a dot"],["ok-ish dot"],["biiig dot"],["ok-ish dot"],["smol dot"],["biiig dot"],["ok-ish dot"],["smol dot"],["ok-ish dot"],["biiig dot"],["smol dot"],["smol dot"],["ok-ish dot"],["ok-ish dot"],["biiig dot"],["ok-ish dot"],["a dot"],["biiig dot"],["a dot"],["biiig dot"],["smol dot"],["smol dot"],["a dot"],["biiig dot"],["biiig dot"],["smol dot"],["biiig dot"],["biiig dot"],["a dot"],["biiig dot"],["ok-ish dot"],["ok-ish dot"],["smol dot"],["smol dot"],["a dot"],["biiig dot"],["biiig dot"],["a dot"],["ok-ish dot"],["ok-ish dot"],["biiig dot"],["biiig dot"],["biiig dot"],["ok-ish dot"],["a dot"],["biiig dot"],["smol dot"],["smol dot"],["a dot"],["smol dot"],["a dot"],["smol dot"],["biiig dot"],["ok-ish dot"],["a dot"],["ok-ish dot"],["a dot"],["ok-ish dot"],["a dot"],["ok-ish dot"],["ok-ish dot"],["smol dot"],["a dot"],["ok-ish dot"],["smol dot"],["ok-ish dot"],["biiig dot"],["biiig dot"],["smol dot"],["smol dot"],["smol dot"],["a dot"],["a dot"],["biiig dot"],["ok-ish dot"],["a dot"],["a dot"],["a dot"],["smol dot"],["ok-ish dot"],["biiig dot"],["ok-ish dot"],["biiig dot"],["a dot"],["ok-ish dot"],["smol dot"],["ok-ish dot"],["ok-ish dot"],["smol dot"],["ok-ish dot"],["smol dot"],["ok-ish dot"],["biiig dot"],["a dot"],["ok-ish dot"],["biiig dot"],["smol dot"],["biiig dot"],["a dot"],["biiig dot"],["a dot"],["a dot"],["ok-ish dot"],["a dot"],["smol dot"],["smol dot"],["a dot"],["biiig dot"],["smol dot"],["ok-ish dot"],["biiig dot"],["ok-ish dot"],["smol dot"],["smol dot"],["ok-ish dot"],["a dot"],["a dot"],["biiig dot"],["ok-ish dot"],["biiig dot"],["smol dot"],["smol dot"],["smol dot"],["a dot"],["biiig dot"],["smol dot"],["ok-ish dot"],["biiig dot"],["smol dot"],["smol dot"],["biiig dot"],["smol dot"],["smol dot"],["biiig dot"],["a dot"],["a dot"],["a dot"],["biiig dot"],["biiig dot"],["a dot"],["a dot"],["a dot"],["a dot"],["biiig dot"],["a dot"],["biiig dot"],["ok-ish dot"],["biiig dot"],["a dot"],["ok-ish dot"],["biiig dot"],["smol dot"],["a dot"],["smol dot"],["a dot"],["ok-ish dot"],["biiig dot"],["ok-ish dot"],["smol dot"],["biiig dot"],["ok-ish dot"],["smol dot"],["a dot"],["ok-ish dot"],["biiig dot"],["smol dot"],["a dot"],["biiig dot"],["biiig dot"],["smol dot"],["smol dot"],["smol dot"],["biiig dot"],["smol dot"],["smol dot"],["smol dot"],["a dot"],["ok-ish dot"],["a dot"],["biiig dot"],["smol dot"],["smol dot"],["biiig dot"],["a dot"],["biiig dot"],["a dot"],["biiig dot"],["a dot"],["smol dot"],["a dot"],["smol dot"],["ok-ish dot"],["biiig dot"],["smol dot"],["smol dot"],["ok-ish dot"],["biiig dot"],["a dot"],["smol dot"],["ok-ish dot"],["smol dot"],["ok-ish dot"],["a dot"],["a dot"],["biiig dot"],["ok-ish dot"],["ok-ish dot"],["ok-ish dot"],["a dot"],["smol dot"],["ok-ish dot"],["smol dot"],["smol dot"],["smol dot"],["a dot"],["ok-ish dot"],["ok-ish dot"],["biiig dot"],["biiig dot"],["a dot"],["ok-ish dot"],["a dot"],["smol dot"],["biiig dot"],["a dot"],["a dot"],["a dot"],["smol dot"],["a dot"],["smol dot"],["biiig dot"],["smol dot"],["smol dot"],["smol dot"],["ok-ish dot"],["biiig dot"],["ok-ish dot"],["ok-ish dot"],["biiig dot"],["biiig dot"],["a dot"],["smol dot"],["ok-ish dot"],["a dot"],["ok-ish dot"],["a dot"],["ok-ish dot"],["a dot"],["ok-ish dot"],["ok-ish dot"],["a dot"],["a dot"],["a dot"],["ok-ish dot"],["a dot"],["biiig dot"],["smol dot"],["smol dot"],["a dot"],["ok-ish dot"],["a dot"],["smol dot"],["smol dot"],["biiig dot"],["smol dot"],["smol dot"],["biiig dot"],["ok-ish dot"],["smol dot"],["a dot"],["ok-ish dot"],["smol dot"],["a dot"],["a dot"],["smol dot"],["a dot"],["ok-ish dot"],["smol dot"],["ok-ish dot"],["a dot"],["smol dot"],["ok-ish dot"],["ok-ish dot"],["ok-ish dot"],["smol dot"],["ok-ish dot"],["smol dot"],["smol dot"],["a dot"],["ok-ish dot"],["ok-ish dot"],["ok-ish dot"],["ok-ish dot"],["a dot"],["ok-ish dot"],["biiig dot"],["a dot"],["biiig dot"],["biiig dot"],["smol dot"],["biiig dot"],["a dot"],["a dot"],["smol dot"],["biiig dot"],["a dot"],["a dot"],["smol dot"],["ok-ish dot"],["smol dot"],["a dot"],["ok-ish dot"],["a dot"],["a dot"],["a dot"],["smol dot"],["ok-ish dot"],["ok-ish dot"],["a dot"],["biiig dot"],["smol dot"],["a dot"],["smol dot"],["ok-ish dot"],["biiig dot"],["biiig dot"],["biiig dot"],["ok-ish dot"],["biiig dot"],["ok-ish dot"],["smol dot"],["ok-ish dot"],["ok-ish dot"],["biiig dot"],["smol dot"],["smol dot"],["ok-ish dot"],["ok-ish dot"],["ok-ish dot"],["biiig dot"],["ok-ish dot"],["ok-ish dot"],["ok-ish dot"],["ok-ish dot"],["smol dot"],["ok-ish dot"],["smol dot"],["smol dot"],["smol dot"],["a dot"],["a dot"],["a dot"],["a dot"],["smol dot"],["a dot"],["a dot"],["biiig dot"],["ok-ish dot"],["ok-ish dot"],["smol dot"],["a dot"],["ok-ish dot"],["smol dot"],["smol dot"],["ok-ish dot"],["a dot"]],"hoverlabel":{"namelength":0},"hovertemplate":"Dot category: %{customdata[0]}","marker":{"color":{"dtype":"f8","bdata":"NSuP9zcJI0DzwFYGn0gaQLZzmkZHByFA+P+aDpbfHkCwxIAvKbMiQLxO7ywDOSFAkrQilW9qGkA42CJpbWcUQBsoeWzpYyBAQjfOKzOZHkAirl5Y47kZQHo9O10vahVA9tL3qjtYIEDDffk1l7UjQO4gDWWi9RhAbl4O21pGHEC2NukPe44cQEBdAmUkehhAaO9vYTNPGEAkKuTDTKQeQCwfNyVGoRZA7AXlPkQVFUBBS1LPtqEhQHjQxmS+fSBAzIXDZfEoGEDwmCYU3FQjQNBMefzIwx5APDOqSw25FUCJceoo8vQeQP7yGSlEKyNAQTLBxzJtHUDBmCp6+7oiQBoQiWhE8yFA8nURfbx6IUBmXmxn2ucZQCV81yB8MR1APe9aowa5IED5TDni5\u002fMiQE8Bv4u6zhxAuahCRUDHGUAuRAEnvokVQK9yTdCItxdAS9Rm\u002fEJxFUAkVNMOqH0XQLUcVRC3QhVAr9JC7d6BFECx6VQKAsMZQJiTu8jHAiBASNpbXE+iIEBuNZJTAhkjQDf4LRmEZRxAAc\u002fXZA4iGEBoI3er6SwbQDzSHSGc\u002fiBALJKQx8UFGUBnxhDVR5kZQNxdroKZTRVAmIzuoRi0G0DXi1x1ruEXQH4OD4n5ghhA2GYHR1L1HkBGY0\u002fvvsMgQFQklZCQ6x5AUoEYHYbFFEC9SZgflvgbQDubyvxZJR9Al0AR8t+YG0BjEwai1O8jQIm9fGx9DB9AJ\u002fViqUi9FkCP01kOAW8ZQF\u002fwJr5pmCBAsN8BVrmfHECAA0NocFweQEK4F2+EXxtA6ie8cVJaIUADJI6NekMdQB2bndnvwxdAjTccD2guFUBBMr6RvxEhQHnH3ClkthtASKBd2FYgIkAOCefEnMwiQB71sK8twR5AKix3z3LvIECge2K95\u002fQjQKKwiqhdnhdApBqwIX9fGkBjBAhPEWAbQNxyN8qJEBZAFRHupGPBHEBM28zFik0gQHsg478ihhlAx1QGiQRgIkB193H6ybMjQAAXe2\u002fpiiJAFofW\u002fyalIEAvhq2s3p8bQCAw71hf8SNAOICnOo\u002fwGUABkT73IBIUQDqKRFRq7iFAWuOaJsGiGkCdNtGT1T0eQPwUzYZdhhxA226rezPlIEAK2rKtZCYhQM6e29Px6RlALDlNZmBcIkBF51ygESIbQARdbLsmGiBAcPPGqWmoI0AA9vbuxj0fQFLy6PqqKSJAX2nTXbL2FkDk6uyOBjsgQBIKnNMwzSFADkUJljHpFUBRWFONQhkfQF7mpoKcuR5AS6GPzhyaIECWv1cvA3QiQCTJYvjoRh5AFSMr+II+I0DAVgHpCp0ZQDLZRRIwCx5AToyx0piwIECo1p8Myr0fQOSPrLgYpBZAM1UrnbGlIEDy552AKr4YQE8C39hIuiJAmPTyZN1jG0B9p4nAEOscQDBHdvtiPBZAATP1FnlSIUC9lsGRS2sUQHTHAX8sViBAxsOQHXPOFUD8JQEmPBgcQPrz2LeufxZAhNMjenowHUBT+7k7bO8YQASsYAOMfhlAef5a6KU8F0BMqKTYMfQZQIBwogq5mh5AsyQGXIyzHECm+hE9MjwiQLKGFnhocB1ADTMLPgRxFUCO9NdLNYEhQMy\u002fTuw\u002fwxpAeqAQGXgoGEDg1wTLsCwcQMC+x+kqLCNAQy3+JdgJFkDajzrsrUUUQCi4qErh6RxASsFgzXDsGkAtogkxtf8jQPLsdJgkxhxAKjnbSbMrIECWefuN6fMiQKbpd2eHxh9AbEz3spPGI0CA3RaqTq4UQJhk6nfe8xRAWbiqkx5oHkBbUE1wC4MjQLr9sg+1+CFAaXQ1CDK4GEBi8yblEzcjQOUy6xGLuyJA2vM+aS8eIUBDzKhWJ4UjQM2IV\u002fUqhxpAgZjH335aGUC\u002fCbav8yQYQBN9THrmgxZAykj1ErUCHkANqY5Z1e4hQPjVtvSFrSFAIIEFQhyPHkCW76\u002fZUPEaQEo256Q8ORxAblmeP4aiIkCueeDEm3UjQEXS4+kRdiJAjeNXaVm9GkDYxejIj50fQKO+bcSJdiNAr5cChkQcF0Dvt+qkTPcVQPShkHR4MR9AonlqcvnTGEBvTbvwc\u002fggQB\u002fmUxg9DBRA36HYc0QHI0A\u002fOMEnWU4cQI\u002f5IjbVpSBADkL2EA7EGkA7x\u002f0ADSEhQGOxHli7QBpAElvxWLuVIEAEPS8PokEaQGoP7+CiqhtAelK\u002fncH7FUC6io+f4MgeQI8O806nQR1AAJb6Oue1FEDH30BZI3gZQLSKZLXNqSNAt0nJeolIIkBu3y6iCqsXQLINEatrzRZAIvn0fdw4FEDA2IWf83UeQKEy+iUjBCBArE1hmsbGI0BklE0I9kwZQGM7pV3HHCBA\u002fhDNXYsOH0AhE22eArIgQIYTUgxigRdAXitsu7eYGkA9RXE5IPYjQDxwmauxkRlAfDYQL6rVI0BUgV+Ks3kgQG6AnJhXXR1A8or\u002f2hwSFUAIgfUlIwsZQDTcZVLumhxA6QjirXc3FUA10SE2ppkaQG\u002fchBsurRZAEKqnjjRRG0DbXS+XVOMjQCR+klfDvCBAcvT5sjPmG0AuEZiM9VkiQE7\u002f9nPMwxhAzUNaGpsYIkAMFl+iM4geQAbi0IVJkSJAXxa114t\u002fH0A4LZs7SxMhQCPR49QEkxtA1Jk77+3lIECI0aR07toVQIsuW1P1hBdAR4MfDSR2IECzOdYNDI4hQOpD59D\u002fEBZAiERnFZezHECEaCFOJH0iQJ6LD6MStRpAIPg8FEDaFkC77pOHxHsXQIKYaKpe0hxACHYgLLeVH0CMjGdWsu0gQAp8kRBXiiFABnaqzwIiHED4o8ohaUYjQB6ez25KZhVAODyihKQZGEAVl18M61sVQGTFlTpa1iBANQ9RIDogI0A0b3ba87kVQAxFuokw1xlAVn0YdOV5IkCZF1qq3xEYQF70crxSgRhAmjv\u002fDYqqI0A2Tf\u002f9wFcUQF0zMnYpyBVAp+Va3dnnI0DrnbA6MMwgQK2PaoBKXx9AvDk4yCcMHkCMHtGK3hUiQOBifysgRyJATjUTFEe3IEAgYn34c1QgQEvJtnn6nB9ArB0t3VRRHkCp64NUq68jQOwK\u002foo\u002fQiFAfOBrePfqI0DdiBMwugIdQOw4g57jZiJA\u002fmRSrrobIUDgwI46TQMcQLbiHMtOfyNA\u002fKVcwF8gFEC+RWrXTB8eQOZRReUF0xRAJG5asaiVIEDakyC8ttwcQGbJjrgiCyJAFUL1ynSOHEBCG81wE28UQCrioPQFRyNA2PjDcDBsGUB4mAX+diMVQFdwfQyOxiBA9OHZFopzHUDVGh8\u002frCMjQMrNAYaphxVAaZIfeQx9HkBDOPxNx0IjQE9mdsWUJyJAX\u002fQc\u002f+2mFEAfi1InuigXQFDGmnvezBZA6PU\u002fkkx1IkCftvrATAcYQJhU2m0ywhhAa+DPV2YBFkBjxzSzkyYhQAdYVtH4tBtAQnTQUPrvH0Blcff07xgjQOomWdPlGRVAHftwUzmWF0Cu3UU8\u002fQMjQGNEGODnGyFAI9CO58OXIkC4uGz9+GkeQL9mwKcK2SFAiCDh8DB0HkA\u002ffgMr1AoXQCqEn79enR5A5\u002f4QbQF\u002fFUDo63HexOYbQKGlPexgAiNA7w5z3\u002fe7FUD7w0ihBSwWQPObsdTv1xlARBUncGKNIkBziwdUxmAhQIJc4VeMchZAyNWlhm3JGkD1cSz66tgWQH3dkv6F9hlAnJZtjukzHkC+gt+\u002fyXceQGuFps\u002fhsyJA5RxPbzDJHUBngvyx1AkcQPq8r9pjxhlA6lCggb+pIEAkljYd5PkUQAoR+A4uZBxAepH5CPOWF0D2xOpi0owXQOzzMRoF\u002fRVAimoq9\u002fDjHkBCnaUXrbgbQBvV9Dlc+RxAzhvtX0bDIUBeDedw3NgiQI9ETzyi9h9ASi4PNTLaHUCaJ5fLdK8gQMj5UGSR6xVAUmrCJdbYIkAZMNcR95keQKkPSNYCCCBAXFQQu7B0IUDjTOlgmtAVQDw40sx3lyBAHcAuNNttF0Cz7w7mKb0hQH2I2dYpoBVArnEF66FxFUDiyzTxOfsVQD10LW8mDRpApmGJNN4AIkDWaZ3JVGcdQDlgj\u002fvhrhxAso53sF8EIkB4DnaJeHkiQIxYL6+8JyBAWHPi7bYIFUBHMGhAEY4dQH\u002fu5BowQCBA3KbYZVFMGkDeyXgP+iwhQCIhfAhIyBlApYieNe1bHkBjEq8JlEcZQIsVC53D8x1AKiHzjaXiHkAcYAKff1EeQOIpVwfIViFArTruYthVG0DnODsdngYeQIqpzBDaiSJAetNrtQUVF0Cs5RrCv1MXQPy\u002fs1wPBx9AcVZi3gmMG0AAcSNO7GAhQO7DrwgelBdA4I9\u002fCbbRFUD02rnmUXYiQA0nAofWmxZA9lpQ2knmGEBIzd08bWMjQHRwoyEaExxAnDR0LaLVFUANmCOh5FUhQJSfonbdfxtAUjbX2LijF0DaEYkjMwgfQKvTLhO59x9AAdAuHiVqFkD+wSLgFSYhQM4UO56kBBpAwBrh7AVxFEBnSg21iIIdQEU3tP5vTiFA\u002fH7caY8FGEA1ZCiKw6MaQNRXiCFsAhxAHKfehkc5G0CMiz3qO38XQCjWrBo0vB1AHUNHqkXNGED42SOQswwWQINqqlljQCFASO0cCsKpHUDXqSebilQZQDc5kkkoYxlAQvk0RIkGG0CCYbCOrWgeQIA\u002fqE12PhtAcXFUY4VkI0AOyXgkkBIeQKJvQPHYtiFAe0EZtDaYIUD8\u002fQlH7V0WQHM7peTEYyJAijXMZdN3H0C+JmlrDr8eQB5qjSf0CxhA9uOfyXiRIUDyfwSTBPkgQEfe0AG38CBAZrKTYsMAFUBS7s8vM4YdQP8kigLgehdAKT1cehiLIECGFsISqFcZQEtuU8KPBSFADHYpuE\u002fUH0Bzii+nNJogQBKLWoeEPRZA6JhcA0n8HECo043e+PsaQKiwTkQ\u002fhiBAYIbQ2Ag+I0BAUs3h0hUVQFMSLlE9sCBAa858w+OnFEBUZbCvilMZQLXSXhGnYiNAXrgm5HR0I0AtIFA36OMiQL7jBurC8BpA+IKdy5PfIUCyBWnn55gaQCBSf8phhhdA0Wa39GivHUAWl8V29VsdQN0iq6mx7CFAgOs7Eh6sFED2xGaxhMkVQMnA1q8QoBtAKjYhWCp0GkD+bx8TmFMZQI3ZsRBNnCJAqemy4d0BHEAm4+7jnmAbQC7zeSXclRpA6d6Q6mIlG0Cp0Zf1k10WQNQhSlVqBxxAHAGbxJfFFUC2J3iKX1cYQJwxxnwaMRRA7ZKHBBWwIEDKKimWtkohQHaPE1U0cyFAM3OghPxkHkAgTj3nsEoYQNTXKBxhYh9AOxQ8ny92IUAM9NL\u002fsfIjQHSF5K38QBpA2FyPhk6SGkAk3MpoeFAUQAPDfMo61B9AuD4ny5LdHEBcOdiDMTgUQEageTdcbBVAkiFjKsK8GkDdjkY8xpkgQA=="},"colorscale":[[0,"rgb(78, 165, 183)"],[0.5,"rgb(206, 192, 250)"],[1,"rgb(232, 137, 171)"]],"opacity":0.9,"size":{"dtype":"f8","bdata":"NSuP9zcJI0DzwFYGn0gaQLZzmkZHByFA+P+aDpbfHkCwxIAvKbMiQLxO7ywDOSFAkrQilW9qGkA42CJpbWcUQBsoeWzpYyBAQjfOKzOZHkAirl5Y47kZQHo9O10vahVA9tL3qjtYIEDDffk1l7UjQO4gDWWi9RhAbl4O21pGHEC2NukPe44cQEBdAmUkehhAaO9vYTNPGEAkKuTDTKQeQCwfNyVGoRZA7AXlPkQVFUBBS1LPtqEhQHjQxmS+fSBAzIXDZfEoGEDwmCYU3FQjQNBMefzIwx5APDOqSw25FUCJceoo8vQeQP7yGSlEKyNAQTLBxzJtHUDBmCp6+7oiQBoQiWhE8yFA8nURfbx6IUBmXmxn2ucZQCV81yB8MR1APe9aowa5IED5TDni5\u002fMiQE8Bv4u6zhxAuahCRUDHGUAuRAEnvokVQK9yTdCItxdAS9Rm\u002fEJxFUAkVNMOqH0XQLUcVRC3QhVAr9JC7d6BFECx6VQKAsMZQJiTu8jHAiBASNpbXE+iIEBuNZJTAhkjQDf4LRmEZRxAAc\u002fXZA4iGEBoI3er6SwbQDzSHSGc\u002fiBALJKQx8UFGUBnxhDVR5kZQNxdroKZTRVAmIzuoRi0G0DXi1x1ruEXQH4OD4n5ghhA2GYHR1L1HkBGY0\u002fvvsMgQFQklZCQ6x5AUoEYHYbFFEC9SZgflvgbQDubyvxZJR9Al0AR8t+YG0BjEwai1O8jQIm9fGx9DB9AJ\u002fViqUi9FkCP01kOAW8ZQF\u002fwJr5pmCBAsN8BVrmfHECAA0NocFweQEK4F2+EXxtA6ie8cVJaIUADJI6NekMdQB2bndnvwxdAjTccD2guFUBBMr6RvxEhQHnH3ClkthtASKBd2FYgIkAOCefEnMwiQB71sK8twR5AKix3z3LvIECge2K95\u002fQjQKKwiqhdnhdApBqwIX9fGkBjBAhPEWAbQNxyN8qJEBZAFRHupGPBHEBM28zFik0gQHsg478ihhlAx1QGiQRgIkB193H6ybMjQAAXe2\u002fpiiJAFofW\u002fyalIEAvhq2s3p8bQCAw71hf8SNAOICnOo\u002fwGUABkT73IBIUQDqKRFRq7iFAWuOaJsGiGkCdNtGT1T0eQPwUzYZdhhxA226rezPlIEAK2rKtZCYhQM6e29Px6RlALDlNZmBcIkBF51ygESIbQARdbLsmGiBAcPPGqWmoI0AA9vbuxj0fQFLy6PqqKSJAX2nTXbL2FkDk6uyOBjsgQBIKnNMwzSFADkUJljHpFUBRWFONQhkfQF7mpoKcuR5AS6GPzhyaIECWv1cvA3QiQCTJYvjoRh5AFSMr+II+I0DAVgHpCp0ZQDLZRRIwCx5AToyx0piwIECo1p8Myr0fQOSPrLgYpBZAM1UrnbGlIEDy552AKr4YQE8C39hIuiJAmPTyZN1jG0B9p4nAEOscQDBHdvtiPBZAATP1FnlSIUC9lsGRS2sUQHTHAX8sViBAxsOQHXPOFUD8JQEmPBgcQPrz2LeufxZAhNMjenowHUBT+7k7bO8YQASsYAOMfhlAef5a6KU8F0BMqKTYMfQZQIBwogq5mh5AsyQGXIyzHECm+hE9MjwiQLKGFnhocB1ADTMLPgRxFUCO9NdLNYEhQMy\u002fTuw\u002fwxpAeqAQGXgoGEDg1wTLsCwcQMC+x+kqLCNAQy3+JdgJFkDajzrsrUUUQCi4qErh6RxASsFgzXDsGkAtogkxtf8jQPLsdJgkxhxAKjnbSbMrIECWefuN6fMiQKbpd2eHxh9AbEz3spPGI0CA3RaqTq4UQJhk6nfe8xRAWbiqkx5oHkBbUE1wC4MjQLr9sg+1+CFAaXQ1CDK4GEBi8yblEzcjQOUy6xGLuyJA2vM+aS8eIUBDzKhWJ4UjQM2IV\u002fUqhxpAgZjH335aGUC\u002fCbav8yQYQBN9THrmgxZAykj1ErUCHkANqY5Z1e4hQPjVtvSFrSFAIIEFQhyPHkCW76\u002fZUPEaQEo256Q8ORxAblmeP4aiIkCueeDEm3UjQEXS4+kRdiJAjeNXaVm9GkDYxejIj50fQKO+bcSJdiNAr5cChkQcF0Dvt+qkTPcVQPShkHR4MR9AonlqcvnTGEBvTbvwc\u002fggQB\u002fmUxg9DBRA36HYc0QHI0A\u002fOMEnWU4cQI\u002f5IjbVpSBADkL2EA7EGkA7x\u002f0ADSEhQGOxHli7QBpAElvxWLuVIEAEPS8PokEaQGoP7+CiqhtAelK\u002fncH7FUC6io+f4MgeQI8O806nQR1AAJb6Oue1FEDH30BZI3gZQLSKZLXNqSNAt0nJeolIIkBu3y6iCqsXQLINEatrzRZAIvn0fdw4FEDA2IWf83UeQKEy+iUjBCBArE1hmsbGI0BklE0I9kwZQGM7pV3HHCBA\u002fhDNXYsOH0AhE22eArIgQIYTUgxigRdAXitsu7eYGkA9RXE5IPYjQDxwmauxkRlAfDYQL6rVI0BUgV+Ks3kgQG6AnJhXXR1A8or\u002f2hwSFUAIgfUlIwsZQDTcZVLumhxA6QjirXc3FUA10SE2ppkaQG\u002fchBsurRZAEKqnjjRRG0DbXS+XVOMjQCR+klfDvCBAcvT5sjPmG0AuEZiM9VkiQE7\u002f9nPMwxhAzUNaGpsYIkAMFl+iM4geQAbi0IVJkSJAXxa114t\u002fH0A4LZs7SxMhQCPR49QEkxtA1Jk77+3lIECI0aR07toVQIsuW1P1hBdAR4MfDSR2IECzOdYNDI4hQOpD59D\u002fEBZAiERnFZezHECEaCFOJH0iQJ6LD6MStRpAIPg8FEDaFkC77pOHxHsXQIKYaKpe0hxACHYgLLeVH0CMjGdWsu0gQAp8kRBXiiFABnaqzwIiHED4o8ohaUYjQB6ez25KZhVAODyihKQZGEAVl18M61sVQGTFlTpa1iBANQ9RIDogI0A0b3ba87kVQAxFuokw1xlAVn0YdOV5IkCZF1qq3xEYQF70crxSgRhAmjv\u002fDYqqI0A2Tf\u002f9wFcUQF0zMnYpyBVAp+Va3dnnI0DrnbA6MMwgQK2PaoBKXx9AvDk4yCcMHkCMHtGK3hUiQOBifysgRyJATjUTFEe3IEAgYn34c1QgQEvJtnn6nB9ArB0t3VRRHkCp64NUq68jQOwK\u002foo\u002fQiFAfOBrePfqI0DdiBMwugIdQOw4g57jZiJA\u002fmRSrrobIUDgwI46TQMcQLbiHMtOfyNA\u002fKVcwF8gFEC+RWrXTB8eQOZRReUF0xRAJG5asaiVIEDakyC8ttwcQGbJjrgiCyJAFUL1ynSOHEBCG81wE28UQCrioPQFRyNA2PjDcDBsGUB4mAX+diMVQFdwfQyOxiBA9OHZFopzHUDVGh8\u002frCMjQMrNAYaphxVAaZIfeQx9HkBDOPxNx0IjQE9mdsWUJyJAX\u002fQc\u002f+2mFEAfi1InuigXQFDGmnvezBZA6PU\u002fkkx1IkCftvrATAcYQJhU2m0ywhhAa+DPV2YBFkBjxzSzkyYhQAdYVtH4tBtAQnTQUPrvH0Blcff07xgjQOomWdPlGRVAHftwUzmWF0Cu3UU8\u002fQMjQGNEGODnGyFAI9CO58OXIkC4uGz9+GkeQL9mwKcK2SFAiCDh8DB0HkA\u002ffgMr1AoXQCqEn79enR5A5\u002f4QbQF\u002fFUDo63HexOYbQKGlPexgAiNA7w5z3\u002fe7FUD7w0ihBSwWQPObsdTv1xlARBUncGKNIkBziwdUxmAhQIJc4VeMchZAyNWlhm3JGkD1cSz66tgWQH3dkv6F9hlAnJZtjukzHkC+gt+\u002fyXceQGuFps\u002fhsyJA5RxPbzDJHUBngvyx1AkcQPq8r9pjxhlA6lCggb+pIEAkljYd5PkUQAoR+A4uZBxAepH5CPOWF0D2xOpi0owXQOzzMRoF\u002fRVAimoq9\u002fDjHkBCnaUXrbgbQBvV9Dlc+RxAzhvtX0bDIUBeDedw3NgiQI9ETzyi9h9ASi4PNTLaHUCaJ5fLdK8gQMj5UGSR6xVAUmrCJdbYIkAZMNcR95keQKkPSNYCCCBAXFQQu7B0IUDjTOlgmtAVQDw40sx3lyBAHcAuNNttF0Cz7w7mKb0hQH2I2dYpoBVArnEF66FxFUDiyzTxOfsVQD10LW8mDRpApmGJNN4AIkDWaZ3JVGcdQDlgj\u002fvhrhxAso53sF8EIkB4DnaJeHkiQIxYL6+8JyBAWHPi7bYIFUBHMGhAEY4dQH\u002fu5BowQCBA3KbYZVFMGkDeyXgP+iwhQCIhfAhIyBlApYieNe1bHkBjEq8JlEcZQIsVC53D8x1AKiHzjaXiHkAcYAKff1EeQOIpVwfIViFArTruYthVG0DnODsdngYeQIqpzBDaiSJAetNrtQUVF0Cs5RrCv1MXQPy\u002fs1wPBx9AcVZi3gmMG0AAcSNO7GAhQO7DrwgelBdA4I9\u002fCbbRFUD02rnmUXYiQA0nAofWmxZA9lpQ2knmGEBIzd08bWMjQHRwoyEaExxAnDR0LaLVFUANmCOh5FUhQJSfonbdfxtAUjbX2LijF0DaEYkjMwgfQKvTLhO59x9AAdAuHiVqFkD+wSLgFSYhQM4UO56kBBpAwBrh7AVxFEBnSg21iIIdQEU3tP5vTiFA\u002fH7caY8FGEA1ZCiKw6MaQNRXiCFsAhxAHKfehkc5G0CMiz3qO38XQCjWrBo0vB1AHUNHqkXNGED42SOQswwWQINqqlljQCFASO0cCsKpHUDXqSebilQZQDc5kkkoYxlAQvk0RIkGG0CCYbCOrWgeQIA\u002fqE12PhtAcXFUY4VkI0AOyXgkkBIeQKJvQPHYtiFAe0EZtDaYIUD8\u002fQlH7V0WQHM7peTEYyJAijXMZdN3H0C+JmlrDr8eQB5qjSf0CxhA9uOfyXiRIUDyfwSTBPkgQEfe0AG38CBAZrKTYsMAFUBS7s8vM4YdQP8kigLgehdAKT1cehiLIECGFsISqFcZQEtuU8KPBSFADHYpuE\u002fUH0Bzii+nNJogQBKLWoeEPRZA6JhcA0n8HECo043e+PsaQKiwTkQ\u002fhiBAYIbQ2Ag+I0BAUs3h0hUVQFMSLlE9sCBAa858w+OnFEBUZbCvilMZQLXSXhGnYiNAXrgm5HR0I0AtIFA36OMiQL7jBurC8BpA+IKdy5PfIUCyBWnn55gaQCBSf8phhhdA0Wa39GivHUAWl8V29VsdQN0iq6mx7CFAgOs7Eh6sFED2xGaxhMkVQMnA1q8QoBtAKjYhWCp0GkD+bx8TmFMZQI3ZsRBNnCJAqemy4d0BHEAm4+7jnmAbQC7zeSXclRpA6d6Q6mIlG0Cp0Zf1k10WQNQhSlVqBxxAHAGbxJfFFUC2J3iKX1cYQJwxxnwaMRRA7ZKHBBWwIEDKKimWtkohQHaPE1U0cyFAM3OghPxkHkAgTj3nsEoYQNTXKBxhYh9AOxQ8ny92IUAM9NL\u002fsfIjQHSF5K38QBpA2FyPhk6SGkAk3MpoeFAUQAPDfMo61B9AuD4ny5LdHEBcOdiDMTgUQEageTdcbBVAkiFjKsK8GkDdjkY8xpkgQA=="}},"mode":"markers","showlegend":false,"x":{"dtype":"f8","bdata":"ZaZJYVHlA0DmuLID8asAQLKPz+BPTdA\u002f5f5a+KGH6j80JHzLf1HsP2LrYV2TLwNAzIeQle4k8T+KwUeudAkEQH6AjqCL0QVAaja7ae7M\u002fj+a\u002fNdmJ+r0P6oQfmdibfg\u002fynzdDP+b2z9k07IszIvlPwL6k9GlJQNADJnTzltQB0AVAYZ2zG4DQDIaHeZofQRAE6ge4I1d+z+UCLa4WIHzPwjSkmGUDrA\u002fFP\u002fx8mgF8z+A0ydwAF7PP8+iGoSjlwNAXXn9e8rl5z\u002fgCExiRNIBQONJe\u002fR0+eg\u002f1LRMUcCR9T+57Wnf+0cBQOxPFX6s8gJA3YnoU3hsAkCqzPCNmPz1P179nEf9P\u002f8\u002f1+uIDtAVAkBqDbwN8eMFQMBw0mhajPM\u002fLJMqd6O+8T8Ea\u002fPpNGnCPyDwxEL+vgRATJnORxoBBUA6JZUI\u002f8v2P\u002flJS3BKWwBAbUkXrjhJ5z+O8Mtkq\u002fj5P9pKkjl3cPQ\u002fqFbcxuHy3D82CojwFjcGQIatSsHD5+0\u002fUyL0tx40\u002fz\u002fm8YEgKnH+P0zlKQoR\u002fgJA2GY9A3PI\u002fz+m+zSwPu\u002f+P7mcLhcCdf0\u002f4UxKp4IVAEBeBvaxAZEGQKD8JBQ+b+M\u002fMI9ipyinrj+G8JNULukFQES\u002fNrlICf0\u002fijKrOxMK6D909Bir4Rn8P4UTS8y0BgFA3O0Dbt209z9hG0KsbnQCQJ7QJYDoOwZA59YRMkP6AUBDY1\u002fPlOgGQPDJnszvLuM\u002fkPnByo5l8T+FqLY241r\u002fPzQaYz5vxAZAtmf\u002f9SfX\u002fj+8M1SHJrUFQHT45EAd9QJA8X8jCst5\u002fD9rW6dEZz4AQGqsYNCPUQBAIglU6pkc+j8O0Lx8tdDjPyKuYuGqftk\u002fEWhIG\u002fkS6j+nubUD7nIEQKxJsvivteQ\u002fZh5lXNUQ9D+Ma4fa7ET3P\u002fRJzCV1Vfg\u002fPXNWwDyX+T\u002fZcT\u002f9z\u002f\u002f9P5oQ+dID5vI\u002fcHO5LZ\u002fB+j+u2ezZnxf0P6p4eMrFh\u002fs\u002fvhwSxSV21T+GT9OIjx0AQDu+GFcwzwJAuuSNDdef+T+vGUl9XZPsPxqOXI4fk\u002fk\u002fEt3q5iAlAkCaUelOu48GQKFruUniL+o\u002fOHPEHrGc\u002fj81roPeI5vzP4CXfoUegPo\u002fFl4eqEOb8j\u002fyYx1BWBnjP0mh5fVJevI\u002fmt84a6b6B0DmaDZRgOfxP7yQ+ZKK\u002fQJAhR6q4jXUA0AwUHFh9ZrmP6bEkeD24PI\u002fpfqGG0ro6D9SrmZZvfn5P617cj6t+Pk\u002fAtKPombCAEAIz7j+22kHQIjwJicF0v8\u002f9FIlLAGaBkA4ae+Vo+XsP5kRN6ETqQFAQ+ZMMmqR8D\u002fpc+KGkxHzP0mupFe20eQ\u002fRycll1mH6D8bcc7al3PsPzoCPtsuFgNAUPC20QQjAUCfG2ekqeYDQMPopcVK4QFAavX9OsN88z\u002fOPq1Xqfj+P\u002fuZAfDKRgRAfI7EhOMVBUD4\u002f07cyQy4P\u002fuaEjay3gVAGtOVviZU9T9Skr0xq6HiP2gHXiBiFwJAtJdZLE+34j81FmF63Z3oPykCwpQxWfQ\u002fEIQ6gkeB+D+AMmtJGi32P7WGN9J1rPY\u002fGcDF2Drb5z8igcQg9pPwPyhaBeicsrY\u002fwM37H1fQoz9cHNBkuLHIP+Q4u+aKBfs\u002fN5SBADuYB0DN\u002fBCTwZcGQES7vYWJXMY\u002fVtFO6L0g+T\u002fmxu9uy+baP12ufoz5n+k\u002frKQNF67tBkCCo75Ksm\u002f+P3p0cBefIOI\u002fQKq1f9q3ij8jHp9hryT6P2+VWVrCDvk\u002f+FsJwpvt+j\u002fOzCYOeoYDQCaqMtD+LPg\u002fJrPNvMZk0z9wCfpDsbiqP6qCqyYCbAJAaCutMuUb9D9cLTnOZb8HQDg+gHvAsNk\u002fjF0bhK0Y1z+yZCGPqfIFQPRsXUxxo8w\u002fRriHyWe4+T+QLhULnafOP64IXtao\u002fQFASuS4VtNf2T+Q3HPLLNUGQDh1Dgu8Rek\u002fHkeoCKPU0T8IQhqMit\u002fdP4cO8zGUdeo\u002f+clG2KP1AkCslyTMFWP\u002fP32GaiCo7ABA0H5Q8uGBsz9GyJDmg9YBQCCPlLZDogRAVki1\u002fTXFAUC8oS+JjQb0PyDi+08GybU\u002fipMGiOd9BkAgZizYBgj4P+y78YN0HdQ\u002fg\u002f8UgZy\u002fAkAVLYKorQnrP16zwP1WWANAvTDFI4nB+j8QrgzrNt6pP5kD0z\u002feNAdAbPge8aX6AUD5O6mjSdnrP8h9Wcuz9Mk\u002fLIV4NetsA0Ds0wRETFr\u002fP0555dkm\u002fAdA\u002fPtRWHT28j\u002f6bPRHyP4CQLB+lKbkdeo\u002fvfpAOHly8z9NoZMPtj7pP8oXWpPrbf0\u002fz6M6CLQ3AkDuSDgupw0FQLNtC\u002fcUbeI\u002fSLoRyYIn2z8rmTuT6hMBQAlENslpRek\u002fPM2USDkR4T86FStCNKwHQKq88OSwaPY\u002fA+e1HcyNAUDAhKPPzqXeP\u002foNZOIssPs\u002fg+Zj1RFPA0AAdIQK522dP\u002fJOKJKKNfI\u002fYLIkqq4KA0BeBsTEPYn3PzTg\u002f+87PsI\u002fFpdHapie8z8uv3FZ8Q8EQEr9\u002frd\u002f+fY\u002fkJz\u002f9xquBkBpFwBsgtzqPwjV2j7ei9E\u002ffyNZyKYk\u002fz+y4nNxi0YGQN0eeVwVRAFAyyLKWh6o6j8esYZ5hDUBQJWaq\u002flDTO8\u002fhHEqIgKd\u002fz+OHcs6Sp74P\u002fEoJXwjKgNAOsV1hUXi8D9J9PSp1er9PydnLA8O4AZACPMIgnQEzD9b6zAb023qPw3pYE5o0fo\u002fVtNm\u002ffuu1T9yPp2fAr\u002fkP09hxV5pD\u002fw\u002fBnImapkf3T\u002f4LIEgbn7APygmZ\u002fCNu88\u002fmI7faU1A0T\u002fmupa79zXxP5GkSZ9FMPE\u002fQFE+uLiZqT9\u002f2dbLXX72P8oZF2c5Lfg\u002feLsMehR5B0CSjvD007UCQNmd1FisI+Y\u002f8m5qYyTu9j9KWWwqZD7aPwzh7jbxegZAcMNvLry6pD98f6w0Fp0CQK56fBZo7\u002fA\u002fOJ4B0UPOyj9MdghIojDqPwiA2KmYbLc\u002f9PHUk15+3D8uxGCJUjoEQE1JNnuEzeQ\u002f+h17pQDn5T9o+Ikb\u002fdTwPynhfnTnm\u002f4\u002fxwYRUcAp6T+u3v12g8PyP3JiXGiQgv8\u002fGOo\u002fxnVi4z+UqqfijHAEQMzNCL56gv0\u002fShUSt7LmAkDN5MfgXPP1P3N5aEoeYPI\u002fZCashnM2+j9OTBnRqsHhP782zjXuMgZAQoomp\u002fxC9z+gEqhOFisFQLBsDwOvRO8\u002fHmTPoaZS9j+sNWdC7AvMPwsO0L+ix+Y\u002fMVF5sL4A5j+CnBF9Q2f9PzPgSzI6hgZAFrS5UvdC0z+U+2DouW3qP1IOZfE0lO8\u002f9FpG5gIS9T\u002fLsS7iLMz4P1peCFO4wQJAOLZuHf6o2j\u002fsNYwTLSEDQPjA30y\u002fFQRAtVOFZEAG6D9YLls6YbrwPyM7oRBXsOI\u002fMlddUCKcA0D4nHf+iebkP+\u002fAQhmt+\u002f0\u002fY41NdjuUAkDcKRCr5Vr7P4gjP4CYv\u002fU\u002fDiA6xnouAUADgVH8mg3sP2QyTLegi\u002fQ\u002fXAWrj4pW\u002fT9beN3LcjwHQCszw3hjVANAQKfKhlmeqj+FHx2URL\u002flP55Mnu7b9fE\u002f2eqQqlkP8z+iLCRTK8LUP+0M6Hf5YABARCxLaXoe8D+EOdTVgEz5P7wKgJa3\u002fwFAcHBGqWGy8D8i4Y9e4SD\u002fP63jGAbHseA\u002fCviL2Sw30D+khCIrLqsDQGBlwLQdQwRAWFXE3YabAECq72ZYCtvlPxSQjSeXO\u002fo\u002fEEZZEInb8z8HO4ZR1EzsP2j0r\u002fEuowVAggrmnJow1j8IZ2gUas38PyaZ\u002fbKxIQFAWh2VLtyP4D\u002fWl31g8lvzP3wnt7\u002fQePk\u002foiet2Y6y1j8LxqFBAjP\u002fPz5FUoX73wJAUr5qqjiVAUBKb5R43af\u002fP5qBTZbguARAgZCvVAW39T8mvYLxJy\u002f4P8hzIOTqzeg\u002fI\u002ffr81iR9T\u002fSfVPvAjrrPyhJOprDhP8\u002fdsolZwIL9T9omss5bbX7P1oQj1FJVvU\u002fjSTnPjk3\u002fj9l6cZYsMTpP3Rp8FK7egRA0l8u2Vcf3z99KxuGsSDqP0fl3OaDdAFAzuB1\u002feUMBkCOVnwZvgftP7LD92e0Ods\u002fHulOdAKlA0Akmu9SeSLwPwC3q53ldME\u002faK3P5wF+AUAspXt4xzL8PzR0KxEgMPQ\u002fEOnVj1TfAkB9WTlMeTblP1xK5GkCFPw\u002fiCRctYE8A0Dq+zywrlftP0CCKEgUgdY\u002fynt13lPj5j8X0B3X+QP0PwpPSjtzV\u002f4\u002fAF2zRIHtkT+SoCw5TgYAQGYyiUpOD\u002fU\u002fTLCxrbOABkCmlLv8qhLcP2CezFgBEAVAIKW3NV+GAkDgVaiqNwfXP0KLh8LBzvw\u002f87WfwG6pB0CcVXKTnMPzP+blzMYJRARA\u002fZJxBmDP6T+QsiBIf7aoP1y1gTxv\u002f\u002fE\u002fiouTYkOZAEDR2XLfTfvmP60psePbuwJARgypgY1fAEBwucPgLuIAQH49bZjbzwBALecwHu\u002f+BkD093bca1vaP3Q2rYG+Mto\u002fKDuV2kWJ3j9COoWSEbsAQPhEdxHmZrc\u002fej4QF71Q8T9yrx183WEFQIq7+I5Qj\u002fo\u002fxJmZYqnm1T\u002fUAVD74jDfP\u002fRUaSjRQ\u002fA\u002foeL9w4FR4T9waOCchiH8P9yC2Tz2SQdAqFCvs\u002fKyAUBPoDpMEdvlPwAtf\u002fOeLN8\u002fMqEGusm4AkDIaOIln1jKPxI3j1Tr\u002fOE\u002fKzASUGRGAEB\u002fdTTfozIBQPD4EklwJgRAxAE8m2a4AkBWKndw37P2P2p9ffzHBvQ\u002fdg8g7HB63D\u002ftvJc3H1H8P+J3mo3ANwRAT5MUl3\u002feAUAmcEUrIvPxP8rrFYUoZ98\u002f0zAMR1OJ\u002fT+CPWZRIjXsP+hHML9YPMU\u002fxK5SKoZrBkCWahlpkN3kP\u002fIfpXo4d\u002fA\u002f0EOUVbjZB0ApWEMiyiEBQGWsC2FRhgVAOOaDmen72j8W7rd\u002fUTAHQCIB68ebPfA\u002fcWv47kEg4z8YKyDH+WgCQKjllCbQmbM\u002fV9QlCQu7BkBAAJ+s4BQBQGJL+EvHjgRAYqgl\u002f96fAkAr5J1taEr3PzAHwmpZv\u002fA\u002f9L7Tu0Uczz+9Vf6h6z3hPzcMkKLz2ARACGw1Mxl\u002fxT+UxnlotAj5P5Dj2GnbJso\u002fSk4gOj0yBkD4UJLHEn8GQFzq40Y3pvg\u002fBBCl1pghBUAE6XAEyWj8P3xKlFEJIMs\u002fCUOD5Z2k+j9aj6Dfx3HhPxBXLjdYias\u002fD7JdzbwV4T8+PsttGwQHQCD\u002fTb8NCdE\u002fNbRQAzrtBUAsWSyrK7HRP0B79I8oNY0\u002fdFb4B0PKxT8Cx6nZrd7dP75VB2h8u\u002f4\u002fHmwpK9Pp+D\u002f0S7xenYgDQDrD6a1bXeg\u002fbbcRR34r9T\u002fTSbkebCnyPwUmQObrE+g\u002fXv5T4U5u9j964epGVXvnP6H1O7DRrQRA2\u002ftfX8VB5D9094tpfITgP\u002ft14u6zwO4\u002feJnMO\u002f0N7T81G8OpMkfhP4xrBTCPTNI\u002fyg79\u002f8gc9z9TXlNcxDEDQA=="},"y":{"dtype":"f8","bdata":"cH9iISbfuD+Gjzada6LYP1SVraWrBOQ\u002fECAcF5Ozoz+Il+rvJGPBP5jdCsyiguw\u002f\u002fpXtDf1J7T8toa9MU0HlP9Bl09IPQMo\u002frJaFQHoB2T8EfztEDL7fP1B4biaYqKs\u002f8Y5ciKeu4T+BmmzIr\u002fnjP\u002f17i9rnHuQ\u002fYJ6kIeG45D9YM\u002fkgeKfpPwJEWuewVdE\u002fk9sZfvw37z9UTeu+\u002fKfPP1TKGwjRM+I\u002fdNlJRqr67j+oT7Pb+erNP7UrdQP5j+4\u002fqHphX9b82z\u002fe7m4XazLaP4uHWj6bSeE\u002fWN0LYH5O5j\u002f7h6NNjVTjP2LNky4\u002f0OI\u002fAOSoFsH+4T9MEdoEV7HdP4zdo5o8acs\u002fWLXLEtb71D\u002fcUdd\u002fiInoPyIYSEPvH+E\u002fzl+Vsd\u002fs3z\u002faVgqX193UP+e3xz2mauo\u002fmlqZDd533D\u002ffLu3FKQzkP+AQ74rtHq0\u002fyErpWFMi2D9cQrY7AbnvPzfEmPqu6+4\u002f8AFGDuCL0T9MwLmT2CDbPwIH4KACqdc\u002f+5uufM7d6j84+1xqANPLP+i2Mfi7o7Y\u002fm2Kn6Jz07j9809mlCPjOP54\u002fIDVCieI\u002fmDDJiWvT0T8UW1KYRN7FP+CjraUE67Y\u002ftKYkr2Oq7z+CdXpZ4ZDQP9pyrPmXj9k\u002fZCVaCHVI5j+UuIKpHrPNP8FJh4XB+eE\u002fWFrH33rawz91vyrPiQPnP060T1Ztd+o\u002fkD2WzmWY7z92xccJ6OzuPxjlL+h\u002fILg\u002f3jfF3XJA7j\u002fQBAairMjGP\u002fyppg9LtuI\u002f1gtecfMu2z9grdrfO++3P1rp\u002fL4E3NE\u002ficU\u002f9sDl4j\u002fI\u002fkfy5NPLP9ASF3TbpLE\u002f1xnsJqN26j\u002fh\u002f8fVBCPoPzzzFtLmWu4\u002fONxzPlIAvz9EE3j3ml3KP4ioPoq+lLY\u002fHGUtBbx2wz+8TDIpyQjPP+TvEU\u002f13Mo\u002fztcZP8v71z8kebOR93nfP\u002f4MxL09I9M\u002fwPGgtHS67T9Aud3x86m7P\u002fENg7ueveU\u002fftxPg45h2j\u002fnKhdSL\u002fnkP\u002fDIcMkyI8s\u002flJYcUnB67z\u002fAEmujlo6GPzYIGwfLgdg\u002fuESpPDAQ7z+A6X0SLZN2P07yR25fO+4\u002f57ZHr7Q26j8hTqrmioLpP2RGC86\u002fJOs\u002fEKJAXARZ4z\u002f0Nrxn7mrRPwTnRDGi8so\u002fSks1\u002fVHp4z9ZHnJOCvPoPzCvYHSL18g\u002fEm37s7wI7z+TOF2Qor\u002frP9MKm1HRPuA\u002fmoI6\u002fCAh0D9yupQQ4v7aP7hdpAPzcN0\u002fzBLTXgEI2j9Qk6ity4S0P\u002fBAycg7P+s\u002fUK3jxFxWuj+UVccMwdDcP+DqHB6Cfpg\u002fEKy5CQCyuT9yU2MzVTToP9myjRnB0uM\u002fyLFVl1Wm0D\u002fgDcPiu9GfPxH9fO8zruo\u002frggXDEvr6j8g67riwCebP1I1ud+piO8\u002fWIKX21Uf4j9GmSLnQ7vdP5geXK1iKdU\u002friR6MrW70j8ifdiFQTDiP6QWLSZuMN0\u002fdA7Ugoff6D\u002fS5KvSFlrdP+FylSZtMuI\u002ftlJUCo2P5z\u002f8ezEULwbuP2BayNz4qZ4\u002fp1UXea4g7z+dk5g0dnXhP7CwkMi+Yus\u002fXsyhqhia1j\u002fSSWnRyxzcP3A2uJ\u002f9b9I\u002f4MVYpjxs4D9rpsVMgz\u002fvP7ue6UoXWug\u002fjYWHKkVm5D9QBeQ5OTHMP+0UktAT4eM\u002fGtxIwRGJ2j\u002f+eAKX0d7iPyHVIt4\u002fuOs\u002fu+HucEwi7z\u002fQsI13+L7YPwUsDru6keI\u002fBNfetb8+7j+JinLERcvgPyppFoPWz9o\u002fwMnh6ho+0z8xiddjfj\u002fqP\u002fiHRnJzTrA\u002fyGrZCaSQ4T++qptXy7XnP8CbBlsaJN0\u002fUyDnGQrN7z8gD4gxk1GZP96CTrjbouc\u002feLc17B5m5T\u002fg0BSUYFnjP9QZzqkKycY\u002fRTi6X3cK4D+M7mg+qWLQP9SiS3EVr8A\u002fDfe+V\u002fDx5z9OsA8satfhP24zYk5BNtc\u002fuQHk1sOA6T+YC1MIh9+1P3Gkh8mZDOQ\u002fiBC\u002fI30nzT8PwGvS2aXvP8q13tQj\u002f+o\u002fQf7MoTNq5z8wd6Ve8IrLPxMD+kAaoeU\u002fEKsRujmfpD8IjAKNKeLcP3bbgmh9ct0\u002f7CP0SutO3T+6L0FewzraP94r23k4Gug\u002fkA7Zs4D01z966TgQoJXZP\u002fQOOWuAt8o\u002flH2MHOm3xj985pPXuQrqPz1bPZ6jGeU\u002fuAJJtvpv2D8UgiSPNo\u002fhP9h3D6rGqL8\u002femiHLsqG1T+QjyuG7kiuPyhROUq46s0\u002fxideE6GQ6z+MMh\u002fXaTHUP9+Ar+yC6Os\u002fABDP+TiDVD9wpfLBg36oP6snBDazge0\u002fYbDcu0Kx5z\u002fEX6O3epLuP6ch2UUe1uI\u002fmBJDPCd\u002f5j8hdewsyxTrPyAx+0UR2t0\u002f8Hi0XGUlqD90A1XI\u002f47uPzjoENg4Suc\u002f2+FEXcL97T8nsWUbyNzoP1\u002fSgreNIeY\u002fAYhGXhAY4T9wBJim1hKlPzS3xEW8se8\u002fh3K5iGei5D8hiNiqc0XrPzyVssiuVdY\u002fC+uxdA0Q5T\u002foygoEQK+4P5jKznpljME\u002f8C6L61U4vz+IXlE9P8nZP1hSbOM\u002fneI\u002ffPNQ66Pt4D8aejNXdtHTP3uClad6kOE\u002fyLfTs6Gt0T9nmWXJJzzoPyhKT1wBlNI\u002fwI\u002fNFdX2qD84nvHFGUPkPylxyUVyYOI\u002fDRZKQz045z+sqZ7+7WjcPwCMdWKOssA\u002fhLJXtHdM6D\u002flKuNZTybqP4ShawGGAdQ\u002fBGRNkQ\u002fd5T+8ud+JkoTGP0SPwP6ZBe8\u002ffldNWju+0D8wuXC7pXe2P4ANy+iR73Y\u002fOPEeEL3J0D9ovzKSle+xP3zhxpYR2Ng\u002fqHSQD4rrtD\u002fIdod774TKP60+bRLAou4\u002fHJPFTgLH1j\u002fo36zGcti5P5J6QiV9IO4\u002f8I7UIBAHsT\u002f3oXyXEFvjP3Yi4lCrS9s\u002fPXF4AAwn5D834YLRLefsPwAtfN5KvL8\u002fxg2AXmhx0D\u002fwtaT1GGTaP9fqpbQjA+M\u002fKTe+UfOO6T+6VsN4r4nVP9gYYrSI5dc\u002fdD9XVU0LyD+WyLbUe6jlP6pC7bx7gOM\u002fNnyTie3s0j9llOohdgXmPwzHPhqb7dc\u002fymccnhfW4j+WPguxVNfQPzTJENPY1u8\u002f7EjLTNid6T8Gmn1jCk\u002fXP7yUknDeat4\u002fvLU+7Vw5zz9g0nw9KHmkP1jNvtYGJ9s\u002fhPL5Xydz0T8tq8zuPyHsP9WorCa\u002fXek\u002f+yp9nS+D4z+L5nsCTCzqP1x+ul+5S9U\u002fGOGvX4ss2D\u002fNlsUH86zlP6GEdPrT9+g\u002fxzc2vxjB6j\u002f3TZXHGvfiP\u002fqS2rRvlNU\u002fS2pHvDZw6z+7GMLh3gjnP6a9x2Pkpus\u002f1JmpT1pEyT82h2kTdEjvPwRhR9aBC+8\u002ft+6jIIBI5D+1bAwI8K\u002fqP8JAPZFRT+c\u002fF9c7tfGS4j94Y9BCflrHP\u002fAu4cshTtY\u002fQCfDNwoyzj+ECan+0k7TPz7jqM8VBus\u002fsry0Btuj2T+xEei0XG7hP2+p5pw8We8\u002fCvTCJcvQ2T\u002fwfouU627bPzbL0rxeSNE\u002fSvZDmHWd2T\u002fOPaySqGDsP\u002fDMavHKVaI\u002fotuclkDR4z+7658CHo7oPzoYtbNUdu0\u002fLYe\u002fKfc45D8tN2BsZTXnP7Qt+yVxA+4\u002fktIsLr3r0D9LPsWs7NLgPynH+wIJDek\u002fEz4WIunp5D\u002fMJsCb3wPaP+BdtHmEka8\u002f1KKh5c4w4T9Qz1Pi9Q7NPyCSkwTGiKI\u002fAHd1k59Jmj\u002fETynUKPfQP8yuZX4PL8Q\u002fO34TJFJl5T\u002f493MDCMzlP\u002fP8xwg5ue4\u002f4kwd07cs7z8R4i9OWOPlP2zQQhWUPMk\u002fgZDjica65j\u002f5buYTMw\u002fqP1wWrHambsg\u002fBmmP26wL6z8IOCL6pqHAP+l99vABZ+E\u002fpn0TJ4f95j+o\u002fE+7ary7P+48GjvJ2e0\u002f7KoB3R317D+4Fk13HA7eP+KPVIJYB9Y\u002frMwFaa0v1j8YJEwtU1G8P+ANgKuJzJc\u002fhNeUC3cA7z9ouBLOqhHIPwwWxX87bMs\u002ftNqvNXZTzD8T+0lIctjtP+VSqo8GF+s\u002fLCmwuSJS2j+4BaK5THXNP2AMmPbWMaM\u002fEMWQhPFLsD+GsvylVHTfP3Avp7NfE+s\u002fuGlXKH5B1z9CH+Z8AZXpPyQlnXKBTdY\u002fknVPVpdZ7T8GCn73ApjmP76+hgRAHOc\u002fQFh\u002fVeUMlj968sO6rhHoP2okUF9bEOc\u002faF4i93G42T9sjO9tATvjP0j30CRKfr8\u002fmItrKbjVtT9EMx5fNGLYP1MCmbF8beI\u002fEN9P0nKp7j\u002fokXOePEPrPzQLIcE\u002fysY\u002fAdPXAz8B6T+oUQMEaQDjP8SwQhBw6c4\u002fHK826V3M1j8g2A3piu7HP7z+tsYFKOY\u002fcMY32CBdyj9Q1f\u002fP8cTPP7pT3dtmxe8\u002fuOkUhe4Xwj9IZ79sJ5jcP6agvaiiUeA\u002fQJTXOuicxT8oI53C2N\u002fBP6onWAhfWdY\u002firqTNAz57z\u002fA1W6genWbP9kF6+qQQek\u002fqzmLJeAk5T+gt63ohemjP4BDryTkCag\u002fQJbkO5iFmT\u002fgUkSQHtWaPxxFMGEpEMs\u002fFiyHyWiE2T+Mmq6o4sHZP1cp8LhVZOw\u002fgc9gLcDT4j9Z2o0L2E3kPyZciQJCg9Y\u002fuMj3k8Kgsz+2hlIUbdvWP+RQp+OPPcY\u002fTt6vELP27T9WrCHNd1rrP5rAm9VuYtQ\u002fsori2n2L1D9y54Y7rX7eP2JaN3aTGe8\u002fyDFd\u002fdAszj8QdeVhZ7W5P0gFtI0k988\u002feo85vECE7j9inVyB81LuP3aVZMxledw\u002fIITWrUyl7T+vCO68XrziP3itbaIKA8Y\u002f3XgAe1vg6T8NQQDyv\u002fvoP9dJLYOM4+s\u002fw\u002f\u002fi7cYB4j9cD0m9c8nOP\u002feO9umqvOE\u002fZv8cZmZK5z9zeJFnt2PtP9vWRWTGEeM\u002fqMiVW22o2D8tvsAd\u002f4TkPzx385td19M\u002f4OqVv3zL7j8K6vYrATnrPzVfIJsis+w\u002f0T3W94ln5j+mwAcnzd3cP9hFs4pCO7w\u002fgik4z5Aq0j+4uMxSGuHAP6vFk0IUzug\u002fnHA74II2zD+oajhg2ZDfP0ire\u002fEy\u002fe4\u002fYIfRmK4Psj8KtKBD77XSP5AcJF5CFbU\u002fXDgDiW7l0T8YgkZUOWDKP75viylnQNc\u002fmODKQLSItT\u002f1GidFidTuPxNjBwjXN+k\u002feJoeeJs7vT+CHWu7SCLTPyCuqVjs\u002f5A\u002f3snCbF3+0z+AMzads2vRP\u002fpBsnkcsdY\u002fdG0oeU+AzD8ubss5DivdP83wQNRTQuw\u002f0HMxgeyxqT\u002fAVCzJ\u002fgC6Px\u002fF0Hf\u002fh+s\u002fDCqSMDpm1T+iU4Lx4gffP4ZdjcLVmdM\u002fuMyyKr1DyT\u002fIEwq60TXoP2aog8ykL9A\u002fzYZh8G1K6D\u002fpO4sG7CvrP9RIFLkqYdE\u002fcGClag11zT8AiLuKcY3ZP3htdc47Kss\u002fnqaBMP2C1T+qTzG670nrPx5wmwWI9do\u002fQDGvQ0Vjij+n6KEqWC\u002fnPw=="},"type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermap":[{"type":"scattermap","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"margin":{"l":0,"r":0,"t":0,"b":0},"xaxis":{"showgrid":false,"zeroline":false,"showticklabels":false,"range":[0,3]},"yaxis":{"showgrid":false,"zeroline":false,"showticklabels":false,"scaleanchor":"x","scaleratio":1,"range":[0,1]},"width":1200,"height":400,"paper_bgcolor":"white","plot_bgcolor":"white","showlegend":false},                        {"displayModeBar": false, "responsive": true, "scrollZoom": false}                    )                };            </script>        </div>

src/index.html

@@ -0,0 +1,454 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+    <script src="distill.bundle.js" type="module" fetchpriority="high" blocking></script>
+    <script src="main.bundle.js" type="module" fetchpriority="low" defer></script>
+    <script src="https://cdn.plot.ly/plotly-3.0.0.min.js" charset="utf-8"></script>
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <meta charset="utf8">
+    <base target="_blank">
+    <title>A Primer on LLM Embeddings: Understanding the Semantic Backbone of AI</title>
+    <link rel="stylesheet" href="style.css">
+</head>
+
+<body>
+    <d-front-matter>
+        <script id='distill-front-matter' type="text/json">{
+        "title": "A Primer on LLM Embeddings: Understanding the Semantic Backbone of AI",
+        "description": "The role of embeddings in traditional NLP, and their application to LLMs.",
+        "published": "Mar 18, 2025",
+        "affiliation": {
+            "name": "University of Bologna"
+        },
+        "authors": [
+        {
+            "author":"Hesam Sheikh Hassani",
+            "authorURL":"https://huggingface.co/hesamation"
+        }
+        ],
+        "katex": {
+        "delimiters": [
+            {"left": "$$", "right": "$$", "display": false}
+        ]
+        }
+    }
+    </script>
+    </d-front-matter>
+    <d-title>
+        <h1 class="l-page" style="text-align: center;">A Primer on LLM Embeddings:<br>Understanding the Semantic Backbone of AI</h1>
+        <div id="title-plot" class="main-plot-container l-screen" style="overflow-x: hidden; width: 100%; text-align: center;">
+            <div style="display: flex; justify-content: center; position: relative;">
+                {{{fragment-banner}}}
+            </div>
+            <p style="text-align: cekter; font-style: italic; margin-top: 10px; max-width: 900px; margin-left: auto; margin-right: auto;">It's nice to have a cute interactive banner!</p>
+  
+        </div>
+    </d-title>
+    <d-byline></d-byline>
+      <d-article>
+        <d-contents>
+        </d-contents>
+        
+        <p>Welcome to the Distill Blog Template! This framework is based on the <a href="https://distill.pub/">distill.pub</a> design and has been adapted to make it easy for you to create beautiful, interactive technical blogs. Whether you're writing about machine learning, data science, physics, or any technical topic, this template provides the structure and components you need.</p>
+    
+        <aside>Reading time: 10-15 minutes.</aside>
+    
+        <p>This template includes support for mathematics, interactive visualizations, citations, footnotes, and more. In this guide, we'll walk you through the structure of the template and how to customize it for your own blog.</p>
+        
+        <aside>If you have questions or remarks open a discussion on the <a href="https://huggingface.co/spaces/lvwerra/distill-blog-template/discussions?status=open&type=discussion">Community tab</a>!</aside>
+        
+        <p>The template is built with the following features in mind:</p>
+
+        <p><strong>1. Beautiful typography and layout</strong>: The template follows Distill's clean, readable design principles with responsive layouts that work well on desktop and mobile.</p>
+
+        <p><strong>2. Support for interactive components</strong>: You can include interactive visualizations, like the memory widget shown below:</p>
+
+        <h2>Getting Started with the Template</h2>
+
+        <p>Let's walk through how to use this template to create your own blog:</p>
+
+        <h3>Installation and Setup</h3>
+
+        <p>First, duplicate the repository and clone it to your local machine and install the dependencies:</p>
+
+
+        <d-code block language="bash">
+            git clone https://huggingface.co/spaces/lvwerra/distill-blog-template
+            cd distill-blog-template
+            npm install
+        </d-code>
+
+        <p>To run the development server:</p>
+
+        <d-code block language="bash">
+            npm run dev
+        </d-code>
+
+        <p>This will start a local server at <code> http://localhost:8080 </code> where you can preview your blog as you edit it.</p>
+
+        <h3>Editing the Front Matter</h3>
+
+        <p>The front matter contains metadata about your blog post, such as the title, description, and author information. Edit the <code>&lt;d-front-matter&gt;</code> section at the top of <code>src/index.html</code>:</p>
+
+        <d-code block language="html">&lt;d-front-matter&gt;
+    &lt;script id='distill-front-matter' type="text/json"&gt;{
+    "title": "Your Blog Title",
+    "description": "A brief description of your blog",
+    "published": "Month Day, Year",
+    "affiliation": {"name": "Your Organization"},
+    "authors": [
+      {
+        "author":"Your Name",
+        "authorURL":"https://your-website.com"
+      }
+    ],
+    "katex": {
+      "delimiters": [
+        {"left": "$$", "right": "$$", "display": false}
+      ]
+    }
+  }
+    &lt;/script&gt;
+&lt;/d-front-matter&gt;</d-code>
+
+        <p>An the corresponding entries in <code>src/distill.js</code> in the <code>bylineTemplate</code>.</p>
+
+        <h3>Writing Your Content</h3>
+
+        <p>Your main content goes within the <code>&lt;d-article&gt;</code> tags. You can use standard HTML tags with some special Distill components:</p>
+
+        <h4>Mathematical Equations</h4>
+
+        <p>You can include inline math using <code>&lt;d-math&gt;x^2 + y^2 = z^2&lt;/d-math&gt;</code> which renders as <d-math>x^2 + y^2 = z^2</d-math>.</p>
+
+        <p>For block equations, use <code>&lt;d-math block&gt;E = mc^2&lt;/d-math&gt;</code>:</p>
+
+        <d-math block>E = mc^2</d-math>
+
+        <h4>Code</h4>
+
+        <p>It's great to include code samples in your blog post. In-line you can do it via the <code>&lt;code&gt;</code> tags. For code blocks you can use the <code>&lt;d-code block language="clike"&gt;</d-code></code> tags. Take the for example the following block:</p>
+
+        <d-code block language="html">
+            &lt;d-code block language="python"&gt;
+                def print_hello_world():
+                    print("hello world")
+            &lt;/d-code&gt;
+        </d-code>
+
+        <p>It will render as:</p>
+        
+        <d-code block language="python">
+            def print_hello_world():
+                print("hello world")
+        </d-code>
+
+        <p>Finally, if you want to include code from GitHub you can use <a href="www.emgithub.com">emgithub.com</a> and for example create a collapsable widget like this:</p>
+
+        <d-code block language="html">
+            &lt;details style="background: #f6f8fa; border: 1px solid #d0d7de; border-radius: 6px; margin: 1em 0;"&gt;
+                &lt;summary style="padding: 12px; cursor: pointer; user-select: none; background: #f3f4f6; border-bottom: 1px solid #d0d7de;"&gt;
+                    👉 Naive DP implementation with overlap in Picotron (Click to expand)
+                &lt;/summary&gt;
+                    &lt;div class="code-embed-container" style="margin: 0; border-radius: 0; overflow-x: scroll; width: max-content; min-width: 100%; font-size: 8px;"&gt;&lt;/div&gt;
+                        &lt;script 
+                            src="https://emgithub.com/embed-v2.js?target=https%3A%2F%2Fgithub.com%2Fhuggingface%2Fpicotron%2Fblob%2F0035cce0e04afd6192763b11efe50010d8ad0f71%2Fpicotron%2Fdata_parallel%2Fdata_parallel.py%23L10-L60&style=github&type=code&showBorder=off&showLineNumbers=on&showFileMeta=on&showCopy=on&showFullPath=on"&gt;
+                        &lt;/script&gt;
+                    &lt;/div&gt;
+            &lt;/details&gt;
+        </d-code>
+
+        <p>Which will display as follows:</p>
+
+        <details style="background: #f6f8fa; border: 1px solid #d0d7de; border-radius: 6px; margin: 1em 0;">
+            <summary style="padding: 12px; cursor: pointer; user-select: none; background: #f3f4f6; border-bottom: 1px solid #d0d7de;">
+                👉 Naive DP implementation with overlap in Picotron (Click to expand)
+            </summary>
+                <div class="code-embed-container" style="margin: 0; border-radius: 0; overflow-x: scroll; width: max-content; min-width: 100%; font-size: 8px;"></div>
+                    <script 
+                        src="https://emgithub.com/embed-v2.js?target=https%3A%2F%2Fgithub.com%2Fhuggingface%2Fpicotron%2Fblob%2F0035cce0e04afd6192763b11efe50010d8ad0f71%2Fpicotron%2Fdata_parallel%2Fdata_parallel.py%23L10-L60&style=github&type=code&showBorder=off&showLineNumbers=on&showFileMeta=on&showCopy=on&showFullPath=on">
+                    </script>
+                </div>
+        </details>
+
+
+        <h4>Asides</h4>
+
+        <p>You can include side notes using the <code>&lt;aside&gt;...&lt;/aside&gt;</code> tag:</p>
+
+        <aside>This is a side note that will appear in the margin.</aside>
+
+        <h4>Figures and Images</h4>
+
+        <p>Include figures with captions:</p>
+
+        <d-code block language="html">
+            &lt;figure&gt;
+                &lt;img src="/assets/images/placeholder.png" alt="A placeholder image"&gt;
+                &lt;figcaption&gt;Caption for your figure&lt;/figcaption&gt;
+            &lt;/figure&gt;
+        </d-code>
+
+        <p>This will show the following:</p>
+
+        <figure>
+            <img src="/assets/images/placeholder.png" alt="A placeholder image">
+            <figcaption>Caption for your figure</figcaption>
+        </figure>
+
+        <h4>Tables</h4>
+        
+        <p>You can easily add tables with the default HTML formatting:</p>
+
+        <d-code block language="html">
+            &lt;table&gt;
+                &lt;thead&gt;
+                    &lt;tr&gt;
+                        &lt;th&gt;&lt;strong&gt;Model&lt;/strong&gt;&lt;/th&gt;
+                        &lt;th&gt;&lt;strong&gt;Accuracy&lt;/strong&gt;&lt;/th&gt;
+                        &lt;th&gt;&lt;strong&gt;Speed&lt;/strong&gt;&lt;/th&gt;
+                    &lt;/tr&gt;
+                &lt;/thead&gt;
+                &lt;tbody&gt;
+                    &lt;tr&gt;
+                        &lt;td&gt;Model A&lt;/td&gt;
+                        &lt;td&gt;95%&lt;/td&gt;
+                        &lt;td&gt;Fast&lt;/td&gt;
+                    &lt;/tr&gt;
+                    &lt;tr&gt;
+                        &lt;td&gt;Model B&lt;/td&gt;
+                        &lt;td&gt;98%&lt;/td&gt;
+                        &lt;td&gt;Medium&lt;/td&gt;
+                    &lt;/tr&gt;
+                    &lt;tr&gt;
+                        &lt;td&gt;Model C&lt;/td&gt;
+                        &lt;td&gt;99%&lt;/td&gt;
+                        &lt;td&gt;Slow&lt;/td&gt;
+                    &lt;/tr&gt;
+                &lt;/tbody&gt;
+            &lt;/table&gt;
+        </d-code>
+        
+        <p>This will render as a table as expected:</p>
+
+        <table>
+            <thead>
+                <tr>
+                    <th><strong>Model</strong></th>
+                    <th><strong>Accuracy</strong></th>
+                    <th><strong>Speed</strong></th>
+                </tr>
+            </thead>
+            <tbody>
+                <tr>
+                    <td>Model A</td>
+                    <td>95%</td>
+                    <td>Fast</td>
+                </tr>
+                <tr>
+                    <td>Model B</td>
+                    <td>98%</td>
+                    <td>Medium</td>
+                </tr>
+                <tr>
+                    <td>Model C</td>
+                    <td>99%</td>
+                    <td>Slow</td>
+                </tr>
+            </tbody>
+        </table>
+
+        <h4>Citations</h4>
+
+        <p>Citations can be included using <code>&lt;d-cite bibtex-key="example2023"&gt;&lt;/d-cite&gt;</code> tags and a separate bibliography file:</p>
+
+        <p>As shown in the paper<d-cite bibtex-key="example2023"></d-cite>, this approach has several advantages.</p>
+
+        <h3>Building and Deploying</h3>
+
+        <p>When you're ready to build your blog for production:</p>
+
+        <pre><code>npm run build</code></pre>
+
+        <p>This will create optimized files in the <code>dist/</code> directory. You can then commit and push the changes back to the space and see the result.</p>
+
+        <h2>Interactive Components</h2>
+
+        <p>You can for example include interactive Plotly visualizations in your blog by creating HTML fragments:</p>
+
+        <p>To generate these charts, run the included Python script:</p>
+        
+        <d-code block language="bash">
+            pip install numpy plotly pandas
+            python plotly_example.py
+        </d-code>
+        
+        <p>This creates HTML fragments in the <code>src/fragments/</code> directory that you can include in your blog using the <code>id</code> attribute matching the fragment name.</p>
+
+        <h3>Using HTML Fragments</h3>
+
+        <p>This template supports HTML fragments, which are useful for including complex visualizations or interactive components:</p>
+
+        <ol>
+            <li>Create a new HTML file in the <code>src/fragments/</code> directory, e.g., <code>src/fragments/my-visualization.html</code></li>
+            <li>Include it in your main document using <code>\{{{fragment-my-visualization}}}</code></li>
+        </ol>
+
+        <p>Fragments can contain HTML, CSS, and JavaScript. They're loaded asynchronously, which helps keep your main document lightweight.</p>
+
+
+        <h2>Advanced Customization</h2>
+
+        <h3>Styling</h3>
+
+        <p>You can customize the appearance of your blog by editing <code>src/style.css</code>. The template uses CSS variables for colors, fonts, and spacing, making it easy to adjust the overall look and feel.</p>
+
+        <h3>Adding JavaScript</h3>
+
+        <p>For interactive components, add your JavaScript code to <code>src/index.js</code> or create new modules and import them. The template uses webpack to bundle JavaScript files.</p>
+
+        <h3>Bibliography</h3>
+
+        <p>Edit <code>src/bibliography.bib</code> to include your citations in BibTeX format.</p>
+
+        <h2>Best Practices</h2>
+
+        <ul>
+            <li>Keep your content focused and concise</li>
+            <li>Use visualizations to explain complex concepts</li>
+            <li>Break down long articles into clear sections with descriptive headings</li>
+            <li>Use mathematics sparingly and always explain the notation</li>
+            <li>Optimize images and interactive components for fast loading</li>
+        </ul>
+
+        <h2>Conclusion</h2>
+
+        <p>This template gives you everything you need to create beautiful, interactive technical blogs. By combining clear writing with interactive visualizations and mathematical notation, you can effectively communicate complex ideas to your audience.</p>
+
+        <p>For more examples and inspiration, visit <a href="https://distill.pub/">distill.pub</a>.</p>
+
+
+        
+    </d-article>
+
+    <d-appendix>
+        <d-bibliography src="bibliography.bib"></d-bibliography>
+        <style>
+            d-appendix .citation {
+                font-size: 11px;
+                line-height: 15px;
+                border-left: 1px solid rgba(0, 0, 0, 0.1);
+                padding-left: 18px;
+                border: 1px solid rgba(0, 0, 0, 0.1);
+                background: rgba(0, 0, 0, 0.02);
+                padding: 10px 18px;
+                border-radius: 3px;
+                color: rgba(150, 150, 150, 1);
+                overflow: hidden;
+                margin-top: -12px;
+                white-space: pre-wrap;
+                word-wrap: break-word;
+            }
+        </style>
+
+        <h3 id="citation">Citation</h3>
+        <p>For attribution in academic contexts, please cite this work as</p>
+        <pre
+            class="citation short">"The Distill Blog Template", 2025.</pre>
+        <p>BibTeX citation</p>
+        <pre class="citation long">@misc{distill_blog_template,
+      title={The Distill Blog Template},
+      author={Some Authors et al},
+      year={2025},
+}</pre>
+    </d-appendix>
+    <script>
+      function toggleTOC() {
+          const content = document.querySelector('.toc-content');
+          const icon = document.querySelector('.toggle-icon');
+          
+          content.classList.toggle('collapsed');
+          icon.classList.toggle('collapsed');
+      }
+    </script>
+
+    <script>
+        const article = document.querySelector('d-article');
+        const toc = document.querySelector('d-contents');
+        if (toc) {
+            const headings = article.querySelectorAll('h2, h3, h4');
+            // let ToC = `<nav role="navigation" class="l-text figcaption"><h3>Table of contents</h3>`;
+            let ToC = `<nav role="navigation" class="l-text figcaption"><div class="toc-header" onclick="toggleTOC()">
+            <span class="toc-title">Table of Contents</span>
+            <span class="toggle-icon">▼</span>
+            </div><div class="toc-content">`;
+            let prevLevel = 0;
+
+            for (const el of headings) {
+                // should element be included in TOC?
+                const isInTitle = el.parentElement.tagName == 'D-TITLE';
+                const isException = el.getAttribute('no-toc');
+                if (isInTitle || isException) continue;
+                el.setAttribute('id', el.textContent.toLowerCase().replaceAll(" ", "_"))
+                const link = '<a target="_self" href="' + '#' + el.getAttribute('id') + '">' + el.textContent + '</a>';
+
+                const level = el.tagName === 'H2' ? 0 : (el.tagName === 'H3' ? 1 : 2);
+                while (prevLevel < level) {
+                    ToC += '<ul>'
+                    prevLevel++;
+                }
+                while (prevLevel > level) {
+                    ToC += '</ul>'
+                    prevLevel--;
+                }
+                if (level === 0)
+                    ToC += '<div>' + link + '</div>';
+                else
+                // else if (level === 1)
+                    ToC += '<li>' + link + '</li>';
+            }
+
+            while (prevLevel > 0) {
+                ToC += '</ul>'
+                prevLevel--;
+            }
+            ToC += '</div></nav>';
+            toc.innerHTML = ToC;
+            toc.setAttribute('prerendered', 'true');
+            const toc_links = document.querySelectorAll('d-contents > nav div a');
+
+            window.addEventListener('scroll', (_event) => {
+                if (typeof (headings) != 'undefined' && headings != null && typeof (toc_links) != 'undefined' && toc_links != null) {
+                    find_active: {
+                        for (let i = headings.length - 1; i >= 0; i--) {
+                            const heading = headings[i];
+                            // Skip headings that shouldn't be in TOC
+                            if (heading.parentElement.tagName == 'D-TITLE' || heading.getAttribute('no-toc')) {
+                                continue;
+                            }
+                            
+                            if (heading.getBoundingClientRect().top - 50 <= 0) {
+                                // Find matching TOC link by href
+                                const headingId = heading.getAttribute('id');
+                                const activeLink = Array.from(toc_links).find(link => 
+                                    link.getAttribute('href') === '#' + headingId
+                                );
+                                
+                                if (activeLink && !activeLink.classList.contains("active")) {
+                                    toc_links.forEach(link => link.classList.remove("active"));
+                                    activeLink.classList.add('active');
+                                }
+                                break find_active;
+                            }
+                        }
+                        toc_links.forEach(link => link.classList.remove("active"));
+                    }
+                }
+            });
+        }
+    </script>
+
+</body>
+
+</html>

src/index.js

@@ -0,0 +1,10 @@
+// import { plotClusters } from './clusters'
+import { loadFragments } from './fragmentLoader'
+import { syncHFSpacesURLHash } from './syncHFSpacesURLHash'
+
+document.addEventListener("DOMContentLoaded", () => {
+    console.log("DOMContentLoaded");
+    loadFragments();
+    init_memory_plot();
+    syncHFSpacesURLHash();
+}, { once: true });

src/plotly_example.py

@@ -0,0 +1,82 @@
+import plotly.graph_objects as go
+import numpy as np
+import pandas as pd
+
+data = {
+    "x": 3*np.random.rand(512),
+    "y": np.random.rand(512), 
+    "z": np.random.random(512), 
+}
+
+df = pd.DataFrame(data)
+
+def get_label(z):
+    if z<0.25:
+        return "smol dot"
+    if z<0.5:
+        return "ok-ish dot"
+    if z<0.75:
+        return "a dot"
+    else:
+        return "biiig dot"
+
+df["label"] = df["z"].apply(get_label)
+
+df["z"] = (df["z"]+1)*5
+
+fig = go.Figure()
+
+fig.add_trace(go.Scatter(
+    x=df['x'],
+    y=df['y'],
+    mode='markers',
+    marker=dict(
+        size=df['z'],
+        color=df['z'],
+        colorscale=[
+            [0, 'rgb(78, 165, 183)'],      # Light blue
+            [0.5, 'rgb(206, 192, 250)'],    # Purple
+            [1, 'rgb(232, 137, 171)']       # Pink
+        ],
+        opacity=0.9,
+    ),
+        customdata=df[["label"]],
+    hovertemplate="Dot category: %{customdata[0]}",
+    hoverlabel=dict(namelength=0),
+    showlegend=False
+))
+
+
+fig.update_layout(
+    width=1200,
+    height=400,
+    paper_bgcolor='white',
+    plot_bgcolor='white',
+    showlegend=False,
+    margin=dict(l=0, r=0, t=0, b=0),
+    xaxis=dict(
+        showgrid=False,
+        zeroline=False,
+        showticklabels=False,
+        range=[0, 3]
+    ),
+    yaxis=dict(
+        showgrid=False,
+        zeroline=False,
+        showticklabels=False,
+        scaleanchor="x",
+        scaleratio=1,
+        range=[0, 1]
+    )
+)
+
+fig.show()
+
+fig.write_html("fragments/banner.html", 
+               include_plotlyjs=False, 
+               full_html=False, 
+               config={
+                   'displayModeBar': False,
+                   'responsive': True, 
+                   'scrollZoom': False,
+               })

src/style.css

@@ -0,0 +1,599 @@
+/* style.css */
+/* Define colors */
+:root {
+    --distill-gray: rgb(107, 114, 128);
+    --distill-gray-light: rgb(185, 185, 185);
+    --distill-gray-lighter: rgb(228, 228, 228);
+    --distill-gray-lightest: rgb(245, 245, 245);
+    --distill-blue: #007BFF;
+}
+
+/* Container for the controls */
+[id^="plot-"] {
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+    gap: 15px; /* Adjust the gap between controls as needed */
+}
+[id^="plot-"] figure {
+    margin-bottom: 0px;
+    margin-top: 0px;
+    padding: 0px;
+}
+.plotly_caption {
+    font-style: italic;
+    margin-top: 10px;
+}
+
+.plotly_controls {
+    display: flex;
+    flex-wrap: wrap;
+    flex-direction: row;
+    justify-content: center;
+    align-items: flex-start;
+    gap: 30px;
+}
+
+
+.plotly_input_container {
+    display: flex;
+    align-items: center;
+    flex-direction: column;
+    gap: 10px;
+}
+
+/* Style for the select dropdown */
+.plotly_input_container > select {
+    padding: 2px 4px;
+    /* border: 1px solid #ccc; */
+    line-height: 1.5em;
+    text-align: center;
+    border-radius: 4px;
+    font-size: 12px;
+    background-color: var(--distill-gray-lightest);
+    outline: none;
+}
+
+/* Style for the range input */
+
+.plotly_slider {
+    display: flex;
+    align-items: center;
+    gap: 10px;
+}
+
+.plotly_slider > input[type="range"] {
+    -webkit-appearance: none;
+    height: 2px;
+    background: var(--distill-gray-light);
+    border-radius: 5px;
+    outline: none;
+}
+
+.plotly_slider > span {
+    font-size: 14px;
+    line-height: 1.6em;
+    min-width: 16px;
+}
+
+.plotly_slider > input[type="range"]::-webkit-slider-thumb {
+    -webkit-appearance: none;
+    appearance: none;
+    width: 18px;
+    height: 18px;
+    border-radius: 50%;
+    background: var(--distill-blue);
+    cursor: pointer;
+}
+
+.plotly_slider > input[type="range"]::-moz-range-thumb {
+    width: 18px;
+    height: 18px;
+    border-radius: 50%;
+    background: var(--distill-blue);
+    cursor: pointer;
+}
+
+/* Style for the labels */
+.plotly_input_container > label {
+    font-size: 14px;
+    font-weight: bold;
+}
+
+.main-plot-container {
+    margin-top: 21px;
+    margin-bottom: 35px;
+}
+
+.main-plot-container > figure {
+    display: block !important;
+    /* Let this be handled by graph-container */
+    margin-bottom: 0px;
+    margin-top: 0px;
+}
+.main-plot-container > div {
+    display: none !important;
+}
+
+
+@media (min-width: 768px) {
+    .main-plot-container > figure {
+        display: none !important;
+    }
+    .main-plot-container > div {
+        display: flex !important;
+    }
+}
+
+d-byline .byline {
+  grid-template-columns: 1fr;
+  grid-column: text;
+  font-size: 0.9rem;
+  line-height: 1.8em;
+}
+
+@media (min-width: 768px) {
+  d-byline .byline {
+    grid-template-columns: 5fr 1fr 1fr;
+  }
+}
+
+#title-plot {
+    margin-top: 0px;
+    margin-bottom: 0px;
+}
+
+d-contents > nav a.active {
+    text-decoration: underline;
+}
+
+@media (max-width: 1199px) {
+    d-contents {
+        display: none;
+        background: white;
+        justify-self: start;
+        align-self: start;
+        padding-bottom: 0.5em;
+        margin-bottom: 1em;
+        padding-left: 0.25em;
+        border-bottom: 1px solid rgba(0, 0, 0, 0.1);
+        border-bottom-width: 1px;
+        border-bottom-style: solid;
+        border-bottom-color: rgba(0, 0, 0, 0.1);
+        overflow-y: scroll;
+        height: calc(100vh - 40px);
+        scrollbar-width: none;
+        z-index: -100;
+    }
+}
+
+d-contents a:hover {
+    border-bottom: none;
+}
+
+toc-title {
+    font-weight: bold;
+    font-size: 1.2em;
+    color: #333;
+}
+
+toggle-icon {
+    transition: transform 0.3s;
+}
+
+toggle-icon.collapsed {
+    transform: rotate(90deg);
+}
+
+.toc-content {
+    margin-top: 15px;
+    overflow: hidden;
+    /* max-height: 1000px; */
+    transition: max-height 0.3s ease-out;
+}
+
+.toc-content.collapsed {
+    max-height: 0;
+    margin-top: 0;
+}
+
+@media (min-width: 1200px) {
+    d-article {
+        /* Ensure d-article does not prevent sticky positioning */
+        overflow: visible;
+    }
+
+    d-contents {
+        align-self: start;
+        background: white;
+        grid-column-start: 1 !important;
+        grid-column-end: 4 !important;
+        grid-row: auto / span 6;
+        justify-self: end;
+        margin-top: 0em;
+        padding-right: 3em;
+        padding-left: 2em;
+        /* border-right: 1px solid rgba(0, 0, 0, 0.1);
+        border-right-width: 1px;
+        border-right-style: solid;
+        border-right-color: rgba(0, 0, 0, 0.1); */
+        position: -webkit-sticky; /* For Safari */
+        position: sticky;
+        top: 10px; /* Adjust this value if needed */
+        overflow-y: auto;
+        height: calc(100vh - 40px);
+        scrollbar-width: none;
+        transition: max-height 0.3s ease-out;
+        z-index: -100;
+    }
+}
+
+d-contents nav h3 {
+    margin-top: 0;
+    margin-bottom: 1em;
+}
+
+d-contents nav div div {
+    color: rgba(0, 0, 0, 0.8);
+    font-weight: bold;
+}
+
+d-contents nav a {
+    color: rgba(0, 0, 0, 0.8);
+    border-bottom: none;
+    text-decoration: none;
+}
+
+d-contents li {
+    list-style-type: none;
+}
+
+d-contents ul, d-article d-contents ul {
+    padding-left: 1em;
+}
+
+d-contents nav ul li {
+    margin-bottom: .25em;
+}
+
+d-contents nav a:hover {
+    text-decoration: underline solid rgba(0, 0, 0, 0.6);
+}
+
+d-contents nav ul {
+    margin-top: 0;
+    margin-bottom: 6px;
+}
+
+
+d-contents nav > div {
+    display: block;
+    outline: none;
+    margin-bottom: 0.5em;
+}
+
+d-contents nav > div > a {
+    font-size: 13px;
+    font-weight: 600;
+}
+
+d-article aside {
+    margin-bottom: 1em;
+}
+
+d-article img {
+    max-width: 100%;
+}
+
+@media (min-width: 768px) {
+    d-article aside {
+        margin-bottom: 0;
+    }
+}
+
+d-contents nav > div > a:hover,
+d-contents nav > ul > li > a:hover {
+    text-decoration: none;
+}
+
+.note-box {
+    background-color: #f6f8fa;
+    border-left: 4px solid #444444;
+    padding: 1rem;
+    margin: 1rem 0;  /* Keep this modest margin */
+    border-radius: 6px;
+    /* Add this to ensure the box only takes up needed space */
+    display: inline-block;
+    width: 100%;
+}
+
+.note-box-title {
+    margin: 0;
+    color: #444444;
+    font-weight: 600;
+    font-size: 1em;
+}
+
+.note-box-content {
+    margin-top: 0.5rem;
+    margin-bottom: 0;  /* Ensure no bottom margin */
+    color: #24292f;
+    font-size: 0.9em;
+    line-height: 1.5em;
+}
+
+/* For dark mode support */
+@media (prefers-color-scheme: dark) {
+    .note-box {
+        background-color: #1c1c1c;
+        border-left-color: #888888;
+    }
+    .note-box-title {
+        color: #888888;
+    }
+    .note-box-content {
+        color: #d4d4d4;
+    }
+}
+
+d-article {
+    font-size: 1.0em;
+}
+
+.figure-legend {
+    font-size: 0.9em;
+    font-style: italic;
+    color: var(--distill-gray);
+    line-height: 1.5em;
+}
+
+d-code {
+    font-size: 12px;
+}
+
+.large-image-background {
+        width: 100vw;
+        padding-top: 10px;
+        padding-bottom: 10px;
+        margin-left: calc(-50vw + 50%);
+        margin-right: calc(-50vw + 50%);
+        background: white;
+        height: fit-content; /* This will make it match the image height */
+        display: flex;
+        justify-content: center; /* This will center your image */
+}
+
+.large-image-background-transparent {
+    /* width: 100vw; */
+    padding-top: 10px;
+    padding-bottom: 10px;
+    /* margin-left: calc(-50vw + 50%); */
+    margin-left:-100px;
+    margin-right: -100px;
+    /* margin-right: calc(-50vw + 50%); */
+    /* background: white; */
+    height: fit-content; /* This will make it match the image height */
+    display: flex;
+    justify-content: center; /* This will center your image */
+}
+
+.boxed-image {
+    padding: 0.5rem;
+    background: white;
+    border-radius: 12px;
+    border: 1px solid #e5e7eb;
+    box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
+}
+
+d-article li {
+    margin-bottom: 0.0em;
+}
+
+d-article ul ul {
+    margin-bottom: 0.0em;
+}
+
+d-article ol ol {
+    margin-bottom: 0.0em;
+}
+
+d-article hr {
+    grid-column: text;
+}
+
+/* Memory visualization */
+#graph-all {
+    min-width: 500px;
+    margin-right: 10px;
+    margin-bottom: 2rem;
+    padding: 0.5rem;
+    background: #f9fafb;
+    border-radius: 12px;
+    border: 1px solid #e5e7eb;
+    box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
+}
+
+
+/* Main container styles */
+#controls {
+    max-width: 1200px;
+    /* margin: 2rem auto; */
+    margin-bottom: 2rem;
+    margin-left: 10px;
+    padding: 0.6rem;
+    background: #f9fafb;
+    border-radius: 12px;
+    border: 1px solid #e5e7eb;
+    box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
+}
+
+/* Grid layout */
+#controls {
+    display: grid;
+    grid-template-columns: 1fr 1fr;
+    /* gap: 2rem; */
+}
+
+/* Cell styles */
+.cell {
+    margin-bottom: 0.2rem;
+}
+
+/* Label styles */
+label {
+    display: block;
+    /* margin-bottom: 0.5rem; */
+    font-size: 0.8rem;
+    font-weight: 500;
+    color: #374151;
+}
+
+/* Input container for range + number combination */
+.input-container {
+    display: flex;
+    gap: 1rem;
+    align-items: center;
+}
+
+/* Range input styling */
+input[type="range"] {
+    flex: 1;
+    height: 6px;
+    background: #e5e7eb;
+    border-radius: 3px;
+    appearance: none;
+    outline: none;
+}
+
+input[type="range"]::-webkit-slider-thumb {
+    appearance: none;
+    width: 16px;
+    height: 16px;
+    background: #3b82f6;
+    border-radius: 50%;
+    cursor: pointer;
+    transition: background 0.15s ease;
+}
+
+input[type="range"]::-webkit-slider-thumb:hover {
+    background: #2563eb;
+}
+
+/* Number input styling */
+input[type="number"] {
+    width: 80px;
+    padding: 0.5rem;
+    border: 1px solid #e5e7eb;
+    border-radius: 6px;
+    font-size: 0.9rem;
+    color: #374151;
+}
+
+/* Select styling */
+select {
+    width: 100%;
+    padding: 0.5rem;
+    border: 1px solid #e5e7eb;
+    border-radius: 6px;
+    background: white;
+    font-size: 0.9rem;
+    color: #374151;
+    cursor: pointer;
+}
+
+/* Checkbox styling */
+input[type="checkbox"] {
+    width: 1.2rem;
+    height: 1.2rem;
+    margin-right: 0.5rem;
+    border: 2px solid #e5e7eb;
+    border-radius: 4px;
+    cursor: pointer;
+}
+
+/* Column specific styles */
+.column-1 {
+    padding-right: 0.5rem;
+}
+
+.column-2 {
+    padding-left: 0.5rem;
+}
+
+/* Checkbox container */
+.checkbox-container {
+    display: flex;
+    align-items: center;
+    margin-bottom: 1rem;
+}
+
+/* Memory visualization styles */
+.memory-block {
+    background: #fff;
+    border-radius: 8px;
+    padding: 1rem;
+    margin-bottom: 1rem;
+    box-shadow: 0 2px 4px rgba(0, 0, 0, 0.05);
+}
+
+.memory-title {
+    font-size: 1.1rem;
+    font-weight: 500;
+    color: #374151;
+    margin-bottom: 0.5rem;
+}
+
+.memory-value {
+    font-size: 1.5rem;
+    font-weight: 600;
+    color: #3b82f6;
+}
+
+/* Responsive adjustments */
+@media (max-width: 768px) {
+    #controls {
+        grid-template-columns: 1fr;
+        padding: 1rem;
+    }
+
+    .column-1, .column-2 {
+        padding: 0;
+    }
+}
+
+/* Hover states and transitions */
+input:hover, select:hover {
+    border-color: #3b82f6;
+}
+
+input:focus, select:focus {
+    border-color: #2563eb;
+    outline: none;
+    box-shadow: 0 0 0 2px rgba(59, 130, 246, 0.1);
+}
+
+/* Add smooth transitions */
+input, select, button {
+    transition: all 0.15s ease;
+}
+
+/* Preset dropdown special styling */
+select[name="presets"] {
+    background-color: #f3f4f6;
+    font-weight: 500;
+}
+
+/* Memory graph enhancements */
+.activation-memory {
+    background: #dbeafe;
+    padding: 1rem;
+    border-radius: 8px;
+    margin-bottom: 1rem;
+}
+
+.gradient-memory {
+    background: #ede9fe;
+    padding: 1rem;
+    border-radius: 8px;
+}
+

src/syncHFSpacesURLHash.js

@@ -0,0 +1,124 @@
+const queryArg = "section";
+
+function syncHFSpacesURLHash() {
+  // Handle explicit section requests (don't update hash automatically on load)
+  const hasExplicitRequest = handleExplicitSectionRequest();
+  
+  // Set up hash change monitoring
+  updateHashBasedOnHashChange();
+  
+  // Always set up scroll monitoring to update hash during scrolling
+  setupScrollMonitoring();
+  
+  // If no explicit request, we don't update the hash on initial load
+  // The hash will only start updating when the user scrolls
+}
+
+function handleExplicitSectionRequest() {
+  // Check for section parameter in URL
+  const urlParams = new URLSearchParams(window.location.search);
+  const sectionId = urlParams.get(queryArg);
+  
+  // If we have an explicit section request
+  if (sectionId) {
+    const targetElement = document.getElementById(sectionId);
+    if (targetElement) {
+      // Slight delay to ensure the browser doesn't try to do its own scrolling first
+      setTimeout(() => {
+        targetElement.scrollIntoView();
+        history.replaceState(null, null, `#${sectionId}`);
+      }, 100);
+    }
+    return true;
+  }
+  
+  // No explicit section parameter found
+  return false;
+}
+
+function setupScrollMonitoring() {
+  // Variables to manage throttling
+  let isScrolling = false;
+  let lastKnownScrollPosition = 0;
+  let initialScroll = true;
+  
+  // Add the scroll event listener
+  window.addEventListener('scroll', function() {
+    lastKnownScrollPosition = window.scrollY;
+    
+    if (!isScrolling) {
+      window.requestAnimationFrame(function() {
+        // Skip the first scroll event which might be browser's automatic scroll
+        // to a hash on page load
+        if (initialScroll) {
+          initialScroll = false;
+        } else {
+          updateHashBasedOnScroll(lastKnownScrollPosition);
+        }
+        isScrolling = false;
+      });
+    }
+    
+    isScrolling = true;
+  });
+}
+
+// Function to update the URL hash based on scroll position
+function updateHashBasedOnScroll(scrollPosition) {
+  const closestHeading = findClosestHeading(scrollPosition);
+  
+  // Update the URL hash if we found a closest element
+  if (closestHeading && closestHeading.id) {
+    // Only update if the hash is different to avoid unnecessary operations
+    if (window.location.hash !== `#${closestHeading.id}`) {
+      silentlyUpdateHash(closestHeading.id);
+      postMessageToHFSpaces(closestHeading.id);
+    }
+  }
+}
+
+// Find the closest heading to the current scroll position
+function findClosestHeading(scrollPosition) {
+  // Get only heading elements with IDs that we want to track
+  const headingsWithIds = Array.from(document.querySelectorAll('h1[id], h2[id], h3[id], h4[id], h5[id], h6[id]'));
+
+  // Skip if there are no headings with IDs
+  if (headingsWithIds.length === 0) return null;
+
+  // Find the element closest to the middle of the viewport
+  let closestHeading = null;
+  let closestDistance = Infinity;
+  const viewportMiddle = scrollPosition + window.innerHeight / 2;
+  
+  // Iterate through all headings to find the closest one
+  headingsWithIds.forEach(heading => {
+    const headingTop = heading.getBoundingClientRect().top + scrollPosition;
+    const distance = Math.abs(headingTop - viewportMiddle);
+    
+    if (distance < closestDistance) {
+      closestDistance = distance;
+      closestHeading = heading;
+    }
+  });
+  
+  return closestHeading;
+}
+
+// Update hash without triggering scroll or other side effects
+function silentlyUpdateHash(id) {
+  history.replaceState(null, null, `#${id}`);
+}
+
+function updateHashBasedOnHashChange() {
+  window.addEventListener('hashchange', () => {
+    const elementId = window.location.hash.slice(1);
+    postMessageToHFSpaces(elementId);
+  });
+}
+
+function postMessageToHFSpaces(elementId) {
+  const parentOrigin = "https://huggingface.co";
+  window.parent.postMessage({ queryString: `${queryArg}=${elementId}` }, parentOrigin);
+}
+
+export { syncHFSpacesURLHash };

webpack.config.js

@@ -0,0 +1,153 @@
+const path = require("path");
+const { CleanWebpackPlugin } = require("clean-webpack-plugin");
+const CopyPlugin = require("copy-webpack-plugin");
+const BundleAnalyzerPlugin = require("webpack-bundle-analyzer").BundleAnalyzerPlugin;
+const Handlebars = require("handlebars");
+const fs = require("fs");
+const ImageMinimizerPlugin = require("image-minimizer-webpack-plugin");
+const HtmlMinimizerPlugin = require("html-minimizer-webpack-plugin");
+
+
+const FRAGMENTS_PATH = "src/fragments";
+
+// Load the fragments from the fragments directory and caches it
+const loadFragmentsMap = (() => {
+    let cachedFragments = null;
+    return async () => {
+        if (cachedFragments === null) {
+            cachedFragments = {};
+            const walkDir = async (dir, basePath = '') => {
+                const files = fs.readdirSync(dir);
+                await Promise.all(files.map(async file => {
+                    const filePath = path.join(dir, file);
+                    const relativePath = path.join(basePath, file);
+                    if (fs.statSync(filePath).isDirectory()) {
+                        await walkDir(filePath, relativePath);
+                    } else {
+                        // Remove the .html extension before creating the dotted path
+                        const nameWithoutExt = relativePath.replace(/\.html$/, '');
+                        const dottedPath = 'fragment-' + nameWithoutExt.replace(/\\/g, '-').replace(/\//g, '-').replace(/\./g, '-');
+                        const content = fs.readFileSync(filePath, "utf8");
+                        // Minify the HTML content using swcMinifyFragment
+                        const minifiedRes = await HtmlMinimizerPlugin.swcMinifyFragment({"tmp.html": content})
+                        if (minifiedRes.errors) {
+                            console.error(minifiedRes.errors)
+                        }
+                        const minifiedContent = minifiedRes.code;
+                        cachedFragments[dottedPath] = minifiedContent;
+                    }
+                }));
+            };
+            await walkDir(FRAGMENTS_PATH);
+        }
+        return cachedFragments;
+    };
+})();
+
+const transformHandlebars = async (data, path) => {
+    const fragments = await loadFragmentsMap();
+    console.log(`Available fragments: ${Object.keys(fragments).join(', ')}`);
+    // Load the template file
+    const template = Handlebars.compile(data.toString('utf8'));
+    const html = template(fragments);
+    return html;
+};
+
+module.exports = {
+    entry: {
+        distill: "./src/distill.js",
+        main: "./src/index.js",
+    },
+    output: {
+        filename: "[name].bundle.js", // The output file
+        path: path.resolve(__dirname, "dist"), // Output directory
+    },
+    module: {
+        rules: [
+            { test: /\.css$/, use: ["style-loader", "css-loader"] },
+            {
+                test: /\.(js|mjs)$/,
+                exclude: /node_modules/,
+                use: {
+                    loader: "babel-loader",
+                    options: {
+                        presets: ["@babel/preset-env"],
+                    },
+                },
+            },
+            {}
+        ],
+    },
+    plugins: [
+        new CleanWebpackPlugin(),
+        new CopyPlugin({
+            patterns: [
+                {
+                    from: "assets",
+                    to: "assets",
+                },
+                { from: "src/fragments/*", to: "fragments/[name].html" },
+                { from: "src/style.css", to: "style.css" },
+                { from: "src/bibliography.bib", to: "bibliography.bib" },
+                { 
+                    from: "src/index.html", 
+                    to: "index.html",
+                    transform: transformHandlebars, 
+                },
+            ],
+        }),
+    ],
+    devtool: process.env.NODE_ENV === 'production' ? 'source-map' : 'eval-source-map',
+    devServer: {
+        static: "./dist", // Serve files from the 'dist' directory
+        open: process.env.NODE_ENV !== 'production', // Automatically open the browser unless in production
+        hot: process.env.NODE_ENV !== 'production', // Enable hot module replacement unless in production
+    },
+    mode: process.env.NODE_ENV === 'production' ? 'production' : 'development',
+    optimization: {
+        minimizer: [
+            // Hynek: Ideally we would convert all images to webp and just use webp, but I don't have time
+            // to write script which would also modify html to reflect the new extensions.
+            new ImageMinimizerPlugin({
+                minimizer: [{
+                    implementation: ImageMinimizerPlugin.sharpMinify,
+                    options: {
+                        encodeOptions: {
+                            // For JPG
+                            jpeg: {
+                                quality: 80
+                            },
+                            // For PNG
+                            png: {
+                                quality: 80
+                            },
+                            // For WebP
+                            webp: {
+                                quality: 80
+                            }
+                        }
+                    }
+                },
+                {
+                    implementation: ImageMinimizerPlugin.svgoMinify,
+                    options: {
+                        encodeOptions: {
+                            multipass: true,
+                            plugins: [
+                                'preset-default',
+                            ]
+                        }
+                    }
+                }
+            ]
+            }),
+            //Hynek: Ideally we don't run this twice but we
+            new HtmlMinimizerPlugin({
+                test: /fragments\/.*\.html$/i,
+                minify: HtmlMinimizerPlugin.swcMinifyFragment,
+            })
+        ]
+    },
+};
+
+console.log(process.env.NODE_ENV)