diff --git a/gguf-converter/.gitignore b/gguf-converter/.gitignore
new file mode 100644
index 0000000..9d001c1
--- /dev/null
+++ b/gguf-converter/.gitignore
@@ -0,0 +1,2 @@
+params.json
+.venv
diff --git a/gguf-converter/Dockerfile b/gguf-converter/Dockerfile
new file mode 100644
index 0000000..77eafdd
--- /dev/null
+++ b/gguf-converter/Dockerfile
@@ -0,0 +1,34 @@
+ARG BASE_IMAGE=substratusai/base:latest
+FROM nvidia/cuda:12.2.0-devel-ubuntu22.04 as build
+
+RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \
+    apt-get update && \
+    apt-get -y --no-install-recommends install \
+      python3 python3-pip python3-venv git build-essential gcc wget && \
+    mkdir -p /etc/OpenCL/vendors && echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN mkdir /build
+WORKDIR /build
+RUN git clone https://github.com/ggerganov/llama.cpp.git
+RUN cd llama.cpp  && make quantize
+
+FROM ${BASE_IMAGE}
+
+RUN mkdir -p /content/src /content/data /content/bin
+WORKDIR /content
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+RUN git clone https://github.com/ggerganov/llama.cpp.git
+RUN pip install --no-cache-dir -r llama.cpp/requirements.txt
+COPY --from=build /build/llama.cpp/quantize /content/bin
+
+ENV PATH="$PATH:/content/scripts:/content/bin"
+
+# Copy in build dependencies only since the build will take a while.
+COPY ./scripts/ ./scripts
+COPY ./src/ ./src
+
+ENTRYPOINT ["/tini", "--", "/content/scripts/entrypoint.sh"]
+CMD convert.sh
diff --git a/gguf-converter/README.md b/gguf-converter/README.md
new file mode 100644
index 0000000..1138970
--- /dev/null
+++ b/gguf-converter/README.md
@@ -0,0 +1,24 @@
+# Dataset Loader HTTP
+
+Load existing files into Substratus by downloading using HTTP
+
+## Usage
+
+Build the image locally:
+
+```sh
+docker build -t gguf-converter .
+```
+
+Explore and develop with a Jupyter Lab:
+```sh
+# create test params.json
+cat > params.json <<EOF
+{"name": "test", "download_model_id": "lmsys/vicuna-13b-v1.5"}
+EOF
+# Run a Jupyter Notebook.
+docker run -it -v $(pwd)/src:/content/src \
+  -v $(pwd)/params.json:/content/params.json -p 8888:8888 \
+  --security-opt seccomp=unconfined gguf-converter notebook.sh
+```
+Now open your browser at http://localhost:8888
diff --git a/gguf-converter/requirements.txt b/gguf-converter/requirements.txt
new file mode 100644
index 0000000..6f9d557
--- /dev/null
+++ b/gguf-converter/requirements.txt
@@ -0,0 +1,5 @@
+jupyterlab
+ipywidgets
+jupytext
+
+huggingface_hub
\ No newline at end of file
diff --git a/gguf-converter/scripts/convert.sh b/gguf-converter/scripts/convert.sh
new file mode 100755
index 0000000..55f028a
--- /dev/null
+++ b/gguf-converter/scripts/convert.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env sh
+
+set -xe
+
+jupyter nbconvert --debug --to notebook --execute /content/src/convert.ipynb --output /content/logs/convert.ipynb
diff --git a/gguf-converter/scripts/entrypoint.sh b/gguf-converter/scripts/entrypoint.sh
new file mode 100755
index 0000000..18b13fa
--- /dev/null
+++ b/gguf-converter/scripts/entrypoint.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+export HUGGING_FACE_HUB_TOKEN=$PARAM_HUGGING_FACE_HUB_TOKEN
+
+exec "$@"
diff --git a/gguf-converter/src/convert.ipynb b/gguf-converter/src/convert.ipynb
new file mode 100644
index 0000000..fe31556
--- /dev/null
+++ b/gguf-converter/src/convert.ipynb
@@ -0,0 +1,1721 @@
+{
+ "cells": [
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "ead0b7e2",
+   "metadata": {},
+   "source": [
+    "# Convert HuggingFace models to GGUF format\n",
+    "\n",
+    "This notebook converts HuggingFace models to GGUF format that's supported\n",
+    "by llama.cpp. The notebook also supports downloading a model from HuggingFace\n",
+    "directly by setting the `download_model_id` param in Substratus."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "78d0dede",
+   "metadata": {},
+   "source": [
+    "Load the params provided by substratus"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "f11c4c64",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "from pathlib import Path\n",
+    "\n",
+    "params = {}\n",
+    "params_path = Path(\"/content/params.json\")\n",
+    "if params_path.is_file():\n",
+    "    with params_path.open(\"r\", encoding=\"UTF-8\") as params_file:\n",
+    "        params = json.load(params_file)\n",
+    "if 'name' not in params:\n",
+    "    raise Exception(\"Missing required param `name`\")\n",
+    "\n",
+    "name = params[\"name\"]\n",
+    "\n",
+    "output_path = params.get(\"output_path\", \"/content/model\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a32938b1",
+   "metadata": {},
+   "source": [
+    "Download the model from huggingFace if `download_model_id` params is set. Otherwise\n",
+    "this expects HuggingFace model to be present at `/content/saved-model`"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "38bfe9dd",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "bc0fe83fb8f845b99b5ebb97d8ea8173",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Fetching 10 files:   0%|          | 0/10 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "ed33d03c047e466ebf120e17c23ba93a",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading (…)797effff84/README.md:   0%|          | 0.00/3.35k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "f1e0fadf5e8e4d69b55aa0a6eb696a71",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading (…)fff84/.gitattributes:   0%|          | 0.00/1.52k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "4762d7c419884d1abab2419775d8ccdd",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading (…)cial_tokens_map.json:   0%|          | 0.00/438 [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "4bf428bb910440158993b64063b5a6ea",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading (…)7effff84/config.json:   0%|          | 0.00/631 [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "4ef837e86cb94f6da68103bf2d634d1d",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading (…)model.bin.index.json:   0%|          | 0.00/23.9k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "d0e5fcc3f61c46669dccbab3f08151af",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading (…)fff84/tokenizer.json:   0%|          | 0.00/1.84M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "5221fe9da60345c8b584585536c3fcc8",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading (…)okenizer_config.json:   0%|          | 0.00/762 [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "da2dc8f70b2844b7b32c6b816f7a416d",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading tokenizer.model:   0%|          | 0.00/500k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "c8f6e3bb0d074fcbad350e8a1543c7dc",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading (…)l-00001-of-00002.bin:   0%|          | 0.00/9.98G [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "3ee5e7ed16a5434a95cdfe823e0ad43c",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading (…)l-00002-of-00002.bin:   0%|          | 0.00/3.50G [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "from huggingface_hub import snapshot_download\n",
+    "\n",
+    "model_path = \"/content/saved-model\"\n",
+    "\n",
+    "download_model_id = params.get(\"download_model_id\")\n",
+    "if download_model_id:\n",
+    "    model_path = \"/content/downloaded-model\"\n",
+    "    snapshot_download(repo_id=download_model_id, local_dir=model_path,\n",
+    "                      local_dir_use_symlinks=False, revision=\"main\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7b44d82d",
+   "metadata": {},
+   "source": [
+    "Convert the model to GGUF 16 bit so it can be further used with `llama.cpp/example/quantize` tool"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "4fa56848-1d74-4c36-b34f-cda126fffd47",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "total 13G\n",
+      "4.0K drwxr-xr-x 2 root root 4.0K Sep  4 03:43 .\n",
+      "8.0K drwxr-xr-x 1 root root 4.0K Sep  4 03:43 ..\n",
+      "4.0K -rw-r--r-- 1 root root 1.5K Sep  4 03:40 .gitattributes\n",
+      "4.0K -rw-r--r-- 1 root root 3.3K Sep  4 03:40 README.md\n",
+      "4.0K -rw-r--r-- 1 root root  631 Sep  4 03:40 config.json\n",
+      "9.3G -rw-r--r-- 1 root root 9.3G Sep  4 03:43 pytorch_model-00001-of-00002.bin\n",
+      "3.3G -rw-r--r-- 1 root root 3.3G Sep  4 03:41 pytorch_model-00002-of-00002.bin\n",
+      " 24K -rw-r--r-- 1 root root  24K Sep  4 03:40 pytorch_model.bin.index.json\n",
+      "4.0K -rw-r--r-- 1 root root  438 Sep  4 03:40 special_tokens_map.json\n",
+      "1.8M -rw-r--r-- 1 root root 1.8M Sep  4 03:40 tokenizer.json\n",
+      "492K -rw-r--r-- 1 root root 489K Sep  4 03:40 tokenizer.model\n",
+      "4.0K -rw-r--r-- 1 root root  762 Sep  4 03:40 tokenizer_config.json\n",
+      "Loading model file /content/downloaded-model/pytorch_model-00001-of-00002.bin\n",
+      "Loading model file /content/downloaded-model/pytorch_model-00001-of-00002.bin\n",
+      "Loading model file /content/downloaded-model/pytorch_model-00002-of-00002.bin\n",
+      "params = Params(n_vocab=32000, n_embd=4096, n_mult=5504, n_layer=32, n_ctx=4096, n_ff=11008, n_head=32, n_head_kv=32, f_norm_eps=1e-05, f_rope_freq_base=10000.0, f_rope_scale=None, ftype=<GGMLFileType.MostlyF16: 1>, path_model=PosixPath('/content/downloaded-model'))\n",
+      "Loading vocab file '/content/downloaded-model/tokenizer.model', type 'spm'\n",
+      "Permuting layer 0\n",
+      "Permuting layer 1\n",
+      "Permuting layer 2\n",
+      "Permuting layer 3\n",
+      "Permuting layer 4\n",
+      "Permuting layer 5\n",
+      "Permuting layer 6\n",
+      "Permuting layer 7\n",
+      "Permuting layer 8\n",
+      "Permuting layer 9\n",
+      "Permuting layer 10\n",
+      "Permuting layer 11\n",
+      "Permuting layer 12\n",
+      "Permuting layer 13\n",
+      "Permuting layer 14\n",
+      "Permuting layer 15\n",
+      "Permuting layer 16\n",
+      "Permuting layer 17\n",
+      "Permuting layer 18\n",
+      "Permuting layer 19\n",
+      "Permuting layer 20\n",
+      "Permuting layer 21\n",
+      "Permuting layer 22\n",
+      "Permuting layer 23\n",
+      "Permuting layer 24\n",
+      "Permuting layer 25\n",
+      "Permuting layer 26\n",
+      "Permuting layer 27\n",
+      "Permuting layer 28\n",
+      "Permuting layer 29\n",
+      "Permuting layer 30\n",
+      "Permuting layer 31\n",
+      "model.embed_tokens.weight                        -> token_embd.weight                        | F16    | [32000, 4096]\n",
+      "model.layers.0.self_attn.q_proj.weight           -> blk.0.attn_q.weight                      | F16    | [4096, 4096]\n",
+      "model.layers.0.self_attn.k_proj.weight           -> blk.0.attn_k.weight                      | F16    | [4096, 4096]\n",
+      "model.layers.0.self_attn.v_proj.weight           -> blk.0.attn_v.weight                      | F16    | [4096, 4096]\n",
+      "model.layers.0.self_attn.o_proj.weight           -> blk.0.attn_output.weight                 | F16    | [4096, 4096]\n",
+      "model.layers.0.mlp.gate_proj.weight              -> blk.0.ffn_gate.weight                    | F16    | [11008, 4096]\n",
+      "model.layers.0.mlp.up_proj.weight                -> blk.0.ffn_up.weight                      | F16    | [11008, 4096]\n",
+      "model.layers.0.mlp.down_proj.weight              -> blk.0.ffn_down.weight                    | F16    | [4096, 11008]\n",
+      "model.layers.0.input_layernorm.weight            -> blk.0.attn_norm.weight                   | F16    | [4096]\n",
+      "model.layers.0.post_attention_layernorm.weight   -> blk.0.ffn_norm.weight                    | F16    | [4096]\n",
+      "model.layers.1.self_attn.q_proj.weight           -> blk.1.attn_q.weight                      | F16    | [4096, 4096]\n",
+      "model.layers.1.self_attn.k_proj.weight           -> blk.1.attn_k.weight                      | F16    | [4096, 4096]\n",
+      "model.layers.1.self_attn.v_proj.weight           -> blk.1.attn_v.weight                      | F16    | [4096, 4096]\n",
+      "model.layers.1.self_attn.o_proj.weight           -> blk.1.attn_output.weight                 | F16    | [4096, 4096]\n",
+      "model.layers.1.mlp.gate_proj.weight              -> blk.1.ffn_gate.weight                    | F16    | [11008, 4096]\n",
+      "model.layers.1.mlp.up_proj.weight                -> blk.1.ffn_up.weight                      | F16    | [11008, 4096]\n",
+      "model.layers.1.mlp.down_proj.weight              -> blk.1.ffn_down.weight                    | F16    | [4096, 11008]\n",
+      "model.layers.1.input_layernorm.weight            -> blk.1.attn_norm.weight                   | F16    | [4096]\n",
+      "model.layers.1.post_attention_layernorm.weight   -> blk.1.ffn_norm.weight                    | F16    | [4096]\n",
+      "model.layers.2.self_attn.q_proj.weight           -> blk.2.attn_q.weight                      | F16    | [4096, 4096]\n",
+      "model.layers.2.self_attn.k_proj.weight           -> blk.2.attn_k.weight                      | F16    | [4096, 4096]\n",
+      "model.layers.2.self_attn.v_proj.weight           -> blk.2.attn_v.weight                      | F16    | [4096, 4096]\n",
+      "model.layers.2.self_attn.o_proj.weight           -> blk.2.attn_output.weight                 | F16    | [4096, 4096]\n",
+      "model.layers.2.mlp.gate_proj.weight              -> blk.2.ffn_gate.weight                    | F16    | [11008, 4096]\n",
+      "model.layers.2.mlp.up_proj.weight                -> blk.2.ffn_up.weight                      | F16    | [11008, 4096]\n",
+      "model.layers.2.mlp.down_proj.weight              -> blk.2.ffn_down.weight                    | F16    | [4096, 11008]\n",
+      "model.layers.2.input_layernorm.weight            -> blk.2.attn_norm.weight                   | F16    | [4096]\n",
+      "model.layers.2.post_attention_layernorm.weight   -> blk.2.ffn_norm.weight                    | F16    | [4096]\n",
+      "model.layers.3.self_attn.q_proj.weight           -> blk.3.attn_q.weight                      | F16    | [4096, 4096]\n",
+      "model.layers.3.self_attn.k_proj.weight           -> blk.3.attn_k.weight                      | F16    | [4096, 4096]\n",
+      "model.layers.3.self_attn.v_proj.weight           -> blk.3.attn_v.weight                      | F16    | [4096, 4096]\n",
+      "model.layers.3.self_attn.o_proj.weight           -> blk.3.attn_output.weight                 | F16    | [4096, 4096]\n",
+      "model.layers.3.mlp.gate_proj.weight              -> blk.3.ffn_gate.weight                    | F16    | [11008, 4096]\n",
+      "model.layers.3.mlp.up_proj.weight                -> blk.3.ffn_up.weight                      | F16    | [11008, 4096]\n",
+      "model.layers.3.mlp.down_proj.weight              -> blk.3.ffn_down.weight                    | F16    | [4096, 11008]\n",
+      "model.layers.3.input_layernorm.weight            -> blk.3.attn_norm.weight                   | F16    | [4096]\n",
+      "model.layers.3.post_attention_layernorm.weight   -> blk.3.ffn_norm.weight                    | F16    | [4096]\n",
+      "model.layers.4.self_attn.q_proj.weight           -> blk.4.attn_q.weight                      | F16    | [4096, 4096]\n",
+      "model.layers.4.self_attn.k_proj.weight           -> blk.4.attn_k.weight                      | F16    | [4096, 4096]\n",
+      "model.layers.4.self_attn.v_proj.weight           -> blk.4.attn_v.weight                      | F16    | [4096, 4096]\n",
+      "model.layers.4.self_attn.o_proj.weight           -> blk.4.attn_output.weight                 | F16    | [4096, 4096]\n",
+      "model.layers.4.mlp.gate_proj.weight              -> blk.4.ffn_gate.weight                    | F16    | [11008, 4096]\n",
+      "model.layers.4.mlp.up_proj.weight                -> blk.4.ffn_up.weight                      | F16    | [11008, 4096]\n",
+      "model.layers.4.mlp.down_proj.weight              -> blk.4.ffn_down.weight                    | F16    | [4096, 11008]\n",
+      "model.layers.4.input_layernorm.weight            -> blk.4.attn_norm.weight                   | F16    | [4096]\n",
+      "model.layers.4.post_attention_layernorm.weight   -> blk.4.ffn_norm.weight                    | F16    | [4096]\n",
+      "model.layers.5.self_attn.q_proj.weight           -> blk.5.attn_q.weight                      | F16    | [4096, 4096]\n",
+      "model.layers.5.self_attn.k_proj.weight           -> blk.5.attn_k.weight                      | F16    | [4096, 4096]\n",
+      "model.layers.5.self_attn.v_proj.weight           -> blk.5.attn_v.weight                      | F16    | [4096, 4096]\n",
+      "model.layers.5.self_attn.o_proj.weight           -> blk.5.attn_output.weight                 | F16    | [4096, 4096]\n",
+      "model.layers.5.mlp.gate_proj.weight              -> blk.5.ffn_gate.weight                    | F16    | [11008, 4096]\n",
+      "model.layers.5.mlp.up_proj.weight                -> blk.5.ffn_up.weight                      | F16    | [11008, 4096]\n",
+      "model.layers.5.mlp.down_proj.weight              -> blk.5.ffn_down.weight                    | F16    | [4096, 11008]\n",
+      "model.layers.5.input_layernorm.weight            -> blk.5.attn_norm.weight                   | F16    | [4096]\n",
+      "model.layers.5.post_attention_layernorm.weight   -> blk.5.ffn_norm.weight                    | F16    | [4096]\n",
+      "model.layers.6.self_attn.q_proj.weight           -> blk.6.attn_q.weight                      | F16    | [4096, 4096]\n",
+      "model.layers.6.self_attn.k_proj.weight           -> blk.6.attn_k.weight                      | F16    | [4096, 4096]\n",
+      "model.layers.6.self_attn.v_proj.weight           -> blk.6.attn_v.weight                      | F16    | [4096, 4096]\n",
+      "model.layers.6.self_attn.o_proj.weight           -> blk.6.attn_output.weight                 | F16    | [4096, 4096]\n",
+      "model.layers.6.mlp.gate_proj.weight              -> blk.6.ffn_gate.weight                    | F16    | [11008, 4096]\n",
+      "model.layers.6.mlp.up_proj.weight                -> blk.6.ffn_up.weight                      | F16    | [11008, 4096]\n",
+      "model.layers.6.mlp.down_proj.weight              -> blk.6.ffn_down.weight                    | F16    | [4096, 11008]\n",
+      "model.layers.6.input_layernorm.weight            -> blk.6.attn_norm.weight                   | F16    | [4096]\n",
+      "model.layers.6.post_attention_layernorm.weight   -> blk.6.ffn_norm.weight                    | F16    | [4096]\n",
+      "model.layers.7.self_attn.q_proj.weight           -> blk.7.attn_q.weight                      | F16    | [4096, 4096]\n",
+      "model.layers.7.self_attn.k_proj.weight           -> blk.7.attn_k.weight                      | F16    | [4096, 4096]\n",
+      "model.layers.7.self_attn.v_proj.weight           -> blk.7.attn_v.weight                      | F16    | [4096, 4096]\n",
+      "model.layers.7.self_attn.o_proj.weight           -> blk.7.attn_output.weight                 | F16    | [4096, 4096]\n",
+      "model.layers.7.mlp.gate_proj.weight              -> blk.7.ffn_gate.weight                    | F16    | [11008, 4096]\n",
+      "model.layers.7.mlp.up_proj.weight                -> blk.7.ffn_up.weight                      | F16    | [11008, 4096]\n",
+      "model.layers.7.mlp.down_proj.weight              -> blk.7.ffn_down.weight                    | F16    | [4096, 11008]\n",
+      "model.layers.7.input_layernorm.weight            -> blk.7.attn_norm.weight                   | F16    | [4096]\n",
+      "model.layers.7.post_attention_layernorm.weight   -> blk.7.ffn_norm.weight                    | F16    | [4096]\n",
+      "model.layers.8.self_attn.q_proj.weight           -> blk.8.attn_q.weight                      | F16    | [4096, 4096]\n",
+      "model.layers.8.self_attn.k_proj.weight           -> blk.8.attn_k.weight                      | F16    | [4096, 4096]\n",
+      "model.layers.8.self_attn.v_proj.weight           -> blk.8.attn_v.weight                      | F16    | [4096, 4096]\n",
+      "model.layers.8.self_attn.o_proj.weight           -> blk.8.attn_output.weight                 | F16    | [4096, 4096]\n",
+      "model.layers.8.mlp.gate_proj.weight              -> blk.8.ffn_gate.weight                    | F16    | [11008, 4096]\n",
+      "model.layers.8.mlp.up_proj.weight                -> blk.8.ffn_up.weight                      | F16    | [11008, 4096]\n",
+      "model.layers.8.mlp.down_proj.weight              -> blk.8.ffn_down.weight                    | F16    | [4096, 11008]\n",
+      "model.layers.8.input_layernorm.weight            -> blk.8.attn_norm.weight                   | F16    | [4096]\n",
+      "model.layers.8.post_attention_layernorm.weight   -> blk.8.ffn_norm.weight                    | F16    | [4096]\n",
+      "model.layers.9.self_attn.q_proj.weight           -> blk.9.attn_q.weight                      | F16    | [4096, 4096]\n",
+      "model.layers.9.self_attn.k_proj.weight           -> blk.9.attn_k.weight                      | F16    | [4096, 4096]\n",
+      "model.layers.9.self_attn.v_proj.weight           -> blk.9.attn_v.weight                      | F16    | [4096, 4096]\n",
+      "model.layers.9.self_attn.o_proj.weight           -> blk.9.attn_output.weight                 | F16    | [4096, 4096]\n",
+      "model.layers.9.mlp.gate_proj.weight              -> blk.9.ffn_gate.weight                    | F16    | [11008, 4096]\n",
+      "model.layers.9.mlp.up_proj.weight                -> blk.9.ffn_up.weight                      | F16    | [11008, 4096]\n",
+      "model.layers.9.mlp.down_proj.weight              -> blk.9.ffn_down.weight                    | F16    | [4096, 11008]\n",
+      "model.layers.9.input_layernorm.weight            -> blk.9.attn_norm.weight                   | F16    | [4096]\n",
+      "model.layers.9.post_attention_layernorm.weight   -> blk.9.ffn_norm.weight                    | F16    | [4096]\n",
+      "model.layers.10.self_attn.q_proj.weight          -> blk.10.attn_q.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.10.self_attn.k_proj.weight          -> blk.10.attn_k.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.10.self_attn.v_proj.weight          -> blk.10.attn_v.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.10.self_attn.o_proj.weight          -> blk.10.attn_output.weight                | F16    | [4096, 4096]\n",
+      "model.layers.10.mlp.gate_proj.weight             -> blk.10.ffn_gate.weight                   | F16    | [11008, 4096]\n",
+      "model.layers.10.mlp.up_proj.weight               -> blk.10.ffn_up.weight                     | F16    | [11008, 4096]\n",
+      "model.layers.10.mlp.down_proj.weight             -> blk.10.ffn_down.weight                   | F16    | [4096, 11008]\n",
+      "model.layers.10.input_layernorm.weight           -> blk.10.attn_norm.weight                  | F16    | [4096]\n",
+      "model.layers.10.post_attention_layernorm.weight  -> blk.10.ffn_norm.weight                   | F16    | [4096]\n",
+      "model.layers.11.self_attn.q_proj.weight          -> blk.11.attn_q.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.11.self_attn.k_proj.weight          -> blk.11.attn_k.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.11.self_attn.v_proj.weight          -> blk.11.attn_v.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.11.self_attn.o_proj.weight          -> blk.11.attn_output.weight                | F16    | [4096, 4096]\n",
+      "model.layers.11.mlp.gate_proj.weight             -> blk.11.ffn_gate.weight                   | F16    | [11008, 4096]\n",
+      "model.layers.11.mlp.up_proj.weight               -> blk.11.ffn_up.weight                     | F16    | [11008, 4096]\n",
+      "model.layers.11.mlp.down_proj.weight             -> blk.11.ffn_down.weight                   | F16    | [4096, 11008]\n",
+      "model.layers.11.input_layernorm.weight           -> blk.11.attn_norm.weight                  | F16    | [4096]\n",
+      "model.layers.11.post_attention_layernorm.weight  -> blk.11.ffn_norm.weight                   | F16    | [4096]\n",
+      "model.layers.12.self_attn.q_proj.weight          -> blk.12.attn_q.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.12.self_attn.k_proj.weight          -> blk.12.attn_k.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.12.self_attn.v_proj.weight          -> blk.12.attn_v.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.12.self_attn.o_proj.weight          -> blk.12.attn_output.weight                | F16    | [4096, 4096]\n",
+      "model.layers.12.mlp.gate_proj.weight             -> blk.12.ffn_gate.weight                   | F16    | [11008, 4096]\n",
+      "model.layers.12.mlp.up_proj.weight               -> blk.12.ffn_up.weight                     | F16    | [11008, 4096]\n",
+      "model.layers.12.mlp.down_proj.weight             -> blk.12.ffn_down.weight                   | F16    | [4096, 11008]\n",
+      "model.layers.12.input_layernorm.weight           -> blk.12.attn_norm.weight                  | F16    | [4096]\n",
+      "model.layers.12.post_attention_layernorm.weight  -> blk.12.ffn_norm.weight                   | F16    | [4096]\n",
+      "model.layers.13.self_attn.q_proj.weight          -> blk.13.attn_q.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.13.self_attn.k_proj.weight          -> blk.13.attn_k.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.13.self_attn.v_proj.weight          -> blk.13.attn_v.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.13.self_attn.o_proj.weight          -> blk.13.attn_output.weight                | F16    | [4096, 4096]\n",
+      "model.layers.13.mlp.gate_proj.weight             -> blk.13.ffn_gate.weight                   | F16    | [11008, 4096]\n",
+      "model.layers.13.mlp.up_proj.weight               -> blk.13.ffn_up.weight                     | F16    | [11008, 4096]\n",
+      "model.layers.13.mlp.down_proj.weight             -> blk.13.ffn_down.weight                   | F16    | [4096, 11008]\n",
+      "model.layers.13.input_layernorm.weight           -> blk.13.attn_norm.weight                  | F16    | [4096]\n",
+      "model.layers.13.post_attention_layernorm.weight  -> blk.13.ffn_norm.weight                   | F16    | [4096]\n",
+      "model.layers.14.self_attn.q_proj.weight          -> blk.14.attn_q.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.14.self_attn.k_proj.weight          -> blk.14.attn_k.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.14.self_attn.v_proj.weight          -> blk.14.attn_v.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.14.self_attn.o_proj.weight          -> blk.14.attn_output.weight                | F16    | [4096, 4096]\n",
+      "model.layers.14.mlp.gate_proj.weight             -> blk.14.ffn_gate.weight                   | F16    | [11008, 4096]\n",
+      "model.layers.14.mlp.up_proj.weight               -> blk.14.ffn_up.weight                     | F16    | [11008, 4096]\n",
+      "model.layers.14.mlp.down_proj.weight             -> blk.14.ffn_down.weight                   | F16    | [4096, 11008]\n",
+      "model.layers.14.input_layernorm.weight           -> blk.14.attn_norm.weight                  | F16    | [4096]\n",
+      "model.layers.14.post_attention_layernorm.weight  -> blk.14.ffn_norm.weight                   | F16    | [4096]\n",
+      "model.layers.15.self_attn.q_proj.weight          -> blk.15.attn_q.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.15.self_attn.k_proj.weight          -> blk.15.attn_k.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.15.self_attn.v_proj.weight          -> blk.15.attn_v.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.15.self_attn.o_proj.weight          -> blk.15.attn_output.weight                | F16    | [4096, 4096]\n",
+      "model.layers.15.mlp.gate_proj.weight             -> blk.15.ffn_gate.weight                   | F16    | [11008, 4096]\n",
+      "model.layers.15.mlp.up_proj.weight               -> blk.15.ffn_up.weight                     | F16    | [11008, 4096]\n",
+      "model.layers.15.mlp.down_proj.weight             -> blk.15.ffn_down.weight                   | F16    | [4096, 11008]\n",
+      "model.layers.15.input_layernorm.weight           -> blk.15.attn_norm.weight                  | F16    | [4096]\n",
+      "model.layers.15.post_attention_layernorm.weight  -> blk.15.ffn_norm.weight                   | F16    | [4096]\n",
+      "model.layers.16.self_attn.q_proj.weight          -> blk.16.attn_q.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.16.self_attn.k_proj.weight          -> blk.16.attn_k.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.16.self_attn.v_proj.weight          -> blk.16.attn_v.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.16.self_attn.o_proj.weight          -> blk.16.attn_output.weight                | F16    | [4096, 4096]\n",
+      "model.layers.16.mlp.gate_proj.weight             -> blk.16.ffn_gate.weight                   | F16    | [11008, 4096]\n",
+      "model.layers.16.mlp.up_proj.weight               -> blk.16.ffn_up.weight                     | F16    | [11008, 4096]\n",
+      "model.layers.16.mlp.down_proj.weight             -> blk.16.ffn_down.weight                   | F16    | [4096, 11008]\n",
+      "model.layers.16.input_layernorm.weight           -> blk.16.attn_norm.weight                  | F16    | [4096]\n",
+      "model.layers.16.post_attention_layernorm.weight  -> blk.16.ffn_norm.weight                   | F16    | [4096]\n",
+      "model.layers.17.self_attn.q_proj.weight          -> blk.17.attn_q.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.17.self_attn.k_proj.weight          -> blk.17.attn_k.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.17.self_attn.v_proj.weight          -> blk.17.attn_v.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.17.self_attn.o_proj.weight          -> blk.17.attn_output.weight                | F16    | [4096, 4096]\n",
+      "model.layers.17.mlp.gate_proj.weight             -> blk.17.ffn_gate.weight                   | F16    | [11008, 4096]\n",
+      "model.layers.17.mlp.up_proj.weight               -> blk.17.ffn_up.weight                     | F16    | [11008, 4096]\n",
+      "model.layers.17.mlp.down_proj.weight             -> blk.17.ffn_down.weight                   | F16    | [4096, 11008]\n",
+      "model.layers.17.input_layernorm.weight           -> blk.17.attn_norm.weight                  | F16    | [4096]\n",
+      "model.layers.17.post_attention_layernorm.weight  -> blk.17.ffn_norm.weight                   | F16    | [4096]\n",
+      "model.layers.18.self_attn.q_proj.weight          -> blk.18.attn_q.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.18.self_attn.k_proj.weight          -> blk.18.attn_k.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.18.self_attn.v_proj.weight          -> blk.18.attn_v.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.18.self_attn.o_proj.weight          -> blk.18.attn_output.weight                | F16    | [4096, 4096]\n",
+      "model.layers.18.mlp.gate_proj.weight             -> blk.18.ffn_gate.weight                   | F16    | [11008, 4096]\n",
+      "model.layers.18.mlp.up_proj.weight               -> blk.18.ffn_up.weight                     | F16    | [11008, 4096]\n",
+      "model.layers.18.mlp.down_proj.weight             -> blk.18.ffn_down.weight                   | F16    | [4096, 11008]\n",
+      "model.layers.18.input_layernorm.weight           -> blk.18.attn_norm.weight                  | F16    | [4096]\n",
+      "model.layers.18.post_attention_layernorm.weight  -> blk.18.ffn_norm.weight                   | F16    | [4096]\n",
+      "model.layers.19.self_attn.q_proj.weight          -> blk.19.attn_q.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.19.self_attn.k_proj.weight          -> blk.19.attn_k.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.19.self_attn.v_proj.weight          -> blk.19.attn_v.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.19.self_attn.o_proj.weight          -> blk.19.attn_output.weight                | F16    | [4096, 4096]\n",
+      "model.layers.19.mlp.gate_proj.weight             -> blk.19.ffn_gate.weight                   | F16    | [11008, 4096]\n",
+      "model.layers.19.mlp.up_proj.weight               -> blk.19.ffn_up.weight                     | F16    | [11008, 4096]\n",
+      "model.layers.19.mlp.down_proj.weight             -> blk.19.ffn_down.weight                   | F16    | [4096, 11008]\n",
+      "model.layers.19.input_layernorm.weight           -> blk.19.attn_norm.weight                  | F16    | [4096]\n",
+      "model.layers.19.post_attention_layernorm.weight  -> blk.19.ffn_norm.weight                   | F16    | [4096]\n",
+      "model.layers.20.self_attn.q_proj.weight          -> blk.20.attn_q.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.20.self_attn.k_proj.weight          -> blk.20.attn_k.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.20.self_attn.v_proj.weight          -> blk.20.attn_v.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.20.self_attn.o_proj.weight          -> blk.20.attn_output.weight                | F16    | [4096, 4096]\n",
+      "model.layers.20.mlp.gate_proj.weight             -> blk.20.ffn_gate.weight                   | F16    | [11008, 4096]\n",
+      "model.layers.20.mlp.up_proj.weight               -> blk.20.ffn_up.weight                     | F16    | [11008, 4096]\n",
+      "model.layers.20.mlp.down_proj.weight             -> blk.20.ffn_down.weight                   | F16    | [4096, 11008]\n",
+      "model.layers.20.input_layernorm.weight           -> blk.20.attn_norm.weight                  | F16    | [4096]\n",
+      "model.layers.20.post_attention_layernorm.weight  -> blk.20.ffn_norm.weight                   | F16    | [4096]\n",
+      "model.layers.21.self_attn.q_proj.weight          -> blk.21.attn_q.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.21.self_attn.k_proj.weight          -> blk.21.attn_k.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.21.self_attn.v_proj.weight          -> blk.21.attn_v.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.21.self_attn.o_proj.weight          -> blk.21.attn_output.weight                | F16    | [4096, 4096]\n",
+      "model.layers.21.mlp.gate_proj.weight             -> blk.21.ffn_gate.weight                   | F16    | [11008, 4096]\n",
+      "model.layers.21.mlp.up_proj.weight               -> blk.21.ffn_up.weight                     | F16    | [11008, 4096]\n",
+      "model.layers.21.mlp.down_proj.weight             -> blk.21.ffn_down.weight                   | F16    | [4096, 11008]\n",
+      "model.layers.21.input_layernorm.weight           -> blk.21.attn_norm.weight                  | F16    | [4096]\n",
+      "model.layers.21.post_attention_layernorm.weight  -> blk.21.ffn_norm.weight                   | F16    | [4096]\n",
+      "model.layers.22.self_attn.q_proj.weight          -> blk.22.attn_q.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.22.self_attn.k_proj.weight          -> blk.22.attn_k.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.22.self_attn.v_proj.weight          -> blk.22.attn_v.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.22.self_attn.o_proj.weight          -> blk.22.attn_output.weight                | F16    | [4096, 4096]\n",
+      "model.layers.22.mlp.gate_proj.weight             -> blk.22.ffn_gate.weight                   | F16    | [11008, 4096]\n",
+      "model.layers.22.mlp.up_proj.weight               -> blk.22.ffn_up.weight                     | F16    | [11008, 4096]\n",
+      "model.layers.22.mlp.down_proj.weight             -> blk.22.ffn_down.weight                   | F16    | [4096, 11008]\n",
+      "model.layers.22.input_layernorm.weight           -> blk.22.attn_norm.weight                  | F16    | [4096]\n",
+      "model.layers.22.post_attention_layernorm.weight  -> blk.22.ffn_norm.weight                   | F16    | [4096]\n",
+      "model.layers.23.self_attn.q_proj.weight          -> blk.23.attn_q.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.23.self_attn.k_proj.weight          -> blk.23.attn_k.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.23.self_attn.v_proj.weight          -> blk.23.attn_v.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.23.self_attn.o_proj.weight          -> blk.23.attn_output.weight                | F16    | [4096, 4096]\n",
+      "model.layers.23.mlp.gate_proj.weight             -> blk.23.ffn_gate.weight                   | F16    | [11008, 4096]\n",
+      "model.layers.23.mlp.up_proj.weight               -> blk.23.ffn_up.weight                     | F16    | [11008, 4096]\n",
+      "model.layers.23.mlp.down_proj.weight             -> blk.23.ffn_down.weight                   | F16    | [4096, 11008]\n",
+      "model.layers.23.input_layernorm.weight           -> blk.23.attn_norm.weight                  | F16    | [4096]\n",
+      "model.layers.23.post_attention_layernorm.weight  -> blk.23.ffn_norm.weight                   | F16    | [4096]\n",
+      "model.layers.24.self_attn.q_proj.weight          -> blk.24.attn_q.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.24.self_attn.k_proj.weight          -> blk.24.attn_k.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.24.self_attn.v_proj.weight          -> blk.24.attn_v.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.24.self_attn.o_proj.weight          -> blk.24.attn_output.weight                | F16    | [4096, 4096]\n",
+      "model.layers.24.mlp.gate_proj.weight             -> blk.24.ffn_gate.weight                   | F16    | [11008, 4096]\n",
+      "model.layers.24.mlp.up_proj.weight               -> blk.24.ffn_up.weight                     | F16    | [11008, 4096]\n",
+      "model.layers.24.mlp.down_proj.weight             -> blk.24.ffn_down.weight                   | F16    | [4096, 11008]\n",
+      "model.layers.24.input_layernorm.weight           -> blk.24.attn_norm.weight                  | F16    | [4096]\n",
+      "model.layers.24.post_attention_layernorm.weight  -> blk.24.ffn_norm.weight                   | F16    | [4096]\n",
+      "model.layers.25.self_attn.q_proj.weight          -> blk.25.attn_q.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.25.self_attn.k_proj.weight          -> blk.25.attn_k.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.25.self_attn.v_proj.weight          -> blk.25.attn_v.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.25.self_attn.o_proj.weight          -> blk.25.attn_output.weight                | F16    | [4096, 4096]\n",
+      "model.layers.25.mlp.gate_proj.weight             -> blk.25.ffn_gate.weight                   | F16    | [11008, 4096]\n",
+      "model.layers.25.mlp.up_proj.weight               -> blk.25.ffn_up.weight                     | F16    | [11008, 4096]\n",
+      "model.layers.25.mlp.down_proj.weight             -> blk.25.ffn_down.weight                   | F16    | [4096, 11008]\n",
+      "model.layers.25.input_layernorm.weight           -> blk.25.attn_norm.weight                  | F16    | [4096]\n",
+      "model.layers.25.post_attention_layernorm.weight  -> blk.25.ffn_norm.weight                   | F16    | [4096]\n",
+      "model.layers.26.self_attn.q_proj.weight          -> blk.26.attn_q.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.26.self_attn.k_proj.weight          -> blk.26.attn_k.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.26.self_attn.v_proj.weight          -> blk.26.attn_v.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.26.self_attn.o_proj.weight          -> blk.26.attn_output.weight                | F16    | [4096, 4096]\n",
+      "model.layers.26.mlp.gate_proj.weight             -> blk.26.ffn_gate.weight                   | F16    | [11008, 4096]\n",
+      "model.layers.26.mlp.up_proj.weight               -> blk.26.ffn_up.weight                     | F16    | [11008, 4096]\n",
+      "model.layers.26.mlp.down_proj.weight             -> blk.26.ffn_down.weight                   | F16    | [4096, 11008]\n",
+      "model.layers.26.input_layernorm.weight           -> blk.26.attn_norm.weight                  | F16    | [4096]\n",
+      "model.layers.26.post_attention_layernorm.weight  -> blk.26.ffn_norm.weight                   | F16    | [4096]\n",
+      "model.layers.27.self_attn.q_proj.weight          -> blk.27.attn_q.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.27.self_attn.k_proj.weight          -> blk.27.attn_k.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.27.self_attn.v_proj.weight          -> blk.27.attn_v.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.27.self_attn.o_proj.weight          -> blk.27.attn_output.weight                | F16    | [4096, 4096]\n",
+      "model.layers.27.mlp.gate_proj.weight             -> blk.27.ffn_gate.weight                   | F16    | [11008, 4096]\n",
+      "model.layers.27.mlp.up_proj.weight               -> blk.27.ffn_up.weight                     | F16    | [11008, 4096]\n",
+      "model.layers.27.mlp.down_proj.weight             -> blk.27.ffn_down.weight                   | F16    | [4096, 11008]\n",
+      "model.layers.27.input_layernorm.weight           -> blk.27.attn_norm.weight                  | F16    | [4096]\n",
+      "model.layers.27.post_attention_layernorm.weight  -> blk.27.ffn_norm.weight                   | F16    | [4096]\n",
+      "model.layers.28.self_attn.q_proj.weight          -> blk.28.attn_q.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.28.self_attn.k_proj.weight          -> blk.28.attn_k.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.28.self_attn.v_proj.weight          -> blk.28.attn_v.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.28.self_attn.o_proj.weight          -> blk.28.attn_output.weight                | F16    | [4096, 4096]\n",
+      "model.layers.28.mlp.gate_proj.weight             -> blk.28.ffn_gate.weight                   | F16    | [11008, 4096]\n",
+      "model.layers.28.mlp.up_proj.weight               -> blk.28.ffn_up.weight                     | F16    | [11008, 4096]\n",
+      "model.layers.28.mlp.down_proj.weight             -> blk.28.ffn_down.weight                   | F16    | [4096, 11008]\n",
+      "model.layers.28.input_layernorm.weight           -> blk.28.attn_norm.weight                  | F16    | [4096]\n",
+      "model.layers.28.post_attention_layernorm.weight  -> blk.28.ffn_norm.weight                   | F16    | [4096]\n",
+      "model.layers.29.self_attn.q_proj.weight          -> blk.29.attn_q.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.29.self_attn.k_proj.weight          -> blk.29.attn_k.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.29.self_attn.v_proj.weight          -> blk.29.attn_v.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.29.self_attn.o_proj.weight          -> blk.29.attn_output.weight                | F16    | [4096, 4096]\n",
+      "model.layers.29.mlp.gate_proj.weight             -> blk.29.ffn_gate.weight                   | F16    | [11008, 4096]\n",
+      "model.layers.29.mlp.up_proj.weight               -> blk.29.ffn_up.weight                     | F16    | [11008, 4096]\n",
+      "model.layers.29.mlp.down_proj.weight             -> blk.29.ffn_down.weight                   | F16    | [4096, 11008]\n",
+      "model.layers.29.input_layernorm.weight           -> blk.29.attn_norm.weight                  | F16    | [4096]\n",
+      "model.layers.29.post_attention_layernorm.weight  -> blk.29.ffn_norm.weight                   | F16    | [4096]\n",
+      "model.layers.30.self_attn.q_proj.weight          -> blk.30.attn_q.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.30.self_attn.k_proj.weight          -> blk.30.attn_k.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.30.self_attn.v_proj.weight          -> blk.30.attn_v.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.30.self_attn.o_proj.weight          -> blk.30.attn_output.weight                | F16    | [4096, 4096]\n",
+      "model.layers.30.mlp.gate_proj.weight             -> blk.30.ffn_gate.weight                   | F16    | [11008, 4096]\n",
+      "model.layers.30.mlp.up_proj.weight               -> blk.30.ffn_up.weight                     | F16    | [11008, 4096]\n",
+      "model.layers.30.mlp.down_proj.weight             -> blk.30.ffn_down.weight                   | F16    | [4096, 11008]\n",
+      "model.layers.30.input_layernorm.weight           -> blk.30.attn_norm.weight                  | F16    | [4096]\n",
+      "model.layers.30.post_attention_layernorm.weight  -> blk.30.ffn_norm.weight                   | F16    | [4096]\n",
+      "model.layers.31.self_attn.q_proj.weight          -> blk.31.attn_q.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.31.self_attn.k_proj.weight          -> blk.31.attn_k.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.31.self_attn.v_proj.weight          -> blk.31.attn_v.weight                     | F16    | [4096, 4096]\n",
+      "model.layers.31.self_attn.o_proj.weight          -> blk.31.attn_output.weight                | F16    | [4096, 4096]\n",
+      "model.layers.31.mlp.gate_proj.weight             -> blk.31.ffn_gate.weight                   | F16    | [11008, 4096]\n",
+      "model.layers.31.mlp.up_proj.weight               -> blk.31.ffn_up.weight                     | F16    | [11008, 4096]\n",
+      "model.layers.31.mlp.down_proj.weight             -> blk.31.ffn_down.weight                   | F16    | [4096, 11008]\n",
+      "model.layers.31.input_layernorm.weight           -> blk.31.attn_norm.weight                  | F16    | [4096]\n",
+      "model.layers.31.post_attention_layernorm.weight  -> blk.31.ffn_norm.weight                   | F16    | [4096]\n",
+      "model.norm.weight                                -> output_norm.weight                       | F16    | [4096]\n",
+      "lm_head.weight                                   -> output.weight                            | F16    | [32000, 4096]\n",
+      "Writing /content/model/weaviate-gorilla-random-split-f16.gguf, format 1\n",
+      "gguf: Setting special token type bos to 1\n",
+      "gguf: Setting special token type eos to 2\n",
+      "gguf: Setting special token type unk to 0\n",
+      "[  1/291] Writing tensor token_embd.weight                      | size  32000 x   4096  | type F16  | T+   0\n",
+      "[  2/291] Writing tensor blk.0.attn_q.weight                    | size   4096 x   4096  | type F16  | T+   0\n",
+      "[  3/291] Writing tensor blk.0.attn_k.weight                    | size   4096 x   4096  | type F16  | T+   0\n",
+      "[  4/291] Writing tensor blk.0.attn_v.weight                    | size   4096 x   4096  | type F16  | T+   0\n",
+      "[  5/291] Writing tensor blk.0.attn_output.weight               | size   4096 x   4096  | type F16  | T+   0\n",
+      "[  6/291] Writing tensor blk.0.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+   0\n",
+      "[  7/291] Writing tensor blk.0.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+   1\n",
+      "[  8/291] Writing tensor blk.0.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+   1\n",
+      "[  9/291] Writing tensor blk.0.attn_norm.weight                 | size   4096           | type F32  | T+   1\n",
+      "[ 10/291] Writing tensor blk.0.ffn_norm.weight                  | size   4096           | type F32  | T+   1\n",
+      "[ 11/291] Writing tensor blk.1.attn_q.weight                    | size   4096 x   4096  | type F16  | T+   1\n",
+      "[ 12/291] Writing tensor blk.1.attn_k.weight                    | size   4096 x   4096  | type F16  | T+   1\n",
+      "[ 13/291] Writing tensor blk.1.attn_v.weight                    | size   4096 x   4096  | type F16  | T+   1\n",
+      "[ 14/291] Writing tensor blk.1.attn_output.weight               | size   4096 x   4096  | type F16  | T+   1\n",
+      "[ 15/291] Writing tensor blk.1.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+   1\n",
+      "[ 16/291] Writing tensor blk.1.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+   1\n",
+      "[ 17/291] Writing tensor blk.1.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+   1\n",
+      "[ 18/291] Writing tensor blk.1.attn_norm.weight                 | size   4096           | type F32  | T+   1\n",
+      "[ 19/291] Writing tensor blk.1.ffn_norm.weight                  | size   4096           | type F32  | T+   1\n",
+      "[ 20/291] Writing tensor blk.2.attn_q.weight                    | size   4096 x   4096  | type F16  | T+   1\n",
+      "[ 21/291] Writing tensor blk.2.attn_k.weight                    | size   4096 x   4096  | type F16  | T+   1\n",
+      "[ 22/291] Writing tensor blk.2.attn_v.weight                    | size   4096 x   4096  | type F16  | T+   1\n",
+      "[ 23/291] Writing tensor blk.2.attn_output.weight               | size   4096 x   4096  | type F16  | T+   1\n",
+      "[ 24/291] Writing tensor blk.2.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+   1\n",
+      "[ 25/291] Writing tensor blk.2.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+   1\n",
+      "[ 26/291] Writing tensor blk.2.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+   1\n",
+      "[ 27/291] Writing tensor blk.2.attn_norm.weight                 | size   4096           | type F32  | T+   1\n",
+      "[ 28/291] Writing tensor blk.2.ffn_norm.weight                  | size   4096           | type F32  | T+   1\n",
+      "[ 29/291] Writing tensor blk.3.attn_q.weight                    | size   4096 x   4096  | type F16  | T+   1\n",
+      "[ 30/291] Writing tensor blk.3.attn_k.weight                    | size   4096 x   4096  | type F16  | T+   1\n",
+      "[ 31/291] Writing tensor blk.3.attn_v.weight                    | size   4096 x   4096  | type F16  | T+   1\n",
+      "[ 32/291] Writing tensor blk.3.attn_output.weight               | size   4096 x   4096  | type F16  | T+   1\n",
+      "[ 33/291] Writing tensor blk.3.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+   1\n",
+      "[ 34/291] Writing tensor blk.3.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+   1\n",
+      "[ 35/291] Writing tensor blk.3.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+   2\n",
+      "[ 36/291] Writing tensor blk.3.attn_norm.weight                 | size   4096           | type F32  | T+   2\n",
+      "[ 37/291] Writing tensor blk.3.ffn_norm.weight                  | size   4096           | type F32  | T+   2\n",
+      "[ 38/291] Writing tensor blk.4.attn_q.weight                    | size   4096 x   4096  | type F16  | T+   2\n",
+      "[ 39/291] Writing tensor blk.4.attn_k.weight                    | size   4096 x   4096  | type F16  | T+   2\n",
+      "[ 40/291] Writing tensor blk.4.attn_v.weight                    | size   4096 x   4096  | type F16  | T+   2\n",
+      "[ 41/291] Writing tensor blk.4.attn_output.weight               | size   4096 x   4096  | type F16  | T+   2\n",
+      "[ 42/291] Writing tensor blk.4.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+   2\n",
+      "[ 43/291] Writing tensor blk.4.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+   2\n",
+      "[ 44/291] Writing tensor blk.4.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+   2\n",
+      "[ 45/291] Writing tensor blk.4.attn_norm.weight                 | size   4096           | type F32  | T+   2\n",
+      "[ 46/291] Writing tensor blk.4.ffn_norm.weight                  | size   4096           | type F32  | T+   2\n",
+      "[ 47/291] Writing tensor blk.5.attn_q.weight                    | size   4096 x   4096  | type F16  | T+   2\n",
+      "[ 48/291] Writing tensor blk.5.attn_k.weight                    | size   4096 x   4096  | type F16  | T+   2\n",
+      "[ 49/291] Writing tensor blk.5.attn_v.weight                    | size   4096 x   4096  | type F16  | T+   2\n",
+      "[ 50/291] Writing tensor blk.5.attn_output.weight               | size   4096 x   4096  | type F16  | T+   2\n",
+      "[ 51/291] Writing tensor blk.5.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+   2\n",
+      "[ 52/291] Writing tensor blk.5.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+   2\n",
+      "[ 53/291] Writing tensor blk.5.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+   3\n",
+      "[ 54/291] Writing tensor blk.5.attn_norm.weight                 | size   4096           | type F32  | T+   3\n",
+      "[ 55/291] Writing tensor blk.5.ffn_norm.weight                  | size   4096           | type F32  | T+   3\n",
+      "[ 56/291] Writing tensor blk.6.attn_q.weight                    | size   4096 x   4096  | type F16  | T+   3\n",
+      "[ 57/291] Writing tensor blk.6.attn_k.weight                    | size   4096 x   4096  | type F16  | T+   3\n",
+      "[ 58/291] Writing tensor blk.6.attn_v.weight                    | size   4096 x   4096  | type F16  | T+   3\n",
+      "[ 59/291] Writing tensor blk.6.attn_output.weight               | size   4096 x   4096  | type F16  | T+   3\n",
+      "[ 60/291] Writing tensor blk.6.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+   3\n",
+      "[ 61/291] Writing tensor blk.6.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+   3\n",
+      "[ 62/291] Writing tensor blk.6.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+   3\n",
+      "[ 63/291] Writing tensor blk.6.attn_norm.weight                 | size   4096           | type F32  | T+   3\n",
+      "[ 64/291] Writing tensor blk.6.ffn_norm.weight                  | size   4096           | type F32  | T+   3\n",
+      "[ 65/291] Writing tensor blk.7.attn_q.weight                    | size   4096 x   4096  | type F16  | T+   3\n",
+      "[ 66/291] Writing tensor blk.7.attn_k.weight                    | size   4096 x   4096  | type F16  | T+   3\n",
+      "[ 67/291] Writing tensor blk.7.attn_v.weight                    | size   4096 x   4096  | type F16  | T+   3\n",
+      "[ 68/291] Writing tensor blk.7.attn_output.weight               | size   4096 x   4096  | type F16  | T+   3\n",
+      "[ 69/291] Writing tensor blk.7.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+   3\n",
+      "[ 70/291] Writing tensor blk.7.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+   3\n",
+      "[ 71/291] Writing tensor blk.7.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+   4\n",
+      "[ 72/291] Writing tensor blk.7.attn_norm.weight                 | size   4096           | type F32  | T+   4\n",
+      "[ 73/291] Writing tensor blk.7.ffn_norm.weight                  | size   4096           | type F32  | T+   4\n",
+      "[ 74/291] Writing tensor blk.8.attn_q.weight                    | size   4096 x   4096  | type F16  | T+   4\n",
+      "[ 75/291] Writing tensor blk.8.attn_k.weight                    | size   4096 x   4096  | type F16  | T+   4\n",
+      "[ 76/291] Writing tensor blk.8.attn_v.weight                    | size   4096 x   4096  | type F16  | T+   4\n",
+      "[ 77/291] Writing tensor blk.8.attn_output.weight               | size   4096 x   4096  | type F16  | T+   4\n",
+      "[ 78/291] Writing tensor blk.8.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+   4\n",
+      "[ 79/291] Writing tensor blk.8.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+   4\n",
+      "[ 80/291] Writing tensor blk.8.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+   4\n",
+      "[ 81/291] Writing tensor blk.8.attn_norm.weight                 | size   4096           | type F32  | T+   4\n",
+      "[ 82/291] Writing tensor blk.8.ffn_norm.weight                  | size   4096           | type F32  | T+   4\n",
+      "[ 83/291] Writing tensor blk.9.attn_q.weight                    | size   4096 x   4096  | type F16  | T+   4\n",
+      "[ 84/291] Writing tensor blk.9.attn_k.weight                    | size   4096 x   4096  | type F16  | T+   4\n",
+      "[ 85/291] Writing tensor blk.9.attn_v.weight                    | size   4096 x   4096  | type F16  | T+   4\n",
+      "[ 86/291] Writing tensor blk.9.attn_output.weight               | size   4096 x   4096  | type F16  | T+   4\n",
+      "[ 87/291] Writing tensor blk.9.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+   4\n",
+      "[ 88/291] Writing tensor blk.9.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+   4\n",
+      "[ 89/291] Writing tensor blk.9.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+   4\n",
+      "[ 90/291] Writing tensor blk.9.attn_norm.weight                 | size   4096           | type F32  | T+   5\n",
+      "[ 91/291] Writing tensor blk.9.ffn_norm.weight                  | size   4096           | type F32  | T+   5\n",
+      "[ 92/291] Writing tensor blk.10.attn_q.weight                   | size   4096 x   4096  | type F16  | T+   5\n",
+      "[ 93/291] Writing tensor blk.10.attn_k.weight                   | size   4096 x   4096  | type F16  | T+   5\n",
+      "[ 94/291] Writing tensor blk.10.attn_v.weight                   | size   4096 x   4096  | type F16  | T+   5\n",
+      "[ 95/291] Writing tensor blk.10.attn_output.weight              | size   4096 x   4096  | type F16  | T+   5\n",
+      "[ 96/291] Writing tensor blk.10.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+   5\n",
+      "[ 97/291] Writing tensor blk.10.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+   5\n",
+      "[ 98/291] Writing tensor blk.10.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+   5\n",
+      "[ 99/291] Writing tensor blk.10.attn_norm.weight                | size   4096           | type F32  | T+   5\n",
+      "[100/291] Writing tensor blk.10.ffn_norm.weight                 | size   4096           | type F32  | T+   5\n",
+      "[101/291] Writing tensor blk.11.attn_q.weight                   | size   4096 x   4096  | type F16  | T+   5\n",
+      "[102/291] Writing tensor blk.11.attn_k.weight                   | size   4096 x   4096  | type F16  | T+   5\n",
+      "[103/291] Writing tensor blk.11.attn_v.weight                   | size   4096 x   4096  | type F16  | T+   5\n",
+      "[104/291] Writing tensor blk.11.attn_output.weight              | size   4096 x   4096  | type F16  | T+   5\n",
+      "[105/291] Writing tensor blk.11.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+   5\n",
+      "[106/291] Writing tensor blk.11.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+   5\n",
+      "[107/291] Writing tensor blk.11.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+   6\n",
+      "[108/291] Writing tensor blk.11.attn_norm.weight                | size   4096           | type F32  | T+   6\n",
+      "[109/291] Writing tensor blk.11.ffn_norm.weight                 | size   4096           | type F32  | T+   6\n",
+      "[110/291] Writing tensor blk.12.attn_q.weight                   | size   4096 x   4096  | type F16  | T+   6\n",
+      "[111/291] Writing tensor blk.12.attn_k.weight                   | size   4096 x   4096  | type F16  | T+   6\n",
+      "[112/291] Writing tensor blk.12.attn_v.weight                   | size   4096 x   4096  | type F16  | T+   6\n",
+      "[113/291] Writing tensor blk.12.attn_output.weight              | size   4096 x   4096  | type F16  | T+   6\n",
+      "[114/291] Writing tensor blk.12.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+   6\n",
+      "[115/291] Writing tensor blk.12.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+   6\n",
+      "[116/291] Writing tensor blk.12.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+   6\n",
+      "[117/291] Writing tensor blk.12.attn_norm.weight                | size   4096           | type F32  | T+   6\n",
+      "[118/291] Writing tensor blk.12.ffn_norm.weight                 | size   4096           | type F32  | T+   6\n",
+      "[119/291] Writing tensor blk.13.attn_q.weight                   | size   4096 x   4096  | type F16  | T+   6\n",
+      "[120/291] Writing tensor blk.13.attn_k.weight                   | size   4096 x   4096  | type F16  | T+   6\n",
+      "[121/291] Writing tensor blk.13.attn_v.weight                   | size   4096 x   4096  | type F16  | T+   6\n",
+      "[122/291] Writing tensor blk.13.attn_output.weight              | size   4096 x   4096  | type F16  | T+   7\n",
+      "[123/291] Writing tensor blk.13.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+   7\n",
+      "[124/291] Writing tensor blk.13.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+   7\n",
+      "[125/291] Writing tensor blk.13.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+   7\n",
+      "[126/291] Writing tensor blk.13.attn_norm.weight                | size   4096           | type F32  | T+   7\n",
+      "[127/291] Writing tensor blk.13.ffn_norm.weight                 | size   4096           | type F32  | T+   7\n",
+      "[128/291] Writing tensor blk.14.attn_q.weight                   | size   4096 x   4096  | type F16  | T+   7\n",
+      "[129/291] Writing tensor blk.14.attn_k.weight                   | size   4096 x   4096  | type F16  | T+   7\n",
+      "[130/291] Writing tensor blk.14.attn_v.weight                   | size   4096 x   4096  | type F16  | T+   7\n",
+      "[131/291] Writing tensor blk.14.attn_output.weight              | size   4096 x   4096  | type F16  | T+   7\n",
+      "[132/291] Writing tensor blk.14.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+   7\n",
+      "[133/291] Writing tensor blk.14.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+   8\n",
+      "[134/291] Writing tensor blk.14.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+   8\n",
+      "[135/291] Writing tensor blk.14.attn_norm.weight                | size   4096           | type F32  | T+   8\n",
+      "[136/291] Writing tensor blk.14.ffn_norm.weight                 | size   4096           | type F32  | T+   8\n",
+      "[137/291] Writing tensor blk.15.attn_q.weight                   | size   4096 x   4096  | type F16  | T+   8\n",
+      "[138/291] Writing tensor blk.15.attn_k.weight                   | size   4096 x   4096  | type F16  | T+   8\n",
+      "[139/291] Writing tensor blk.15.attn_v.weight                   | size   4096 x   4096  | type F16  | T+   8\n",
+      "[140/291] Writing tensor blk.15.attn_output.weight              | size   4096 x   4096  | type F16  | T+   8\n",
+      "[141/291] Writing tensor blk.15.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+   8\n",
+      "[142/291] Writing tensor blk.15.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+   8\n",
+      "[143/291] Writing tensor blk.15.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+   9\n",
+      "[144/291] Writing tensor blk.15.attn_norm.weight                | size   4096           | type F32  | T+   9\n",
+      "[145/291] Writing tensor blk.15.ffn_norm.weight                 | size   4096           | type F32  | T+   9\n",
+      "[146/291] Writing tensor blk.16.attn_q.weight                   | size   4096 x   4096  | type F16  | T+   9\n",
+      "[147/291] Writing tensor blk.16.attn_k.weight                   | size   4096 x   4096  | type F16  | T+   9\n",
+      "[148/291] Writing tensor blk.16.attn_v.weight                   | size   4096 x   4096  | type F16  | T+   9\n",
+      "[149/291] Writing tensor blk.16.attn_output.weight              | size   4096 x   4096  | type F16  | T+   9\n",
+      "[150/291] Writing tensor blk.16.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+   9\n",
+      "[151/291] Writing tensor blk.16.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+   9\n",
+      "[152/291] Writing tensor blk.16.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+   9\n",
+      "[153/291] Writing tensor blk.16.attn_norm.weight                | size   4096           | type F32  | T+  10\n",
+      "[154/291] Writing tensor blk.16.ffn_norm.weight                 | size   4096           | type F32  | T+  10\n",
+      "[155/291] Writing tensor blk.17.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  10\n",
+      "[156/291] Writing tensor blk.17.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  10\n",
+      "[157/291] Writing tensor blk.17.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  10\n",
+      "[158/291] Writing tensor blk.17.attn_output.weight              | size   4096 x   4096  | type F16  | T+  10\n",
+      "[159/291] Writing tensor blk.17.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  10\n",
+      "[160/291] Writing tensor blk.17.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  11\n",
+      "[161/291] Writing tensor blk.17.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  11\n",
+      "[162/291] Writing tensor blk.17.attn_norm.weight                | size   4096           | type F32  | T+  11\n",
+      "[163/291] Writing tensor blk.17.ffn_norm.weight                 | size   4096           | type F32  | T+  11\n",
+      "[164/291] Writing tensor blk.18.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  11\n",
+      "[165/291] Writing tensor blk.18.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  11\n",
+      "[166/291] Writing tensor blk.18.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  11\n",
+      "[167/291] Writing tensor blk.18.attn_output.weight              | size   4096 x   4096  | type F16  | T+  11\n",
+      "[168/291] Writing tensor blk.18.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  11\n",
+      "[169/291] Writing tensor blk.18.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  11\n",
+      "[170/291] Writing tensor blk.18.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  11\n",
+      "[171/291] Writing tensor blk.18.attn_norm.weight                | size   4096           | type F32  | T+  12\n",
+      "[172/291] Writing tensor blk.18.ffn_norm.weight                 | size   4096           | type F32  | T+  12\n",
+      "[173/291] Writing tensor blk.19.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  12\n",
+      "[174/291] Writing tensor blk.19.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  12\n",
+      "[175/291] Writing tensor blk.19.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  12\n",
+      "[176/291] Writing tensor blk.19.attn_output.weight              | size   4096 x   4096  | type F16  | T+  12\n",
+      "[177/291] Writing tensor blk.19.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  12\n",
+      "[178/291] Writing tensor blk.19.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  12\n",
+      "[179/291] Writing tensor blk.19.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  12\n",
+      "[180/291] Writing tensor blk.19.attn_norm.weight                | size   4096           | type F32  | T+  13\n",
+      "[181/291] Writing tensor blk.19.ffn_norm.weight                 | size   4096           | type F32  | T+  13\n",
+      "[182/291] Writing tensor blk.20.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  13\n",
+      "[183/291] Writing tensor blk.20.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  13\n",
+      "[184/291] Writing tensor blk.20.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  13\n",
+      "[185/291] Writing tensor blk.20.attn_output.weight              | size   4096 x   4096  | type F16  | T+  13\n",
+      "[186/291] Writing tensor blk.20.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  13\n",
+      "[187/291] Writing tensor blk.20.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  13\n",
+      "[188/291] Writing tensor blk.20.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  13\n",
+      "[189/291] Writing tensor blk.20.attn_norm.weight                | size   4096           | type F32  | T+  13\n",
+      "[190/291] Writing tensor blk.20.ffn_norm.weight                 | size   4096           | type F32  | T+  13\n",
+      "[191/291] Writing tensor blk.21.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  13\n",
+      "[192/291] Writing tensor blk.21.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  13\n",
+      "[193/291] Writing tensor blk.21.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  13\n",
+      "[194/291] Writing tensor blk.21.attn_output.weight              | size   4096 x   4096  | type F16  | T+  13\n",
+      "[195/291] Writing tensor blk.21.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  13\n",
+      "[196/291] Writing tensor blk.21.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  14\n",
+      "[197/291] Writing tensor blk.21.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  14\n",
+      "[198/291] Writing tensor blk.21.attn_norm.weight                | size   4096           | type F32  | T+  14\n",
+      "[199/291] Writing tensor blk.21.ffn_norm.weight                 | size   4096           | type F32  | T+  14\n",
+      "[200/291] Writing tensor blk.22.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  14\n",
+      "[201/291] Writing tensor blk.22.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  14\n",
+      "[202/291] Writing tensor blk.22.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  14\n",
+      "[203/291] Writing tensor blk.22.attn_output.weight              | size   4096 x   4096  | type F16  | T+  14\n",
+      "[204/291] Writing tensor blk.22.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  14\n",
+      "[205/291] Writing tensor blk.22.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  14\n",
+      "[206/291] Writing tensor blk.22.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  14\n",
+      "[207/291] Writing tensor blk.22.attn_norm.weight                | size   4096           | type F32  | T+  14\n",
+      "[208/291] Writing tensor blk.22.ffn_norm.weight                 | size   4096           | type F32  | T+  14\n",
+      "[209/291] Writing tensor blk.23.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  14\n",
+      "[210/291] Writing tensor blk.23.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  14\n",
+      "[211/291] Writing tensor blk.23.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  14\n",
+      "[212/291] Writing tensor blk.23.attn_output.weight              | size   4096 x   4096  | type F16  | T+  14\n",
+      "[213/291] Writing tensor blk.23.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  15\n",
+      "[214/291] Writing tensor blk.23.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  15\n",
+      "[215/291] Writing tensor blk.23.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  15\n",
+      "[216/291] Writing tensor blk.23.attn_norm.weight                | size   4096           | type F32  | T+  15\n",
+      "[217/291] Writing tensor blk.23.ffn_norm.weight                 | size   4096           | type F32  | T+  15\n",
+      "[218/291] Writing tensor blk.24.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  15\n",
+      "[219/291] Writing tensor blk.24.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  15\n",
+      "[220/291] Writing tensor blk.24.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  15\n",
+      "[221/291] Writing tensor blk.24.attn_output.weight              | size   4096 x   4096  | type F16  | T+  15\n",
+      "[222/291] Writing tensor blk.24.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  16\n",
+      "[223/291] Writing tensor blk.24.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  16\n",
+      "[224/291] Writing tensor blk.24.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  16\n",
+      "[225/291] Writing tensor blk.24.attn_norm.weight                | size   4096           | type F32  | T+  16\n",
+      "[226/291] Writing tensor blk.24.ffn_norm.weight                 | size   4096           | type F32  | T+  16\n",
+      "[227/291] Writing tensor blk.25.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  16\n",
+      "[228/291] Writing tensor blk.25.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  16\n",
+      "[229/291] Writing tensor blk.25.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  16\n",
+      "[230/291] Writing tensor blk.25.attn_output.weight              | size   4096 x   4096  | type F16  | T+  16\n",
+      "[231/291] Writing tensor blk.25.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  16\n",
+      "[232/291] Writing tensor blk.25.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  16\n",
+      "[233/291] Writing tensor blk.25.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  16\n",
+      "[234/291] Writing tensor blk.25.attn_norm.weight                | size   4096           | type F32  | T+  16\n",
+      "[235/291] Writing tensor blk.25.ffn_norm.weight                 | size   4096           | type F32  | T+  16\n",
+      "[236/291] Writing tensor blk.26.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  17\n",
+      "[237/291] Writing tensor blk.26.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  17\n",
+      "[238/291] Writing tensor blk.26.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  17\n",
+      "[239/291] Writing tensor blk.26.attn_output.weight              | size   4096 x   4096  | type F16  | T+  17\n",
+      "[240/291] Writing tensor blk.26.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  17\n",
+      "[241/291] Writing tensor blk.26.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  17\n",
+      "[242/291] Writing tensor blk.26.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  17\n",
+      "[243/291] Writing tensor blk.26.attn_norm.weight                | size   4096           | type F32  | T+  17\n",
+      "[244/291] Writing tensor blk.26.ffn_norm.weight                 | size   4096           | type F32  | T+  17\n",
+      "[245/291] Writing tensor blk.27.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  17\n",
+      "[246/291] Writing tensor blk.27.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  17\n",
+      "[247/291] Writing tensor blk.27.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  17\n",
+      "[248/291] Writing tensor blk.27.attn_output.weight              | size   4096 x   4096  | type F16  | T+  17\n",
+      "[249/291] Writing tensor blk.27.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  17\n",
+      "[250/291] Writing tensor blk.27.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  18\n",
+      "[251/291] Writing tensor blk.27.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  18\n",
+      "[252/291] Writing tensor blk.27.attn_norm.weight                | size   4096           | type F32  | T+  18\n",
+      "[253/291] Writing tensor blk.27.ffn_norm.weight                 | size   4096           | type F32  | T+  18\n",
+      "[254/291] Writing tensor blk.28.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  18\n",
+      "[255/291] Writing tensor blk.28.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  18\n",
+      "[256/291] Writing tensor blk.28.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  18\n",
+      "[257/291] Writing tensor blk.28.attn_output.weight              | size   4096 x   4096  | type F16  | T+  18\n",
+      "[258/291] Writing tensor blk.28.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  18\n",
+      "[259/291] Writing tensor blk.28.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  18\n",
+      "[260/291] Writing tensor blk.28.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  18\n",
+      "[261/291] Writing tensor blk.28.attn_norm.weight                | size   4096           | type F32  | T+  18\n",
+      "[262/291] Writing tensor blk.28.ffn_norm.weight                 | size   4096           | type F32  | T+  18\n",
+      "[263/291] Writing tensor blk.29.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  18\n",
+      "[264/291] Writing tensor blk.29.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  18\n",
+      "[265/291] Writing tensor blk.29.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  18\n",
+      "[266/291] Writing tensor blk.29.attn_output.weight              | size   4096 x   4096  | type F16  | T+  18\n",
+      "[267/291] Writing tensor blk.29.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  19\n",
+      "[268/291] Writing tensor blk.29.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  19\n",
+      "[269/291] Writing tensor blk.29.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  19\n",
+      "[270/291] Writing tensor blk.29.attn_norm.weight                | size   4096           | type F32  | T+  19\n",
+      "[271/291] Writing tensor blk.29.ffn_norm.weight                 | size   4096           | type F32  | T+  19\n",
+      "[272/291] Writing tensor blk.30.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  19\n",
+      "[273/291] Writing tensor blk.30.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  19\n",
+      "[274/291] Writing tensor blk.30.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  19\n",
+      "[275/291] Writing tensor blk.30.attn_output.weight              | size   4096 x   4096  | type F16  | T+  19\n",
+      "[276/291] Writing tensor blk.30.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  19\n",
+      "[277/291] Writing tensor blk.30.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  19\n",
+      "[278/291] Writing tensor blk.30.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  19\n",
+      "[279/291] Writing tensor blk.30.attn_norm.weight                | size   4096           | type F32  | T+  20\n",
+      "[280/291] Writing tensor blk.30.ffn_norm.weight                 | size   4096           | type F32  | T+  20\n",
+      "[281/291] Writing tensor blk.31.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  20\n",
+      "[282/291] Writing tensor blk.31.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  20\n",
+      "[283/291] Writing tensor blk.31.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  20\n",
+      "[284/291] Writing tensor blk.31.attn_output.weight              | size   4096 x   4096  | type F16  | T+  20\n",
+      "[285/291] Writing tensor blk.31.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  20\n",
+      "[286/291] Writing tensor blk.31.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  20\n",
+      "[287/291] Writing tensor blk.31.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  20\n",
+      "[288/291] Writing tensor blk.31.attn_norm.weight                | size   4096           | type F32  | T+  20\n",
+      "[289/291] Writing tensor blk.31.ffn_norm.weight                 | size   4096           | type F32  | T+  20\n",
+      "[290/291] Writing tensor output_norm.weight                     | size   4096           | type F32  | T+  20\n",
+      "[291/291] Writing tensor output.weight                          | size  32000 x   4096  | type F16  | T+  21\n",
+      "Wrote /content/model/weaviate-gorilla-random-split-f16.gguf\n",
+      "total 13G\n",
+      "4.0K drwxr-xr-x 2 root root 4.0K Sep  4 03:43 .\n",
+      "8.0K drwxr-xr-x 1 root root 4.0K Sep  4 03:43 ..\n",
+      " 13G -rw-r--r-- 1 root root  13G Sep  4 03:43 weaviate-gorilla-random-split-f16.gguf\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "# have to use this hack otherwise the python3 command won't work\n",
+    "os.environ[\"MODEL_PATH\"] = model_path\n",
+    "outfile = f\"{output_path}/{name}-f16.gguf\"\n",
+    "os.environ[\"OUTFILE\"] = outfile\n",
+    "\n",
+    "! mkdir -p {output_path}\n",
+    "! ls -lash {model_path}\n",
+    "! python3 /content/llama.cpp/convert.py \\\n",
+    "  --outfile $OUTFILE \\\n",
+    "  --outtype f16 $MODEL_PATH\n",
+    "\n",
+    "! ls -lash {output_path}"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f3ef3201-3508-4042-afae-a4a252b1f263",
+   "metadata": {},
+   "source": [
+    "Upload the model if param `push_to_hub` was set"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "5acda74c-56bb-462d-bc8d-a616a615eaea",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Creating HuggingFace repo substratusai/weaviate-gorilla-v4-random-split-gguf\n"
+     ]
+    }
+   ],
+   "source": [
+    "from huggingface_hub import HfApi\n",
+    "from pathlib import Path\n",
+    "\n",
+    "push_to_hub = params.get(\"push_to_hub\")\n",
+    "if push_to_hub:\n",
+    "    hf_api = HfApi()\n",
+    "    model_id = push_to_hub\n",
+    "    print(f\"Creating HuggingFace repo {model_id}\")\n",
+    "    hf_api.create_repo(model_id, exist_ok=True, repo_type=\"model\")\n",
+    "\n",
+    "def push_to_huggingface(file):\n",
+    "    hf_api.upload_file(\n",
+    "        path_or_fileobj=file,\n",
+    "        path_in_repo=Path(file).name,\n",
+    "        repo_id=model_id,\n",
+    "    )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "515857d2-0d49-4091-9d08-01ecc94dbdba",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "d4f1ada0d1864a028f3f79532ef3a77b",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "weaviate-gorilla-random-split-f16.gguf:   0%|          | 0.00/13.5G [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "if push_to_hub:\n",
+    "    push_to_huggingface(outfile)\n",
+    "    readme_path = Path(model_path) / \"README.md\"\n",
+    "    if readme_path.exists():\n",
+    "        push_to_huggingface(readme_path)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7ce44075",
+   "metadata": {},
+   "source": [
+    "Optionally create additional quantized models"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "1d52a2b8-70cc-4d49-bf35-f4af88a45c1d",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "usage: quantize [--help] [--allow-requantize] [--leave-output-tensor] model-f32.gguf [model-quant.gguf] type [nthreads]\n",
+      "\n",
+      "  --allow-requantize: Allows requantizing tensors that have already been quantized. Warning: This can severely reduce quality compared to quantizing from 16bit or 32bit\n",
+      "  --leave-output-tensor: Will leave output.weight un(re)quantized. Increases model size but may also increase quality, especially when requantizing\n",
+      "\n",
+      "Allowed quantization types:\n",
+      "   2  or  Q4_0   :  3.56G, +0.2166 ppl @ LLaMA-v1-7B\n",
+      "   3  or  Q4_1   :  3.90G, +0.1585 ppl @ LLaMA-v1-7B\n",
+      "   8  or  Q5_0   :  4.33G, +0.0683 ppl @ LLaMA-v1-7B\n",
+      "   9  or  Q5_1   :  4.70G, +0.0349 ppl @ LLaMA-v1-7B\n",
+      "  10  or  Q2_K   :  2.63G, +0.6717 ppl @ LLaMA-v1-7B\n",
+      "  12  or  Q3_K   : alias for Q3_K_M\n",
+      "  11  or  Q3_K_S :  2.75G, +0.5551 ppl @ LLaMA-v1-7B\n",
+      "  12  or  Q3_K_M :  3.07G, +0.2496 ppl @ LLaMA-v1-7B\n",
+      "  13  or  Q3_K_L :  3.35G, +0.1764 ppl @ LLaMA-v1-7B\n",
+      "  15  or  Q4_K   : alias for Q4_K_M\n",
+      "  14  or  Q4_K_S :  3.59G, +0.0992 ppl @ LLaMA-v1-7B\n",
+      "  15  or  Q4_K_M :  3.80G, +0.0532 ppl @ LLaMA-v1-7B\n",
+      "  17  or  Q5_K   : alias for Q5_K_M\n",
+      "  16  or  Q5_K_S :  4.33G, +0.0400 ppl @ LLaMA-v1-7B\n",
+      "  17  or  Q5_K_M :  4.45G, +0.0122 ppl @ LLaMA-v1-7B\n",
+      "  18  or  Q6_K   :  5.15G, -0.0008 ppl @ LLaMA-v1-7B\n",
+      "   7  or  Q8_0   :  6.70G, +0.0004 ppl @ LLaMA-v1-7B\n",
+      "   1  or  F16    : 13.00G              @ 7B\n",
+      "   0  or  F32    : 26.00G              @ 7B\n",
+      "          COPY   : only copy tensors, no quantizing\n"
+     ]
+    }
+   ],
+   "source": [
+    "! quantize -h"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "2e4fa933",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Running Q4_K_M quantization and writing to /content/model/weaviate-gorilla-random-split-Q4_K_M.gguf\n",
+      "main: build = 1156 (2753415)\n",
+      "main: quantizing '/content/model/weaviate-gorilla-random-split-f16.gguf' to '/content/model/weaviate-gorilla-random-split-Q4_K_M.gguf' as Q4_K_M\n",
+      "llama_model_loader: loaded meta data with 19 key-value pairs and 291 tensors from /content/model/weaviate-gorilla-random-split-f16.gguf (version GGUF V2 (latest))\n",
+      "llama_model_loader: - tensor    0:                token_embd.weight f16      [  4096, 32000,     1,     1 ]\n",
+      "llama_model_loader: - tensor    1:              blk.0.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor    2:              blk.0.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor    3:              blk.0.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor    4:         blk.0.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor    5:            blk.0.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor    6:              blk.0.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor    7:            blk.0.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor    8:           blk.0.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor    9:            blk.0.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor   10:              blk.1.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   11:              blk.1.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   12:              blk.1.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   13:         blk.1.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   14:            blk.1.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor   15:              blk.1.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor   16:            blk.1.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   17:           blk.1.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor   18:            blk.1.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor   19:              blk.2.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   20:              blk.2.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   21:              blk.2.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   22:         blk.2.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   23:            blk.2.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor   24:              blk.2.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor   25:            blk.2.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   26:           blk.2.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor   27:            blk.2.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor   28:              blk.3.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   29:              blk.3.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   30:              blk.3.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   31:         blk.3.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   32:            blk.3.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor   33:              blk.3.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor   34:            blk.3.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   35:           blk.3.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor   36:            blk.3.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor   37:              blk.4.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   38:              blk.4.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   39:              blk.4.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   40:         blk.4.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   41:            blk.4.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor   42:              blk.4.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor   43:            blk.4.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   44:           blk.4.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor   45:            blk.4.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor   46:              blk.5.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   47:              blk.5.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   48:              blk.5.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   49:         blk.5.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   50:            blk.5.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor   51:              blk.5.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor   52:            blk.5.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   53:           blk.5.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor   54:            blk.5.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor   55:              blk.6.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   56:              blk.6.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   57:              blk.6.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   58:         blk.6.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   59:            blk.6.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor   60:              blk.6.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor   61:            blk.6.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   62:           blk.6.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor   63:            blk.6.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor   64:              blk.7.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   65:              blk.7.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   66:              blk.7.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   67:         blk.7.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   68:            blk.7.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor   69:              blk.7.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor   70:            blk.7.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   71:           blk.7.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor   72:            blk.7.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor   73:              blk.8.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   74:              blk.8.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   75:              blk.8.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   76:         blk.8.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   77:            blk.8.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor   78:              blk.8.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor   79:            blk.8.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   80:           blk.8.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor   81:            blk.8.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor   82:              blk.9.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   83:              blk.9.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   84:              blk.9.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   85:         blk.9.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   86:            blk.9.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor   87:              blk.9.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor   88:            blk.9.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   89:           blk.9.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor   90:            blk.9.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor   91:             blk.10.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   92:             blk.10.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   93:             blk.10.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   94:        blk.10.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   95:           blk.10.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor   96:             blk.10.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor   97:           blk.10.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor   98:          blk.10.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor   99:           blk.10.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  100:             blk.11.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  101:             blk.11.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  102:             blk.11.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  103:        blk.11.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  104:           blk.11.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  105:             blk.11.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  106:           blk.11.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  107:          blk.11.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  108:           blk.11.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  109:             blk.12.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  110:             blk.12.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  111:             blk.12.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  112:        blk.12.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  113:           blk.12.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  114:             blk.12.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  115:           blk.12.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  116:          blk.12.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  117:           blk.12.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  118:             blk.13.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  119:             blk.13.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  120:             blk.13.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  121:        blk.13.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  122:           blk.13.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  123:             blk.13.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  124:           blk.13.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  125:          blk.13.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  126:           blk.13.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  127:             blk.14.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  128:             blk.14.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  129:             blk.14.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  130:        blk.14.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  131:           blk.14.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  132:             blk.14.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  133:           blk.14.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  134:          blk.14.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  135:           blk.14.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  136:             blk.15.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  137:             blk.15.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  138:             blk.15.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  139:        blk.15.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  140:           blk.15.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  141:             blk.15.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  142:           blk.15.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  143:          blk.15.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  144:           blk.15.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  145:             blk.16.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  146:             blk.16.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  147:             blk.16.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  148:        blk.16.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  149:           blk.16.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  150:             blk.16.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  151:           blk.16.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  152:          blk.16.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  153:           blk.16.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  154:             blk.17.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  155:             blk.17.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  156:             blk.17.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  157:        blk.17.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  158:           blk.17.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  159:             blk.17.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  160:           blk.17.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  161:          blk.17.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  162:           blk.17.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  163:             blk.18.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  164:             blk.18.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  165:             blk.18.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  166:        blk.18.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  167:           blk.18.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  168:             blk.18.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  169:           blk.18.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  170:          blk.18.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  171:           blk.18.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  172:             blk.19.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  173:             blk.19.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  174:             blk.19.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  175:        blk.19.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  176:           blk.19.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  177:             blk.19.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  178:           blk.19.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  179:          blk.19.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  180:           blk.19.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  181:             blk.20.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  182:             blk.20.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  183:             blk.20.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  184:        blk.20.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  185:           blk.20.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  186:             blk.20.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  187:           blk.20.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  188:          blk.20.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  189:           blk.20.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  190:             blk.21.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  191:             blk.21.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  192:             blk.21.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  193:        blk.21.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  194:           blk.21.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  195:             blk.21.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  196:           blk.21.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  197:          blk.21.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  198:           blk.21.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  199:             blk.22.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  200:             blk.22.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  201:             blk.22.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  202:        blk.22.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  203:           blk.22.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  204:             blk.22.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  205:           blk.22.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  206:          blk.22.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  207:           blk.22.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  208:             blk.23.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  209:             blk.23.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  210:             blk.23.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  211:        blk.23.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  212:           blk.23.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  213:             blk.23.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  214:           blk.23.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  215:          blk.23.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  216:           blk.23.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  217:             blk.24.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  218:             blk.24.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  219:             blk.24.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  220:        blk.24.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  221:           blk.24.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  222:             blk.24.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  223:           blk.24.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  224:          blk.24.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  225:           blk.24.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  226:             blk.25.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  227:             blk.25.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  228:             blk.25.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  229:        blk.25.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  230:           blk.25.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  231:             blk.25.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  232:           blk.25.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  233:          blk.25.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  234:           blk.25.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  235:             blk.26.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  236:             blk.26.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  237:             blk.26.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  238:        blk.26.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  239:           blk.26.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  240:             blk.26.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  241:           blk.26.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  242:          blk.26.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  243:           blk.26.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  244:             blk.27.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  245:             blk.27.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  246:             blk.27.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  247:        blk.27.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  248:           blk.27.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  249:             blk.27.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  250:           blk.27.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  251:          blk.27.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  252:           blk.27.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  253:             blk.28.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  254:             blk.28.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  255:             blk.28.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  256:        blk.28.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  257:           blk.28.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  258:             blk.28.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  259:           blk.28.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  260:          blk.28.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  261:           blk.28.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  262:             blk.29.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  263:             blk.29.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  264:             blk.29.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  265:        blk.29.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  266:           blk.29.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  267:             blk.29.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  268:           blk.29.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  269:          blk.29.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  270:           blk.29.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  271:             blk.30.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  272:             blk.30.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  273:             blk.30.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  274:        blk.30.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  275:           blk.30.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  276:             blk.30.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  277:           blk.30.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  278:          blk.30.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  279:           blk.30.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  280:             blk.31.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  281:             blk.31.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  282:             blk.31.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  283:        blk.31.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  284:           blk.31.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  285:             blk.31.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
+      "llama_model_loader: - tensor  286:           blk.31.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
+      "llama_model_loader: - tensor  287:          blk.31.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  288:           blk.31.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  289:               output_norm.weight f32      [  4096,     1,     1,     1 ]\n",
+      "llama_model_loader: - tensor  290:                    output.weight f16      [  4096, 32000,     1,     1 ]\n",
+      "llama_model_loader: - kv   0:                       general.architecture str     \n",
+      "llama_model_loader: - kv   1:                               general.name str     \n",
+      "llama_model_loader: - kv   2:                       llama.context_length u32     \n",
+      "llama_model_loader: - kv   3:                     llama.embedding_length u32     \n",
+      "llama_model_loader: - kv   4:                          llama.block_count u32     \n",
+      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32     \n",
+      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32     \n",
+      "llama_model_loader: - kv   7:                 llama.attention.head_count u32     \n",
+      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32     \n",
+      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32     \n",
+      "llama_model_loader: - kv  10:                       llama.rope.freq_base f32     \n",
+      "llama_model_loader: - kv  11:                          general.file_type u32     \n",
+      "llama_model_loader: - kv  12:                       tokenizer.ggml.model str     \n",
+      "llama_model_loader: - kv  13:                      tokenizer.ggml.tokens arr     \n",
+      "llama_model_loader: - kv  14:                      tokenizer.ggml.scores arr     \n",
+      "llama_model_loader: - kv  15:                  tokenizer.ggml.token_type arr     \n",
+      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32     \n",
+      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32     \n",
+      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32     \n",
+      "llama_model_loader: - type  f32:   65 tensors\n",
+      "llama_model_loader: - type  f16:  226 tensors\n",
+      "llama_model_quantize_internal: meta size = 741088 bytes\n",
+      "[   1/ 291]                    token_embd.weight - [ 4096, 32000,     1,     1], type =    f16, quantizing to q4_K .. size =   250.00 MB ->    70.31 MB | hist: \n",
+      "[   2/ 291]                  blk.0.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[   3/ 291]                  blk.0.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[   4/ 291]                  blk.0.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
+      "[   5/ 291]             blk.0.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[   6/ 291]                blk.0.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[   7/ 291]                  blk.0.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[   8/ 291]                blk.0.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
+      "[   9/ 291]               blk.0.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[  10/ 291]                blk.0.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[  11/ 291]                  blk.1.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[  12/ 291]                  blk.1.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[  13/ 291]                  blk.1.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
+      "[  14/ 291]             blk.1.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[  15/ 291]                blk.1.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[  16/ 291]                  blk.1.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[  17/ 291]                blk.1.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
+      "[  18/ 291]               blk.1.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[  19/ 291]                blk.1.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[  20/ 291]                  blk.2.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[  21/ 291]                  blk.2.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[  22/ 291]                  blk.2.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
+      "[  23/ 291]             blk.2.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[  24/ 291]                blk.2.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[  25/ 291]                  blk.2.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[  26/ 291]                blk.2.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
+      "[  27/ 291]               blk.2.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[  28/ 291]                blk.2.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[  29/ 291]                  blk.3.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[  30/ 291]                  blk.3.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[  31/ 291]                  blk.3.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
+      "[  32/ 291]             blk.3.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[  33/ 291]                blk.3.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[  34/ 291]                  blk.3.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[  35/ 291]                blk.3.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
+      "[  36/ 291]               blk.3.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[  37/ 291]                blk.3.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[  38/ 291]                  blk.4.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[  39/ 291]                  blk.4.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[  40/ 291]                  blk.4.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[  41/ 291]             blk.4.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[  42/ 291]                blk.4.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[  43/ 291]                  blk.4.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[  44/ 291]                blk.4.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[  45/ 291]               blk.4.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[  46/ 291]                blk.4.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[  47/ 291]                  blk.5.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[  48/ 291]                  blk.5.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[  49/ 291]                  blk.5.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[  50/ 291]             blk.5.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[  51/ 291]                blk.5.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[  52/ 291]                  blk.5.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[  53/ 291]                blk.5.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[  54/ 291]               blk.5.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[  55/ 291]                blk.5.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[  56/ 291]                  blk.6.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[  57/ 291]                  blk.6.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[  58/ 291]                  blk.6.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
+      "[  59/ 291]             blk.6.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[  60/ 291]                blk.6.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[  61/ 291]                  blk.6.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[  62/ 291]                blk.6.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
+      "[  63/ 291]               blk.6.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[  64/ 291]                blk.6.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[  65/ 291]                  blk.7.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[  66/ 291]                  blk.7.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[  67/ 291]                  blk.7.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[  68/ 291]             blk.7.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[  69/ 291]                blk.7.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[  70/ 291]                  blk.7.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[  71/ 291]                blk.7.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[  72/ 291]               blk.7.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[  73/ 291]                blk.7.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[  74/ 291]                  blk.8.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[  75/ 291]                  blk.8.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[  76/ 291]                  blk.8.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[  77/ 291]             blk.8.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[  78/ 291]                blk.8.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[  79/ 291]                  blk.8.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[  80/ 291]                blk.8.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[  81/ 291]               blk.8.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[  82/ 291]                blk.8.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[  83/ 291]                  blk.9.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[  84/ 291]                  blk.9.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[  85/ 291]                  blk.9.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
+      "[  86/ 291]             blk.9.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[  87/ 291]                blk.9.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[  88/ 291]                  blk.9.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[  89/ 291]                blk.9.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
+      "[  90/ 291]               blk.9.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[  91/ 291]                blk.9.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[  92/ 291]                 blk.10.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[  93/ 291]                 blk.10.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[  94/ 291]                 blk.10.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[  95/ 291]            blk.10.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[  96/ 291]               blk.10.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[  97/ 291]                 blk.10.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[  98/ 291]               blk.10.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[  99/ 291]              blk.10.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 100/ 291]               blk.10.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 101/ 291]                 blk.11.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 102/ 291]                 blk.11.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 103/ 291]                 blk.11.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 104/ 291]            blk.11.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 105/ 291]               blk.11.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 106/ 291]                 blk.11.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 107/ 291]               blk.11.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 108/ 291]              blk.11.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 109/ 291]               blk.11.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 110/ 291]                 blk.12.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 111/ 291]                 blk.12.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 112/ 291]                 blk.12.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
+      "[ 113/ 291]            blk.12.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 114/ 291]               blk.12.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 115/ 291]                 blk.12.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 116/ 291]               blk.12.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
+      "[ 117/ 291]              blk.12.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 118/ 291]               blk.12.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 119/ 291]                 blk.13.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 120/ 291]                 blk.13.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 121/ 291]                 blk.13.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 122/ 291]            blk.13.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 123/ 291]               blk.13.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 124/ 291]                 blk.13.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 125/ 291]               blk.13.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 126/ 291]              blk.13.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 127/ 291]               blk.13.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 128/ 291]                 blk.14.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 129/ 291]                 blk.14.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 130/ 291]                 blk.14.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 131/ 291]            blk.14.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 132/ 291]               blk.14.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 133/ 291]                 blk.14.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 134/ 291]               blk.14.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 135/ 291]              blk.14.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 136/ 291]               blk.14.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 137/ 291]                 blk.15.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 138/ 291]                 blk.15.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 139/ 291]                 blk.15.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
+      "[ 140/ 291]            blk.15.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 141/ 291]               blk.15.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 142/ 291]                 blk.15.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 143/ 291]               blk.15.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
+      "[ 144/ 291]              blk.15.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 145/ 291]               blk.15.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 146/ 291]                 blk.16.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 147/ 291]                 blk.16.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 148/ 291]                 blk.16.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 149/ 291]            blk.16.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 150/ 291]               blk.16.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 151/ 291]                 blk.16.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 152/ 291]               blk.16.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 153/ 291]              blk.16.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 154/ 291]               blk.16.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 155/ 291]                 blk.17.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 156/ 291]                 blk.17.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 157/ 291]                 blk.17.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 158/ 291]            blk.17.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 159/ 291]               blk.17.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 160/ 291]                 blk.17.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 161/ 291]               blk.17.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 162/ 291]              blk.17.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 163/ 291]               blk.17.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 164/ 291]                 blk.18.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 165/ 291]                 blk.18.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 166/ 291]                 blk.18.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
+      "[ 167/ 291]            blk.18.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 168/ 291]               blk.18.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 169/ 291]                 blk.18.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 170/ 291]               blk.18.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
+      "[ 171/ 291]              blk.18.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 172/ 291]               blk.18.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 173/ 291]                 blk.19.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 174/ 291]                 blk.19.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 175/ 291]                 blk.19.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 176/ 291]            blk.19.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 177/ 291]               blk.19.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 178/ 291]                 blk.19.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 179/ 291]               blk.19.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 180/ 291]              blk.19.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 181/ 291]               blk.19.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 182/ 291]                 blk.20.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 183/ 291]                 blk.20.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 184/ 291]                 blk.20.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 185/ 291]            blk.20.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 186/ 291]               blk.20.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 187/ 291]                 blk.20.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 188/ 291]               blk.20.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 189/ 291]              blk.20.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 190/ 291]               blk.20.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 191/ 291]                 blk.21.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 192/ 291]                 blk.21.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 193/ 291]                 blk.21.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
+      "[ 194/ 291]            blk.21.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 195/ 291]               blk.21.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 196/ 291]                 blk.21.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 197/ 291]               blk.21.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
+      "[ 198/ 291]              blk.21.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 199/ 291]               blk.21.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 200/ 291]                 blk.22.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 201/ 291]                 blk.22.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 202/ 291]                 blk.22.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 203/ 291]            blk.22.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 204/ 291]               blk.22.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 205/ 291]                 blk.22.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 206/ 291]               blk.22.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 207/ 291]              blk.22.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 208/ 291]               blk.22.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 209/ 291]                 blk.23.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 210/ 291]                 blk.23.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 211/ 291]                 blk.23.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 212/ 291]            blk.23.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 213/ 291]               blk.23.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 214/ 291]                 blk.23.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 215/ 291]               blk.23.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 216/ 291]              blk.23.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 217/ 291]               blk.23.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 218/ 291]                 blk.24.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 219/ 291]                 blk.24.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 220/ 291]                 blk.24.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
+      "[ 221/ 291]            blk.24.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 222/ 291]               blk.24.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 223/ 291]                 blk.24.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 224/ 291]               blk.24.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
+      "[ 225/ 291]              blk.24.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 226/ 291]               blk.24.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 227/ 291]                 blk.25.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 228/ 291]                 blk.25.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 229/ 291]                 blk.25.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 230/ 291]            blk.25.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 231/ 291]               blk.25.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 232/ 291]                 blk.25.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 233/ 291]               blk.25.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 234/ 291]              blk.25.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 235/ 291]               blk.25.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 236/ 291]                 blk.26.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 237/ 291]                 blk.26.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 238/ 291]                 blk.26.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 239/ 291]            blk.26.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 240/ 291]               blk.26.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 241/ 291]                 blk.26.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 242/ 291]               blk.26.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 243/ 291]              blk.26.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 244/ 291]               blk.26.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 245/ 291]                 blk.27.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 246/ 291]                 blk.27.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 247/ 291]                 blk.27.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
+      "[ 248/ 291]            blk.27.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 249/ 291]               blk.27.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 250/ 291]                 blk.27.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 251/ 291]               blk.27.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
+      "[ 252/ 291]              blk.27.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 253/ 291]               blk.27.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 254/ 291]                 blk.28.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 255/ 291]                 blk.28.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 256/ 291]                 blk.28.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
+      "[ 257/ 291]            blk.28.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 258/ 291]               blk.28.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 259/ 291]                 blk.28.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 260/ 291]               blk.28.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
+      "[ 261/ 291]              blk.28.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 262/ 291]               blk.28.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 263/ 291]                 blk.29.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 264/ 291]                 blk.29.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 265/ 291]                 blk.29.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
+      "[ 266/ 291]            blk.29.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 267/ 291]               blk.29.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 268/ 291]                 blk.29.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 269/ 291]               blk.29.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
+      "[ 270/ 291]              blk.29.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 271/ 291]               blk.29.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 272/ 291]                 blk.30.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 273/ 291]                 blk.30.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 274/ 291]                 blk.30.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
+      "[ 275/ 291]            blk.30.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 276/ 291]               blk.30.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 277/ 291]                 blk.30.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 278/ 291]               blk.30.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
+      "[ 279/ 291]              blk.30.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 280/ 291]               blk.30.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 281/ 291]                 blk.31.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 282/ 291]                 blk.31.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 283/ 291]                 blk.31.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
+      "[ 284/ 291]            blk.31.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
+      "[ 285/ 291]               blk.31.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 286/ 291]                 blk.31.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
+      "[ 287/ 291]               blk.31.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
+      "[ 288/ 291]              blk.31.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 289/ 291]               blk.31.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 290/ 291]                   output_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
+      "[ 291/ 291]                        output.weight - [ 4096, 32000,     1,     1], type =    f16, quantizing to q6_K .. size =   250.00 MB ->   102.54 MB | hist: \n",
+      "llama_model_quantize_internal: model size  = 12853.02 MB\n",
+      "llama_model_quantize_internal: quant size  =  3891.24 MB\n",
+      "\n",
+      "main: quantize time = 117198.26 ms\n",
+      "main:    total time = 117198.26 ms\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "f17c40e1e25740d8aafdfe3778c98110",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "weaviate-gorilla-random-split-Q4_K_M.gguf:   0%|          | 0.00/4.08G [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "total 17G\n",
+      "4.0K drwxr-xr-x 2 root root 4.0K Sep  4 03:51 .\n",
+      "8.0K drwxr-xr-x 1 root root 4.0K Sep  4 03:44 ..\n",
+      "3.9G -rw-r--r-- 1 root root 3.9G Sep  4 03:53 weaviate-gorilla-random-split-Q4_K_M.gguf\n",
+      " 13G -rw-r--r-- 1 root root  13G Sep  4 03:43 weaviate-gorilla-random-split-f16.gguf\n"
+     ]
+    }
+   ],
+   "source": [
+    "quantize = params.get(\"quantize\")\n",
+    "if quantize:\n",
+    "    quantize = [q.strip() for q in quantize.split(\",\")]\n",
+    "    for quantize_type in quantize:\n",
+    "        filename = f\"{output_path}/{name}-{quantize_type}.gguf\"\n",
+    "        os.environ[\"filename\"] = filename\n",
+    "        os.environ[\"quantize_type\"] = quantize_type\n",
+    "        print(f\"Running {quantize_type} quantization and writing to {filename}\")\n",
+    "        ! quantize $OUTFILE $filename $quantize_type\n",
+    "        if push_to_hub:\n",
+    "            push_to_huggingface(filename)\n",
+    "    ! ls -lash {output_path}"
+   ]
+  }
+ ],
+ "metadata": {
+  "jupytext": {
+   "cell_metadata_filter": "-all",
+   "main_language": "python",
+   "notebook_metadata_filter": "-all"
+  },
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/model-server-llama-cpp/scripts/serve.sh b/model-server-llama-cpp/scripts/serve.sh
index 1908567..c2fcbae 100755
--- a/model-server-llama-cpp/scripts/serve.sh
+++ b/model-server-llama-cpp/scripts/serve.sh
@@ -3,7 +3,11 @@
 set -xe
 
 ls ${MODEL_DIR}
-export MODEL=$(find "${MODEL_DIR}" -type f -iname "*.bin" | head -n 1)
+if [[ -z "$PARAM_MODEL" ]]; then
+  export MODEL=$(find "${MODEL_DIR}" -type f -iname "*.bin" | head -n 1)
+else
+  export MODEL=${PARAM_MODEL}
+fi
 # TODO figure out how to automatically set N_GPU_LAYERS when a GPU is available
 export N_GPU_LAYERS="${PARAM_N_GPU_LAYERS:-${N_GPU_LAYERS:-0}}"
 PYTHONUNBUFFERED=1 python3 -m llama_cpp.server