From f0d6ead877b01d12fcbdf7028954d17fd094771d Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Mon, 18 Dec 2023 01:51:58 -0300
Subject: [PATCH] llama.cpp: read instruction template from GGUF metadata
 (#4975)

---
 modules/metadata_gguf.py   |  5 +++--
 modules/models_settings.py | 11 ++++++++++-
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/modules/metadata_gguf.py b/modules/metadata_gguf.py
index 0ea41a2a..70ad41dc 100644
--- a/modules/metadata_gguf.py
+++ b/modules/metadata_gguf.py
@@ -82,8 +82,9 @@ def load_metadata(fname):
             if value_type == GGUFValueType.ARRAY:
                 ltype = GGUFValueType(struct.unpack("<I", file.read(4))[0])
                 length = struct.unpack("<Q", file.read(8))[0]
-                for j in range(length):
-                    _ = get_single(ltype, file)
+
+                arr = [get_single(ltype, file) for _ in range(length)]
+                metadata[key.decode()] = arr
             else:
                 value = get_single(value_type, file)
                 metadata[key.decode()] = value
diff --git a/modules/models_settings.py b/modules/models_settings.py
index 8a4febdc..156c05d9 100644
--- a/modules/models_settings.py
+++ b/modules/models_settings.py
@@ -64,6 +64,16 @@ def get_model_metadata(model):
             model_settings['compress_pos_emb'] = metadata['llama.rope.scale_linear']
         if 'llama.rope.freq_base' in metadata:
             model_settings['rope_freq_base'] = metadata['llama.rope.freq_base']
+        if 'tokenizer.chat_template' in metadata:
+            template = metadata['tokenizer.chat_template']
+            eos_token = metadata['tokenizer.ggml.tokens'][metadata['tokenizer.ggml.eos_token_id']]
+            bos_token = metadata['tokenizer.ggml.tokens'][metadata['tokenizer.ggml.bos_token_id']]
+            template = template.replace('eos_token', "'{}'".format(eos_token))
+            template = template.replace('bos_token', "'{}'".format(bos_token))
+
+            template = re.sub(r'raise_exception\([^)]*\)', "''", template)
+            model_settings['instruction_template'] = 'Custom (obtained from model metadata)'
+            model_settings['instruction_template_str'] = template
 
     else:
         # Transformers metadata
@@ -114,7 +124,6 @@ def get_model_metadata(model):
                     template = template.replace(k, "'{}'".format(value))
 
             template = re.sub(r'raise_exception\([^)]*\)', "''", template)
-
             model_settings['instruction_template'] = 'Custom (obtained from model metadata)'
             model_settings['instruction_template_str'] = template