AI/turbopilot
1
0
Fork 0
mirror of https://github.com/ravenscroftj/turbopilot.git synced 2024-10-01 01:06:01 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #5 from swanserquack/fix_vocab_encoding

Browse Source 
Fix inability to load vocab.json on converting the 16B model due to encoding of the file not being set
This commit is contained in:
James Ravenscroft 2023-04-12 20:50:33 +01:00 committed by GitHub
commit 5d2ea0a841
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
convert-codegen-to-ggml.py
View File

@ -59,7 +59,7 @@ if len(sys.argv) < 3:
dir_model = sys.argv[1] dir_model = sys.argv[1]
fname_out = sys.argv[1] + "/ggml-model.bin" fname_out = sys.argv[1] + "/ggml-model.bin"
with open(dir_model + "/vocab.json", "r") as f: with open(dir_model + "/vocab.json", "r", encoding="utf8") as f:
encoder = json.load(f) encoder = json.load(f)
with open(dir_model + "/added_tokens.json", "r") as f: with open(dir_model + "/added_tokens.json", "r") as f: