diff --git a/download-model.py b/download-model.py index d9b21d3a..d37ae32c 100644 --- a/download-model.py +++ b/download-model.py @@ -91,7 +91,8 @@ class ModelDownloader: is_safetensors = re.match(r".*\.safetensors", fname) is_pt = re.match(r".*\.pt", fname) is_gguf = re.match(r'.*\.gguf', fname) - is_tokenizer = re.match(r"(tokenizer|ice|spiece).*\.model", fname) + is_tiktoken = re.match(r".*\.tiktoken", fname) + is_tokenizer = re.match(r"(tokenizer|ice|spiece).*\.model", fname) or is_tiktoken is_text = re.match(r".*\.(txt|json|py|md)", fname) or is_tokenizer if any((is_pytorch, is_safetensors, is_pt, is_gguf, is_tokenizer, is_text)): if 'lfs' in dict[i]: