Let model downloader download *.tiktoken as well (#4121)

This commit is contained in:
快乐的我531 2023-09-29 05:03:18 +08:00 committed by GitHub
parent 822ba7fcbb
commit 4e56ad55e1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

View File

@ -91,7 +91,8 @@ class ModelDownloader:
is_safetensors = re.match(r".*\.safetensors", fname) is_safetensors = re.match(r".*\.safetensors", fname)
is_pt = re.match(r".*\.pt", fname) is_pt = re.match(r".*\.pt", fname)
is_gguf = re.match(r'.*\.gguf', fname) is_gguf = re.match(r'.*\.gguf', fname)
is_tokenizer = re.match(r"(tokenizer|ice|spiece).*\.model", fname) is_tiktoken = re.match(r".*\.tiktoken", fname)
is_tokenizer = re.match(r"(tokenizer|ice|spiece).*\.model", fname) or is_tiktoken
is_text = re.match(r".*\.(txt|json|py|md)", fname) or is_tokenizer is_text = re.match(r".*\.(txt|json|py|md)", fname) or is_tokenizer
if any((is_pytorch, is_safetensors, is_pt, is_gguf, is_tokenizer, is_text)): if any((is_pytorch, is_safetensors, is_pt, is_gguf, is_tokenizer, is_text)):
if 'lfs' in dict[i]: if 'lfs' in dict[i]: