mirror of
https://github.com/oobabooga/text-generation-webui.git
synced 2024-10-01 01:26:03 -04:00
Bump llama-cpp-python, +tensor_split by @shouyiwang, +mul_mat_q (#3610)
This commit is contained in:
parent
4b69f4f6ae
commit
7cba000421
@ -261,7 +261,9 @@ Optionally, you can use the following command-line flags:
|
|||||||
|-------------|-------------|
|
|-------------|-------------|
|
||||||
| `--no-mmap` | Prevent mmap from being used. |
|
| `--no-mmap` | Prevent mmap from being used. |
|
||||||
| `--mlock` | Force the system to keep the model in RAM. |
|
| `--mlock` | Force the system to keep the model in RAM. |
|
||||||
|
| `--mul_mat_q` | Activate new mulmat kernels. |
|
||||||
| `--cache-capacity CACHE_CAPACITY` | Maximum cache capacity. Examples: 2000MiB, 2GiB. When provided without units, bytes will be assumed. |
|
| `--cache-capacity CACHE_CAPACITY` | Maximum cache capacity. Examples: 2000MiB, 2GiB. When provided without units, bytes will be assumed. |
|
||||||
|
| `--tensor_split TENSOR_SPLIT` | Split the model across multiple GPUs, comma-separated list of proportions, e.g. 18,17 |
|
||||||
| `--llama_cpp_seed SEED` | Seed for llama-cpp models. Default 0 (random). |
|
| `--llama_cpp_seed SEED` | Seed for llama-cpp models. Default 0 (random). |
|
||||||
| `--n_gqa N_GQA` | grouped-query attention. Must be 8 for llama-2 70b. |
|
| `--n_gqa N_GQA` | grouped-query attention. Must be 8 for llama-2 70b. |
|
||||||
| `--rms_norm_eps RMS_NORM_EPS` | 5e-6 is a good value for llama-2 models. |
|
| `--rms_norm_eps RMS_NORM_EPS` | 5e-6 is a good value for llama-2 models. |
|
||||||
|
@ -102,6 +102,12 @@ class LlamacppHF(PreTrainedModel):
|
|||||||
model_file = list(path.glob('*ggml*.bin'))[0]
|
model_file = list(path.glob('*ggml*.bin'))[0]
|
||||||
|
|
||||||
logger.info(f"llama.cpp weights detected: {model_file}\n")
|
logger.info(f"llama.cpp weights detected: {model_file}\n")
|
||||||
|
|
||||||
|
if shared.args.tensor_split is None or shared.args.tensor_split.strip() == '':
|
||||||
|
tensor_split_list = None
|
||||||
|
else:
|
||||||
|
tensor_split_list = [float(x) for x in shared.args.tensor_split.strip().split(",")]
|
||||||
|
|
||||||
params = {
|
params = {
|
||||||
'model_path': str(model_file),
|
'model_path': str(model_file),
|
||||||
'n_ctx': shared.args.n_ctx,
|
'n_ctx': shared.args.n_ctx,
|
||||||
@ -110,9 +116,11 @@ class LlamacppHF(PreTrainedModel):
|
|||||||
'n_batch': shared.args.n_batch,
|
'n_batch': shared.args.n_batch,
|
||||||
'use_mmap': not shared.args.no_mmap,
|
'use_mmap': not shared.args.no_mmap,
|
||||||
'use_mlock': shared.args.mlock,
|
'use_mlock': shared.args.mlock,
|
||||||
|
'mul_mat_q': shared.args.mul_mat_q,
|
||||||
'low_vram': shared.args.low_vram,
|
'low_vram': shared.args.low_vram,
|
||||||
'n_gpu_layers': shared.args.n_gpu_layers,
|
'n_gpu_layers': shared.args.n_gpu_layers,
|
||||||
'rope_freq_base': 10000 * shared.args.alpha_value ** (64 / 63.),
|
'rope_freq_base': 10000 * shared.args.alpha_value ** (64 / 63.),
|
||||||
|
'tensor_split': tensor_split_list,
|
||||||
'rope_freq_scale': 1.0 / shared.args.compress_pos_emb,
|
'rope_freq_scale': 1.0 / shared.args.compress_pos_emb,
|
||||||
'n_gqa': shared.args.n_gqa or None,
|
'n_gqa': shared.args.n_gqa or None,
|
||||||
'rms_norm_eps': shared.args.rms_norm_eps or None,
|
'rms_norm_eps': shared.args.rms_norm_eps or None,
|
||||||
|
@ -55,6 +55,12 @@ class LlamaCppModel:
|
|||||||
cache_capacity = int(shared.args.cache_capacity)
|
cache_capacity = int(shared.args.cache_capacity)
|
||||||
|
|
||||||
logger.info("Cache capacity is " + str(cache_capacity) + " bytes")
|
logger.info("Cache capacity is " + str(cache_capacity) + " bytes")
|
||||||
|
|
||||||
|
if shared.args.tensor_split is None or shared.args.tensor_split.strip() == '':
|
||||||
|
tensor_split_list = None
|
||||||
|
else:
|
||||||
|
tensor_split_list = [float(x) for x in shared.args.tensor_split.strip().split(",")]
|
||||||
|
|
||||||
params = {
|
params = {
|
||||||
'model_path': str(path),
|
'model_path': str(path),
|
||||||
'n_ctx': shared.args.n_ctx,
|
'n_ctx': shared.args.n_ctx,
|
||||||
@ -63,9 +69,11 @@ class LlamaCppModel:
|
|||||||
'n_batch': shared.args.n_batch,
|
'n_batch': shared.args.n_batch,
|
||||||
'use_mmap': not shared.args.no_mmap,
|
'use_mmap': not shared.args.no_mmap,
|
||||||
'use_mlock': shared.args.mlock,
|
'use_mlock': shared.args.mlock,
|
||||||
|
'mul_mat_q': shared.args.mul_mat_q,
|
||||||
'low_vram': shared.args.low_vram,
|
'low_vram': shared.args.low_vram,
|
||||||
'n_gpu_layers': shared.args.n_gpu_layers,
|
'n_gpu_layers': shared.args.n_gpu_layers,
|
||||||
'rope_freq_base': 10000 * shared.args.alpha_value ** (64 / 63.),
|
'rope_freq_base': 10000 * shared.args.alpha_value ** (64 / 63.),
|
||||||
|
'tensor_split': tensor_split_list,
|
||||||
'rope_freq_scale': 1.0 / shared.args.compress_pos_emb,
|
'rope_freq_scale': 1.0 / shared.args.compress_pos_emb,
|
||||||
'n_gqa': shared.args.n_gqa or None,
|
'n_gqa': shared.args.n_gqa or None,
|
||||||
'rms_norm_eps': shared.args.rms_norm_eps or None,
|
'rms_norm_eps': shared.args.rms_norm_eps or None,
|
||||||
|
@ -67,11 +67,13 @@ loaders_and_params = OrderedDict({
|
|||||||
'n_gqa',
|
'n_gqa',
|
||||||
'rms_norm_eps',
|
'rms_norm_eps',
|
||||||
'n_gpu_layers',
|
'n_gpu_layers',
|
||||||
|
'tensor_split',
|
||||||
'n_batch',
|
'n_batch',
|
||||||
'threads',
|
'threads',
|
||||||
'no_mmap',
|
'no_mmap',
|
||||||
'low_vram',
|
'low_vram',
|
||||||
'mlock',
|
'mlock',
|
||||||
|
'mul_mat_q',
|
||||||
'llama_cpp_seed',
|
'llama_cpp_seed',
|
||||||
'alpha_value',
|
'alpha_value',
|
||||||
'compress_pos_emb',
|
'compress_pos_emb',
|
||||||
@ -82,11 +84,13 @@ loaders_and_params = OrderedDict({
|
|||||||
'n_gqa',
|
'n_gqa',
|
||||||
'rms_norm_eps',
|
'rms_norm_eps',
|
||||||
'n_gpu_layers',
|
'n_gpu_layers',
|
||||||
|
'tensor_split',
|
||||||
'n_batch',
|
'n_batch',
|
||||||
'threads',
|
'threads',
|
||||||
'no_mmap',
|
'no_mmap',
|
||||||
'low_vram',
|
'low_vram',
|
||||||
'mlock',
|
'mlock',
|
||||||
|
'mul_mat_q',
|
||||||
'alpha_value',
|
'alpha_value',
|
||||||
'compress_pos_emb',
|
'compress_pos_emb',
|
||||||
'cpu',
|
'cpu',
|
||||||
|
@ -119,8 +119,10 @@ parser.add_argument('--n_batch', type=int, default=512, help='Maximum number of
|
|||||||
parser.add_argument('--no-mmap', action='store_true', help='Prevent mmap from being used.')
|
parser.add_argument('--no-mmap', action='store_true', help='Prevent mmap from being used.')
|
||||||
parser.add_argument('--low-vram', action='store_true', help='Low VRAM Mode')
|
parser.add_argument('--low-vram', action='store_true', help='Low VRAM Mode')
|
||||||
parser.add_argument('--mlock', action='store_true', help='Force the system to keep the model in RAM.')
|
parser.add_argument('--mlock', action='store_true', help='Force the system to keep the model in RAM.')
|
||||||
|
parser.add_argument('--mul_mat_q', action='store_true', help='Activate new mulmat kernels.')
|
||||||
parser.add_argument('--cache-capacity', type=str, help='Maximum cache capacity. Examples: 2000MiB, 2GiB. When provided without units, bytes will be assumed.')
|
parser.add_argument('--cache-capacity', type=str, help='Maximum cache capacity. Examples: 2000MiB, 2GiB. When provided without units, bytes will be assumed.')
|
||||||
parser.add_argument('--n-gpu-layers', type=int, default=0, help='Number of layers to offload to the GPU.')
|
parser.add_argument('--n-gpu-layers', type=int, default=0, help='Number of layers to offload to the GPU.')
|
||||||
|
parser.add_argument('--tensor_split', type=str, default=None, help="Split the model across multiple GPUs, comma-separated list of proportions, e.g. 18,17")
|
||||||
parser.add_argument('--n_ctx', type=int, default=2048, help='Size of the prompt context.')
|
parser.add_argument('--n_ctx', type=int, default=2048, help='Size of the prompt context.')
|
||||||
parser.add_argument('--llama_cpp_seed', type=int, default=0, help='Seed for llama-cpp models. Default 0 (random)')
|
parser.add_argument('--llama_cpp_seed', type=int, default=0, help='Seed for llama-cpp models. Default 0 (random)')
|
||||||
parser.add_argument('--n_gqa', type=int, default=0, help='grouped-query attention. Must be 8 for llama-2 70b.')
|
parser.add_argument('--n_gqa', type=int, default=0, help='grouped-query attention. Must be 8 for llama-2 70b.')
|
||||||
|
@ -68,7 +68,9 @@ def list_model_elements():
|
|||||||
'no_mmap',
|
'no_mmap',
|
||||||
'low_vram',
|
'low_vram',
|
||||||
'mlock',
|
'mlock',
|
||||||
|
'mul_mat_q',
|
||||||
'n_gpu_layers',
|
'n_gpu_layers',
|
||||||
|
'tensor_split',
|
||||||
'n_ctx',
|
'n_ctx',
|
||||||
'n_gqa',
|
'n_gqa',
|
||||||
'rms_norm_eps',
|
'rms_norm_eps',
|
||||||
|
@ -33,6 +33,7 @@ def create_ui():
|
|||||||
default_gpu_mem.append(int(re.sub('[a-zA-Z ]', '', i)))
|
default_gpu_mem.append(int(re.sub('[a-zA-Z ]', '', i)))
|
||||||
else:
|
else:
|
||||||
default_gpu_mem.append(int(re.sub('[a-zA-Z ]', '', i)) * 1000)
|
default_gpu_mem.append(int(re.sub('[a-zA-Z ]', '', i)) * 1000)
|
||||||
|
|
||||||
while len(default_gpu_mem) < len(total_mem):
|
while len(default_gpu_mem) < len(total_mem):
|
||||||
default_gpu_mem.append(0)
|
default_gpu_mem.append(0)
|
||||||
|
|
||||||
@ -109,6 +110,8 @@ def create_ui():
|
|||||||
shared.gradio['no_mmap'] = gr.Checkbox(label="no-mmap", value=shared.args.no_mmap)
|
shared.gradio['no_mmap'] = gr.Checkbox(label="no-mmap", value=shared.args.no_mmap)
|
||||||
shared.gradio['low_vram'] = gr.Checkbox(label="low-vram", value=shared.args.low_vram)
|
shared.gradio['low_vram'] = gr.Checkbox(label="low-vram", value=shared.args.low_vram)
|
||||||
shared.gradio['mlock'] = gr.Checkbox(label="mlock", value=shared.args.mlock)
|
shared.gradio['mlock'] = gr.Checkbox(label="mlock", value=shared.args.mlock)
|
||||||
|
shared.gradio['mul_mat_q'] = gr.Checkbox(label="mul_mat_q", value=shared.args.mul_mat_q)
|
||||||
|
shared.gradio['tensor_split'] = gr.Textbox(label='tensor_split', info='Split the model across multiple GPUs, comma-separated list of proportions, e.g. 18,17')
|
||||||
shared.gradio['llama_cpp_seed'] = gr.Number(label='Seed (0 for random)', value=shared.args.llama_cpp_seed)
|
shared.gradio['llama_cpp_seed'] = gr.Number(label='Seed (0 for random)', value=shared.args.llama_cpp_seed)
|
||||||
shared.gradio['trust_remote_code'] = gr.Checkbox(label="trust-remote-code", value=shared.args.trust_remote_code, info='Make sure to inspect the .py files inside the model folder before loading it with this option enabled.')
|
shared.gradio['trust_remote_code'] = gr.Checkbox(label="trust-remote-code", value=shared.args.trust_remote_code, info='Make sure to inspect the .py files inside the model folder before loading it with this option enabled.')
|
||||||
shared.gradio['gptq_for_llama_info'] = gr.Markdown('GPTQ-for-LLaMa support is currently only kept for compatibility with older GPUs. AutoGPTQ or ExLlama is preferred when compatible. GPTQ-for-LLaMa is installed by default with the webui on supported systems. Otherwise, it has to be installed manually following the instructions here: [instructions](https://github.com/oobabooga/text-generation-webui/blob/main/docs/GPTQ-models-(4-bit-mode).md#installation-1).')
|
shared.gradio['gptq_for_llama_info'] = gr.Markdown('GPTQ-for-LLaMa support is currently only kept for compatibility with older GPUs. AutoGPTQ or ExLlama is preferred when compatible. GPTQ-for-LLaMa is installed by default with the webui on supported systems. Otherwise, it has to be installed manually following the instructions here: [instructions](https://github.com/oobabooga/text-generation-webui/blob/main/docs/GPTQ-models-(4-bit-mode).md#installation-1).')
|
||||||
|
@ -31,8 +31,8 @@ https://github.com/jllllll/exllama/releases/download/0.0.10/exllama-0.0.10+cu117
|
|||||||
https://github.com/jllllll/exllama/releases/download/0.0.10/exllama-0.0.10+cu117-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
|
https://github.com/jllllll/exllama/releases/download/0.0.10/exllama-0.0.10+cu117-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
|
||||||
|
|
||||||
# llama-cpp-python without GPU support
|
# llama-cpp-python without GPU support
|
||||||
llama-cpp-python==0.1.77; platform_system != "Windows"
|
llama-cpp-python==0.1.78; platform_system != "Windows"
|
||||||
https://github.com/abetlen/llama-cpp-python/releases/download/v0.1.77/llama_cpp_python-0.1.77-cp310-cp310-win_amd64.whl; platform_system == "Windows"
|
https://github.com/abetlen/llama-cpp-python/releases/download/v0.1.78/llama_cpp_python-0.1.78-cp310-cp310-win_amd64.whl; platform_system == "Windows"
|
||||||
# llama-cpp-python with CUDA support
|
# llama-cpp-python with CUDA support
|
||||||
https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.1.77+cu117-cp310-cp310-win_amd64.whl; platform_system == "Windows"
|
https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.1.77+cu117-cp310-cp310-win_amd64.whl; platform_system == "Windows"
|
||||||
https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.1.77+cu117-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
|
https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.1.77+cu117-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
|
||||||
|
Loading…
Reference in New Issue
Block a user