AI/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-10-01 01:26:03 -04:00
Code Issues Packages Projects Releases Wiki Activity

Revert "Add tensor split support for llama.cpp (#3171)"

Browse Source 
This reverts commit 031fe7225e.
This commit is contained in:
oobabooga 2023-07-26 07:06:01 -07:00
parent 517d40cffe
commit b17893a58f
6 changed files with 0 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
README.md
View File

@ -247,7 +247,6 @@ Optionally, you can use the following command-line flags:
| `--mlock` | Force the system to keep the model in RAM. | | `--mlock` | Force the system to keep the model in RAM. |
| `--cache-capacity CACHE_CAPACITY` | Maximum cache capacity. Examples: 2000MiB, 2GiB. When provided without units, bytes will be assumed. | | `--cache-capacity CACHE_CAPACITY` | Maximum cache capacity. Examples: 2000MiB, 2GiB. When provided without units, bytes will be assumed. |
| `--n-gpu-layers N_GPU_LAYERS` | Number of layers to offload to the GPU. Only works if llama-cpp-python was compiled with BLAS. Set this to 1000000000 to offload all layers to the GPU. | | `--n-gpu-layers N_GPU_LAYERS` | Number of layers to offload to the GPU. Only works if llama-cpp-python was compiled with BLAS. Set this to 1000000000 to offload all layers to the GPU. |
| `--tensor_split TENSOR_SPLIT` | Split the model across multiple GPUs, comma-separated list of proportions, e.g. 18,17 |
| `--n_ctx N_CTX` | Size of the prompt context. | | `--n_ctx N_CTX` | Size of the prompt context. |
| `--llama_cpp_seed SEED` | Seed for llama-cpp models. Default 0 (random). | | `--llama_cpp_seed SEED` | Seed for llama-cpp models. Default 0 (random). |
| `--n_gqa N_GQA` | grouped-query attention. Must be 8 for llama2 70b. | | `--n_gqa N_GQA` | grouped-query attention. Must be 8 for llama2 70b. |

7
modules/llamacpp_hf.py
View File

@ -94,12 +94,6 @@ class LlamacppHF(PreTrainedModel):
model_file = list(path.glob('*ggml*.bin'))[0] model_file = list(path.glob('*ggml*.bin'))[0]
logger.info(f"llama.cpp weights detected: {model_file}\n") logger.info(f"llama.cpp weights detected: {model_file}\n")
if shared.args.tensor_split is None or shared.args.tensor_split.strip() == '':
tensor_split_list = None
else:
tensor_split_list = [float(x) for x in shared.args.tensor_split.strip().split(",")]
params = { params = {
'model_path': str(model_file), 'model_path': str(model_file),
'n_ctx': shared.args.n_ctx, 'n_ctx': shared.args.n_ctx,
@ -110,7 +104,6 @@ class LlamacppHF(PreTrainedModel):
'use_mlock': shared.args.mlock, 'use_mlock': shared.args.mlock,
'low_vram': shared.args.low_vram, 'low_vram': shared.args.low_vram,
'n_gpu_layers': shared.args.n_gpu_layers, 'n_gpu_layers': shared.args.n_gpu_layers,
'tensor_split': tensor_split_list,
'rope_freq_base': 10000 * shared.args.alpha_value ** (64/63.), 'rope_freq_base': 10000 * shared.args.alpha_value ** (64/63.),
'rope_freq_scale': 1.0 / shared.args.compress_pos_emb, 'rope_freq_scale': 1.0 / shared.args.compress_pos_emb,
'n_gqa': shared.args.n_gqa or None, 'n_gqa': shared.args.n_gqa or None,

7
modules/llamacpp_model.py
View File

@ -41,12 +41,6 @@ class LlamaCppModel:
cache_capacity = int(shared.args.cache_capacity) cache_capacity = int(shared.args.cache_capacity)
logger.info("Cache capacity is " + str(cache_capacity) + " bytes") logger.info("Cache capacity is " + str(cache_capacity) + " bytes")
if shared.args.tensor_split is None or shared.args.tensor_split.strip() == '':
tensor_split_list = None
else:
tensor_split_list = [float(x) for x in shared.args.tensor_split.strip().split(",")]
params = { params = {
'model_path': str(path), 'model_path': str(path),
'n_ctx': shared.args.n_ctx, 'n_ctx': shared.args.n_ctx,
@ -57,7 +51,6 @@ class LlamaCppModel:
'use_mlock': shared.args.mlock, 'use_mlock': shared.args.mlock,
'low_vram': shared.args.low_vram, 'low_vram': shared.args.low_vram,
'n_gpu_layers': shared.args.n_gpu_layers, 'n_gpu_layers': shared.args.n_gpu_layers,
'tensor_split': tensor_split_list,
'rope_freq_base': 10000 * shared.args.alpha_value ** (64/63.), 'rope_freq_base': 10000 * shared.args.alpha_value ** (64/63.),
'rope_freq_scale': 1.0 / shared.args.compress_pos_emb, 'rope_freq_scale': 1.0 / shared.args.compress_pos_emb,
'n_gqa': shared.args.n_gqa or None, 'n_gqa': shared.args.n_gqa or None,

2
modules/loaders.py
View File

@ -33,7 +33,6 @@ loaders_and_params = {
'n_gqa', 'n_gqa',
'rms_norm_eps', 'rms_norm_eps',
'n_gpu_layers', 'n_gpu_layers',
'tensor_split',
'n_batch', 'n_batch',
'threads', 'threads',
'no_mmap', 'no_mmap',
@ -48,7 +47,6 @@ loaders_and_params = {
'n_gqa', 'n_gqa',
'rms_norm_eps', 'rms_norm_eps',
'n_gpu_layers', 'n_gpu_layers',
'tensor_split',
'n_batch', 'n_batch',
'threads', 'threads',
'no_mmap', 'no_mmap',

1
modules/shared.py
View File

@ -125,7 +125,6 @@ parser.add_argument('--low-vram', action='store_true', help='Low VRAM Mode')
parser.add_argument('--mlock', action='store_true', help='Force the system to keep the model in RAM.') parser.add_argument('--mlock', action='store_true', help='Force the system to keep the model in RAM.')
parser.add_argument('--cache-capacity', type=str, help='Maximum cache capacity. Examples: 2000MiB, 2GiB. When provided without units, bytes will be assumed.') parser.add_argument('--cache-capacity', type=str, help='Maximum cache capacity. Examples: 2000MiB, 2GiB. When provided without units, bytes will be assumed.')
parser.add_argument('--n-gpu-layers', type=int, default=0, help='Number of layers to offload to the GPU.') parser.add_argument('--n-gpu-layers', type=int, default=0, help='Number of layers to offload to the GPU.')
parser.add_argument('--tensor_split', type=str, default=None, help="Split the model across multiple GPUs, comma-separated list of proportions, e.g. 18,17")
parser.add_argument('--n_ctx', type=int, default=2048, help='Size of the prompt context.') parser.add_argument('--n_ctx', type=int, default=2048, help='Size of the prompt context.')
parser.add_argument('--llama_cpp_seed', type=int, default=0, help='Seed for llama-cpp models. Default 0 (random)') parser.add_argument('--llama_cpp_seed', type=int, default=0, help='Seed for llama-cpp models. Default 0 (random)')
parser.add_argument('--n_gqa', type=int, default=0, help='grouped-query attention. Must be 8 for llama2 70b.') parser.add_argument('--n_gqa', type=int, default=0, help='grouped-query attention. Must be 8 for llama2 70b.')

1
modules/ui.py
View File

@ -60,7 +60,6 @@ def list_model_elements():
'low_vram', 'low_vram',
'mlock', 'mlock',
'n_gpu_layers', 'n_gpu_layers',
'tensor_split',
'n_ctx', 'n_ctx',
'n_gqa', 'n_gqa',
'rms_norm_eps', 'rms_norm_eps',