From 2b7ba9586fb80cfbc47c77ad7bbbb03f7d6bc0df Mon Sep 17 00:00:00 2001 From: matatonic <73265741+matatonic@users.noreply.github.com> Date: Mon, 29 May 2023 13:19:57 -0400 Subject: [PATCH 1/4] Fixes #2326, KeyError: 'assistant' (#2382) --- extensions/openai/script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/openai/script.py b/extensions/openai/script.py index d41592a3..b865fe7e 100644 --- a/extensions/openai/script.py +++ b/extensions/openai/script.py @@ -259,7 +259,7 @@ class Handler(BaseHTTPRequestHandler): role_formats = { 'user': 'user: {message}\n', - 'bot': 'assistant: {message}\n', + 'assistant': 'assistant: {message}\n', 'system': '{message}', 'context': 'You are a helpful assistant. Answer as concisely as possible.', 'prompt': 'assistant:', From 4a190a98fd50e07fa011b17daf6a78ae1de15c4a Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Mon, 29 May 2023 14:56:05 -0300 Subject: [PATCH 2/4] Update GPTQ-models-(4-bit-mode).md --- docs/GPTQ-models-(4-bit-mode).md | 67 +++++++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 10 deletions(-) diff --git a/docs/GPTQ-models-(4-bit-mode).md b/docs/GPTQ-models-(4-bit-mode).md index 0ec28fa6..deb69555 100644 --- a/docs/GPTQ-models-(4-bit-mode).md +++ b/docs/GPTQ-models-(4-bit-mode).md @@ -1,12 +1,59 @@ In 4-bit mode, models are loaded with just 25% of their regular VRAM usage. So LLaMA-7B fits into a 6GB GPU, and LLaMA-30B fits into a 24GB GPU. -This is possible thanks to [@qwopqwop200](https://github.com/qwopqwop200/GPTQ-for-LLaMa)'s adaptation of the GPTQ algorithm for LLaMA: https://github.com/qwopqwop200/GPTQ-for-LLaMa GPTQ is a clever quantization algorithm that lightly reoptimizes the weights during quantization so that the accuracy loss is compensated relative to a round-to-nearest quantization. See the paper for more details: https://arxiv.org/abs/2210.17323 -## GPTQ-for-LLaMa branches +## AutoGPTQ -Different branches of GPTQ-for-LLaMa are available: +AutoGPTQ is the recommended way to create new quantized models: https://github.com/PanQiWei/AutoGPTQ + +#### Installation + +To load a model quantized with AutoGPTQ in the web UI, manual installation is currently necessary: + +``` +conda activate textgen +git clone https://github.com/PanQiWei/AutoGPTQ.git && cd AutoGPTQ +pip install . +``` + +You are going to need to have `nvcc` installed (see the [instructions below](https://github.com/oobabooga/text-generation-webui/blob/main/docs/GPTQ-models-(4-bit-mode).md#step-0-install-nvcc)). + +#### Usage + +Place the output folder generated by AutoGPTQ in your `models/` folder and load it with the `--autogptq` flag: + +``` +python server.py --autogptq --model model_name +``` + +Alternatively, check the `autogptq` box in the "Model" tab of the UI before loading the model. + +#### Offloading + +In order to do CPU offloading or multi-cpu inference with AutoGPTQ, use the `--gpu-memory` flag. It is currently somewhat slower than offloading with the `--pre_layer` option in GPTQ-for-LLaMA. + +For CPU offloading: + +``` +python server.py --autogptq --gpu-memory 3000MiB --model model_name +``` + +For multi-GPU: + +``` +python server.py --autogptq --gpu-memory 3000MiB 6000MiB --model model_name +``` + +#### Applying LoRAs + +Not supported yet. + +## GPTQ-for-LLaMa + +GPTQ-for-LLaMa is the original adaptation of GPTQ for the LLaMA model. It was made by [@qwopqwop200](https://github.com/qwopqwop200/GPTQ-for-LLaMa) in this repository: https://github.com/qwopqwop200/GPTQ-for-LLaMa + +Different branches of GPTQ-for-LLaMa are currently available, including: | Branch | Comment | |----|----| @@ -16,9 +63,9 @@ Different branches of GPTQ-for-LLaMa are available: Overall, I recommend using the old CUDA branch. It is included by default in the one-click-installer for this web UI. -## Installation +### Installation -### Step 0: install nvcc +#### Step 0: install nvcc ``` conda activate textgen @@ -29,7 +76,7 @@ The command above takes some 10 minutes to run and shows no progress bar or upda See this issue for more details: https://github.com/oobabooga/text-generation-webui/issues/416#issuecomment-1475078571 -### Step 1: install GPTQ-for-LLaMa +#### Step 1: install GPTQ-for-LLaMa Clone the GPTQ-for-LLaMa repository into the `text-generation-webui/repositories` subfolder and install it: @@ -64,14 +111,14 @@ git clone https://github.com/qwopqwop200/GPTQ-for-LLaMa.git -b triton https://github.com/qwopqwop200/GPTQ-for-LLaMa -### Step 2: get the pre-converted weights +#### Step 2: get the pre-converted weights * Converted without `group-size` (better for the 7b model): https://github.com/oobabooga/text-generation-webui/pull/530#issuecomment-1483891617 * Converted with `group-size` (better from 13b upwards): https://github.com/oobabooga/text-generation-webui/pull/530#issuecomment-1483941105 ⚠️ The tokenizer files in the sources above may be outdated. Make sure to obtain the universal LLaMA tokenizer as described [here](https://github.com/oobabooga/text-generation-webui/blob/main/docs/LLaMA-model.md#option-1-pre-converted-weights). -### Step 3: Start the web UI: +#### Step 3: Start the web UI: For the models converted without `group-size`: @@ -91,7 +138,7 @@ The command-line flags `--wbits` and `--groupsize` are automatically detected ba python server.py --model llama-13b-4bit-128g --wbits 4 --groupsize 128 ``` -## CPU offloading +### CPU offloading It is possible to offload part of the layers of the 4-bit model to the CPU with the `--pre_layer` flag. The higher the number after `--pre_layer`, the more layers will be allocated to the GPU. @@ -109,7 +156,7 @@ Output generated in 123.79 seconds (1.61 tokens/s, 199 tokens) You can also use multiple GPUs with `pre_layer` if using the oobabooga fork of GPTQ, eg `--pre_layer 30 60` will load a LLaMA-30B model half onto your first GPU and half onto your second, or `--pre_layer 20 40` will load 20 layers onto GPU-0, 20 layers onto GPU-1, and 20 layers offloaded to CPU. -## Using LoRAs in 4-bit mode +### Using LoRAs with GPTQ-for-LLaMa At the moment, this feature is not officially supported by the relevant libraries, but a patch exists and is supported by this web UI: https://github.com/johnsmith0031/alpaca_lora_4bit From 962d05ca7e6d7551495a638e789f4ff7e46b50c8 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Mon, 29 May 2023 14:56:55 -0300 Subject: [PATCH 3/4] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b71db0d1..c794fabd 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ cd text-generation-webui pip install -r requirements.txt ``` -#### 4. Install GPTQ-for-LLaMa and the monkey patch +#### 4. Install GPTQ The base installation covers [transformers](https://github.com/huggingface/transformers) models (`AutoModelForCausalLM` and `AutoModelForSeq2SeqLM` specifically) and [llama.cpp](https://github.com/ggerganov/llama.cpp) (GGML) models. From 166a0d9893ab8f39625eaeb3ad126e360da79eea Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Mon, 29 May 2023 15:07:59 -0300 Subject: [PATCH 4/4] Update GPTQ-models-(4-bit-mode).md --- docs/GPTQ-models-(4-bit-mode).md | 36 +++++++++++++------------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/docs/GPTQ-models-(4-bit-mode).md b/docs/GPTQ-models-(4-bit-mode).md index deb69555..8eaf86ca 100644 --- a/docs/GPTQ-models-(4-bit-mode).md +++ b/docs/GPTQ-models-(4-bit-mode).md @@ -1,13 +1,10 @@ -In 4-bit mode, models are loaded with just 25% of their regular VRAM usage. So LLaMA-7B fits into a 6GB GPU, and LLaMA-30B fits into a 24GB GPU. - - GPTQ is a clever quantization algorithm that lightly reoptimizes the weights during quantization so that the accuracy loss is compensated relative to a round-to-nearest quantization. See the paper for more details: https://arxiv.org/abs/2210.17323 ## AutoGPTQ AutoGPTQ is the recommended way to create new quantized models: https://github.com/PanQiWei/AutoGPTQ -#### Installation +### Installation To load a model quantized with AutoGPTQ in the web UI, manual installation is currently necessary: @@ -19,7 +16,7 @@ pip install . You are going to need to have `nvcc` installed (see the [instructions below](https://github.com/oobabooga/text-generation-webui/blob/main/docs/GPTQ-models-(4-bit-mode).md#step-0-install-nvcc)). -#### Usage +### Usage Place the output folder generated by AutoGPTQ in your `models/` folder and load it with the `--autogptq` flag: @@ -29,9 +26,9 @@ python server.py --autogptq --model model_name Alternatively, check the `autogptq` box in the "Model" tab of the UI before loading the model. -#### Offloading +### Offloading -In order to do CPU offloading or multi-cpu inference with AutoGPTQ, use the `--gpu-memory` flag. It is currently somewhat slower than offloading with the `--pre_layer` option in GPTQ-for-LLaMA. +In order to do CPU offloading or multi-gpu inference with AutoGPTQ, use the `--gpu-memory` flag. It is currently somewhat slower than offloading with the `--pre_layer` option in GPTQ-for-LLaMA (more on that below). For CPU offloading: @@ -45,7 +42,7 @@ For multi-GPU: python server.py --autogptq --gpu-memory 3000MiB 6000MiB --model model_name ``` -#### Applying LoRAs +### Using LoRAs with AutoGPTQ Not supported yet. @@ -63,7 +60,11 @@ Different branches of GPTQ-for-LLaMa are currently available, including: Overall, I recommend using the old CUDA branch. It is included by default in the one-click-installer for this web UI. -### Installation +### Installation using precompiled wheels + +https://github.com/jllllll/GPTQ-for-LLaMa-Wheels + +### Manual installation #### Step 0: install nvcc @@ -158,18 +159,11 @@ You can also use multiple GPUs with `pre_layer` if using the oobabooga fork of G ### Using LoRAs with GPTQ-for-LLaMa -At the moment, this feature is not officially supported by the relevant libraries, but a patch exists and is supported by this web UI: https://github.com/johnsmith0031/alpaca_lora_4bit +This requires using a monkey patch that is supported by this web UI: https://github.com/johnsmith0031/alpaca_lora_4bit -In order to use it: +To use it: -1. Make sure that your requirements are up to date: - -``` -cd text-generation-webui -pip install -r requirements.txt --upgrade -``` - -2. Clone `johnsmith0031/alpaca_lora_4bit` into the repositories folder: +1. Clone `johnsmith0031/alpaca_lora_4bit` into the repositories folder: ``` cd text-generation-webui/repositories @@ -178,13 +172,13 @@ git clone https://github.com/johnsmith0031/alpaca_lora_4bit ⚠️ I have tested it with the following commit specifically: `2f704b93c961bf202937b10aac9322b092afdce0` -3. Install https://github.com/sterlind/GPTQ-for-LLaMa with this command: +2. Install https://github.com/sterlind/GPTQ-for-LLaMa with this command: ``` pip install git+https://github.com/sterlind/GPTQ-for-LLaMa.git@lora_4bit ``` -4. Start the UI with the `--monkey-patch` flag: +3. Start the UI with the `--monkey-patch` flag: ``` python server.py --model llama-7b-4bit-128g --listen --lora tloen_alpaca-lora-7b --monkey-patch