mirror of
https://github.com/ravenscroftj/turbopilot.git
synced 2024-07-01 00:31:27 +00:00
Compare commits
53 Commits
5b561f7b7e
...
8fd357e0a5
Author | SHA1 | Date | |
---|---|---|---|
|
8fd357e0a5 | ||
|
30c437700a | ||
|
86f07745bb | ||
|
7bb93b6f4e | ||
|
2b27760a7f | ||
|
a00de2a332 | ||
|
215a69b5af | ||
|
91639b8fc0 | ||
|
0b408510f4 | ||
|
604183380d | ||
|
88683abe50 | ||
|
326e76c9bb | ||
|
23c0a3d19e | ||
|
31bb33c731 | ||
|
d4989b543c | ||
|
e9dc6a304a | ||
|
97a0377cd6 | ||
|
6d26c9b064 | ||
|
63b554793d | ||
|
0cf7a9c341 | ||
|
356a83c5fd | ||
|
a5517b0fcd | ||
|
8fa70e1518 | ||
|
b79ab46b50 | ||
|
4a47251822 | ||
|
b2b4a1480f | ||
|
5f7155a314 | ||
|
77cde95cb9 | ||
|
bea7ebdb34 | ||
|
812bbea9d7 | ||
|
08e8834390 | ||
|
25680e64d8 | ||
|
dca25d8456 | ||
|
1f6f84a783 | ||
|
0183b30502 | ||
|
b465eae818 | ||
|
e8adff5339 | ||
|
f12dacaa15 | ||
|
0b0b914f92 | ||
|
b21dd0799d | ||
|
c73c196364 | ||
|
30834e3121 | ||
|
39c3182a3a | ||
|
2abdcabf02 | ||
|
6877542ad8 | ||
|
c164deb042 | ||
|
b3d8d996a0 | ||
|
1e14d91bc3 | ||
|
cf7c5285e6 | ||
|
5f5e9f90be | ||
|
68760434b2 | ||
|
364168dda7 | ||
|
f818e2d09f |
1
.github/workflows/build-commit.yml
vendored
1
.github/workflows/build-commit.yml
vendored
|
@ -25,7 +25,6 @@ jobs:
|
|||
- name: Dependencies
|
||||
id: depends
|
||||
run: |
|
||||
brew update
|
||||
brew install cmake boost asio
|
||||
- name: Build
|
||||
id: make_build
|
||||
|
|
69
.github/workflows/docker-image.yml
vendored
69
.github/workflows/docker-image.yml
vendored
|
@ -16,9 +16,52 @@ jobs:
|
|||
strategy:
|
||||
matrix:
|
||||
config:
|
||||
- {tag: "", dockerfile: "./Dockerfile.default", platforms: "linux/amd64,linux/arm64"}
|
||||
- {tag: "-cuda11", dockerfile: "./Dockerfile.cuda11", platforms: "linux/amd64"}
|
||||
- {tag: "-cuda12", dockerfile: "./Dockerfile.cuda12", platforms: "linux/amd64"}
|
||||
- tag:
|
||||
dockerfile: ./Dockerfile.default
|
||||
platforms: linux/amd64,linux/arm64
|
||||
build_base: ubuntu:22.04
|
||||
runtime_base: ubuntu:22.04
|
||||
|
||||
|
||||
- tag: -openblas
|
||||
dockerfile: ./Dockerfile.default
|
||||
platforms: linux/amd64,linux/arm64
|
||||
build_base: ubuntu:22.04
|
||||
runtime_base: ubuntu:22.04
|
||||
extra_deps: libopenblas-dev
|
||||
cmake_args: -DGGML_OPENBLAS=On
|
||||
|
||||
|
||||
- tag: -cuda11-7
|
||||
dockerfile: ./Dockerfile.default
|
||||
platforms: linux/amd64
|
||||
build_base: nvidia/cuda:11.7.1-devel-ubuntu22.04
|
||||
runtime_base: nvidia/cuda:11.7.1-cudnn8-runtime-ubuntu22.04
|
||||
cmake_args: -DGGML_CUBLAS=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc
|
||||
|
||||
- tag: -cuda12-0
|
||||
dockerfile: ./Dockerfile.default
|
||||
platforms: linux/amd64
|
||||
build_base: nvidia/cuda:12.0.0-devel-ubuntu22.04
|
||||
runtime_base: nvidia/cuda:12.0.0-runtime-ubuntu22.04
|
||||
cmake_args: -DGGML_CUBLAS=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc
|
||||
|
||||
- tag: -cuda12-2
|
||||
dockerfile: ./Dockerfile.default
|
||||
platforms: linux/amd64
|
||||
build_base: nvidia/cuda:12.2.0-devel-ubuntu22.04
|
||||
runtime_base: nvidia/cuda:12.2.0-runtime-ubuntu22.04
|
||||
cmake_args: -DGGML_CUBLAS=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc
|
||||
|
||||
# - tag: -clblast
|
||||
# dockerfile: ./Dockerfile.default
|
||||
# platforms: linux/amd64
|
||||
# build_base: ubuntu:22.04
|
||||
# runtime_base: ubuntu:22.04
|
||||
# runtime_deps: libclblast1
|
||||
# extra_deps: libclblast-dev
|
||||
# cmake_args: -DGGML_CLBLAST=On
|
||||
|
||||
|
||||
steps:
|
||||
|
||||
|
@ -45,7 +88,7 @@ jobs:
|
|||
password: ${{ secrets.GH_TOKEN }}
|
||||
|
||||
- name: Build and push incremental
|
||||
uses: docker/build-push-action@v4
|
||||
uses: docker/build-push-action@v4.1.1
|
||||
if: (!startsWith(github.ref, 'refs/tags/'))
|
||||
with:
|
||||
file: ${{matrix.config.dockerfile}}
|
||||
|
@ -53,6 +96,12 @@ jobs:
|
|||
tags: ghcr.io/ravenscroftj/turbopilot:nightly${{matrix.config.tag}}-${{ github.sha }}
|
||||
context: ${{github.workspace}}
|
||||
platforms: ${{matrix.config.platforms}}
|
||||
build-args: |
|
||||
EXTRA_DEPS=${{matrix.config.extra_deps}}
|
||||
CMAKE_ARGS=${{matrix.config.cmake_args}}
|
||||
BUILD_BASE=${{matrix.config.build_base}}
|
||||
RUNTIME_BASE=${{matrix.config.runtime_base}}
|
||||
RUNTIME_DEPS=${{matrix.config.runtime_deps}}
|
||||
|
||||
|
||||
- name: Build and push release (Main Latest Build)
|
||||
|
@ -64,6 +113,12 @@ jobs:
|
|||
tags: ghcr.io/ravenscroftj/turbopilot:${{ github.ref_name }}, ghcr.io/ravenscroftj/turbopilot:latest
|
||||
context: ${{github.workspace}}
|
||||
platforms: ${{matrix.config.platforms}}
|
||||
build-args: |
|
||||
EXTRA_DEPS=${{matrix.config.extra_deps}}
|
||||
CMAKE_ARGS=${{matrix.config.cmake_args}}
|
||||
BUILD_BASE=${{matrix.config.build_base}}
|
||||
RUNTIME_BASE=${{matrix.config.runtime_base}}
|
||||
RUNTIME_DEPS=${{matrix.config.runtime_deps}}
|
||||
|
||||
|
||||
- name: Build and push release (Accelerated Builds)
|
||||
|
@ -75,3 +130,9 @@ jobs:
|
|||
tags: ghcr.io/ravenscroftj/turbopilot:${{ github.ref_name }}${{matrix.config.tag}}
|
||||
context: ${{github.workspace}}
|
||||
platforms: ${{matrix.config.platforms}}
|
||||
build-args: |
|
||||
EXTRA_DEPS=${{matrix.config.extra_deps}}
|
||||
CMAKE_ARGS=${{matrix.config.cmake_args}}
|
||||
BUILD_BASE=${{matrix.config.build_base}}
|
||||
RUNTIME_BASE=${{matrix.config.runtime_base}}
|
||||
RUNTIME_DEPS=${{matrix.config.runtime_deps}}
|
||||
|
|
2
.gitignore
vendored
Normal file
2
.gitignore
vendored
Normal file
|
@ -0,0 +1,2 @@
|
|||
build/
|
||||
models/
|
5
.gitmodules
vendored
5
.gitmodules
vendored
|
@ -1,9 +1,12 @@
|
|||
[submodule "ggml"]
|
||||
path = extern/ggml
|
||||
url = git@github.com:ravenscroftj/ggml.git
|
||||
url = git@github.com:ggerganov/ggml.git
|
||||
[submodule "extern/argparse"]
|
||||
path = extern/argparse
|
||||
url = https://github.com/p-ranav/argparse.git
|
||||
[submodule "extern/sbdlog"]
|
||||
path = extern/spdlog
|
||||
url = https://github.com/gabime/spdlog.git
|
||||
[submodule "extern/ggml"]
|
||||
path = extern/ggml
|
||||
url = https://github.com/ggerganov/ggml
|
||||
|
|
20
.vscode/c_cpp_properties.json
vendored
Normal file
20
.vscode/c_cpp_properties.json
vendored
Normal file
|
@ -0,0 +1,20 @@
|
|||
{
|
||||
"configurations": [
|
||||
{
|
||||
"name": "Linux",
|
||||
"includePath": [
|
||||
"${workspaceFolder}/**",
|
||||
"${workspaceFolder}/extern/crow/include",
|
||||
"${workspaceFolder}/include",
|
||||
"${workspaceFolder}/include"
|
||||
],
|
||||
"defines": [],
|
||||
"compilerPath": "/usr/bin/gcc",
|
||||
"cStandard": "c17",
|
||||
"cppStandard": "gnu++17",
|
||||
"intelliSenseMode": "linux-gcc-x64",
|
||||
"configurationProvider": "ms-vscode.cmake-tools"
|
||||
}
|
||||
],
|
||||
"version": 4
|
||||
}
|
70
.vscode/launch.json
vendored
Normal file
70
.vscode/launch.json
vendored
Normal file
|
@ -0,0 +1,70 @@
|
|||
{
|
||||
// Use IntelliSense to learn about possible attributes.
|
||||
// Hover to view descriptions of existing attributes.
|
||||
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
|
||||
"version": "0.2.0",
|
||||
"configurations": [
|
||||
{
|
||||
"name": "(gdb) Launch TBP",
|
||||
"type": "cppdbg",
|
||||
"request": "launch",
|
||||
"program": "/home/james/workspace/rafael-llm/turbopilot/build/bin/turbopilot",
|
||||
"args": [
|
||||
//TBP ARGS
|
||||
"-v",
|
||||
"-f",
|
||||
"/home/james/Downloads/replit-code-v1-3b-q4_0.bin",
|
||||
"-m",
|
||||
"replit",
|
||||
],
|
||||
"stopAtEntry": false,
|
||||
"cwd": "${workspaceFolder}",
|
||||
"environment": [],
|
||||
"externalConsole": false,
|
||||
"MIMode": "gdb",
|
||||
"setupCommands": [
|
||||
{
|
||||
"description": "Enable pretty-printing for gdb",
|
||||
"text": "-enable-pretty-printing",
|
||||
"ignoreFailures": true
|
||||
},
|
||||
{
|
||||
"description": "Set Disassembly Flavor to Intel",
|
||||
"text": "-gdb-set disassembly-flavor intel",
|
||||
"ignoreFailures": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "(gdb) Launch Replut",
|
||||
"type": "cppdbg",
|
||||
"request": "launch",
|
||||
"program": "/home/james/workspace/rafael-llm/turbopilot/extern/ggml/build/bin/replit",
|
||||
"args": [
|
||||
// REPLIT ARGS
|
||||
"-m",
|
||||
"/home/james/Downloads/replit-code-v1-3b-q4_0.bin",
|
||||
"-f",
|
||||
"/home/james/workspace/rafael-llm/turbopilot/test.txt"
|
||||
],
|
||||
"stopAtEntry": false,
|
||||
"cwd": "${workspaceFolder}",
|
||||
"environment": [],
|
||||
"externalConsole": false,
|
||||
"MIMode": "gdb",
|
||||
"setupCommands": [
|
||||
{
|
||||
"description": "Enable pretty-printing for gdb",
|
||||
"text": "-enable-pretty-printing",
|
||||
"ignoreFailures": true
|
||||
},
|
||||
{
|
||||
"description": "Set Disassembly Flavor to Intel",
|
||||
"text": "-gdb-set disassembly-flavor intel",
|
||||
"ignoreFailures": true
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
]
|
||||
}
|
28
.vscode/tasks.json
vendored
Normal file
28
.vscode/tasks.json
vendored
Normal file
|
@ -0,0 +1,28 @@
|
|||
{
|
||||
"tasks": [
|
||||
{
|
||||
"type": "cppbuild",
|
||||
"label": "C/C++: g++ build active file",
|
||||
"command": "/usr/bin/g++",
|
||||
"args": [
|
||||
"-fdiagnostics-color=always",
|
||||
"-g",
|
||||
"${file}",
|
||||
"-o",
|
||||
"${fileDirname}/${fileBasenameNoExtension}"
|
||||
],
|
||||
"options": {
|
||||
"cwd": "${fileDirname}"
|
||||
},
|
||||
"problemMatcher": [
|
||||
"$gcc"
|
||||
],
|
||||
"group": {
|
||||
"kind": "build",
|
||||
"isDefault": true
|
||||
},
|
||||
"detail": "Task generated by Debugger."
|
||||
}
|
||||
],
|
||||
"version": "2.0.0"
|
||||
}
|
|
@ -15,6 +15,11 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS "on")
|
|||
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
|
||||
set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib")
|
||||
|
||||
option(GGML_CLBLAST "ggml: use clBLAST" OFF)
|
||||
option(GGML_CUBLAS "ggml: use cuBLAS" OFF)
|
||||
|
||||
|
||||
|
||||
if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64")
|
||||
message(STATUS "ARM detected")
|
||||
if (MSVC)
|
||||
|
@ -48,12 +53,20 @@ if (GGML_STATIC)
|
|||
SET(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
|
||||
SET(BUILD_SHARED_LIBS OFF)
|
||||
SET(CMAKE_EXE_LINKER_FLAGS "-static")
|
||||
|
||||
# if(GGML_OPENBLAS)
|
||||
# set(BLA_STATIC ON)
|
||||
# endif()
|
||||
endif()
|
||||
|
||||
if (GGML_CUBLAS)
|
||||
cmake_minimum_required(VERSION 3.17)
|
||||
|
||||
find_package(CUDAToolkit)
|
||||
if (CUDAToolkit_FOUND)
|
||||
add_compile_definitions(GGML_USE_CUBLAS)
|
||||
else()
|
||||
message(WARNING "cuBLAS not found")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
|
||||
|
||||
add_subdirectory(src)
|
||||
|
||||
|
|
|
@ -1,39 +0,0 @@
|
|||
FROM nvidia/cuda:11.7.1-devel-ubuntu22.04 AS build
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
# inlude kitware apt repo to allow us to grab latest cmake
|
||||
RUN apt-get update && apt-get install ca-certificates gpg wget
|
||||
RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
|
||||
RUN echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ jammy main' | tee /etc/apt/sources.list.d/kitware.list >/dev/null
|
||||
|
||||
RUN apt-get update && apt-get install -y build-essential cmake libboost-dev libboost-thread-dev
|
||||
|
||||
|
||||
ADD ./ /turbopilot
|
||||
|
||||
RUN mkdir /turbopilot/build
|
||||
|
||||
WORKDIR /turbopilot/build
|
||||
|
||||
RUN cmake -DGGML_CUBLAS=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc ..
|
||||
RUN make turbopilot
|
||||
|
||||
FROM nvidia/cuda:11.7.1-cudnn8-runtime-ubuntu22.04 AS runtime
|
||||
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY --from=build /turbopilot/build/bin/turbopilot /app/turbopilot
|
||||
|
||||
ENV THREADS=4
|
||||
|
||||
ENV MODEL="/models/codegen-2B-multi-ggml-4bit-quant.bin"
|
||||
|
||||
ENV BATCHSIZE=64
|
||||
|
||||
COPY ./run.sh /app/
|
||||
|
||||
EXPOSE 18080
|
||||
|
||||
CMD /app/run.sh
|
|
@ -1,37 +0,0 @@
|
|||
FROM nvidia/cuda:12.2.0-devel-ubuntu20.04 AS build
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
# inlude kitware apt repo to allow us to grab latest cmake
|
||||
RUN apt-get update && apt-get install ca-certificates gpg wget
|
||||
RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
|
||||
RUN echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ focal main' | tee /etc/apt/sources.list.d/kitware.list >/dev/null
|
||||
|
||||
RUN apt-get update && apt-get install -y build-essential cmake libboost-dev libboost-thread-dev
|
||||
|
||||
ADD ./ /turbopilot
|
||||
|
||||
RUN mkdir /turbopilot/build
|
||||
|
||||
WORKDIR /turbopilot/build
|
||||
|
||||
RUN cmake -DGGML_CUBLAS=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc ..
|
||||
RUN make turbopilot
|
||||
|
||||
FROM nvidia/cuda:12.2.0-runtime-ubuntu20.04 AS runtime
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY --from=build /turbopilot/build/bin/turbopilot /app/turbopilot
|
||||
|
||||
ENV THREADS=4
|
||||
|
||||
ENV MODEL="/models/codegen-2B-multi-ggml-4bit-quant.bin"
|
||||
|
||||
ENV BATCHSIZE=64
|
||||
|
||||
COPY ./run.sh /app/
|
||||
|
||||
EXPOSE 18080
|
||||
|
||||
CMD /app/run.sh
|
|
@ -1,4 +1,13 @@
|
|||
FROM ubuntu:22.04 AS build
|
||||
ARG BUILD_BASE="ubuntu:22.04"
|
||||
ARG RUNTIME_BASE="ubuntu:22.04"
|
||||
|
||||
FROM ${BUILD_BASE} AS build
|
||||
|
||||
ARG EXTRA_DEPS=""
|
||||
ARG CMAKE_ARGS=""
|
||||
|
||||
RUN echo "CMAKE_ARGS: ${CMAKE_ARGS}"
|
||||
RUN echo "EXTRA_DEPS: ${EXTRA_DEPS}"
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
|
@ -7,7 +16,7 @@ RUN apt-get update && apt-get install -y ca-certificates gpg wget
|
|||
RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
|
||||
RUN echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ jammy main' | tee /etc/apt/sources.list.d/kitware.list >/dev/null
|
||||
|
||||
RUN apt-get update && apt-get install -y build-essential cmake libboost-dev libboost-thread-dev
|
||||
RUN apt-get update && apt-get install -y build-essential cmake libboost-dev libboost-thread-dev ${EXTRA_DEPS}
|
||||
|
||||
ADD ./ /turbopilot
|
||||
|
||||
|
@ -15,10 +24,14 @@ RUN mkdir /turbopilot/build
|
|||
|
||||
WORKDIR /turbopilot/build
|
||||
|
||||
RUN cmake ..
|
||||
RUN cmake .. ${CMAKE_ARGS}
|
||||
RUN make turbopilot
|
||||
|
||||
FROM ubuntu:22.04 AS runtime
|
||||
FROM ${RUNTIME_BASE} AS runtime
|
||||
|
||||
ARG RUNTIME_DEPS=""
|
||||
|
||||
RUN if [[ -z "${RUNTIME_DEPS}" ]] ; then echo "No runtime libs required" ; else apt-get update && apt-get install -y ${RUNTIME_DEPS} ; fi
|
||||
|
||||
|
||||
WORKDIR /app
|
||||
|
|
16
README.md
16
README.md
|
@ -94,11 +94,14 @@ docker run --gpus=all --rm -it \
|
|||
-e THREADS=6 \
|
||||
-e MODEL_TYPE=starcoder \
|
||||
-e MODEL="/models/santacoder-q4_0.bin" \
|
||||
-e GPU_LAYERS=32 \
|
||||
-p 18080:18080 \
|
||||
ghcr.io/ravenscroftj/turbopilot:v0.1.0-cuda11
|
||||
ghcr.io/ravenscroftj/turbopilot:v0.2.0-cuda11-7
|
||||
```
|
||||
|
||||
Swap `ghcr.io/ravenscroftj/turbopilot:v0.1.0-cuda11` for `ghcr.io/ravenscroftj/turbopilot:v0.1.0-cuda12` if you are using CUDA 12 or later.
|
||||
If you have a big enough GPU then setting `GPU_LAYERS` will allow turbopilot to fully offload computation onto your GPU rather than copying data backwards and forwards, dramatically speeding up inference.
|
||||
|
||||
Swap `ghcr.io/ravenscroftj/turbopilot:v0.1.0-cuda11` for `ghcr.io/ravenscroftj/turbopilot:v0.2.0-cuda12-0` or `ghcr.io/ravenscroftj/turbopilot:v0.2.0-cuda12-2` if you are using CUDA 12.0 or 12.2 respectively.
|
||||
|
||||
You will need CUDA 11 or CUDA 12 later to run this container. You should be able to see `/app/turbopilot` listed when you run `nvidia-smi`.
|
||||
|
||||
|
@ -107,6 +110,8 @@ You will need CUDA 11 or CUDA 12 later to run this container. You should be able
|
|||
|
||||
As of v0.0.5 a CUDA version of the linux executable is available - it requires that libcublas 11 be installed on the machine - I might build ubuntu debs at some point but for now running in docker may be more convenient if you want to use a CUDA GPU.
|
||||
|
||||
You can use GPU offloading via the `--ngl` option.
|
||||
|
||||
### 🌐 Using the API
|
||||
|
||||
#### Support for the official Copilot Plugin
|
||||
|
@ -177,12 +182,7 @@ Should get you something like this:
|
|||
|
||||
## 👉 Known Limitations
|
||||
|
||||
Again I want to set expectations around this being a proof-of-concept project. With that in mind. Here are some current known limitations.
|
||||
|
||||
As of **v0.0.2**:
|
||||
- The models can be quite slow - especially the 6B ones. It can take ~30-40s to make suggestions across 4 CPU cores.
|
||||
- I've only tested the system on Ubuntu 22.04 but I am now supplying ARM docker images and soon I'll be providing ARM binary releases.
|
||||
- Sometimes suggestions get truncated in nonsensical places - e.g. part way through a variable name or string name. This is due to a hard limit of 2048 on the context length (prompt + suggestion).
|
||||
- Currently Turbopilot only supports one GPU device at a time (it will not try to make use of multiple devices).
|
||||
|
||||
## 👏 Acknowledgements
|
||||
|
||||
|
|
2
extern/ggml
vendored
2
extern/ggml
vendored
|
@ -1 +1 @@
|
|||
Subproject commit f6365c0605ac86c6ab106cda0e8d6650e54097a7
|
||||
Subproject commit 1a5d5f331de1d3c7ace40d86fe2373021a42f9ce
|
|
@ -44,6 +44,7 @@ struct ModelConfig
|
|||
int32_t seed = -1; // RNG seed
|
||||
int32_t n_ctx = 512; // context size
|
||||
int32_t n_batch = 512; // batch size for prompt processing (must be >=32 to use BLAS)
|
||||
int32_t n_gpu_layers = 0;
|
||||
};
|
||||
|
||||
class TurbopilotModel
|
||||
|
@ -67,4 +68,5 @@ protected:
|
|||
std::mutex model_lock;
|
||||
};
|
||||
|
||||
#endif //__TURBOPILOT_MODEL_H
|
||||
|
||||
#endif //__TURBOPILOT_MODEL_H
|
||||
|
|
7
run.sh
7
run.sh
|
@ -1,3 +1,6 @@
|
|||
#!/bin/sh
|
||||
|
||||
/app/turbopilot -t $THREADS -m $MODEL_TYPE -f $MODEL
|
||||
if [ -z "$GPU_LAYERS" ]; then
|
||||
/app/turbopilot -t $THREADS -m $MODEL_TYPE -f $MODEL
|
||||
else
|
||||
/app/turbopilot -t $THREADS -m $MODEL_TYPE -f $MODEL --ngl $GPU_LAYERS
|
||||
fi
|
52
src/gptj.cpp
52
src/gptj.cpp
|
@ -6,6 +6,14 @@
|
|||
#include <iostream>
|
||||
#include <fstream>
|
||||
|
||||
|
||||
#ifdef GGML_USE_CLBLAST
|
||||
#include "ggml-opencl.h"
|
||||
#endif
|
||||
#ifdef GGML_USE_CUBLAS
|
||||
#include "ggml-cuda.h"
|
||||
#endif
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
#pragma warning(disable: 4244 4267) // possible loss of data
|
||||
#endif
|
||||
|
@ -455,6 +463,9 @@ bool GPTJModel::load_model(std::string fname) {
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
// key + value memory
|
||||
{
|
||||
const auto & hparams = model->hparams;
|
||||
|
@ -553,6 +564,47 @@ bool GPTJModel::load_model(std::string fname) {
|
|||
|
||||
fin.close();
|
||||
|
||||
|
||||
|
||||
#if defined(GGML_USE_CLBLAST) || defined(GGML_USE_CUBLAS)
|
||||
|
||||
if(config.n_gpu_layers > 0){
|
||||
size_t vram_total = 0;
|
||||
int gpu_layers = std::min(config.n_gpu_layers, model->hparams.n_layer);
|
||||
spdlog::info("Attempting to offload {} layers to GPU", gpu_layers);
|
||||
|
||||
for(int i=0; i < gpu_layers; i++) {
|
||||
const auto & layer = model->layers[i];
|
||||
layer.c_attn_q_proj_w->backend = GGML_BACKEND_GPU;
|
||||
layer.c_attn_k_proj_w->backend = GGML_BACKEND_GPU;
|
||||
layer.c_attn_v_proj_w->backend = GGML_BACKEND_GPU;
|
||||
|
||||
layer.c_attn_proj_w->backend = GGML_BACKEND_GPU;
|
||||
layer.c_mlp_fc_w->backend = GGML_BACKEND_GPU;
|
||||
layer.c_mlp_proj_w->backend = GGML_BACKEND_GPU;
|
||||
|
||||
#if defined(GGML_USE_CLBLAST)
|
||||
ggml_cl_transform_tensor(layer.c_attn_q_proj_w->data,layer.c_attn_q_proj_w); vram_total += ggml_nbytes(layer.c_attn_q_proj_w);
|
||||
ggml_cl_transform_tensor(layer.c_attn_k_proj_w->data,layer.c_attn_k_proj_w); vram_total += ggml_nbytes(layer.c_attn_k_proj_w);
|
||||
ggml_cl_transform_tensor(layer.c_attn_v_proj_w->data,layer.c_attn_v_proj_w); vram_total += ggml_nbytes(layer.c_attn_v_proj_w);
|
||||
ggml_cl_transform_tensor(layer.c_attn_proj_w->data,layer.c_attn_proj_w); vram_total += ggml_nbytes(layer.c_attn_proj_w);
|
||||
ggml_cl_transform_tensor(layer.c_mlp_fc_w->data,layer.c_mlp_fc_w); vram_total += ggml_nbytes(layer.c_mlp_fc_w);
|
||||
ggml_cl_transform_tensor(layer.c_mlp_proj_w->data,layer.c_mlp_proj_w); vram_total += ggml_nbytes(layer.c_mlp_proj_w);
|
||||
#else
|
||||
ggml_cuda_transform_tensor(layer.c_attn_q_proj_w->data,layer.c_attn_q_proj_w); vram_total += ggml_nbytes(layer.c_attn_q_proj_w);
|
||||
ggml_cuda_transform_tensor(layer.c_attn_k_proj_w->data,layer.c_attn_k_proj_w); vram_total += ggml_nbytes(layer.c_attn_k_proj_w);
|
||||
ggml_cuda_transform_tensor(layer.c_attn_v_proj_w->data,layer.c_attn_v_proj_w); vram_total += ggml_nbytes(layer.c_attn_v_proj_w);
|
||||
ggml_cuda_transform_tensor(layer.c_attn_proj_w->data,layer.c_attn_proj_w); vram_total += ggml_nbytes(layer.c_attn_proj_w);
|
||||
ggml_cuda_transform_tensor(layer.c_mlp_fc_w->data,layer.c_mlp_fc_w); vram_total += ggml_nbytes(layer.c_mlp_fc_w);
|
||||
ggml_cuda_transform_tensor(layer.c_mlp_proj_w->data,layer.c_mlp_proj_w); vram_total += ggml_nbytes(layer.c_mlp_proj_w);
|
||||
#endif
|
||||
}
|
||||
|
||||
spdlog::info("{}: [GPU] total VRAM used: {} MB\n", __func__, vram_total / 1024 / 1024);
|
||||
}
|
||||
|
||||
#endif // defined(GGML_USE_CLBLAST) || defined(GGML_USE_CUBLAS)
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
|
@ -3,6 +3,13 @@
|
|||
|
||||
#include <ggml/ggml.h>
|
||||
|
||||
#ifdef GGML_USE_CLBLAST
|
||||
#include "ggml-opencl.h"
|
||||
#endif
|
||||
#ifdef GGML_USE_CUBLAS
|
||||
#include "ggml-cuda.h"
|
||||
#endif
|
||||
|
||||
#include <cinttypes>
|
||||
|
||||
#include <iostream>
|
||||
|
@ -50,6 +57,7 @@ ggml_tensor * gpt_neox_ff(
|
|||
}
|
||||
|
||||
|
||||
|
||||
// evaluate the transformer
|
||||
//
|
||||
// - model: the model
|
||||
|
@ -612,9 +620,42 @@ bool GPTNEOXModel::load_model(std::string fname) {
|
|||
|
||||
printf("%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size/1024.0/1024.0, n_tensors);
|
||||
}
|
||||
|
||||
fin.close();
|
||||
|
||||
#if defined(GGML_USE_CLBLAST) || defined(GGML_USE_CUBLAS)
|
||||
|
||||
|
||||
if(config.n_gpu_layers > 0){
|
||||
size_t vram_total = 0;
|
||||
int gpu_layers = std::min(config.n_gpu_layers, model->hparams.n_layer);
|
||||
spdlog::info("Attempting to offload {} layers to GPU", gpu_layers);
|
||||
|
||||
|
||||
for(int i=0; i < gpu_layers; i++) {
|
||||
const auto & layer = model->layers[i];
|
||||
layer.c_attn_attn_w->backend = GGML_BACKEND_GPU;
|
||||
layer.c_attn_proj_w->backend = GGML_BACKEND_GPU;
|
||||
layer.c_mlp_fc_w->backend = GGML_BACKEND_GPU;
|
||||
layer.c_mlp_proj_w->backend = GGML_BACKEND_GPU;
|
||||
|
||||
#if defined(GGML_USE_CLBLAST)
|
||||
ggml_cl_transform_tensor(layer.c_attn_attn_w->data,layer.c_attn_attn_w); vram_total += ggml_nbytes(layer.c_attn_attn_w);
|
||||
ggml_cl_transform_tensor(layer.c_attn_proj_w->data,layer.c_attn_proj_w); vram_total += ggml_nbytes(layer.c_attn_proj_w);
|
||||
ggml_cl_transform_tensor(layer.c_mlp_fc_w->data,layer.c_mlp_fc_w); vram_total += ggml_nbytes(layer.c_mlp_fc_w);
|
||||
ggml_cl_transform_tensor(layer.c_mlp_proj_w->data,layer.c_mlp_proj_w); vram_total += ggml_nbytes(layer.c_mlp_proj_w);
|
||||
#else
|
||||
ggml_cuda_transform_tensor(layer.c_attn_attn_w->data,layer.c_attn_attn_w); vram_total += ggml_nbytes(layer.c_attn_attn_w);
|
||||
ggml_cuda_transform_tensor(layer.c_attn_proj_w->data,layer.c_attn_proj_w); vram_total += ggml_nbytes(layer.c_attn_proj_w);
|
||||
ggml_cuda_transform_tensor(layer.c_mlp_fc_w->data,layer.c_mlp_fc_w); vram_total += ggml_nbytes(layer.c_mlp_fc_w);
|
||||
ggml_cuda_transform_tensor(layer.c_mlp_proj_w->data,layer.c_mlp_proj_w); vram_total += ggml_nbytes(layer.c_mlp_proj_w);
|
||||
#endif
|
||||
}
|
||||
|
||||
spdlog::info("{}: [GPU] total VRAM used: {} MB\n", __func__, vram_total / 1024 / 1024);
|
||||
}
|
||||
|
||||
#endif // defined(GGML_USE_CLBLAST) || defined(GGML_USE_CUBLAS)
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
12
src/main.cpp
12
src/main.cpp
|
@ -40,6 +40,13 @@ int main(int argc, char **argv)
|
|||
.default_value(4)
|
||||
.scan<'i', int>();
|
||||
|
||||
|
||||
program.add_argument("--ngl", "--n-gpu-layers")
|
||||
.help("The number of layers to offload to GPU")
|
||||
.default_value(0)
|
||||
.scan<'i', int>();
|
||||
|
||||
|
||||
program.add_argument("-p", "--port")
|
||||
.help("The tcp port that turbopilot should listen on")
|
||||
.default_value(18080)
|
||||
|
@ -102,6 +109,7 @@ int main(int argc, char **argv)
|
|||
config.temp = program.get<float>("--temperature");
|
||||
config.top_p = program.get<float>("--top-p");
|
||||
config.n_batch = program.get<int>("--batch-size");
|
||||
config.n_gpu_layers = program.get<int>("--ngl");
|
||||
|
||||
if(model_type.compare("codegen") == 0) {
|
||||
spdlog::info("Initializing GPT-J type model for '{}' model", model_type);
|
||||
|
@ -183,4 +191,6 @@ int main(int argc, char **argv)
|
|||
|
||||
|
||||
free(model);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
|
|
@ -5,6 +5,12 @@
|
|||
#include <ggml/ggml.h>
|
||||
#include <spdlog/spdlog.h>
|
||||
|
||||
#ifdef GGML_USE_CLBLAST
|
||||
#include "ggml-opencl.h"
|
||||
#endif
|
||||
#ifdef GGML_USE_CUBLAS
|
||||
#include "ggml-cuda.h"
|
||||
#endif
|
||||
|
||||
// evaluate the transformer
|
||||
//
|
||||
|
@ -36,10 +42,10 @@ bool starcoder_eval(
|
|||
|
||||
// use 2 scratch buffers
|
||||
// TODO: very hacky solution - reimplement in a more elegant way
|
||||
static size_t scr0_size = 256u*1024*1024;
|
||||
static size_t scr0_size = 512u*1024*1024;
|
||||
static void * scr0 = malloc(scr0_size);
|
||||
|
||||
static size_t scr1_size = 256u*1024*1024;
|
||||
static size_t scr1_size = 512u*1024*1024;
|
||||
static void * scr1 = malloc(scr1_size);
|
||||
|
||||
if (mem_per_token > 0 && mem_per_token*N > buf_size) {
|
||||
|
@ -677,6 +683,41 @@ bool StarcoderModel::load_model(std::string fname) {
|
|||
|
||||
fin.close();
|
||||
|
||||
|
||||
#if defined(GGML_USE_CLBLAST) || defined(GGML_USE_CUBLAS)
|
||||
|
||||
if(config.n_gpu_layers > 0){
|
||||
size_t vram_total = 0;
|
||||
int gpu_layers = std::min(config.n_gpu_layers, model->hparams.n_layer);
|
||||
spdlog::info("Attempting to offload {} layers to GPU", gpu_layers);
|
||||
|
||||
|
||||
for(int i=0; i < gpu_layers; i++) {
|
||||
const auto & layer = model->layers[i];
|
||||
layer.c_attn_attn_w->backend = GGML_BACKEND_GPU;
|
||||
layer.c_attn_proj_w->backend = GGML_BACKEND_GPU;
|
||||
layer.c_mlp_fc_w->backend = GGML_BACKEND_GPU;
|
||||
layer.c_mlp_proj_w->backend = GGML_BACKEND_GPU;
|
||||
|
||||
#if defined(GGML_USE_CLBLAST)
|
||||
ggml_cl_transform_tensor(layer.c_attn_attn_w->data,layer.c_attn_attn_w); vram_total += ggml_nbytes(layer.c_attn_attn_w);
|
||||
ggml_cl_transform_tensor(layer.c_attn_proj_w->data,layer.c_attn_proj_w); vram_total += ggml_nbytes(layer.c_attn_proj_w);
|
||||
ggml_cl_transform_tensor(layer.c_mlp_fc_w->data,layer.c_mlp_fc_w); vram_total += ggml_nbytes(layer.c_mlp_fc_w);
|
||||
ggml_cl_transform_tensor(layer.c_mlp_proj_w->data,layer.c_mlp_proj_w); vram_total += ggml_nbytes(layer.c_mlp_proj_w);
|
||||
#else
|
||||
ggml_cuda_transform_tensor(layer.c_attn_attn_w->data,layer.c_attn_attn_w); vram_total += ggml_nbytes(layer.c_attn_attn_w);
|
||||
ggml_cuda_transform_tensor(layer.c_attn_proj_w->data,layer.c_attn_proj_w); vram_total += ggml_nbytes(layer.c_attn_proj_w);
|
||||
ggml_cuda_transform_tensor(layer.c_mlp_fc_w->data,layer.c_mlp_fc_w); vram_total += ggml_nbytes(layer.c_mlp_fc_w);
|
||||
ggml_cuda_transform_tensor(layer.c_mlp_proj_w->data,layer.c_mlp_proj_w); vram_total += ggml_nbytes(layer.c_mlp_proj_w);
|
||||
#endif
|
||||
}
|
||||
|
||||
spdlog::info("{}: [GPU] total VRAM used: {} MB\n", __func__, vram_total / 1024 / 1024);
|
||||
}
|
||||
|
||||
#endif // defined(GGML_USE_CLBLAST) || defined(GGML_USE_CUBLAS)
|
||||
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
65
test_codegen2.py
Normal file
65
test_codegen2.py
Normal file
|
@ -0,0 +1,65 @@
|
|||
#%%
|
||||
from transformers import AutoTokenizer, AutoModelForCausalLM
|
||||
tokenizer = AutoTokenizer.from_pretrained("Salesforce/codegen2-1B")
|
||||
model = AutoModelForCausalLM.from_pretrained("Salesforce/codegen2-1B", trust_remote_code=True, revision="main")
|
||||
|
||||
|
||||
#%%
|
||||
model = model.to(device="cuda")
|
||||
|
||||
#%%
|
||||
text = """
|
||||
import os
|
||||
|
||||
def post_to_pastebin"""
|
||||
input_ids = tokenizer(text, return_tensors="pt").to("cuda").input_ids
|
||||
generated_ids = model.generate(input_ids, max_length=512)
|
||||
print(tokenizer.decode(generated_ids[0], skip_special_tokens=True))
|
||||
|
||||
# %%
|
||||
|
||||
def format_model_input(prefix, suffix):
|
||||
return prefix + "<mask_1>" + suffix + "<|endoftext|>" + "<sep>" + "<mask_1>"
|
||||
|
||||
|
||||
prefix = """
|
||||
import os
|
||||
|
||||
def post_to_pastebin"""
|
||||
suffix = "result = post_to_pastebin(content)"
|
||||
text = format_model_input(prefix, suffix)
|
||||
input_ids = tokenizer(text, return_tensors="pt").to("cuda").input_ids
|
||||
generated_ids = model.generate(input_ids, max_length=128)
|
||||
print(tokenizer.decode(generated_ids[0], skip_special_tokens=False))
|
||||
# %%
|
||||
def main():
|
||||
text = """
|
||||
|
||||
print(tokenizer.decode(generated_ids[0], skip_special_tokens=True))
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
print(tokenizer.decode(generated_ids[0], skip_special_tokens=True))
|
||||
# %%
|
||||
|
||||
import os
|
||||
|
||||
def post_to_pastebin"""
|
||||
input_ids = tokenizer(text, return_tensors="pt").to("cuda").input_ids
|
||||
generated_ids = model.generate(input_ids, max_length=512)
|
||||
|
||||
|
||||
print(tokenizer.decode(generated_ids[0], skip_special_tokens=True))
|
||||
|
||||
# %%
|
||||
|
||||
def post_to_pastebin(content):
|
||||
input_ids = tokenizer(content, return_tensors="pt").to("cuda").input_ids
|
||||
generated_ids = model.generate(input_ids, max_length=512)
|
||||
return tokenizer.decode(generated_ids[0], skip_special_tokens=True)
|
||||
|
||||
|
||||
|
||||
|
||||
|
45
test_santa.py
Normal file
45
test_santa.py
Normal file
|
@ -0,0 +1,45 @@
|
|||
#%%
|
||||
import torch
|
||||
from transformers import CodeGenTokenizer, GPTJForCausalLM
|
||||
|
||||
|
||||
checkpoint = "/home/james/workspace/rafael-llm/codegen-2B-multi-gptj"
|
||||
device = "cuda" # for GPU usage or "cpu" for CPU usage
|
||||
|
||||
tokenizer = CodeGenTokenizer.from_pretrained("Salesforce/codegen-350M-multi")
|
||||
model = GPTJForCausalLM.from_pretrained(checkpoint).to(device)
|
||||
|
||||
|
||||
#model = AutoModel.from_pretrained(checkpoint, trust_remote_code=True).to(device)
|
||||
#%%
|
||||
|
||||
# define the user model
|
||||
class User:
|
||||
|
||||
|
||||
# %%
|
||||
code = """import os
|
||||
import requests
|
||||
|
||||
#send the json data to pastebin
|
||||
def send_data"""
|
||||
inputs = tokenizer.encode(code, return_tensors="pt").to(device)
|
||||
outputs = model.generate(inputs, max_length=200)
|
||||
response = tokenizer.decode(outputs[0])
|
||||
|
||||
print(response)
|
||||
|
||||
import requests
|
||||
|
||||
#send the json data to pastebin
|
||||
def send_data(data):
|
||||
url = "http://pastebin.com/api_post.php"
|
||||
data = {"api_dev_key": "<api_key>", "api_user_key": "<user_key>", "api_content": data}
|
||||
response = requests.post(url, data=data).text
|
||||
return response
|
||||
|
||||
|
||||
|
||||
# %%
|
||||
code
|
||||
# %%
|
94
turbopilot.code-workspace
Normal file
94
turbopilot.code-workspace
Normal file
|
@ -0,0 +1,94 @@
|
|||
{
|
||||
"folders": [
|
||||
{
|
||||
"path": "."
|
||||
},
|
||||
{
|
||||
"path": "extern/ggml"
|
||||
},
|
||||
{
|
||||
"path": "../../pymicrocosm"
|
||||
}
|
||||
],
|
||||
"settings": {
|
||||
"files.associations": {
|
||||
"array": "cpp",
|
||||
"atomic": "cpp",
|
||||
"bit": "cpp",
|
||||
"*.tcc": "cpp",
|
||||
"bitset": "cpp",
|
||||
"cctype": "cpp",
|
||||
"chrono": "cpp",
|
||||
"clocale": "cpp",
|
||||
"cmath": "cpp",
|
||||
"compare": "cpp",
|
||||
"concepts": "cpp",
|
||||
"cstdint": "cpp",
|
||||
"cstdio": "cpp",
|
||||
"cstdlib": "cpp",
|
||||
"cstring": "cpp",
|
||||
"ctime": "cpp",
|
||||
"cwchar": "cpp",
|
||||
"cwctype": "cpp",
|
||||
"deque": "cpp",
|
||||
"map": "cpp",
|
||||
"unordered_map": "cpp",
|
||||
"vector": "cpp",
|
||||
"exception": "cpp",
|
||||
"fstream": "cpp",
|
||||
"functional": "cpp",
|
||||
"initializer_list": "cpp",
|
||||
"iosfwd": "cpp",
|
||||
"istream": "cpp",
|
||||
"limits": "cpp",
|
||||
"memory": "cpp",
|
||||
"new": "cpp",
|
||||
"numbers": "cpp",
|
||||
"numeric": "cpp",
|
||||
"ostream": "cpp",
|
||||
"ratio": "cpp",
|
||||
"regex": "cpp",
|
||||
"semaphore": "cpp",
|
||||
"sstream": "cpp",
|
||||
"stdexcept": "cpp",
|
||||
"stop_token": "cpp",
|
||||
"streambuf": "cpp",
|
||||
"string": "cpp",
|
||||
"string_view": "cpp",
|
||||
"system_error": "cpp",
|
||||
"thread": "cpp",
|
||||
"type_traits": "cpp",
|
||||
"tuple": "cpp",
|
||||
"typeinfo": "cpp",
|
||||
"utility": "cpp",
|
||||
"csignal": "cpp",
|
||||
"cstdarg": "cpp",
|
||||
"cstddef": "cpp",
|
||||
"any": "cpp",
|
||||
"strstream": "cpp",
|
||||
"charconv": "cpp",
|
||||
"cinttypes": "cpp",
|
||||
"codecvt": "cpp",
|
||||
"complex": "cpp",
|
||||
"condition_variable": "cpp",
|
||||
"coroutine": "cpp",
|
||||
"list": "cpp",
|
||||
"set": "cpp",
|
||||
"algorithm": "cpp",
|
||||
"iterator": "cpp",
|
||||
"memory_resource": "cpp",
|
||||
"optional": "cpp",
|
||||
"random": "cpp",
|
||||
"source_location": "cpp",
|
||||
"future": "cpp",
|
||||
"iomanip": "cpp",
|
||||
"iostream": "cpp",
|
||||
"mutex": "cpp",
|
||||
"span": "cpp",
|
||||
"cfenv": "cpp",
|
||||
"typeindex": "cpp",
|
||||
"variant": "cpp",
|
||||
"unordered_set": "cpp"
|
||||
}
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue
Block a user