From ed3b40d6437f9f92c9d2ea2e33df92e23a8d76a1 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Mon, 8 May 2023 14:21:18 +0100
Subject: [PATCH 01/56] add cuda stuff to readme

---
 README.md | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/README.md b/README.md
index 0e3c818..0edb8d2 100644
--- a/README.md
+++ b/README.md
@@ -9,6 +9,9 @@ TurboPilot is a self-hosted [copilot](https://github.com/features/copilot) clone
 
 ![a screen recording of turbopilot running through fauxpilot plugin](assets/vscode-status.gif)
 
+
+**NEW:** As of v0.0.5 turbopilot supports cuda inference which greatly accelerates suggestions when working with longer prompts (i.e. longer existing code files).
+
 ## 🤝 Contributing
 
 PRs to this project and the corresponding [GGML fork](https://github.com/ravenscroftj/ggml) are very welcome.
@@ -81,6 +84,21 @@ docker run --rm -it \
   ghcr.io/ravenscroftj/turbopilot:latest
 ```
 
+#### Docker and CUDA
+
+As of release v0.0.5 turbocode now supports CUDA inference. In order to run the cuda-enabled container you will need to have [nvidia-docker](https://github.com/NVIDIA/nvidia-docker) enabled, use the cuda tagged versions and pass in `--gpus=all` to docker with access to your GPU like so:
+
+```bash
+docker run --gpus=all --rm -it \
+  -v ./models:/models \
+  -e THREADS=6 \
+  -e MODEL="/models/codegen-2B-multi-ggml-4bit-quant.bin" \
+  -p 18080:18080 \
+  ghcr.io/ravenscroftj/turbopilot:v0.0.5-cuda
+```
+
+You should be able to see `/app/codegen-serve` listed when you run `nvidia-smi`.
+
 ### 🌐 Using the API
 
 #### Support for the official Copilot Plugin

From 78c989076f83161d2a100b25a9d4f48c84d85bca Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Mon, 8 May 2023 14:24:49 +0100
Subject: [PATCH 02/56] update batch size arg in run.sh

---
 Dockerfile.cuda    | 2 ++
 Dockerfile.default | 2 ++
 run.sh             | 2 +-
 3 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/Dockerfile.cuda b/Dockerfile.cuda
index ac72a88..89ef32d 100644
--- a/Dockerfile.cuda
+++ b/Dockerfile.cuda
@@ -22,6 +22,8 @@ ENV THREADS=4
 
 ENV MODEL="/models/codegen-2B-multi-ggml-4bit-quant.bin"
 
+ENV BATCHSIZE=64
+
 COPY ./run.sh /app/
 
 EXPOSE 18080
diff --git a/Dockerfile.default b/Dockerfile.default
index bad770b..d6db39c 100644
--- a/Dockerfile.default
+++ b/Dockerfile.default
@@ -22,6 +22,8 @@ ENV THREADS=4
 
 ENV MODEL="/models/codegen-2B-multi-ggml-4bit-quant.bin"
 
+ENV BATCHSIZE=8
+
 COPY ./run.sh /app/
 
 EXPOSE 18080
diff --git a/run.sh b/run.sh
index 8db96c8..ef1339f 100755
--- a/run.sh
+++ b/run.sh
@@ -1,3 +1,3 @@
 #!/bin/sh
 
-/app/codegen-serve -t $THREADS -m $MODEL
\ No newline at end of file
+/app/codegen-serve -t $THREADS -m $MODEL -b $BATCHSIZE
\ No newline at end of file

From 75547aa4f9d54694e9f635b073462f791cf14deb Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Mon, 8 May 2023 14:27:11 +0100
Subject: [PATCH 03/56] run actions on branches

---
 .github/workflows/build-commit.yml | 2 +-
 .github/workflows/docker-image.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-commit.yml b/.github/workflows/build-commit.yml
index af5f90d..db1d661 100644
--- a/.github/workflows/build-commit.yml
+++ b/.github/workflows/build-commit.yml
@@ -2,7 +2,7 @@ name: Build on Commit
 
 on:
   push:
-    branches: [ "main" ]
+    branches: [ "*" ]
     tags: ['*']
   pull_request:
     branches: [ "main" ]
diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml
index 9f2351b..37ae3d0 100644
--- a/.github/workflows/docker-image.yml
+++ b/.github/workflows/docker-image.yml
@@ -2,7 +2,7 @@ name: Docker Image CI
 
 on:
   push:
-    branches: [ "main" ]
+    branches: [ "*" ]
     tags: ['*']
   pull_request:
     branches: [ "main" ]

From 6358835d986061d6540607ddd083e931ff51bac8 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Mon, 8 May 2023 14:28:10 +0100
Subject: [PATCH 04/56] singlequotes

---
 .github/workflows/build-commit.yml | 2 +-
 .github/workflows/docker-image.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-commit.yml b/.github/workflows/build-commit.yml
index db1d661..808b99e 100644
--- a/.github/workflows/build-commit.yml
+++ b/.github/workflows/build-commit.yml
@@ -2,7 +2,7 @@ name: Build on Commit
 
 on:
   push:
-    branches: [ "*" ]
+    branches: [ '*' ]
     tags: ['*']
   pull_request:
     branches: [ "main" ]
diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml
index 37ae3d0..a0df837 100644
--- a/.github/workflows/docker-image.yml
+++ b/.github/workflows/docker-image.yml
@@ -2,7 +2,7 @@ name: Docker Image CI
 
 on:
   push:
-    branches: [ "*" ]
+    branches: [ '*' ]
     tags: ['*']
   pull_request:
     branches: [ "main" ]

From 161884f495977cf55f7fa53afd7650959538d267 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Mon, 8 May 2023 14:53:36 +0100
Subject: [PATCH 05/56] temp disable avx stuff

---
 .github/workflows/build-commit.yml | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/build-commit.yml b/.github/workflows/build-commit.yml
index 808b99e..40490ad 100644
--- a/.github/workflows/build-commit.yml
+++ b/.github/workflows/build-commit.yml
@@ -75,15 +75,15 @@ jobs:
     # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix
     runs-on: ubuntu-latest
 
-    strategy:
-          matrix:
-            include:
-            - build: 'avx2'
-              defines: ''
-            - build: 'avx'
-              defines: '-DLLAMA_AVX2=OFF'
-            - build: 'avx512'
-              defines: '-DLLAMA_AVX512=ON'
+    # strategy:
+    #       matrix:
+    #         include:
+    #         - build: 'avx2'
+    #           defines: ''
+    #         - build: 'avx'
+    #           defines: '-DLLAMA_AVX2=OFF'
+    #         - build: 'avx512'
+    #           defines: '-DLLAMA_AVX512=ON'
 
 
     steps:

From 79b2f5aae9fec71cb33135595e51a099b47ddf4c Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Mon, 8 May 2023 14:55:33 +0100
Subject: [PATCH 06/56] add builds for all tags and branches

---
 .github/workflows/build-commit.yml | 4 ++--
 .github/workflows/docker-image.yml | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build-commit.yml b/.github/workflows/build-commit.yml
index 40490ad..6feb93c 100644
--- a/.github/workflows/build-commit.yml
+++ b/.github/workflows/build-commit.yml
@@ -2,8 +2,8 @@ name: Build on Commit
 
 on:
   push:
-    branches: [ '*' ]
-    tags: ['*']
+    branches: [ '**' ]
+    tags: ['**']
   pull_request:
     branches: [ "main" ]
 
diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml
index a0df837..d2e4d5d 100644
--- a/.github/workflows/docker-image.yml
+++ b/.github/workflows/docker-image.yml
@@ -2,7 +2,7 @@ name: Docker Image CI
 
 on:
   push:
-    branches: [ '*' ]
+    branches: [ '**' ]
     tags: ['*']
   pull_request:
     branches: [ "main" ]

From 3b7770dbce165b9f93b651ed24dbcffbd571d126 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Sun, 14 May 2023 09:18:40 +0100
Subject: [PATCH 07/56] add cuda to dependencies

---
 .github/workflows/build-cuda.yml | 84 ++++++++++++++++++++++++++++++++
 1 file changed, 84 insertions(+)
 create mode 100644 .github/workflows/build-cuda.yml

diff --git a/.github/workflows/build-cuda.yml b/.github/workflows/build-cuda.yml
new file mode 100644
index 0000000..f815bda
--- /dev/null
+++ b/.github/workflows/build-cuda.yml
@@ -0,0 +1,84 @@
+name: Build CUDA binary on Ubuntu
+
+on:
+  push:
+    branches: [ '**' ]
+    tags: ['**']
+  pull_request:
+    branches: [ "main" ]
+
+env:
+  # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
+  BUILD_TYPE: Release
+
+jobs:
+
+  build-ubuntu:
+    # The CMake configure and build commands are platform agnostic and should work equally well on Windows or Mac.
+    # You can convert this to a matrix build if you need cross-platform coverage.
+    # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix
+    runs-on: ubuntu-latest
+
+    # strategy:
+    #       matrix:
+    #         include:
+    #         - build: 'avx2'
+    #           defines: ''
+    #         - build: 'avx'
+    #           defines: '-DLLAMA_AVX2=OFF'
+    #         - build: 'avx512'
+    #           defines: '-DLLAMA_AVX512=ON'
+
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: true
+
+    - name: Install Dependencies
+      run: |
+          wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.0-1_all.deb
+          sudo dpkg -i cuda-keyring_1.0-1_all.deb
+          sudo apt-get update && sudo apt-get install -yq libboost-dev cuda
+
+    - name: Build
+      # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
+      # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
+      run: |
+        cd ${{github.workspace}}/ggml
+        cmake -B ${{github.workspace}}/ggml/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -D CMAKE_EXE_LINKER_FLAGS="-static"
+        cd ${{github.workspace}}/ggml/build 
+        make codegen codegen-serve codegen-quantize
+        chmod +x ${{github.workspace}}/ggml/build/bin/codegen 
+        chmod +x ${{github.workspace}}/ggml/build/bin/codegen-serve 
+        chmod +x ${{github.workspace}}/ggml/build/bin/codegen-quantize
+        
+        
+    - uses: benjlevesque/short-sha@v2.2
+      id: short-sha
+      with:
+        length: 6
+
+
+    - name: Upload Build Artifacts
+      uses: actions/upload-artifact@v3.1.2
+      with:
+        # Artifact name
+        name: turbopilot-${{ runner.os }}-${{ runner.arch }}-${{ steps.short-sha.outputs.sha }} # optional, default is artifact
+        # A file, directory or wildcard pattern that describes what to upload
+        path: ${{github.workspace}}/ggml/build/bin/codegen*
+        # The desired behavior if no files are found using the provided path.
+        
+    - name: package artifacts for release
+      if: startsWith(github.ref, 'refs/tags/')
+
+      run: |
+        cd ${{github.workspace}}/ggml/build/bin 
+        zip turbopilot-${{ runner.os }}-${{ runner.arch }}.zip ./codegen*
+
+    - name: Upload binaries to release
+      uses: softprops/action-gh-release@v1
+      if: startsWith(github.ref, 'refs/tags/')
+      with:
+        token: ${{ secrets.PUBLISH_TOKEN }}
+        files: ${{github.workspace}}/ggml/build/bin/turbopilot-${{ runner.os }}-${{ runner.arch }}.zip

From cb3fa20f1387bbdf67b8704eef1ee370cae5c332 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Sun, 14 May 2023 09:24:21 +0100
Subject: [PATCH 08/56] try to be more targetted about cuda deps

---
 .github/workflows/build-cuda.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-cuda.yml b/.github/workflows/build-cuda.yml
index f815bda..0572149 100644
--- a/.github/workflows/build-cuda.yml
+++ b/.github/workflows/build-cuda.yml
@@ -39,7 +39,7 @@ jobs:
       run: |
           wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.0-1_all.deb
           sudo dpkg -i cuda-keyring_1.0-1_all.deb
-          sudo apt-get update && sudo apt-get install -yq libboost-dev cuda
+          sudo apt-get update && sudo apt-get install -yq libboost-dev cuda-nvcc-11-7
 
     - name: Build
       # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.

From ecbfed4fc5b86297a1a795403efd7c39cadeb017 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Sun, 14 May 2023 09:29:57 +0100
Subject: [PATCH 09/56] add cuda to cmake args

---
 .github/workflows/build-cuda.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-cuda.yml b/.github/workflows/build-cuda.yml
index 0572149..8117ecc 100644
--- a/.github/workflows/build-cuda.yml
+++ b/.github/workflows/build-cuda.yml
@@ -46,7 +46,7 @@ jobs:
       # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
       run: |
         cd ${{github.workspace}}/ggml
-        cmake -B ${{github.workspace}}/ggml/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -D CMAKE_EXE_LINKER_FLAGS="-static"
+        cmake -B ${{github.workspace}}/ggml/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -D CMAKE_EXE_LINKER_FLAGS="-static" -DGGML_CUBLAS=ON 
         cd ${{github.workspace}}/ggml/build 
         make codegen codegen-serve codegen-quantize
         chmod +x ${{github.workspace}}/ggml/build/bin/codegen 
@@ -64,7 +64,7 @@ jobs:
       uses: actions/upload-artifact@v3.1.2
       with:
         # Artifact name
-        name: turbopilot-${{ runner.os }}-${{ runner.arch }}-${{ steps.short-sha.outputs.sha }} # optional, default is artifact
+        name: turbopilot-${{ runner.os }}-${{ runner.arch }}-${{ steps.short-sha.outputs.sha }}-cuda # optional, default is artifact
         # A file, directory or wildcard pattern that describes what to upload
         path: ${{github.workspace}}/ggml/build/bin/codegen*
         # The desired behavior if no files are found using the provided path.

From 20ca21a5396a2cc0ba31c6c7b7f848a707f627fc Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Sun, 14 May 2023 09:32:31 +0100
Subject: [PATCH 10/56] add nvcc location to cmake args

---
 .github/workflows/build-cuda.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-cuda.yml b/.github/workflows/build-cuda.yml
index 8117ecc..658adbb 100644
--- a/.github/workflows/build-cuda.yml
+++ b/.github/workflows/build-cuda.yml
@@ -46,7 +46,7 @@ jobs:
       # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
       run: |
         cd ${{github.workspace}}/ggml
-        cmake -B ${{github.workspace}}/ggml/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -D CMAKE_EXE_LINKER_FLAGS="-static" -DGGML_CUBLAS=ON 
+        cmake -B ${{github.workspace}}/ggml/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -D CMAKE_EXE_LINKER_FLAGS="-static" -DGGML_CUBLAS=ON  -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc
         cd ${{github.workspace}}/ggml/build 
         make codegen codegen-serve codegen-quantize
         chmod +x ${{github.workspace}}/ggml/build/bin/codegen 

From c355763419f72c0e86c9ea613aafb7067708edf7 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Sun, 14 May 2023 09:34:26 +0100
Subject: [PATCH 11/56] add  libcublas to deps

---
 .github/workflows/build-cuda.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-cuda.yml b/.github/workflows/build-cuda.yml
index 658adbb..730a900 100644
--- a/.github/workflows/build-cuda.yml
+++ b/.github/workflows/build-cuda.yml
@@ -39,7 +39,7 @@ jobs:
       run: |
           wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.0-1_all.deb
           sudo dpkg -i cuda-keyring_1.0-1_all.deb
-          sudo apt-get update && sudo apt-get install -yq libboost-dev cuda-nvcc-11-7
+          sudo apt-get update && sudo apt-get install -yq libboost-dev cuda-nvcc-11-7 libcublas-dev-11-7
 
     - name: Build
       # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.

From 0c737550559119e0891d343613ae63a8f373fe03 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Sun, 14 May 2023 10:04:22 +0100
Subject: [PATCH 12/56] turn on static properly

---
 .github/workflows/build-cuda.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-cuda.yml b/.github/workflows/build-cuda.yml
index 730a900..f46229a 100644
--- a/.github/workflows/build-cuda.yml
+++ b/.github/workflows/build-cuda.yml
@@ -46,7 +46,7 @@ jobs:
       # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
       run: |
         cd ${{github.workspace}}/ggml
-        cmake -B ${{github.workspace}}/ggml/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -D CMAKE_EXE_LINKER_FLAGS="-static" -DGGML_CUBLAS=ON  -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc
+        cmake -B ${{github.workspace}}/ggml/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -D GGML_STATIC=ON -DGGML_CUBLAS=ON  -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc
         cd ${{github.workspace}}/ggml/build 
         make codegen codegen-serve codegen-quantize
         chmod +x ${{github.workspace}}/ggml/build/bin/codegen 

From 222641c959eff8fb8a7dbc0fb548d9dc0f73589d Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Sun, 14 May 2023 10:10:33 +0100
Subject: [PATCH 13/56] turn on static properly and retain static flag

---
 .github/workflows/build-cuda.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-cuda.yml b/.github/workflows/build-cuda.yml
index f46229a..0ed7cc4 100644
--- a/.github/workflows/build-cuda.yml
+++ b/.github/workflows/build-cuda.yml
@@ -46,7 +46,7 @@ jobs:
       # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
       run: |
         cd ${{github.workspace}}/ggml
-        cmake -B ${{github.workspace}}/ggml/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -D GGML_STATIC=ON -DGGML_CUBLAS=ON  -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc
+        cmake -B ${{github.workspace}}/ggml/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -D CMAKE_EXE_LINKER_FLAGS="-static" -DGGML_STATIC=ON -DGGML_CUBLAS=ON  -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc
         cd ${{github.workspace}}/ggml/build 
         make codegen codegen-serve codegen-quantize
         chmod +x ${{github.workspace}}/ggml/build/bin/codegen 

From e424931e2e6f306e4f6bc2e71c5346e458ee635f Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Tue, 16 May 2023 20:16:47 +0100
Subject: [PATCH 14/56] turn off static linking in cuda builds

---
 .github/workflows/build-cuda.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-cuda.yml b/.github/workflows/build-cuda.yml
index 0ed7cc4..29784c3 100644
--- a/.github/workflows/build-cuda.yml
+++ b/.github/workflows/build-cuda.yml
@@ -46,7 +46,7 @@ jobs:
       # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
       run: |
         cd ${{github.workspace}}/ggml
-        cmake -B ${{github.workspace}}/ggml/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -D CMAKE_EXE_LINKER_FLAGS="-static" -DGGML_STATIC=ON -DGGML_CUBLAS=ON  -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc
+        cmake -B ${{github.workspace}}/ggml/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DGGML_CUBLAS=ON  -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc
         cd ${{github.workspace}}/ggml/build 
         make codegen codegen-serve codegen-quantize
         chmod +x ${{github.workspace}}/ggml/build/bin/codegen 

From ec1d897a00240b2f6a54b2750875848c4b9032b8 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Tue, 16 May 2023 20:23:57 +0100
Subject: [PATCH 15/56] fix name of build artifact for release

---
 .github/workflows/build-cuda.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-cuda.yml b/.github/workflows/build-cuda.yml
index 29784c3..ccb301c 100644
--- a/.github/workflows/build-cuda.yml
+++ b/.github/workflows/build-cuda.yml
@@ -81,4 +81,4 @@ jobs:
       if: startsWith(github.ref, 'refs/tags/')
       with:
         token: ${{ secrets.PUBLISH_TOKEN }}
-        files: ${{github.workspace}}/ggml/build/bin/turbopilot-${{ runner.os }}-${{ runner.arch }}.zip
+        files: ${{github.workspace}}/ggml/build/bin/turbopilot-${{ runner.os }}-${{ runner.arch }}-cuda.zip

From d3f92176ca80f1e396ade4272141eab668326a7c Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Tue, 16 May 2023 20:27:16 +0100
Subject: [PATCH 16/56] add cuda stuff to readme

---
 README.md | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 0edb8d2..263d58d 100644
--- a/README.md
+++ b/README.md
@@ -97,7 +97,12 @@ docker run --gpus=all --rm -it \
   ghcr.io/ravenscroftj/turbopilot:v0.0.5-cuda
 ```
 
-You should be able to see `/app/codegen-serve` listed when you run `nvidia-smi`.
+You will need CUDA 11 or later to run this container. You should be able to see `/app/codegen-serve` listed when you run `nvidia-smi`.
+
+
+#### Executable and CUDA
+
+As of v0.0.5 a CUDA version of the linux executable is available - it requires that libcublas 11 be installed on the machine - I might build ubuntu debs at some point but for now running in docker may be more convenient if you want to use a CUDA GPU.
 
 ### 🌐 Using the API
 

From dd29522184f3f1949f74fd99c47ebe58875821ad Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Sun, 28 May 2023 15:34:50 +0100
Subject: [PATCH 17/56] test windows build

---
 .github/workflows/build-commit.yml | 98 ++++++++++++++++++++++++++++++
 1 file changed, 98 insertions(+)

diff --git a/.github/workflows/build-commit.yml b/.github/workflows/build-commit.yml
index 6feb93c..10f7846 100644
--- a/.github/workflows/build-commit.yml
+++ b/.github/workflows/build-commit.yml
@@ -135,3 +135,101 @@ jobs:
       with:
         token: ${{ secrets.PUBLISH_TOKEN }}
         files: ${{github.workspace}}/ggml/build/bin/turbopilot-${{ runner.os }}-${{ runner.arch }}.zip
+
+  windows-latest-cmake:
+    runs-on: windows-latest
+    env:
+      OPENBLAS_VERSION: 0.3.23
+      OPENCL_VERSION: 2023.04.17
+      CLBLAST_VERSION: 1.6.0
+    
+    strategy:
+      matrix:
+        include:
+          - build: 'avx2'
+            defines: ''
+          - build: 'avx'
+            defines: '-GGML_AVX2=OFF'
+          - build: 'avx512'
+            defines: '-GGML_AVX512=ON -DBUILD_SHARED_LIBS=ON'
+          - build: 'openblas'
+            defines: '-GGML_OPENBLAS=ON -DBLAS_INCLUDE_DIRS="$env:RUNNER_TEMP/openblas/include" -DBLAS_LIBRARIES="$env:RUNNER_TEMP/openblas/lib/openblas.lib"'
+
+
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+
+      - name: Download OpenBLAS
+        id: get_openblas
+        if: ${{ matrix.build == 'openblas' }}
+        run: |
+          curl.exe -o $env:RUNNER_TEMP/openblas.zip -L "https://github.com/xianyi/OpenBLAS/releases/download/v${env:OPENBLAS_VERSION}/OpenBLAS-${env:OPENBLAS_VERSION}-x64.zip"
+          curl.exe -o $env:RUNNER_TEMP/OpenBLAS.LICENSE.txt -L "https://github.com/xianyi/OpenBLAS/raw/v${env:OPENBLAS_VERSION}/LICENSE"
+          mkdir $env:RUNNER_TEMP/openblas
+          tar.exe -xvf $env:RUNNER_TEMP/openblas.zip -C $env:RUNNER_TEMP/openblas
+          $vcdir = $(vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath)
+          $msvc = $(join-path $vcdir $('VC\Tools\MSVC\'+$(gc -raw $(join-path $vcdir 'VC\Auxiliary\Build\Microsoft.VCToolsVersion.default.txt')).Trim()))
+          $lib =  $(join-path $msvc 'bin\Hostx64\x64\lib.exe')
+          & $lib /machine:x64 "/def:${env:RUNNER_TEMP}/openblas/lib/libopenblas.def" "/out:${env:RUNNER_TEMP}/openblas/lib/openblas.lib" /name:openblas.dll
+
+      - name: Build
+        id: cmake_build
+        run: |
+          cd ${{github.workspace}}/ggml
+          mkdir build
+          cd build
+          cmake .. ${{ matrix.defines }}
+          cmake --build . --config Release
+
+      - name: Add libopenblas.dll
+        id: add_libopenblas_dll
+        if: ${{ matrix.build == 'openblas' }}
+        run: |
+          cp $env:RUNNER_TEMP/openblas/bin/libopenblas.dll ./build/bin/Release/openblas.dll
+          cp $env:RUNNER_TEMP/OpenBLAS.LICENSE.txt ./build/bin/Release/OpenBLAS-${env:OPENBLAS_VERSION}.txt
+
+      - name: Check AVX512F support
+        id: check_avx512f
+        if: ${{ matrix.build == 'avx512' }}
+        continue-on-error: true
+        run: |
+          cd build
+          $vcdir = $(vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath)
+          $msvc = $(join-path $vcdir $('VC\Tools\MSVC\'+$(gc -raw $(join-path $vcdir 'VC\Auxiliary\Build\Microsoft.VCToolsVersion.default.txt')).Trim()))
+          $cl =  $(join-path $msvc 'bin\Hostx64\x64\cl.exe')
+          echo 'int main(void){unsigned int a[4];__cpuid(a,7);return !(a[1]&65536);}' >> avx512f.c
+          & $cl /O2 /GS- /kernel avx512f.c /link /nodefaultlib /entry:main
+          .\avx512f.exe && echo "AVX512F: YES" && ( echo HAS_AVX512F=1 >> $env:GITHUB_ENV ) || echo "AVX512F: NO"
+
+          - uses: benjlevesque/short-sha@v2.2
+          id: short-sha
+          with:
+            length: 6
+    
+  
+      - name: Upload Build Artifacts
+        uses: actions/upload-artifact@v3.1.2
+        with:
+          # Artifact name
+          name: turbopilot-${{ runner.os }}-${{ runner.arch }}-${{ steps.short-sha.outputs.sha }}-${{ matrix.build }} # optional, default is artifact
+          # A file, directory or wildcard pattern that describes what to upload
+          path: ${{github.workspace}}/ggml/build/bin/codegen*
+          # The desired behavior if no files are found using the provided path.
+          
+      - name: package artifacts for release
+        if: startsWith(github.ref, 'refs/tags/')
+  
+        run: |
+          cd ${{github.workspace}}/ggml/build/bin 
+          7z a turbopilot-${{ runner.os }}-${{ runner.arch }}.zip ./codegen*
+  
+      - name: Upload binaries to release
+        uses: softprops/action-gh-release@v1
+        if: startsWith(github.ref, 'refs/tags/')
+        with:
+          token: ${{ secrets.PUBLISH_TOKEN }}
+          files: ${{github.workspace}}/ggml/build/bin/turbopilot-${{ runner.os }}-${{ runner.arch }}.zip
\ No newline at end of file

From ecfd61d213893a95390660a99f479920f4005aa7 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Sun, 28 May 2023 15:36:23 +0100
Subject: [PATCH 18/56] correct matrix defines

---
 .github/workflows/build-commit.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build-commit.yml b/.github/workflows/build-commit.yml
index 10f7846..9d0eb3f 100644
--- a/.github/workflows/build-commit.yml
+++ b/.github/workflows/build-commit.yml
@@ -149,11 +149,11 @@ jobs:
           - build: 'avx2'
             defines: ''
           - build: 'avx'
-            defines: '-GGML_AVX2=OFF'
+            defines: '-DGGML_AVX2=OFF'
           - build: 'avx512'
-            defines: '-GGML_AVX512=ON -DBUILD_SHARED_LIBS=ON'
+            defines: '-DGGML_AVX512=ON -DBUILD_SHARED_LIBS=ON'
           - build: 'openblas'
-            defines: '-GGML_OPENBLAS=ON -DBLAS_INCLUDE_DIRS="$env:RUNNER_TEMP/openblas/include" -DBLAS_LIBRARIES="$env:RUNNER_TEMP/openblas/lib/openblas.lib"'
+            defines: '-DGGML_OPENBLAS=ON -DBLAS_INCLUDE_DIRS="$env:RUNNER_TEMP/openblas/include" -DBLAS_LIBRARIES="$env:RUNNER_TEMP/openblas/lib/openblas.lib"'
 
 
     steps:

From 32ec80c628cc1e7182716552342caaff27d51394 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Sun, 28 May 2023 15:40:38 +0100
Subject: [PATCH 19/56] add step to install boot lib

---
 .github/workflows/build-commit.yml | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/.github/workflows/build-commit.yml b/.github/workflows/build-commit.yml
index 9d0eb3f..f310218 100644
--- a/.github/workflows/build-commit.yml
+++ b/.github/workflows/build-commit.yml
@@ -163,6 +163,24 @@ jobs:
         with:
           submodules: true
 
+      - name: Install boost
+        uses: MarkusJx/install-boost@v2.4.1
+        id: install-boost
+        with:
+            # REQUIRED: Specify the required boost version
+            # A list of supported versions can be found here:
+            # https://github.com/MarkusJx/prebuilt-boost/blob/main/versions-manifest.json
+            boost_version: 1.73.0
+            # OPTIONAL: Specify a custon install location
+            boost_install_dir: C:\boost
+            # OPTIONAL: Specify a platform version
+            platform_version: 2019
+            # OPTIONAL: Specify a toolset
+            toolset: msvc
+      
+            # NOTE: If a boost version matching all requirements cannot be found,
+            # this build step will fail
+            
       - name: Download OpenBLAS
         id: get_openblas
         if: ${{ matrix.build == 'openblas' }}
@@ -178,6 +196,8 @@ jobs:
 
       - name: Build
         id: cmake_build
+        env:
+          BOOST_ROOT: ${{ steps.install-boost.outputs.BOOST_ROOT }}
         run: |
           cd ${{github.workspace}}/ggml
           mkdir build

From d1469d1dbb22d60f3d6b713a2446be6b0241c2b3 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Sun, 28 May 2023 15:41:50 +0100
Subject: [PATCH 20/56] use a more recent boost lib

---
 .github/workflows/build-commit.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-commit.yml b/.github/workflows/build-commit.yml
index f310218..45be8ff 100644
--- a/.github/workflows/build-commit.yml
+++ b/.github/workflows/build-commit.yml
@@ -170,11 +170,11 @@ jobs:
             # REQUIRED: Specify the required boost version
             # A list of supported versions can be found here:
             # https://github.com/MarkusJx/prebuilt-boost/blob/main/versions-manifest.json
-            boost_version: 1.73.0
+            boost_version: 1.78.0
             # OPTIONAL: Specify a custon install location
             boost_install_dir: C:\boost
             # OPTIONAL: Specify a platform version
-            platform_version: 2019
+            platform_version: 2022
             # OPTIONAL: Specify a toolset
             toolset: msvc
       

From a15f08d36680095fa9e77b19db0cdb8bcea4638d Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Sun, 28 May 2023 15:53:34 +0100
Subject: [PATCH 21/56] be more specific about what to build on windows

---
 .github/workflows/build-commit.yml | 30 +++++++++++++++++++-----------
 1 file changed, 19 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/build-commit.yml b/.github/workflows/build-commit.yml
index 45be8ff..ac3df52 100644
--- a/.github/workflows/build-commit.yml
+++ b/.github/workflows/build-commit.yml
@@ -74,16 +74,18 @@ jobs:
     # You can convert this to a matrix build if you need cross-platform coverage.
     # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - build: 'avx2'
+            defines: ''
+          - build: 'avx'
+            defines: '-DGGML_AVX2=OFF'
+          - build: 'avx512'
+            defines: '-DGGML_AVX512=ON -DBUILD_SHARED_LIBS=ON'
+          - build: 'openblas'
+            defines: '-DGGML_OPENBLAS=ON -DBLAS_INCLUDE_DIRS="$env:RUNNER_TEMP/openblas/include" -DBLAS_LIBRARIES="$env:RUNNER_TEMP/openblas/lib/openblas.lib"'
 
-    # strategy:
-    #       matrix:
-    #         include:
-    #         - build: 'avx2'
-    #           defines: ''
-    #         - build: 'avx'
-    #           defines: '-DLLAMA_AVX2=OFF'
-    #         - build: 'avx512'
-    #           defines: '-DLLAMA_AVX512=ON'
 
 
     steps:
@@ -94,12 +96,16 @@ jobs:
     - name: Install Dependencies
       run: sudo apt-get update && sudo apt-get install -yq libboost-dev
 
+    - name: Install OpenBlas
+      if: ${{ matrix.build == 'openblas' }}
+      run: sudo apt-get install libopenblas-dev
+
     - name: Build
       # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
       # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
       run: |
         cd ${{github.workspace}}/ggml
-        cmake -B ${{github.workspace}}/ggml/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -D CMAKE_EXE_LINKER_FLAGS="-static"
+        cmake -B ${{github.workspace}}/ggml/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -D CMAKE_EXE_LINKER_FLAGS="-static" ${{ matrix.defines }}
         cd ${{github.workspace}}/ggml/build 
         make codegen codegen-serve codegen-quantize
         chmod +x ${{github.workspace}}/ggml/build/bin/codegen 
@@ -203,7 +209,9 @@ jobs:
           mkdir build
           cd build
           cmake .. ${{ matrix.defines }}
-          cmake --build . --config Release
+          cmake --build . --config Release -t codegen
+          cmake --build . --config Release -t codegen-serve
+          cmake --build . --config Release -t codegen-quantize
 
       - name: Add libopenblas.dll
         id: add_libopenblas_dll

From 1d5c0658611fb3461808c91ead75e8f42852a31e Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Sun, 28 May 2023 15:55:10 +0100
Subject: [PATCH 22/56] be a bit more selective about ubuntu build stuff too

---
 .github/workflows/build-commit.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build-commit.yml b/.github/workflows/build-commit.yml
index ac3df52..9381f22 100644
--- a/.github/workflows/build-commit.yml
+++ b/.github/workflows/build-commit.yml
@@ -123,7 +123,7 @@ jobs:
       uses: actions/upload-artifact@v3.1.2
       with:
         # Artifact name
-        name: turbopilot-${{ runner.os }}-${{ runner.arch }}-${{ steps.short-sha.outputs.sha }} # optional, default is artifact
+        name: turbopilot-${{ runner.os }}-${{ runner.arch }}-${{ matrix.build }}-${{ steps.short-sha.outputs.sha }} # optional, default is artifact
         # A file, directory or wildcard pattern that describes what to upload
         path: ${{github.workspace}}/ggml/build/bin/codegen*
         # The desired behavior if no files are found using the provided path.
@@ -133,14 +133,14 @@ jobs:
 
       run: |
         cd ${{github.workspace}}/ggml/build/bin 
-        zip turbopilot-${{ runner.os }}-${{ runner.arch }}.zip ./codegen*
+        zip turbopilot-${{ runner.os }}-${{ runner.arch }}-${{ matrix.build }}.zip ./codegen*
 
     - name: Upload binaries to release
       uses: softprops/action-gh-release@v1
       if: startsWith(github.ref, 'refs/tags/')
       with:
         token: ${{ secrets.PUBLISH_TOKEN }}
-        files: ${{github.workspace}}/ggml/build/bin/turbopilot-${{ runner.os }}-${{ runner.arch }}.zip
+        files: ${{github.workspace}}/ggml/build/bin/turbopilot-${{ runner.os }}-${{ runner.arch }}-${{ matrix.build }}.zip
 
   windows-latest-cmake:
     runs-on: windows-latest

From d936cdd28ad8fd2059c715b92bc6c04d28eb6335 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Sun, 28 May 2023 15:57:11 +0100
Subject: [PATCH 23/56] attempt to fix

---
 .github/workflows/build-commit.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-commit.yml b/.github/workflows/build-commit.yml
index 9381f22..d98a970 100644
--- a/.github/workflows/build-commit.yml
+++ b/.github/workflows/build-commit.yml
@@ -82,7 +82,7 @@ jobs:
           - build: 'avx'
             defines: '-DGGML_AVX2=OFF'
           - build: 'avx512'
-            defines: '-DGGML_AVX512=ON -DBUILD_SHARED_LIBS=ON'
+            defines: '-DGGML_AVX512=ON'
           - build: 'openblas'
             defines: '-DGGML_OPENBLAS=ON -DBLAS_INCLUDE_DIRS="$env:RUNNER_TEMP/openblas/include" -DBLAS_LIBRARIES="$env:RUNNER_TEMP/openblas/lib/openblas.lib"'
 

From ca86e017c9d489f6476c68580ab063e1cd88cb26 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Sun, 28 May 2023 15:58:44 +0100
Subject: [PATCH 24/56] remove windows paths for ubuntu matrix build

---
 .github/workflows/build-commit.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-commit.yml b/.github/workflows/build-commit.yml
index d98a970..3ca62cb 100644
--- a/.github/workflows/build-commit.yml
+++ b/.github/workflows/build-commit.yml
@@ -84,7 +84,7 @@ jobs:
           - build: 'avx512'
             defines: '-DGGML_AVX512=ON'
           - build: 'openblas'
-            defines: '-DGGML_OPENBLAS=ON -DBLAS_INCLUDE_DIRS="$env:RUNNER_TEMP/openblas/include" -DBLAS_LIBRARIES="$env:RUNNER_TEMP/openblas/lib/openblas.lib"'
+            defines: '-DGGML_OPENBLAS=ON'
 
 
 

From e81221ffaf3b3c099fa4bdd6338549280c2ea486 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Sun, 28 May 2023 16:13:00 +0100
Subject: [PATCH 25/56] turn off dynamic lib ggml in windows avx512

---
 .github/workflows/build-commit.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-commit.yml b/.github/workflows/build-commit.yml
index 3ca62cb..67d8c3d 100644
--- a/.github/workflows/build-commit.yml
+++ b/.github/workflows/build-commit.yml
@@ -157,7 +157,7 @@ jobs:
           - build: 'avx'
             defines: '-DGGML_AVX2=OFF'
           - build: 'avx512'
-            defines: '-DGGML_AVX512=ON -DBUILD_SHARED_LIBS=ON'
+            defines: '-DGGML_AVX512=ON'
           - build: 'openblas'
             defines: '-DGGML_OPENBLAS=ON -DBLAS_INCLUDE_DIRS="$env:RUNNER_TEMP/openblas/include" -DBLAS_LIBRARIES="$env:RUNNER_TEMP/openblas/lib/openblas.lib"'
 

From f6256d67b359a0ec69220e52cba48e47d328fed9 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Sun, 28 May 2023 16:15:02 +0100
Subject: [PATCH 26/56] replace linker flag option with proper GGML_STATIC

---
 .github/workflows/build-commit.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-commit.yml b/.github/workflows/build-commit.yml
index 67d8c3d..0e0a298 100644
--- a/.github/workflows/build-commit.yml
+++ b/.github/workflows/build-commit.yml
@@ -105,7 +105,7 @@ jobs:
       # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
       run: |
         cd ${{github.workspace}}/ggml
-        cmake -B ${{github.workspace}}/ggml/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -D CMAKE_EXE_LINKER_FLAGS="-static" ${{ matrix.defines }}
+        cmake -B ${{github.workspace}}/ggml/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -D GGML_STATIC=On ${{ matrix.defines }}
         cd ${{github.workspace}}/ggml/build 
         make codegen codegen-serve codegen-quantize
         chmod +x ${{github.workspace}}/ggml/build/bin/codegen 

From da44c705d43b645b1e0354257197cd73236123d0 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Sun, 28 May 2023 16:48:09 +0100
Subject: [PATCH 27/56] update ggml - build stuff

---
 ggml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ggml b/ggml
index d3b152b..4b01526 160000
--- a/ggml
+++ b/ggml
@@ -1 +1 @@
-Subproject commit d3b152bf2deeebfe83c56701e9719502e2b331b2
+Subproject commit 4b01526aa55ac080310a555ebb2422680683b787

From e7e77bbc5251a2c82c846b21dad2702113400fb6 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Sun, 28 May 2023 17:29:51 +0100
Subject: [PATCH 28/56] add blas include dirs option which is empty by default

---
 ggml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ggml b/ggml
index 4b01526..9d98c51 160000
--- a/ggml
+++ b/ggml
@@ -1 +1 @@
-Subproject commit 4b01526aa55ac080310a555ebb2422680683b787
+Subproject commit 9d98c51aae9029d7246ea28dfd39b4ba918ae038

From d33e1dc7b6c8273655e19bb6288ec78af63ea7b7 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Sun, 28 May 2023 17:30:43 +0100
Subject: [PATCH 29/56] remove superfluous env var

---
 .github/workflows/build-commit.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-commit.yml b/.github/workflows/build-commit.yml
index 0e0a298..acb362b 100644
--- a/.github/workflows/build-commit.yml
+++ b/.github/workflows/build-commit.yml
@@ -199,7 +199,7 @@ jobs:
           $msvc = $(join-path $vcdir $('VC\Tools\MSVC\'+$(gc -raw $(join-path $vcdir 'VC\Auxiliary\Build\Microsoft.VCToolsVersion.default.txt')).Trim()))
           $lib =  $(join-path $msvc 'bin\Hostx64\x64\lib.exe')
           & $lib /machine:x64 "/def:${env:RUNNER_TEMP}/openblas/lib/libopenblas.def" "/out:${env:RUNNER_TEMP}/openblas/lib/openblas.lib" /name:openblas.dll
-
+          
       - name: Build
         id: cmake_build
         env:

From b92c37adc9d84cf5c81cd321766de93629acc79e Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Sun, 28 May 2023 17:55:48 +0100
Subject: [PATCH 30/56] try using libopenblas instead

---
 .github/workflows/build-commit.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-commit.yml b/.github/workflows/build-commit.yml
index acb362b..2d86070 100644
--- a/.github/workflows/build-commit.yml
+++ b/.github/workflows/build-commit.yml
@@ -159,7 +159,7 @@ jobs:
           - build: 'avx512'
             defines: '-DGGML_AVX512=ON'
           - build: 'openblas'
-            defines: '-DGGML_OPENBLAS=ON -DBLAS_INCLUDE_DIRS="$env:RUNNER_TEMP/openblas/include" -DBLAS_LIBRARIES="$env:RUNNER_TEMP/openblas/lib/openblas.lib"'
+            defines: '-DGGML_OPENBLAS=ON -DBLAS_INCLUDE_DIRS="$env:RUNNER_TEMP/openblas/include" -DBLAS_LIBRARIES="$env:RUNNER_TEMP/openblas/lib/libopenblas.lib"'
 
 
     steps:

From 07ddd8ec6575b98fe041dfb6b4ee455b9143b104 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Sun, 28 May 2023 17:58:45 +0100
Subject: [PATCH 31/56] fix link to openblas

---
 ggml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ggml b/ggml
index 9d98c51..7b567de 160000
--- a/ggml
+++ b/ggml
@@ -1 +1 @@
-Subproject commit 9d98c51aae9029d7246ea28dfd39b4ba918ae038
+Subproject commit 7b567de59817a7b5b92345cf7cceaed15a4e8d0f

From 6723f6131182d924ad90e367eb257b87e92d12b2 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Sun, 28 May 2023 18:11:51 +0100
Subject: [PATCH 32/56] try to fix blas link options

---
 ggml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ggml b/ggml
index 7b567de..edc15ea 160000
--- a/ggml
+++ b/ggml
@@ -1 +1 @@
-Subproject commit 7b567de59817a7b5b92345cf7cceaed15a4e8d0f
+Subproject commit edc15ea4f4444fb18bd98d41a33e8f37524c3f3b

From e22d5610654e3940baadd325b1557fa19e456124 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Sun, 28 May 2023 18:13:14 +0100
Subject: [PATCH 33/56] fix windows blas build - don't copy dll

---
 .github/workflows/build-commit.yml | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/build-commit.yml b/.github/workflows/build-commit.yml
index 2d86070..60ee25c 100644
--- a/.github/workflows/build-commit.yml
+++ b/.github/workflows/build-commit.yml
@@ -159,7 +159,7 @@ jobs:
           - build: 'avx512'
             defines: '-DGGML_AVX512=ON'
           - build: 'openblas'
-            defines: '-DGGML_OPENBLAS=ON -DBLAS_INCLUDE_DIRS="$env:RUNNER_TEMP/openblas/include" -DBLAS_LIBRARIES="$env:RUNNER_TEMP/openblas/lib/libopenblas.lib"'
+            defines: '-DGGML_OPENBLAS=ON -DBLAS_INCLUDE_DIRS="$env:RUNNER_TEMP/openblas/include" -DBLAS_LIBRARIES="$env:RUNNER_TEMP/openblas/lib/openblas.lib"'
 
 
     steps:
@@ -199,7 +199,6 @@ jobs:
           $msvc = $(join-path $vcdir $('VC\Tools\MSVC\'+$(gc -raw $(join-path $vcdir 'VC\Auxiliary\Build\Microsoft.VCToolsVersion.default.txt')).Trim()))
           $lib =  $(join-path $msvc 'bin\Hostx64\x64\lib.exe')
           & $lib /machine:x64 "/def:${env:RUNNER_TEMP}/openblas/lib/libopenblas.def" "/out:${env:RUNNER_TEMP}/openblas/lib/openblas.lib" /name:openblas.dll
-          
       - name: Build
         id: cmake_build
         env:
@@ -213,12 +212,12 @@ jobs:
           cmake --build . --config Release -t codegen-serve
           cmake --build . --config Release -t codegen-quantize
 
-      - name: Add libopenblas.dll
-        id: add_libopenblas_dll
-        if: ${{ matrix.build == 'openblas' }}
-        run: |
-          cp $env:RUNNER_TEMP/openblas/bin/libopenblas.dll ./build/bin/Release/openblas.dll
-          cp $env:RUNNER_TEMP/OpenBLAS.LICENSE.txt ./build/bin/Release/OpenBLAS-${env:OPENBLAS_VERSION}.txt
+      # - name: Add libopenblas.dll
+      #   id: add_libopenblas_dll
+      #   if: ${{ matrix.build == 'openblas' }}
+      #   run: |
+      #     cp $env:RUNNER_TEMP/openblas/bin/libopenblas.dll ./build/bin/Release/openblas.dll
+      #     cp $env:RUNNER_TEMP/OpenBLAS.LICENSE.txt ./build/bin/Release/OpenBLAS-${env:OPENBLAS_VERSION}.txt
 
       - name: Check AVX512F support
         id: check_avx512f

From f999601224a7d7c6ae7220748d9bb45371af5973 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Sun, 28 May 2023 20:14:07 +0100
Subject: [PATCH 34/56] hopefully fix blas linking

---
 ggml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ggml b/ggml
index edc15ea..4f28094 160000
--- a/ggml
+++ b/ggml
@@ -1 +1 @@
-Subproject commit edc15ea4f4444fb18bd98d41a33e8f37524c3f3b
+Subproject commit 4f28094a9c53920b2a6353fb6f3fd66fe113d430

From b9c40508f39c0142b09a4ac71849ae036d9864f7 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Mon, 29 May 2023 07:46:38 +0100
Subject: [PATCH 35/56] fix includes for blas on windows

---
 ggml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ggml b/ggml
index 4f28094..73b6db9 160000
--- a/ggml
+++ b/ggml
@@ -1 +1 @@
-Subproject commit 4f28094a9c53920b2a6353fb6f3fd66fe113d430
+Subproject commit 73b6db9e14ac594218cf6d9a1a09dc92b9b78c04

From b2036f34a5c7b8b2478e777e9bd24e81db88262e Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Mon, 29 May 2023 08:21:28 +0100
Subject: [PATCH 36/56] attempt to fix the paths in the windows build

---
 .github/workflows/build-commit.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-commit.yml b/.github/workflows/build-commit.yml
index 60ee25c..b6e33a7 100644
--- a/.github/workflows/build-commit.yml
+++ b/.github/workflows/build-commit.yml
@@ -244,7 +244,7 @@ jobs:
           # Artifact name
           name: turbopilot-${{ runner.os }}-${{ runner.arch }}-${{ steps.short-sha.outputs.sha }}-${{ matrix.build }} # optional, default is artifact
           # A file, directory or wildcard pattern that describes what to upload
-          path: ${{github.workspace}}/ggml/build/bin/codegen*
+          path: ${{github.workspace}}\\ggml\\build\\bin\\Release\\codegen*
           # The desired behavior if no files are found using the provided path.
           
       - name: package artifacts for release

From 431addfe42a4c0c19498d4f2948ef479cf0f8ad7 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Mon, 29 May 2023 10:18:21 +0100
Subject: [PATCH 37/56] fix cmake target

---
 .github/workflows/build-commit.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/build-commit.yml b/.github/workflows/build-commit.yml
index b6e33a7..ae6ba28 100644
--- a/.github/workflows/build-commit.yml
+++ b/.github/workflows/build-commit.yml
@@ -208,9 +208,7 @@ jobs:
           mkdir build
           cd build
           cmake .. ${{ matrix.defines }}
-          cmake --build . --config Release -t codegen
-          cmake --build . --config Release -t codegen-serve
-          cmake --build . --config Release -t codegen-quantize
+          cmake --build . --config Release --target codegen  codegen-serve codegen-quantize
 
       # - name: Add libopenblas.dll
       #   id: add_libopenblas_dll

From 3b9613d87d036ccd04ae962afee67b426e0507d6 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Mon, 29 May 2023 10:37:53 +0100
Subject: [PATCH 38/56] update cxx standard

---
 ggml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ggml b/ggml
index 73b6db9..f7eead0 160000
--- a/ggml
+++ b/ggml
@@ -1 +1 @@
-Subproject commit 73b6db9e14ac594218cf6d9a1a09dc92b9b78c04
+Subproject commit f7eead06ac3f278cd691993643c3aff50fa0f938

From 94e3b905f6802e34cbf76da1d2a1e65605bcf697 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Mon, 29 May 2023 10:55:29 +0100
Subject: [PATCH 39/56] add posix compatible file descriptors

---
 ggml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ggml b/ggml
index f7eead0..38cb4f3 160000
--- a/ggml
+++ b/ggml
@@ -1 +1 @@
-Subproject commit f7eead06ac3f278cd691993643c3aff50fa0f938
+Subproject commit 38cb4f3de2e46280ba86ce854f38dab7d2299a97

From 346eec482b7ff2725f0a91b5f8cbb3f535ee28a0 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Mon, 29 May 2023 11:18:42 +0100
Subject: [PATCH 40/56] add alias for isatty

---
 ggml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ggml b/ggml
index 38cb4f3..99f187e 160000
--- a/ggml
+++ b/ggml
@@ -1 +1 @@
-Subproject commit 38cb4f3de2e46280ba86ce854f38dab7d2299a97
+Subproject commit 99f187ee209e9e94cec425f0231a4734fafb0601

From fdf6b64aefdaef5afbba5a59b92d8eb4fc2c5dad Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Mon, 29 May 2023 11:45:32 +0100
Subject: [PATCH 41/56] don't overwrite existing isatty alias

---
 ggml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ggml b/ggml
index 99f187e..a0f9f0e 160000
--- a/ggml
+++ b/ggml
@@ -1 +1 @@
-Subproject commit 99f187ee209e9e94cec425f0231a4734fafb0601
+Subproject commit a0f9f0e576c959f056974a2dff199edfad063191

From cacaa56182995c982d7008796696bfef005efae2 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Mon, 29 May 2023 15:01:02 +0100
Subject: [PATCH 42/56] replace POST->Post MSVC is fussy

---
 ggml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ggml b/ggml
index a0f9f0e..1848e6c 160000
--- a/ggml
+++ b/ggml
@@ -1 +1 @@
-Subproject commit a0f9f0e576c959f056974a2dff199edfad063191
+Subproject commit 1848e6cb311b1e535c604d5451972243ac10c728

From b7074ada277014b45ac6588c1d8a32b5ed3485c7 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Mon, 12 Jun 2023 20:21:51 +0100
Subject: [PATCH 43/56] Update ggml dep to avoid crash

---
 ggml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ggml b/ggml
index 1848e6c..6c4fe0e 160000
--- a/ggml
+++ b/ggml
@@ -1 +1 @@
-Subproject commit 1848e6cb311b1e535c604d5451972243ac10c728
+Subproject commit 6c4fe0ef5e50b76dd2539130c109e12179da0bd2

From 3dcd72ba56df9998ffea0c52c656bb14f48e0fbe Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Tue, 13 Jun 2023 20:35:41 +0100
Subject: [PATCH 44/56] attempt to get publish builds working for windows

---
 .github/workflows/build-commit.yml | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-commit.yml b/.github/workflows/build-commit.yml
index ae6ba28..4bcc782 100644
--- a/.github/workflows/build-commit.yml
+++ b/.github/workflows/build-commit.yml
@@ -169,6 +169,11 @@ jobs:
         with:
           submodules: true
 
+      - uses: benjlevesque/short-sha@v2.2
+        id: short-sha
+        with:
+          length: 6
+
       - name: Install boost
         uses: MarkusJx/install-boost@v2.4.1
         id: install-boost
@@ -249,7 +254,7 @@ jobs:
         if: startsWith(github.ref, 'refs/tags/')
   
         run: |
-          cd ${{github.workspace}}/ggml/build/bin 
+          cd ${{github.workspace}}\\ggml\\build\\bin\\Release\\
           7z a turbopilot-${{ runner.os }}-${{ runner.arch }}.zip ./codegen*
   
       - name: Upload binaries to release
@@ -257,4 +262,4 @@ jobs:
         if: startsWith(github.ref, 'refs/tags/')
         with:
           token: ${{ secrets.PUBLISH_TOKEN }}
-          files: ${{github.workspace}}/ggml/build/bin/turbopilot-${{ runner.os }}-${{ runner.arch }}.zip
\ No newline at end of file
+          files: ${{github.workspace}}\\ggml\\build\\bin\\Release\\turbopilot-${{ runner.os }}-${{ runner.arch }}.zip
\ No newline at end of file

From ca7ce2f69553b8d79d86c251edd9394c0893fef2 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Tue, 13 Jun 2023 20:52:51 +0100
Subject: [PATCH 45/56] remove separate cuda build step

---
 .github/workflows/build-cuda.yml | 84 --------------------------------
 1 file changed, 84 deletions(-)
 delete mode 100644 .github/workflows/build-cuda.yml

diff --git a/.github/workflows/build-cuda.yml b/.github/workflows/build-cuda.yml
deleted file mode 100644
index ccb301c..0000000
--- a/.github/workflows/build-cuda.yml
+++ /dev/null
@@ -1,84 +0,0 @@
-name: Build CUDA binary on Ubuntu
-
-on:
-  push:
-    branches: [ '**' ]
-    tags: ['**']
-  pull_request:
-    branches: [ "main" ]
-
-env:
-  # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
-  BUILD_TYPE: Release
-
-jobs:
-
-  build-ubuntu:
-    # The CMake configure and build commands are platform agnostic and should work equally well on Windows or Mac.
-    # You can convert this to a matrix build if you need cross-platform coverage.
-    # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix
-    runs-on: ubuntu-latest
-
-    # strategy:
-    #       matrix:
-    #         include:
-    #         - build: 'avx2'
-    #           defines: ''
-    #         - build: 'avx'
-    #           defines: '-DLLAMA_AVX2=OFF'
-    #         - build: 'avx512'
-    #           defines: '-DLLAMA_AVX512=ON'
-
-
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: true
-
-    - name: Install Dependencies
-      run: |
-          wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.0-1_all.deb
-          sudo dpkg -i cuda-keyring_1.0-1_all.deb
-          sudo apt-get update && sudo apt-get install -yq libboost-dev cuda-nvcc-11-7 libcublas-dev-11-7
-
-    - name: Build
-      # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
-      # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
-      run: |
-        cd ${{github.workspace}}/ggml
-        cmake -B ${{github.workspace}}/ggml/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DGGML_CUBLAS=ON  -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc
-        cd ${{github.workspace}}/ggml/build 
-        make codegen codegen-serve codegen-quantize
-        chmod +x ${{github.workspace}}/ggml/build/bin/codegen 
-        chmod +x ${{github.workspace}}/ggml/build/bin/codegen-serve 
-        chmod +x ${{github.workspace}}/ggml/build/bin/codegen-quantize
-        
-        
-    - uses: benjlevesque/short-sha@v2.2
-      id: short-sha
-      with:
-        length: 6
-
-
-    - name: Upload Build Artifacts
-      uses: actions/upload-artifact@v3.1.2
-      with:
-        # Artifact name
-        name: turbopilot-${{ runner.os }}-${{ runner.arch }}-${{ steps.short-sha.outputs.sha }}-cuda # optional, default is artifact
-        # A file, directory or wildcard pattern that describes what to upload
-        path: ${{github.workspace}}/ggml/build/bin/codegen*
-        # The desired behavior if no files are found using the provided path.
-        
-    - name: package artifacts for release
-      if: startsWith(github.ref, 'refs/tags/')
-
-      run: |
-        cd ${{github.workspace}}/ggml/build/bin 
-        zip turbopilot-${{ runner.os }}-${{ runner.arch }}.zip ./codegen*
-
-    - name: Upload binaries to release
-      uses: softprops/action-gh-release@v1
-      if: startsWith(github.ref, 'refs/tags/')
-      with:
-        token: ${{ secrets.PUBLISH_TOKEN }}
-        files: ${{github.workspace}}/ggml/build/bin/turbopilot-${{ runner.os }}-${{ runner.arch }}-cuda.zip

From 021b3913c1aa6ffb2b72d7407d7e9f42755fdcd9 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Tue, 13 Jun 2023 20:53:06 +0100
Subject: [PATCH 46/56] add cuda build to ubuntu build matrix

---
 .github/workflows/build-commit.yml | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/build-commit.yml b/.github/workflows/build-commit.yml
index 4bcc782..a7d63c7 100644
--- a/.github/workflows/build-commit.yml
+++ b/.github/workflows/build-commit.yml
@@ -83,10 +83,11 @@ jobs:
             defines: '-DGGML_AVX2=OFF'
           - build: 'avx512'
             defines: '-DGGML_AVX512=ON'
-          - build: 'openblas'
+          - build: 'avx2-openblas'
             defines: '-DGGML_OPENBLAS=ON'
-
-
+          - build: 'avx2-cuda'
+            defines: '-DGGML_CUBLAS=ON  -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc'
+            
 
     steps:
     - uses: actions/checkout@v3
@@ -97,9 +98,16 @@ jobs:
       run: sudo apt-get update && sudo apt-get install -yq libboost-dev
 
     - name: Install OpenBlas
-      if: ${{ matrix.build == 'openblas' }}
+      if: ${{ matrix.build == 'avx2-openblas' }}
       run: sudo apt-get install libopenblas-dev
 
+    - name: Install CUDA
+      if: ${{ matrix.build == 'avx2-cuda' }}
+      run: |
+          wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.0-1_all.deb
+          sudo dpkg -i cuda-keyring_1.0-1_all.deb
+          sudo apt-get update && sudo apt-get install -yq libboost-dev cuda-nvcc-11-7 libcublas-dev-11-7
+
     - name: Build
       # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
       # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type

From eb560304110ef89f61185f06a6388ba8a0362c75 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Tue, 13 Jun 2023 20:54:41 +0100
Subject: [PATCH 47/56] fix windows build release paths

---
 .github/workflows/build-commit.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-commit.yml b/.github/workflows/build-commit.yml
index a7d63c7..e9ecdb2 100644
--- a/.github/workflows/build-commit.yml
+++ b/.github/workflows/build-commit.yml
@@ -270,4 +270,4 @@ jobs:
         if: startsWith(github.ref, 'refs/tags/')
         with:
           token: ${{ secrets.PUBLISH_TOKEN }}
-          files: ${{github.workspace}}\\ggml\\build\\bin\\Release\\turbopilot-${{ runner.os }}-${{ runner.arch }}.zip
\ No newline at end of file
+          files: ${{github.workspace}}\ggml\build\bin\Release\turbopilot-${{ runner.os }}-${{ runner.arch }}.zip
\ No newline at end of file

From f43c0d4034612a35230c410d557276c0e64815b0 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Tue, 13 Jun 2023 20:58:36 +0100
Subject: [PATCH 48/56] disable static build for nvidia on ubuntu

---
 .github/workflows/build-commit.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/build-commit.yml b/.github/workflows/build-commit.yml
index e9ecdb2..ea2f55b 100644
--- a/.github/workflows/build-commit.yml
+++ b/.github/workflows/build-commit.yml
@@ -78,15 +78,15 @@ jobs:
       matrix:
         include:
           - build: 'avx2'
-            defines: ''
+            defines: '-DGGML_STATIC=On'
           - build: 'avx'
-            defines: '-DGGML_AVX2=OFF'
+            defines: '-DGGML_AVX2=OFF -DGGML_STATIC=On'
           - build: 'avx512'
-            defines: '-DGGML_AVX512=ON'
+            defines: '-DGGML_AVX512=ON -DGGML_STATIC=On'
           - build: 'avx2-openblas'
-            defines: '-DGGML_OPENBLAS=ON'
+            defines: '-DGGML_OPENBLAS=ON -DGGML_STATIC=On'
           - build: 'avx2-cuda'
-            defines: '-DGGML_CUBLAS=ON  -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc'
+            defines: '-DGGML_CUBLAS=ON  -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc -DGGML_STATIC=Off'
             
 
     steps:
@@ -113,7 +113,7 @@ jobs:
       # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
       run: |
         cd ${{github.workspace}}/ggml
-        cmake -B ${{github.workspace}}/ggml/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -D GGML_STATIC=On ${{ matrix.defines }}
+        cmake -B ${{github.workspace}}/ggml/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} ${{ matrix.defines }}
         cd ${{github.workspace}}/ggml/build 
         make codegen codegen-serve codegen-quantize
         chmod +x ${{github.workspace}}/ggml/build/bin/codegen 

From 0f8ccb9659d1eecd635b408968d38f43aea48426 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Tue, 13 Jun 2023 21:22:46 +0100
Subject: [PATCH 49/56] test

---
 ...{build-commit.yml => build-commit.yml.bak} |  4 +-
 ...{docker-image.yml => docker-image.yml.bak} |  0
 .github/workflows/test.yml                    | 38 +++++++++++++++++++
 3 files changed, 40 insertions(+), 2 deletions(-)
 rename .github/workflows/{build-commit.yml => build-commit.yml.bak} (98%)
 rename .github/workflows/{docker-image.yml => docker-image.yml.bak} (100%)
 create mode 100644 .github/workflows/test.yml

diff --git a/.github/workflows/build-commit.yml b/.github/workflows/build-commit.yml.bak
similarity index 98%
rename from .github/workflows/build-commit.yml
rename to .github/workflows/build-commit.yml.bak
index ea2f55b..a3bacb1 100644
--- a/.github/workflows/build-commit.yml
+++ b/.github/workflows/build-commit.yml.bak
@@ -262,8 +262,8 @@ jobs:
         if: startsWith(github.ref, 'refs/tags/')
   
         run: |
-          cd ${{github.workspace}}\\ggml\\build\\bin\\Release\\
-          7z a turbopilot-${{ runner.os }}-${{ runner.arch }}.zip ./codegen*
+          cd ${{github.workspace}}\ggml\build\bin\\Release\
+          7z a ${{github.workspace}}\ggml\build\bin\\Release\turbopilot-${{ runner.os }}-${{ runner.arch }}.zip ./codegen*
   
       - name: Upload binaries to release
         uses: softprops/action-gh-release@v1
diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml.bak
similarity index 100%
rename from .github/workflows/docker-image.yml
rename to .github/workflows/docker-image.yml.bak
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
new file mode 100644
index 0000000..7881090
--- /dev/null
+++ b/.github/workflows/test.yml
@@ -0,0 +1,38 @@
+name: Build on Commit
+
+on:
+  push:
+    branches: [ '**' ]
+    tags: ['**']
+  pull_request:
+    branches: [ "main" ]
+
+env:
+  # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
+  BUILD_TYPE: Release
+
+jobs:
+
+
+  test:
+    runs-on: windows-latest
+
+
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+
+      - name: package artifacts for release
+
+        run: |
+          cd ${{github.workspace}}\ggml
+          7z a ${{github.workspace}}\ggml\build\bin\\Release\turbopilot-${{ runner.os }}-${{ runner.arch }}.zip *.md
+  
+      - name: Upload binaries to release
+        uses: softprops/action-gh-release@v1
+        with:
+          token: ${{ secrets.PUBLISH_TOKEN }}
+          files: ${{github.workspace}}\ggml\build\bin\Release\turbopilot-${{ runner.os }}-${{ runner.arch }}.zip
\ No newline at end of file

From a24234f1dde94a0f0e3f5b3ac1e74a9ad0e82c83 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Tue, 13 Jun 2023 21:23:53 +0100
Subject: [PATCH 50/56] test

---
 .github/workflows/test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 7881090..334e4e9 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -29,10 +29,10 @@ jobs:
 
         run: |
           cd ${{github.workspace}}\ggml
-          7z a ${{github.workspace}}\ggml\build\bin\\Release\turbopilot-${{ runner.os }}-${{ runner.arch }}.zip *.md
+          7z a ${{github.workspace}}\ggml\turbopilot-${{ runner.os }}-${{ runner.arch }}.zip *.md
   
       - name: Upload binaries to release
         uses: softprops/action-gh-release@v1
         with:
           token: ${{ secrets.PUBLISH_TOKEN }}
-          files: ${{github.workspace}}\ggml\build\bin\Release\turbopilot-${{ runner.os }}-${{ runner.arch }}.zip
\ No newline at end of file
+          files: ${{github.workspace}}\ggml\turbopilot-${{ runner.os }}-${{ runner.arch }}.zip
\ No newline at end of file

From d48be56a8f67724fe5e7e1e6d6c92c9b53b5a891 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Tue, 13 Jun 2023 21:30:39 +0100
Subject: [PATCH 51/56] revert gh release action for windows compat

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 334e4e9..da21eb6 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -32,7 +32,7 @@ jobs:
           7z a ${{github.workspace}}\ggml\turbopilot-${{ runner.os }}-${{ runner.arch }}.zip *.md
   
       - name: Upload binaries to release
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@v0.1.13
         with:
           token: ${{ secrets.PUBLISH_TOKEN }}
           files: ${{github.workspace}}\ggml\turbopilot-${{ runner.os }}-${{ runner.arch }}.zip
\ No newline at end of file

From 480e94a4b1f8f36d26f55c8cc658b70fa256be15 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Tue, 13 Jun 2023 21:38:58 +0100
Subject: [PATCH 52/56] convert slashes instead of using old action

---
 .github/workflows/test.yml | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index da21eb6..6d47934 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -30,9 +30,15 @@ jobs:
         run: |
           cd ${{github.workspace}}\ggml
           7z a ${{github.workspace}}\ggml\turbopilot-${{ runner.os }}-${{ runner.arch }}.zip *.md
+
+      - name: Convert backslashes
+        run: |
+            path="${{ github.workspace }}"
+            path="${path//\\//}"
+            echo "::set-output name=path::$path"
   
       - name: Upload binaries to release
-        uses: softprops/action-gh-release@v0.1.13
+        uses: softprops/action-gh-release@v1
         with:
           token: ${{ secrets.PUBLISH_TOKEN }}
-          files: ${{github.workspace}}\ggml\turbopilot-${{ runner.os }}-${{ runner.arch }}.zip
\ No newline at end of file
+          files: ${{ steps.convert_backslashes.outputs.path }}/ggml/turbopilot-${{ runner.os }}-${{ runner.arch }}.zip
\ No newline at end of file

From 7d5988acbd3dcf4564ff662688e618779047256f Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Tue, 13 Jun 2023 21:45:06 +0100
Subject: [PATCH 53/56] try again

---
 .github/workflows/test.yml | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 6d47934..585dabc 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -32,13 +32,15 @@ jobs:
           7z a ${{github.workspace}}\ggml\turbopilot-${{ runner.os }}-${{ runner.arch }}.zip *.md
 
       - name: Convert backslashes
+        id: convert_backslashes
         run: |
-            path="${{ github.workspace }}"
-            path="${path//\\//}"
-            echo "::set-output name=path::$path"
+            $path = $env:GITHUB_WORKSPACE -replace '\\', '/'
+            echo "Converted path: $path"
+            echo "::set-output name=converted_path::$path"
   
+
       - name: Upload binaries to release
         uses: softprops/action-gh-release@v1
         with:
           token: ${{ secrets.PUBLISH_TOKEN }}
-          files: ${{ steps.convert_backslashes.outputs.path }}/ggml/turbopilot-${{ runner.os }}-${{ runner.arch }}.zip
\ No newline at end of file
+          files: ${{ steps.convert_backslashes.outputs.converted_path }}/ggml/turbopilot-${{ runner.os }}-${{ runner.arch }}.zip
\ No newline at end of file

From 4c3bee1160d431b6ff3ac13687e0d18313d0bb41 Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Tue, 13 Jun 2023 21:52:49 +0100
Subject: [PATCH 54/56] correct build processes

---
 ...{build-commit.yml.bak => build-commit.yml} | 10 +++-
 ...{docker-image.yml.bak => docker-image.yml} |  0
 .github/workflows/test.yml                    | 46 -------------------
 3 files changed, 9 insertions(+), 47 deletions(-)
 rename .github/workflows/{build-commit.yml.bak => build-commit.yml} (96%)
 rename .github/workflows/{docker-image.yml.bak => docker-image.yml} (100%)
 delete mode 100644 .github/workflows/test.yml

diff --git a/.github/workflows/build-commit.yml.bak b/.github/workflows/build-commit.yml
similarity index 96%
rename from .github/workflows/build-commit.yml.bak
rename to .github/workflows/build-commit.yml
index a3bacb1..47144ca 100644
--- a/.github/workflows/build-commit.yml.bak
+++ b/.github/workflows/build-commit.yml
@@ -143,12 +143,20 @@ jobs:
         cd ${{github.workspace}}/ggml/build/bin 
         zip turbopilot-${{ runner.os }}-${{ runner.arch }}-${{ matrix.build }}.zip ./codegen*
 
+    - name: Convert backslashes
+      id: convert_backslashes
+      run: |
+            $path = $env:GITHUB_WORKSPACE -replace '\\', '/' 
+            echo "Converted path: $path"
+            echo "::set-output name=converted_path::$path"
+  
+
     - name: Upload binaries to release
       uses: softprops/action-gh-release@v1
       if: startsWith(github.ref, 'refs/tags/')
       with:
         token: ${{ secrets.PUBLISH_TOKEN }}
-        files: ${{github.workspace}}/ggml/build/bin/turbopilot-${{ runner.os }}-${{ runner.arch }}-${{ matrix.build }}.zip
+        files: ${{ steps.convert_backslashes.outputs.converted_path }}/ggml/build/bin/turbopilot-${{ runner.os }}-${{ runner.arch }}-${{ matrix.build }}.zip
 
   windows-latest-cmake:
     runs-on: windows-latest
diff --git a/.github/workflows/docker-image.yml.bak b/.github/workflows/docker-image.yml
similarity index 100%
rename from .github/workflows/docker-image.yml.bak
rename to .github/workflows/docker-image.yml
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
deleted file mode 100644
index 585dabc..0000000
--- a/.github/workflows/test.yml
+++ /dev/null
@@ -1,46 +0,0 @@
-name: Build on Commit
-
-on:
-  push:
-    branches: [ '**' ]
-    tags: ['**']
-  pull_request:
-    branches: [ "main" ]
-
-env:
-  # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
-  BUILD_TYPE: Release
-
-jobs:
-
-
-  test:
-    runs-on: windows-latest
-
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v3
-        with:
-          submodules: true
-
-      - name: package artifacts for release
-
-        run: |
-          cd ${{github.workspace}}\ggml
-          7z a ${{github.workspace}}\ggml\turbopilot-${{ runner.os }}-${{ runner.arch }}.zip *.md
-
-      - name: Convert backslashes
-        id: convert_backslashes
-        run: |
-            $path = $env:GITHUB_WORKSPACE -replace '\\', '/'
-            echo "Converted path: $path"
-            echo "::set-output name=converted_path::$path"
-  
-
-      - name: Upload binaries to release
-        uses: softprops/action-gh-release@v1
-        with:
-          token: ${{ secrets.PUBLISH_TOKEN }}
-          files: ${{ steps.convert_backslashes.outputs.converted_path }}/ggml/turbopilot-${{ runner.os }}-${{ runner.arch }}.zip
\ No newline at end of file

From 017b1b72aa1a28a9bf64afd5ebe9f9ae0427e38f Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Tue, 13 Jun 2023 22:01:33 +0100
Subject: [PATCH 55/56] correct windows bit

---
 .github/workflows/build-commit.yml | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/build-commit.yml b/.github/workflows/build-commit.yml
index 47144ca..6f05445 100644
--- a/.github/workflows/build-commit.yml
+++ b/.github/workflows/build-commit.yml
@@ -143,20 +143,13 @@ jobs:
         cd ${{github.workspace}}/ggml/build/bin 
         zip turbopilot-${{ runner.os }}-${{ runner.arch }}-${{ matrix.build }}.zip ./codegen*
 
-    - name: Convert backslashes
-      id: convert_backslashes
-      run: |
-            $path = $env:GITHUB_WORKSPACE -replace '\\', '/' 
-            echo "Converted path: $path"
-            echo "::set-output name=converted_path::$path"
-  
 
     - name: Upload binaries to release
       uses: softprops/action-gh-release@v1
       if: startsWith(github.ref, 'refs/tags/')
       with:
         token: ${{ secrets.PUBLISH_TOKEN }}
-        files: ${{ steps.convert_backslashes.outputs.converted_path }}/ggml/build/bin/turbopilot-${{ runner.os }}-${{ runner.arch }}-${{ matrix.build }}.zip
+        files: ${{github.workspace}}/ggml/build/bin/turbopilot-${{ runner.os }}-${{ runner.arch }}-${{ matrix.build }}.zip
 
   windows-latest-cmake:
     runs-on: windows-latest
@@ -255,7 +248,7 @@ jobs:
           id: short-sha
           with:
             length: 6
-    
+
   
       - name: Upload Build Artifacts
         uses: actions/upload-artifact@v3.1.2
@@ -271,11 +264,20 @@ jobs:
   
         run: |
           cd ${{github.workspace}}\ggml\build\bin\\Release\
-          7z a ${{github.workspace}}\ggml\build\bin\\Release\turbopilot-${{ runner.os }}-${{ runner.arch }}.zip ./codegen*
+          7z a ${{github.workspace}}\ggml\build\bin\Release\turbopilot-${{ runner.os }}-${{ runner.arch }}.zip ./codegen*
   
+    
+      - name: Convert backslashes
+        id: convert_backslashes
+        run: |
+              $path = $env:GITHUB_WORKSPACE -replace '\\', '/' 
+              echo "Converted path: $path"
+              echo "::set-output name=converted_path::$path"
+    
+
       - name: Upload binaries to release
         uses: softprops/action-gh-release@v1
         if: startsWith(github.ref, 'refs/tags/')
         with:
           token: ${{ secrets.PUBLISH_TOKEN }}
-          files: ${{github.workspace}}\ggml\build\bin\Release\turbopilot-${{ runner.os }}-${{ runner.arch }}.zip
\ No newline at end of file
+          files: ${{ steps.convert_backslashes.outputs.converted_path }}/ggml/build/bin/Release/turbopilot-${{ runner.os }}-${{ runner.arch }}.zip
\ No newline at end of file

From 1f8a7fd9cde1c1058de7a852cff1445c5b211cef Mon Sep 17 00:00:00 2001
From: James Ravenscroft <ravenscroftj@gmail.com>
Date: Tue, 13 Jun 2023 22:13:18 +0100
Subject: [PATCH 56/56] add matrix build to windows zips

---
 .github/workflows/build-commit.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-commit.yml b/.github/workflows/build-commit.yml
index 6f05445..1083b3d 100644
--- a/.github/workflows/build-commit.yml
+++ b/.github/workflows/build-commit.yml
@@ -264,7 +264,7 @@ jobs:
   
         run: |
           cd ${{github.workspace}}\ggml\build\bin\\Release\
-          7z a ${{github.workspace}}\ggml\build\bin\Release\turbopilot-${{ runner.os }}-${{ runner.arch }}.zip ./codegen*
+          7z a ${{github.workspace}}\ggml\build\bin\Release\turbopilot-${{ runner.os }}-${{ runner.arch }}-${{ matrix.build }}.zip ./codegen*
   
     
       - name: Convert backslashes
@@ -280,4 +280,4 @@ jobs:
         if: startsWith(github.ref, 'refs/tags/')
         with:
           token: ${{ secrets.PUBLISH_TOKEN }}
-          files: ${{ steps.convert_backslashes.outputs.converted_path }}/ggml/build/bin/Release/turbopilot-${{ runner.os }}-${{ runner.arch }}.zip
\ No newline at end of file
+          files: ${{ steps.convert_backslashes.outputs.converted_path }}/ggml/build/bin/Release/turbopilot-${{ runner.os }}-${{ runner.arch }}-${{ matrix.build }}.zip
\ No newline at end of file