AI/gpt4all
1
0
Fork 0
mirror of https://github.com/nomic-ai/gpt4all.git synced 2024-10-01 01:06:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

ci: use ccache to cache compiler output (#2942)

Browse Source 
Signed-off-by: Jared Van Bortel <jared@nomic.ai>
This commit is contained in:
Jared Van Bortel 2024-09-06 16:09:11 -04:00 committed by GitHub
parent facb706211
commit 1aae4ffe0a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 263 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

384
.circleci/continue_config.yml
View File

@ -29,25 +29,22 @@ jobs:
command: | command: |
git submodule sync git submodule sync
git submodule update --init --recursive git submodule update --init --recursive
- restore_cache: # this is the new step to restore cache - restore_cache:
keys: keys:
- macos-qt-cache-v3 - ccache-gpt4all-macos-
- run: - run:
name: Install Rosetta name: Install Rosetta
command: softwareupdate --install-rosetta --agree-to-license # needed for QtIFW command: softwareupdate --install-rosetta --agree-to-license # needed for QtIFW
- run:
name: Install dependencies
command: brew install ccache
- run: - run:
name: Installing Qt name: Installing Qt
command: | command: |
if [ ! -d ~/Qt ]; then curl -o qt-unified-macOS-x64-4.6.0-online.dmg https://gpt4all.io/ci/qt-unified-macOS-x64-4.6.0-online.dmg
curl -o qt-unified-macOS-x64-4.6.0-online.dmg https://gpt4all.io/ci/qt-unified-macOS-x64-4.6.0-online.dmg hdiutil attach qt-unified-macOS-x64-4.6.0-online.dmg
hdiutil attach qt-unified-macOS-x64-4.6.0-online.dmg /Volumes/qt-unified-macOS-x64-4.6.0-online/qt-unified-macOS-x64-4.6.0-online.app/Contents/MacOS/qt-unified-macOS-x64-4.6.0-online --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.clang_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
/Volumes/qt-unified-macOS-x64-4.6.0-online/qt-unified-macOS-x64-4.6.0-online.app/Contents/MacOS/qt-unified-macOS-x64-4.6.0-online --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.clang_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver hdiutil detach /Volumes/qt-unified-macOS-x64-4.6.0-online
hdiutil detach /Volumes/qt-unified-macOS-x64-4.6.0-online
fi
- save_cache: # this is the new step to save cache
key: macos-qt-cache-v3
paths:
- ~/Qt
- run: - run:
name: Setup Keychain name: Setup Keychain
command: | command: |
@ -61,6 +58,7 @@ jobs:
name: Build name: Build
no_output_timeout: 30m no_output_timeout: 30m
command: | command: |
ccache -o "cache_dir=${PWD}/../.ccache" -o max_size=500M -p -z
mkdir build mkdir build
cd build cd build
export PATH=$PATH:$HOME/Qt/Tools/QtInstallerFramework/4.7/bin export PATH=$PATH:$HOME/Qt/Tools/QtInstallerFramework/4.7/bin
@ -69,6 +67,8 @@ jobs:
-DCMAKE_BUILD_TYPE=Release \ -DCMAKE_BUILD_TYPE=Release \
-DCMAKE_PREFIX_PATH:PATH=~/Qt/6.5.1/macos/lib/cmake \ -DCMAKE_PREFIX_PATH:PATH=~/Qt/6.5.1/macos/lib/cmake \
-DCMAKE_MAKE_PROGRAM:FILEPATH=~/Qt/Tools/Ninja/ninja \ -DCMAKE_MAKE_PROGRAM:FILEPATH=~/Qt/Tools/Ninja/ninja \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DBUILD_UNIVERSAL=ON \ -DBUILD_UNIVERSAL=ON \
-DCMAKE_OSX_DEPLOYMENT_TARGET=12.6 \ -DCMAKE_OSX_DEPLOYMENT_TARGET=12.6 \
-DGGML_METAL_MACOSX_VERSION_MIN=12.6 \ -DGGML_METAL_MACOSX_VERSION_MIN=12.6 \
@ -78,11 +78,17 @@ jobs:
~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target all ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target all
~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target install ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target install
~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target package ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target package
ccache -s
mkdir upload mkdir upload
cp gpt4all-installer-* upload cp gpt4all-installer-* upload
# persist the unsigned installer # persist the unsigned installer
- store_artifacts: - store_artifacts:
path: build/upload path: build/upload
- save_cache:
key: ccache-gpt4all-macos-{{ epoch }}
when: always
paths:
- ../.ccache
# add workspace so signing jobs can connect & obtain dmg # add workspace so signing jobs can connect & obtain dmg
- persist_to_workspace: - persist_to_workspace:
root: build root: build
@ -161,25 +167,22 @@ jobs:
command: | command: |
git submodule sync git submodule sync
git submodule update --init --recursive git submodule update --init --recursive
- restore_cache: # this is the new step to restore cache - restore_cache:
keys: keys:
- macos-qt-cache-v3 - ccache-gpt4all-macos-
- run: - run:
name: Install Rosetta name: Install Rosetta
command: softwareupdate --install-rosetta --agree-to-license # needed for QtIFW command: softwareupdate --install-rosetta --agree-to-license # needed for QtIFW
- run:
name: Install dependencies
command: brew install ccache
- run: - run:
name: Installing Qt name: Installing Qt
command: | command: |
if [ ! -d ~/Qt ]; then curl -o qt-unified-macOS-x64-4.6.0-online.dmg https://gpt4all.io/ci/qt-unified-macOS-x64-4.6.0-online.dmg
curl -o qt-unified-macOS-x64-4.6.0-online.dmg https://gpt4all.io/ci/qt-unified-macOS-x64-4.6.0-online.dmg hdiutil attach qt-unified-macOS-x64-4.6.0-online.dmg
hdiutil attach qt-unified-macOS-x64-4.6.0-online.dmg /Volumes/qt-unified-macOS-x64-4.6.0-online/qt-unified-macOS-x64-4.6.0-online.app/Contents/MacOS/qt-unified-macOS-x64-4.6.0-online --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.clang_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
/Volumes/qt-unified-macOS-x64-4.6.0-online/qt-unified-macOS-x64-4.6.0-online.app/Contents/MacOS/qt-unified-macOS-x64-4.6.0-online --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.clang_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver hdiutil detach /Volumes/qt-unified-macOS-x64-4.6.0-online
hdiutil detach /Volumes/qt-unified-macOS-x64-4.6.0-online
fi
- save_cache: # this is the new step to save cache
key: macos-qt-cache-v3
paths:
- ~/Qt
- run: - run:
name: Setup Keychain name: Setup Keychain
command: | command: |
@ -193,6 +196,7 @@ jobs:
name: Build name: Build
no_output_timeout: 30m no_output_timeout: 30m
command: | command: |
ccache -o "cache_dir=${PWD}/../.ccache" -o max_size=500M -p -z
mkdir build mkdir build
cd build cd build
export PATH=$PATH:$HOME/Qt/Tools/QtInstallerFramework/4.7/bin export PATH=$PATH:$HOME/Qt/Tools/QtInstallerFramework/4.7/bin
@ -201,6 +205,8 @@ jobs:
-DCMAKE_BUILD_TYPE=Release \ -DCMAKE_BUILD_TYPE=Release \
-DCMAKE_PREFIX_PATH:PATH=~/Qt/6.5.1/macos/lib/cmake \ -DCMAKE_PREFIX_PATH:PATH=~/Qt/6.5.1/macos/lib/cmake \
-DCMAKE_MAKE_PROGRAM:FILEPATH=~/Qt/Tools/Ninja/ninja \ -DCMAKE_MAKE_PROGRAM:FILEPATH=~/Qt/Tools/Ninja/ninja \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DBUILD_UNIVERSAL=ON \ -DBUILD_UNIVERSAL=ON \
-DCMAKE_OSX_DEPLOYMENT_TARGET=12.6 \ -DCMAKE_OSX_DEPLOYMENT_TARGET=12.6 \
-DGGML_METAL_MACOSX_VERSION_MIN=12.6 \ -DGGML_METAL_MACOSX_VERSION_MIN=12.6 \
@ -210,12 +216,18 @@ jobs:
~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target all ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target all
~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target install ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target install
~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target package ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target package
ccache -s
mkdir upload mkdir upload
cp gpt4all-installer-* upload cp gpt4all-installer-* upload
tar -cvzf upload/repository.tar.gz -C _CPack_Packages/Darwin/IFW/gpt4all-installer-darwin repository tar -cvzf upload/repository.tar.gz -C _CPack_Packages/Darwin/IFW/gpt4all-installer-darwin repository
# persist the unsigned installer # persist the unsigned installer
- store_artifacts: - store_artifacts:
path: build/upload path: build/upload
- save_cache:
key: ccache-gpt4all-macos-{{ epoch }}
when: always
paths:
- ../.ccache
# add workspace so signing jobs can connect & obtain dmg # add workspace so signing jobs can connect & obtain dmg
- persist_to_workspace: - persist_to_workspace:
root: build root: build
@ -294,9 +306,9 @@ jobs:
command: | command: |
git submodule sync git submodule sync
git submodule update --init --recursive git submodule update --init --recursive
- restore_cache: # this is the new step to restore cache - restore_cache:
keys: keys:
- linux-qt-cache-v2 - ccache-gpt4all-linux-amd64-
- run: - run:
name: Setup Linux and Dependencies name: Setup Linux and Dependencies
command: | command: |
@ -305,7 +317,7 @@ jobs:
wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
sudo dpkg -i cuda-keyring_1.1-1_all.deb sudo dpkg -i cuda-keyring_1.1-1_all.deb
packages=( packages=(
bison build-essential cuda-compiler-11-8 flex gperf libcublas-dev-11-8 libfontconfig1 libfreetype6 bison build-essential ccache cuda-compiler-11-8 flex gperf libcublas-dev-11-8 libfontconfig1 libfreetype6
libgl1-mesa-dev libmysqlclient21 libnvidia-compute-550-server libodbc2 libpq5 libwayland-dev libx11-6 libgl1-mesa-dev libmysqlclient21 libnvidia-compute-550-server libodbc2 libpq5 libwayland-dev libx11-6
libx11-xcb1 libxcb-cursor0 libxcb-glx0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0 libx11-xcb1 libxcb-cursor0 libxcb-glx0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0
libxcb-render-util0 libxcb-shape0 libxcb-shm0 libxcb-sync1 libxcb-util1 libxcb-xfixes0 libxcb-xinerama0 libxcb-render-util0 libxcb-shape0 libxcb-shm0 libxcb-sync1 libxcb-util1 libxcb-xfixes0 libxcb-xinerama0
@ -317,15 +329,9 @@ jobs:
- run: - run:
name: Installing Qt name: Installing Qt
command: | command: |
if [ ! -d ~/Qt ]; then wget https://gpt4all.io/ci/qt-unified-linux-x64-4.6.0-online.run
wget https://gpt4all.io/ci/qt-unified-linux-x64-4.6.0-online.run chmod +x qt-unified-linux-x64-4.6.0-online.run
chmod +x qt-unified-linux-x64-4.6.0-online.run ./qt-unified-linux-x64-4.6.0-online.run --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.gcc_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver qt.qt6.651.qtwaylandcompositor
./qt-unified-linux-x64-4.6.0-online.run --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.gcc_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver qt.qt6.651.qtwaylandcompositor
fi
- save_cache: # this is the new step to save cache
key: linux-qt-cache-v2
paths:
- ~/Qt
- run: - run:
name: Build linuxdeployqt name: Build linuxdeployqt
command: | command: |
@ -339,19 +345,30 @@ jobs:
export CMAKE_PREFIX_PATH=~/Qt/6.5.1/gcc_64/lib/cmake export CMAKE_PREFIX_PATH=~/Qt/6.5.1/gcc_64/lib/cmake
export PATH=$PATH:$HOME/Qt/Tools/QtInstallerFramework/4.7/bin export PATH=$PATH:$HOME/Qt/Tools/QtInstallerFramework/4.7/bin
export PATH=$PATH:/usr/local/cuda/bin export PATH=$PATH:/usr/local/cuda/bin
ccache -o "cache_dir=${PWD}/../.ccache" -o max_size=500M -p -z
mkdir build mkdir build
cd build cd build
mkdir upload mkdir upload
~/Qt/Tools/CMake/bin/cmake \ ~/Qt/Tools/CMake/bin/cmake \
-S ../gpt4all-chat -B . \ -S ../gpt4all-chat -B . \
-DCMAKE_BUILD_TYPE=Release \ -DCMAKE_BUILD_TYPE=Release \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_CUDA_COMPILER_LAUNCHER=ccache \
-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON \
-DGPT4ALL_OFFLINE_INSTALLER=ON -DGPT4ALL_OFFLINE_INSTALLER=ON
~/Qt/Tools/CMake/bin/cmake --build . -j$(nproc) --target all ~/Qt/Tools/CMake/bin/cmake --build . -j$(nproc) --target all
~/Qt/Tools/CMake/bin/cmake --build . -j$(nproc) --target install ~/Qt/Tools/CMake/bin/cmake --build . -j$(nproc) --target install
~/Qt/Tools/CMake/bin/cmake --build . -j$(nproc) --target package ~/Qt/Tools/CMake/bin/cmake --build . -j$(nproc) --target package
ccache -s
cp gpt4all-installer-* upload cp gpt4all-installer-* upload
- store_artifacts: - store_artifacts:
path: build/upload path: build/upload
- save_cache:
key: ccache-gpt4all-linux-amd64-{{ epoch }}
when: always
paths:
- ../.ccache
build-online-chat-installer-linux: build-online-chat-installer-linux:
machine: machine:
@ -363,9 +380,9 @@ jobs:
command: | command: |
git submodule sync git submodule sync
git submodule update --init --recursive git submodule update --init --recursive
- restore_cache: # this is the new step to restore cache - restore_cache:
keys: keys:
- linux-qt-cache-v2 - ccache-gpt4all-linux-amd64-
- run: - run:
name: Setup Linux and Dependencies name: Setup Linux and Dependencies
command: | command: |
@ -374,7 +391,7 @@ jobs:
wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
sudo dpkg -i cuda-keyring_1.1-1_all.deb sudo dpkg -i cuda-keyring_1.1-1_all.deb
packages=( packages=(
bison build-essential cuda-compiler-11-8 flex gperf libcublas-dev-11-8 libfontconfig1 libfreetype6 bison build-essential ccache cuda-compiler-11-8 flex gperf libcublas-dev-11-8 libfontconfig1 libfreetype6
libgl1-mesa-dev libmysqlclient21 libnvidia-compute-550-server libodbc2 libpq5 libwayland-dev libx11-6 libgl1-mesa-dev libmysqlclient21 libnvidia-compute-550-server libodbc2 libpq5 libwayland-dev libx11-6
libx11-xcb1 libxcb-cursor0 libxcb-glx0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0 libx11-xcb1 libxcb-cursor0 libxcb-glx0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0
libxcb-render-util0 libxcb-shape0 libxcb-shm0 libxcb-sync1 libxcb-util1 libxcb-xfixes0 libxcb-xinerama0 libxcb-render-util0 libxcb-shape0 libxcb-shm0 libxcb-sync1 libxcb-util1 libxcb-xfixes0 libxcb-xinerama0
@ -386,15 +403,9 @@ jobs:
- run: - run:
name: Installing Qt name: Installing Qt
command: | command: |
if [ ! -d ~/Qt ]; then wget https://gpt4all.io/ci/qt-unified-linux-x64-4.6.0-online.run
wget https://gpt4all.io/ci/qt-unified-linux-x64-4.6.0-online.run chmod +x qt-unified-linux-x64-4.6.0-online.run
chmod +x qt-unified-linux-x64-4.6.0-online.run ./qt-unified-linux-x64-4.6.0-online.run --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.gcc_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver qt.qt6.651.qtwaylandcompositor
./qt-unified-linux-x64-4.6.0-online.run --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.gcc_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver qt.qt6.651.qtwaylandcompositor
fi
- save_cache: # this is the new step to save cache
key: linux-qt-cache-v2
paths:
- ~/Qt
- run: - run:
name: Build linuxdeployqt name: Build linuxdeployqt
command: | command: |
@ -408,20 +419,31 @@ jobs:
export CMAKE_PREFIX_PATH=~/Qt/6.5.1/gcc_64/lib/cmake export CMAKE_PREFIX_PATH=~/Qt/6.5.1/gcc_64/lib/cmake
export PATH=$PATH:$HOME/Qt/Tools/QtInstallerFramework/4.7/bin export PATH=$PATH:$HOME/Qt/Tools/QtInstallerFramework/4.7/bin
export PATH=$PATH:/usr/local/cuda/bin export PATH=$PATH:/usr/local/cuda/bin
ccache -o "cache_dir=${PWD}/../.ccache" -o max_size=500M -p -z
mkdir build mkdir build
cd build cd build
mkdir upload mkdir upload
~/Qt/Tools/CMake/bin/cmake \ ~/Qt/Tools/CMake/bin/cmake \
-S ../gpt4all-chat -B . \ -S ../gpt4all-chat -B . \
-DCMAKE_BUILD_TYPE=Release \ -DCMAKE_BUILD_TYPE=Release \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_CUDA_COMPILER_LAUNCHER=ccache \
-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON \
-DGPT4ALL_OFFLINE_INSTALLER=OFF -DGPT4ALL_OFFLINE_INSTALLER=OFF
~/Qt/Tools/CMake/bin/cmake --build . -j$(nproc) --target all ~/Qt/Tools/CMake/bin/cmake --build . -j$(nproc) --target all
~/Qt/Tools/CMake/bin/cmake --build . -j$(nproc) --target install ~/Qt/Tools/CMake/bin/cmake --build . -j$(nproc) --target install
~/Qt/Tools/CMake/bin/cmake --build . -j$(nproc) --target package ~/Qt/Tools/CMake/bin/cmake --build . -j$(nproc) --target package
ccache -s
cp gpt4all-installer-* upload cp gpt4all-installer-* upload
tar -cvzf upload/repository.tar.gz -C _CPack_Packages/Linux/IFW/gpt4all-installer-linux repository tar -cvzf upload/repository.tar.gz -C _CPack_Packages/Linux/IFW/gpt4all-installer-linux repository
- store_artifacts: - store_artifacts:
path: build/upload path: build/upload
- save_cache:
key: ccache-gpt4all-linux-amd64-{{ epoch }}
when: always
paths:
- ../.ccache
build-offline-chat-installer-windows: build-offline-chat-installer-windows:
machine: machine:
@ -435,20 +457,17 @@ jobs:
command: | command: |
git submodule sync git submodule sync
git submodule update --init --recursive git submodule update --init --recursive
- restore_cache: # this is the new step to restore cache - restore_cache:
keys: keys:
- windows-qt-cache-v2 - ccache-gpt4all-win-amd64-
- run:
name: Install dependencies
command: choco install -y ccache
- run: - run:
name: Installing Qt name: Installing Qt
command: | command: |
if (-not (Test-Path C:\Qt)) { Invoke-WebRequest -Uri https://gpt4all.io/ci/qt-unified-windows-x64-4.6.0-online.exe -OutFile qt-unified-windows-x64-4.6.0-online.exe
Invoke-WebRequest -Uri https://gpt4all.io/ci/qt-unified-windows-x64-4.6.0-online.exe -OutFile qt-unified-windows-x64-4.6.0-online.exe & .\qt-unified-windows-x64-4.6.0-online.exe --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email ${Env:QT_EMAIL} --password ${Env:QT_PASSWORD} install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.win64_msvc2019_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
& .\qt-unified-windows-x64-4.6.0-online.exe --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email ${Env:QT_EMAIL} --password ${Env:QT_PASSWORD} install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.win64_msvc2019_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
}
- save_cache: # this is the new step to save cache
key: windows-qt-cache-v2
paths:
- C:\Qt
- run: - run:
name: Install VulkanSDK name: Install VulkanSDK
command: | command: |
@ -483,29 +502,40 @@ jobs:
$Env:PATH = "${Env:PATH};C:\Qt\Tools\QtInstallerFramework\4.7\bin" $Env:PATH = "${Env:PATH};C:\Qt\Tools\QtInstallerFramework\4.7\bin"
$Env:DOTNET_ROOT="$($(Get-Location).Path)\dotnet\dotnet-sdk-8.0.302-win-x64" $Env:DOTNET_ROOT="$($(Get-Location).Path)\dotnet\dotnet-sdk-8.0.302-win-x64"
$Env:PATH="$Env:DOTNET_ROOT;$Env:PATH" $Env:PATH="$Env:DOTNET_ROOT;$Env:PATH"
ccache -o "cache_dir=${pwd}\..\.ccache" -o max_size=500M -p -z
mkdir build mkdir build
cd build cd build
& "C:\Qt\Tools\CMake_64\bin\cmake.exe" ` & "C:\Qt\Tools\CMake_64\bin\cmake.exe" `
-S ..\gpt4all-chat -B . -G Ninja ` -S ..\gpt4all-chat -B . -G Ninja `
"-DCMAKE_BUILD_TYPE=Release" ` -DCMAKE_BUILD_TYPE=Release `
"-DCMAKE_PREFIX_PATH:PATH=C:\Qt\6.5.1\msvc2019_64" ` "-DCMAKE_PREFIX_PATH:PATH=C:\Qt\6.5.1\msvc2019_64" `
"-DCMAKE_MAKE_PROGRAM:FILEPATH=C:\Qt\Tools\Ninja\ninja.exe" ` "-DCMAKE_MAKE_PROGRAM:FILEPATH=C:\Qt\Tools\Ninja\ninja.exe" `
"-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON" ` -DCMAKE_C_COMPILER_LAUNCHER=ccache `
"-DGPT4ALL_OFFLINE_INSTALLER=ON" -DCMAKE_CXX_COMPILER_LAUNCHER=ccache `
-DCMAKE_CUDA_COMPILER_LAUNCHER=ccache `
-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON `
-DGPT4ALL_OFFLINE_INSTALLER=ON
& "C:\Qt\Tools\Ninja\ninja.exe" & "C:\Qt\Tools\Ninja\ninja.exe"
& "C:\Qt\Tools\Ninja\ninja.exe" install & "C:\Qt\Tools\Ninja\ninja.exe" install
& "C:\Qt\Tools\Ninja\ninja.exe" package & "C:\Qt\Tools\Ninja\ninja.exe" package
ccache -s
mkdir upload mkdir upload
copy gpt4all-installer-win64.exe upload copy gpt4all-installer-win64.exe upload
- store_artifacts: - store_artifacts:
path: build/upload path: build/upload
# add workspace so signing jobs can connect & obtain dmg # add workspace so signing jobs can connect & obtain dmg
- save_cache:
key: ccache-gpt4all-win-amd64-{{ epoch }}
when: always
paths:
- ..\.ccache
- persist_to_workspace: - persist_to_workspace:
root: build root: build
# specify path to only include components we want to persist # specify path to only include components we want to persist
# accross builds # accross builds
paths: paths:
- upload - upload
sign-offline-chat-installer-windows: sign-offline-chat-installer-windows:
machine: machine:
image: 'windows-server-2019-vs2019:2022.08.1' image: 'windows-server-2019-vs2019:2022.08.1'
@ -535,6 +565,7 @@ jobs:
AzureSignTool.exe sign -du "https://gpt4all.io/index.html" -kvu https://gpt4all.vault.azure.net -kvi "$Env:AZSignGUID" -kvs "$Env:AZSignPWD" -kvc "$Env:AZSignCertName" -kvt "$Env:AZSignTID" -tr http://timestamp.digicert.com -v "$($(Get-Location).Path)\build\upload\gpt4all-installer-win64.exe" AzureSignTool.exe sign -du "https://gpt4all.io/index.html" -kvu https://gpt4all.vault.azure.net -kvi "$Env:AZSignGUID" -kvs "$Env:AZSignPWD" -kvc "$Env:AZSignCertName" -kvt "$Env:AZSignTID" -tr http://timestamp.digicert.com -v "$($(Get-Location).Path)\build\upload\gpt4all-installer-win64.exe"
- store_artifacts: - store_artifacts:
path: build/upload path: build/upload
build-online-chat-installer-windows: build-online-chat-installer-windows:
machine: machine:
image: 'windows-server-2019-vs2019:2022.08.1' image: 'windows-server-2019-vs2019:2022.08.1'
@ -547,20 +578,17 @@ jobs:
command: | command: |
git submodule sync git submodule sync
git submodule update --init --recursive git submodule update --init --recursive
- restore_cache: # this is the new step to restore cache - restore_cache:
keys: keys:
- windows-qt-cache-v2 - ccache-gpt4all-win-amd64-
- run:
name: Install dependencies
command: choco install -y ccache
- run: - run:
name: Installing Qt name: Installing Qt
command: | command: |
if (-not (Test-Path C:\Qt)) { Invoke-WebRequest -Uri https://gpt4all.io/ci/qt-unified-windows-x64-4.6.0-online.exe -OutFile qt-unified-windows-x64-4.6.0-online.exe
Invoke-WebRequest -Uri https://gpt4all.io/ci/qt-unified-windows-x64-4.6.0-online.exe -OutFile qt-unified-windows-x64-4.6.0-online.exe & .\qt-unified-windows-x64-4.6.0-online.exe --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email ${Env:QT_EMAIL} --password ${Env:QT_PASSWORD} install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.win64_msvc2019_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
& .\qt-unified-windows-x64-4.6.0-online.exe --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email ${Env:QT_EMAIL} --password ${Env:QT_PASSWORD} install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.win64_msvc2019_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
}
- save_cache: # this is the new step to save cache
key: windows-qt-cache-v2
paths:
- C:\Qt
- run: - run:
name: Install VulkanSDK name: Install VulkanSDK
command: | command: |
@ -600,24 +628,34 @@ jobs:
$Env:PATH = "${Env:PATH};C:\Qt\Tools\QtInstallerFramework\4.7\bin" $Env:PATH = "${Env:PATH};C:\Qt\Tools\QtInstallerFramework\4.7\bin"
$Env:DOTNET_ROOT="$($(Get-Location).Path)\dotnet\dotnet-sdk-8.0.302-win-x64" $Env:DOTNET_ROOT="$($(Get-Location).Path)\dotnet\dotnet-sdk-8.0.302-win-x64"
$Env:PATH="$Env:DOTNET_ROOT;$Env:PATH" $Env:PATH="$Env:DOTNET_ROOT;$Env:PATH"
ccache -o "cache_dir=${pwd}\..\.ccache" -o max_size=500M -p -z
mkdir build mkdir build
cd build cd build
& "C:\Qt\Tools\CMake_64\bin\cmake.exe" ` & "C:\Qt\Tools\CMake_64\bin\cmake.exe" `
-S ..\gpt4all-chat -B . -G Ninja ` -S ..\gpt4all-chat -B . -G Ninja `
"-DCMAKE_BUILD_TYPE=Release" ` -DCMAKE_BUILD_TYPE=Release `
"-DCMAKE_PREFIX_PATH:PATH=C:\Qt\6.5.1\msvc2019_64" ` "-DCMAKE_PREFIX_PATH:PATH=C:\Qt\6.5.1\msvc2019_64" `
"-DCMAKE_MAKE_PROGRAM:FILEPATH=C:\Qt\Tools\Ninja\ninja.exe" ` "-DCMAKE_MAKE_PROGRAM:FILEPATH=C:\Qt\Tools\Ninja\ninja.exe" `
"-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON" ` -DCMAKE_C_COMPILER_LAUNCHER=ccache `
"-DGPT4ALL_OFFLINE_INSTALLER=OFF" -DCMAKE_CXX_COMPILER_LAUNCHER=ccache `
-DCMAKE_CUDA_COMPILER_LAUNCHER=ccache `
-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON `
-DGPT4ALL_OFFLINE_INSTALLER=OFF
& "C:\Qt\Tools\Ninja\ninja.exe" & "C:\Qt\Tools\Ninja\ninja.exe"
& "C:\Qt\Tools\Ninja\ninja.exe" install & "C:\Qt\Tools\Ninja\ninja.exe" install
& "C:\Qt\Tools\Ninja\ninja.exe" package & "C:\Qt\Tools\Ninja\ninja.exe" package
ccache -s
mkdir upload mkdir upload
copy gpt4all-installer-win64.exe upload copy gpt4all-installer-win64.exe upload
Set-Location -Path "_CPack_Packages/win64/IFW/gpt4all-installer-win64" Set-Location -Path "_CPack_Packages/win64/IFW/gpt4all-installer-win64"
Compress-Archive -Path 'repository' -DestinationPath '..\..\..\..\upload\repository.zip' Compress-Archive -Path 'repository' -DestinationPath '..\..\..\..\upload\repository.zip'
- store_artifacts: - store_artifacts:
path: build/upload path: build/upload
- save_cache:
key: ccache-gpt4all-win-amd64-{{ epoch }}
when: always
paths:
- ..\.ccache
# add workspace so signing jobs can connect & obtain dmg # add workspace so signing jobs can connect & obtain dmg
- persist_to_workspace: - persist_to_workspace:
root: build root: build
@ -625,6 +663,7 @@ jobs:
# accross builds # accross builds
paths: paths:
- upload - upload
sign-online-chat-installer-windows: sign-online-chat-installer-windows:
machine: machine:
image: 'windows-server-2019-vs2019:2022.08.1' image: 'windows-server-2019-vs2019:2022.08.1'
@ -670,9 +709,9 @@ jobs:
command: | command: |
git submodule sync git submodule sync
git submodule update --init --recursive git submodule update --init --recursive
- restore_cache: # this is the new step to restore cache - restore_cache:
keys: keys:
- linux-qt-cache-v2 - ccache-gpt4all-linux-amd64-
- run: - run:
name: Setup Linux and Dependencies name: Setup Linux and Dependencies
command: | command: |
@ -681,7 +720,7 @@ jobs:
wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
sudo dpkg -i cuda-keyring_1.1-1_all.deb sudo dpkg -i cuda-keyring_1.1-1_all.deb
packages=( packages=(
bison build-essential cuda-compiler-11-8 flex gperf libcublas-dev-11-8 libfontconfig1 libfreetype6 bison build-essential ccache cuda-compiler-11-8 flex gperf libcublas-dev-11-8 libfontconfig1 libfreetype6
libgl1-mesa-dev libmysqlclient21 libnvidia-compute-550-server libodbc2 libpq5 libwayland-dev libx11-6 libgl1-mesa-dev libmysqlclient21 libnvidia-compute-550-server libodbc2 libpq5 libwayland-dev libx11-6
libx11-xcb1 libxcb-cursor0 libxcb-glx0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0 libx11-xcb1 libxcb-cursor0 libxcb-glx0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0
libxcb-render-util0 libxcb-shape0 libxcb-shm0 libxcb-sync1 libxcb-util1 libxcb-xfixes0 libxcb-xinerama0 libxcb-render-util0 libxcb-shape0 libxcb-shm0 libxcb-sync1 libxcb-util1 libxcb-xfixes0 libxcb-xinerama0
@ -693,24 +732,29 @@ jobs:
- run: - run:
name: Installing Qt name: Installing Qt
command: | command: |
if [ ! -d ~/Qt ]; then wget https://gpt4all.io/ci/qt-unified-linux-x64-4.6.0-online.run
wget https://gpt4all.io/ci/qt-unified-linux-x64-4.6.0-online.run chmod +x qt-unified-linux-x64-4.6.0-online.run
chmod +x qt-unified-linux-x64-4.6.0-online.run ./qt-unified-linux-x64-4.6.0-online.run --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.gcc_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver qt.qt6.651.qtwaylandcompositor
./qt-unified-linux-x64-4.6.0-online.run --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.gcc_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver qt.qt6.651.qtwaylandcompositor
fi
- save_cache: # this is the new step to save cache
key: linux-qt-cache-v2
paths:
- ~/Qt
- run: - run:
name: Build name: Build
command: | command: |
export CMAKE_PREFIX_PATH=~/Qt/6.5.1/gcc_64/lib/cmake export CMAKE_PREFIX_PATH=~/Qt/6.5.1/gcc_64/lib/cmake
export PATH=$PATH:/usr/local/cuda/bin export PATH=$PATH:/usr/local/cuda/bin
ccache -o "cache_dir=${PWD}/../.ccache" -o max_size=500M -p -z
~/Qt/Tools/CMake/bin/cmake \ ~/Qt/Tools/CMake/bin/cmake \
-S gpt4all-chat -B build \ -S gpt4all-chat -B build \
-DCMAKE_BUILD_TYPE=Release -DCMAKE_BUILD_TYPE=Release \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_CUDA_COMPILER_LAUNCHER=ccache \
-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON
~/Qt/Tools/CMake/bin/cmake --build build -j$(nproc) --target all ~/Qt/Tools/CMake/bin/cmake --build build -j$(nproc) --target all
ccache -s
- save_cache:
key: ccache-gpt4all-linux-amd64-{{ epoch }}
when: always
paths:
- ../.ccache
build-gpt4all-chat-windows: build-gpt4all-chat-windows:
machine: machine:
@ -724,20 +768,17 @@ jobs:
command: | command: |
git submodule sync git submodule sync
git submodule update --init --recursive git submodule update --init --recursive
- restore_cache: # this is the new step to restore cache - restore_cache:
keys: keys:
- windows-qt-cache-v2 - ccache-gpt4all-win-amd64-
- run:
name: Install dependencies
command: choco install -y ccache
- run: - run:
name: Installing Qt name: Installing Qt
command: | command: |
if (-not (Test-Path C:\Qt)) { Invoke-WebRequest -Uri https://gpt4all.io/ci/qt-unified-windows-x64-4.6.0-online.exe -OutFile qt-unified-windows-x64-4.6.0-online.exe
Invoke-WebRequest -Uri https://gpt4all.io/ci/qt-unified-windows-x64-4.6.0-online.exe -OutFile qt-unified-windows-x64-4.6.0-online.exe & .\qt-unified-windows-x64-4.6.0-online.exe --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email ${Env:QT_EMAIL} --password ${Env:QT_PASSWORD} install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.win64_msvc2019_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
& .\qt-unified-windows-x64-4.6.0-online.exe --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email ${Env:QT_EMAIL} --password ${Env:QT_PASSWORD} install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.win64_msvc2019_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
}
- save_cache: # this is the new step to save cache
key: windows-qt-cache-v2
paths:
- C:\Qt
- run: - run:
name: Install VulkanSDK name: Install VulkanSDK
command: | command: |
@ -757,13 +798,23 @@ jobs:
$Env:PATH = "${Env:PATH};C:\VulkanSDK\1.3.261.1\bin" $Env:PATH = "${Env:PATH};C:\VulkanSDK\1.3.261.1\bin"
$Env:VULKAN_SDK = "C:\VulkanSDK\1.3.261.1" $Env:VULKAN_SDK = "C:\VulkanSDK\1.3.261.1"
ccache -o "cache_dir=${pwd}\..\.ccache" -o max_size=500M -p -z
& "C:\Qt\Tools\CMake_64\bin\cmake.exe" ` & "C:\Qt\Tools\CMake_64\bin\cmake.exe" `
-S gpt4all-chat -B build -G Ninja ` -S gpt4all-chat -B build -G Ninja `
"-DCMAKE_BUILD_TYPE=Release" ` -DCMAKE_BUILD_TYPE=Release `
"-DCMAKE_PREFIX_PATH:PATH=C:\Qt\6.5.1\msvc2019_64" ` "-DCMAKE_PREFIX_PATH:PATH=C:\Qt\6.5.1\msvc2019_64" `
"-DCMAKE_MAKE_PROGRAM:FILEPATH=C:\Qt\Tools\Ninja\ninja.exe" ` "-DCMAKE_MAKE_PROGRAM:FILEPATH=C:\Qt\Tools\Ninja\ninja.exe" `
"-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON" -DCMAKE_C_COMPILER_LAUNCHER=ccache `
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache `
-DCMAKE_CUDA_COMPILER_LAUNCHER=ccache `
-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON
& "C:\Qt\Tools\Ninja\ninja.exe" -C build & "C:\Qt\Tools\Ninja\ninja.exe" -C build
ccache -s
- save_cache:
key: ccache-gpt4all-win-amd64-{{ epoch }}
when: always
paths:
- ..\.ccache
build-gpt4all-chat-macos: build-gpt4all-chat-macos:
macos: macos:
@ -775,37 +826,44 @@ jobs:
command: | command: |
git submodule sync git submodule sync
git submodule update --init --recursive git submodule update --init --recursive
- restore_cache: # this is the new step to restore cache - restore_cache:
keys: keys:
- macos-qt-cache-v3 - ccache-gpt4all-macos-
- run: - run:
name: Install Rosetta name: Install Rosetta
command: softwareupdate --install-rosetta --agree-to-license # needed for QtIFW command: softwareupdate --install-rosetta --agree-to-license # needed for QtIFW
- run:
name: Install dependencies
command: brew install ccache
- run: - run:
name: Installing Qt name: Installing Qt
command: | command: |
if [ ! -d ~/Qt ]; then curl -o qt-unified-macOS-x64-4.6.0-online.dmg https://gpt4all.io/ci/qt-unified-macOS-x64-4.6.0-online.dmg
curl -o qt-unified-macOS-x64-4.6.0-online.dmg https://gpt4all.io/ci/qt-unified-macOS-x64-4.6.0-online.dmg hdiutil attach qt-unified-macOS-x64-4.6.0-online.dmg
hdiutil attach qt-unified-macOS-x64-4.6.0-online.dmg /Volumes/qt-unified-macOS-x64-4.6.0-online/qt-unified-macOS-x64-4.6.0-online.app/Contents/MacOS/qt-unified-macOS-x64-4.6.0-online --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.clang_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
/Volumes/qt-unified-macOS-x64-4.6.0-online/qt-unified-macOS-x64-4.6.0-online.app/Contents/MacOS/qt-unified-macOS-x64-4.6.0-online --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.clang_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver hdiutil detach /Volumes/qt-unified-macOS-x64-4.6.0-online
hdiutil detach /Volumes/qt-unified-macOS-x64-4.6.0-online
fi
- save_cache: # this is the new step to save cache
key: macos-qt-cache-v3
paths:
- ~/Qt
- run: - run:
name: Build name: Build
command: | command: |
ccache -o "cache_dir=${PWD}/../.ccache" -o max_size=500M -p -z
~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake \ ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake \
-S gpt4all-chat -B build -G Ninja \ -S gpt4all-chat -B build -G Ninja \
-DCMAKE_BUILD_TYPE=Release \ -DCMAKE_BUILD_TYPE=Release \
-DCMAKE_PREFIX_PATH:PATH=~/Qt/6.5.1/macos/lib/cmake \ -DCMAKE_PREFIX_PATH:PATH=~/Qt/6.5.1/macos/lib/cmake \
-DCMAKE_MAKE_PROGRAM:FILEPATH=~/Qt/Tools/Ninja/ninja \ -DCMAKE_MAKE_PROGRAM:FILEPATH=~/Qt/Tools/Ninja/ninja \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DBUILD_UNIVERSAL=ON \ -DBUILD_UNIVERSAL=ON \
-DCMAKE_OSX_DEPLOYMENT_TARGET=12.6 \ -DCMAKE_OSX_DEPLOYMENT_TARGET=12.6 \
-DGGML_METAL_MACOSX_VERSION_MIN=12.6 -DGGML_METAL_MACOSX_VERSION_MIN=12.6
~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build build --target all ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build build --target all
ccache -s
- save_cache:
key: ccache-gpt4all-macos-{{ epoch }}
when: always
paths:
- ../.ccache
build-ts-docs: build-ts-docs:
docker: docker:
- image: cimg/base:stable - image: cimg/base:stable
@ -824,6 +882,7 @@ jobs:
command: | command: |
cd gpt4all-bindings/typescript cd gpt4all-bindings/typescript
npm run docs:build npm run docs:build
build-py-docs: build-py-docs:
docker: docker:
- image: circleci/python:3.8 - image: circleci/python:3.8
@ -855,6 +914,9 @@ jobs:
image: ubuntu-2204:2023.04.2 image: ubuntu-2204:2023.04.2
steps: steps:
- checkout - checkout
- restore_cache:
keys:
- ccache-gpt4all-linux-amd64-
- run: - run:
name: Set Python Version name: Set Python Version
command: pyenv global 3.11.2 command: pyenv global 3.11.2
@ -866,7 +928,7 @@ jobs:
wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
sudo dpkg -i cuda-keyring_1.1-1_all.deb sudo dpkg -i cuda-keyring_1.1-1_all.deb
packages=( packages=(
build-essential cmake cuda-compiler-11-8 libcublas-dev-11-8 libnvidia-compute-550-server vulkan-sdk build-essential ccache cmake cuda-compiler-11-8 libcublas-dev-11-8 libnvidia-compute-550-server vulkan-sdk
) )
sudo apt-get update sudo apt-get update
sudo apt-get install -y "${packages[@]}" sudo apt-get install -y "${packages[@]}"
@ -876,12 +938,17 @@ jobs:
command: | command: |
export PATH=$PATH:/usr/local/cuda/bin export PATH=$PATH:/usr/local/cuda/bin
git submodule update --init --recursive git submodule update --init --recursive
ccache -o "cache_dir=${PWD}/../.ccache" -o max_size=500M -p -z
cd gpt4all-backend cd gpt4all-backend
cmake -B build \ cmake -B build \
-DCMAKE_BUILD_TYPE=Release \ -DCMAKE_BUILD_TYPE=Release \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_CUDA_COMPILER_LAUNCHER=ccache \
-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON \ -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON \
-DCMAKE_CUDA_ARCHITECTURES='52-virtual;61-virtual;70-virtual;75-virtual' -DCMAKE_CUDA_ARCHITECTURES='52-virtual;61-virtual;70-virtual;75-virtual'
cmake --build build -j$(nproc) cmake --build build -j$(nproc)
ccache -s
- run: - run:
name: Build wheel name: Build wheel
command: | command: |
@ -889,6 +956,11 @@ jobs:
python setup.py bdist_wheel --plat-name=manylinux1_x86_64 python setup.py bdist_wheel --plat-name=manylinux1_x86_64
- store_artifacts: - store_artifacts:
path: gpt4all-bindings/python/dist path: gpt4all-bindings/python/dist
- save_cache:
key: ccache-gpt4all-linux-amd64-{{ epoch }}
when: always
paths:
- ../.ccache
- persist_to_workspace: - persist_to_workspace:
root: gpt4all-bindings/python/dist root: gpt4all-bindings/python/dist
paths: paths:
@ -900,22 +972,29 @@ jobs:
resource_class: macos.m1.large.gen1 resource_class: macos.m1.large.gen1
steps: steps:
- checkout - checkout
- restore_cache:
keys:
- ccache-gpt4all-macos-
- run: - run:
name: Install dependencies name: Install dependencies
command: | command: |
brew install cmake brew install ccache cmake
pip install setuptools wheel cmake pip install setuptools wheel cmake
- run: - run:
name: Build C library name: Build C library
command: | command: |
git submodule update --init # don't use --recursive because macOS doesn't use Kompute git submodule update --init # don't use --recursive because macOS doesn't use Kompute
ccache -o "cache_dir=${PWD}/../.ccache" -o max_size=500M -p -z
cd gpt4all-backend cd gpt4all-backend
cmake -B build \ cmake -B build \
-DCMAKE_BUILD_TYPE=Release \ -DCMAKE_BUILD_TYPE=Release \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DBUILD_UNIVERSAL=ON \ -DBUILD_UNIVERSAL=ON \
-DCMAKE_OSX_DEPLOYMENT_TARGET=12.6 \ -DCMAKE_OSX_DEPLOYMENT_TARGET=12.6 \
-DGGML_METAL_MACOSX_VERSION_MIN=12.6 -DGGML_METAL_MACOSX_VERSION_MIN=12.6
cmake --build build --parallel cmake --build build --parallel
ccache -s
- run: - run:
name: Build wheel name: Build wheel
command: | command: |
@ -923,6 +1002,11 @@ jobs:
python setup.py bdist_wheel --plat-name=macosx_10_15_universal2 python setup.py bdist_wheel --plat-name=macosx_10_15_universal2
- store_artifacts: - store_artifacts:
path: gpt4all-bindings/python/dist path: gpt4all-bindings/python/dist
- save_cache:
key: ccache-gpt4all-macos-{{ epoch }}
when: always
paths:
- ../.ccache
- persist_to_workspace: - persist_to_workspace:
root: gpt4all-bindings/python/dist root: gpt4all-bindings/python/dist
paths: paths:
@ -940,6 +1024,9 @@ jobs:
command: | command: |
git submodule sync git submodule sync
git submodule update --init --recursive git submodule update --init --recursive
- restore_cache:
keys:
- ccache-gpt4all-win-amd64-
- run: - run:
name: Install VulkanSDK name: Install VulkanSDK
command: | command: |
@ -953,7 +1040,7 @@ jobs:
- run: - run:
name: Install dependencies name: Install dependencies
command: command:
choco install -y cmake ninja --installargs 'ADD_CMAKE_TO_PATH=System' choco install -y ccache cmake ninja --installargs 'ADD_CMAKE_TO_PATH=System'
- run: - run:
name: Install Python dependencies name: Install Python dependencies
command: pip install setuptools wheel cmake command: pip install setuptools wheel cmake
@ -966,12 +1053,17 @@ jobs:
$Env:PATH += ";C:\VulkanSDK\1.3.261.1\bin" $Env:PATH += ";C:\VulkanSDK\1.3.261.1\bin"
$Env:VULKAN_SDK = "C:\VulkanSDK\1.3.261.1" $Env:VULKAN_SDK = "C:\VulkanSDK\1.3.261.1"
ccache -o "cache_dir=${pwd}\..\.ccache" -o max_size=500M -p -z
cd gpt4all-backend cd gpt4all-backend
cmake -B build -G Ninja ` cmake -B build -G Ninja `
-DCMAKE_BUILD_TYPE=Release ` -DCMAKE_BUILD_TYPE=Release `
-DCMAKE_C_COMPILER_LAUNCHER=ccache `
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache `
-DCMAKE_CUDA_COMPILER_LAUNCHER=ccache `
-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON ` -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON `
-DCMAKE_CUDA_ARCHITECTURES='52-virtual;61-virtual;70-virtual;75-virtual' -DCMAKE_CUDA_ARCHITECTURES='52-virtual;61-virtual;70-virtual;75-virtual'
cmake --build build --parallel cmake --build build --parallel
ccache -s
- run: - run:
name: Build wheel name: Build wheel
command: | command: |
@ -979,6 +1071,11 @@ jobs:
python setup.py bdist_wheel --plat-name=win_amd64 python setup.py bdist_wheel --plat-name=win_amd64
- store_artifacts: - store_artifacts:
path: gpt4all-bindings/python/dist path: gpt4all-bindings/python/dist
- save_cache:
key: ccache-gpt4all-win-amd64-{{ epoch }}
when: always
paths:
- ..\.ccache
- persist_to_workspace: - persist_to_workspace:
root: gpt4all-bindings/python/dist root: gpt4all-bindings/python/dist
paths: paths:
@ -1014,6 +1111,9 @@ jobs:
command: | command: |
git submodule sync git submodule sync
git submodule update --init --recursive git submodule update --init --recursive
- restore_cache:
keys:
- ccache-gpt4all-linux-amd64-
- run: - run:
name: Install dependencies name: Install dependencies
command: | command: |
@ -1022,7 +1122,7 @@ jobs:
wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
sudo dpkg -i cuda-keyring_1.1-1_all.deb sudo dpkg -i cuda-keyring_1.1-1_all.deb
packages=( packages=(
build-essential cmake cuda-compiler-11-8 libcublas-dev-11-8 libnvidia-compute-550-server vulkan-sdk build-essential ccache cmake cuda-compiler-11-8 libcublas-dev-11-8 libnvidia-compute-550-server vulkan-sdk
) )
sudo apt-get update sudo apt-get update
sudo apt-get install -y "${packages[@]}" sudo apt-get install -y "${packages[@]}"
@ -1030,14 +1130,25 @@ jobs:
name: Build Libraries name: Build Libraries
command: | command: |
export PATH=$PATH:/usr/local/cuda/bin export PATH=$PATH:/usr/local/cuda/bin
ccache -o "cache_dir=${PWD}/../.ccache" -o max_size=500M -p -z
cd gpt4all-backend cd gpt4all-backend
mkdir -p runtimes/build mkdir -p runtimes/build
cd runtimes/build cd runtimes/build
cmake ../.. \ cmake ../.. \
-DCMAKE_BUILD_TYPE=Release -DCMAKE_BUILD_TYPE=Release \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_CUDA_COMPILER_LAUNCHER=ccache \
-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON
cmake --build . -j$(nproc) cmake --build . -j$(nproc)
ccache -s
mkdir ../linux-x64 mkdir ../linux-x64
cp -L *.so ../linux-x64 # otherwise persist_to_workspace seems to mess symlinks cp -L *.so ../linux-x64 # otherwise persist_to_workspace seems to mess symlinks
- save_cache:
key: ccache-gpt4all-linux-amd64-{{ epoch }}
when: always
paths:
- ../.ccache
- persist_to_workspace: - persist_to_workspace:
root: gpt4all-backend root: gpt4all-backend
paths: paths:
@ -1053,26 +1164,38 @@ jobs:
command: | command: |
git submodule sync git submodule sync
git submodule update --init --recursive git submodule update --init --recursive
- restore_cache:
keys:
- ccache-gpt4all-macos-
- run: - run:
name: Install dependencies name: Install dependencies
command: | command: |
brew install cmake brew install ccache cmake
- run: - run:
name: Build Libraries name: Build Libraries
command: | command: |
ccache -o "cache_dir=${PWD}/../.ccache" -o max_size=500M -p -z
cd gpt4all-backend cd gpt4all-backend
mkdir -p runtimes/build mkdir -p runtimes/build
cd runtimes/build cd runtimes/build
cmake ../.. \ cmake ../.. \
-DCMAKE_BUILD_TYPE=Release \ -DCMAKE_BUILD_TYPE=Release \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DBUILD_UNIVERSAL=ON \ -DBUILD_UNIVERSAL=ON \
-DCMAKE_OSX_DEPLOYMENT_TARGET=12.6 \ -DCMAKE_OSX_DEPLOYMENT_TARGET=12.6 \
-DGGML_METAL_MACOSX_VERSION_MIN=12.6 -DGGML_METAL_MACOSX_VERSION_MIN=12.6
cmake --build . --parallel cmake --build . --parallel
ccache -s
mkdir ../osx-x64 mkdir ../osx-x64
cp -L *.dylib ../osx-x64 cp -L *.dylib ../osx-x64
cp ../../llama.cpp-mainline/*.metal ../osx-x64 cp ../../llama.cpp-mainline/*.metal ../osx-x64
ls ../osx-x64 ls ../osx-x64
- save_cache:
key: ccache-gpt4all-macos-{{ epoch }}
when: always
paths:
- ../.ccache
- persist_to_workspace: - persist_to_workspace:
root: gpt4all-backend root: gpt4all-backend
paths: paths:
@ -1091,6 +1214,9 @@ jobs:
command: | command: |
git submodule sync git submodule sync
git submodule update --init --recursive git submodule update --init --recursive
- restore_cache:
keys:
- ccache-gpt4all-win-amd64-
- run: - run:
name: Install VulkanSDK name: Install VulkanSDK
command: | command: |
@ -1104,19 +1230,34 @@ jobs:
- run: - run:
name: Install dependencies name: Install dependencies
command: | command: |
choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System' choco install -y ccache cmake ninja --installargs 'ADD_CMAKE_TO_PATH=System'
- run: - run:
name: Build Libraries name: Build Libraries
command: | command: |
$Env:Path += ";C:\Program Files\CMake\bin" $vsInstallPath = & "C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe" -property installationpath
Import-Module "${vsInstallPath}\Common7\Tools\Microsoft.VisualStudio.DevShell.dll"
Enter-VsDevShell -VsInstallPath "$vsInstallPath" -SkipAutomaticLocation -DevCmdArguments '-arch=x64 -no_logo'
$Env:Path += ";C:\VulkanSDK\1.3.261.1\bin" $Env:Path += ";C:\VulkanSDK\1.3.261.1\bin"
$Env:VULKAN_SDK = "C:\VulkanSDK\1.3.261.1" $Env:VULKAN_SDK = "C:\VulkanSDK\1.3.261.1"
ccache -o "cache_dir=${pwd}\..\.ccache" -o max_size=500M -p -z
cd gpt4all-backend cd gpt4all-backend
mkdir runtimes/win-x64_msvc mkdir runtimes/win-x64_msvc
cd runtimes/win-x64_msvc cd runtimes/win-x64_msvc
cmake -G "Visual Studio 17 2022" -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON -A X64 ../.. cmake -S ../.. -B . -G Ninja `
cmake --build . --parallel --config Release -DCMAKE_BUILD_TYPE=Release `
-DCMAKE_C_COMPILER_LAUNCHER=ccache `
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache `
-DCMAKE_CUDA_COMPILER_LAUNCHER=ccache `
-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON
cmake --build . --parallel
ccache -s
cp bin/Release/*.dll . cp bin/Release/*.dll .
- save_cache:
key: ccache-gpt4all-win-amd64-{{ epoch }}
when: always
paths:
- ..\.ccache
- persist_to_workspace: - persist_to_workspace:
root: gpt4all-backend root: gpt4all-backend
paths: paths:
@ -1153,6 +1294,7 @@ jobs:
paths: paths:
- prebuilds/linux-x64/*.node - prebuilds/linux-x64/*.node
- runtimes/linux-x64/*-*.so - runtimes/linux-x64/*-*.so
build-nodejs-macos: build-nodejs-macos:
macos: macos:
xcode: 15.4.0 xcode: 15.4.0