mirror of
https://github.com/nomic-ai/gpt4all.git
synced 2024-10-01 01:06:10 -04:00
8aba2c9009
* feat: local inference server * fix: source to use bash + vars * chore: isort and black * fix: make file + inference mode * chore: logging * refactor: remove old links * fix: add new env vars * feat: hf inference server * refactor: remove old links * test: batch and single response * chore: black + isort * separate gpu and cpu dockerfiles * moved gpu to separate dockerfile * Fixed test endpoints * Edits to API. server won't start due to failed instantiation error * Method signature * fix: gpu_infer * tests: fix tests --------- Co-authored-by: Andriy Mulyar <andriy.mulyar@gmail.com>
62 lines
1.5 KiB
Python
62 lines
1.5 KiB
Python
import logging
|
|
import time
|
|
from typing import Dict, List
|
|
|
|
from api_v1.settings import settings
|
|
from fastapi import APIRouter, Depends, Response, Security, status
|
|
from pydantic import BaseModel, Field
|
|
|
|
logger = logging.getLogger(__name__)
|
|
logger.setLevel(logging.DEBUG)
|
|
|
|
### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
|
|
|
|
|
|
class ChatCompletionMessage(BaseModel):
|
|
role: str
|
|
content: str
|
|
|
|
|
|
class ChatCompletionRequest(BaseModel):
|
|
model: str = Field(..., description='The model to generate a completion from.')
|
|
messages: List[ChatCompletionMessage] = Field(..., description='The model to generate a completion from.')
|
|
|
|
|
|
class ChatCompletionChoice(BaseModel):
|
|
message: ChatCompletionMessage
|
|
index: int
|
|
finish_reason: str
|
|
|
|
|
|
class ChatCompletionUsage(BaseModel):
|
|
prompt_tokens: int
|
|
completion_tokens: int
|
|
total_tokens: int
|
|
|
|
|
|
class ChatCompletionResponse(BaseModel):
|
|
id: str
|
|
object: str = 'text_completion'
|
|
created: int
|
|
model: str
|
|
choices: List[ChatCompletionChoice]
|
|
usage: ChatCompletionUsage
|
|
|
|
|
|
router = APIRouter(prefix="/chat", tags=["Completions Endpoints"])
|
|
|
|
|
|
@router.post("/completions", response_model=ChatCompletionResponse)
|
|
async def chat_completion(request: ChatCompletionRequest):
|
|
'''
|
|
Completes a GPT4All model response.
|
|
'''
|
|
|
|
return ChatCompletionResponse(
|
|
id='asdf',
|
|
created=time.time(),
|
|
model=request.model,
|
|
choices=[{}],
|
|
usage={'prompt_tokens': 0, 'completion_tokens': 0, 'total_tokens': 0},
|
|
)
|