From 5f5a5863ca548de4ff4ca6a0a45c39e6f56d7293 Mon Sep 17 00:00:00 2001 From: hibobmaster <32976627+hibobmaster@users.noreply.github.com> Date: Wed, 13 Sep 2023 14:36:35 +0800 Subject: [PATCH] Introduce pre-commit-hooks --- .env.example | 2 +- .github/workflows/pylint.yml | 31 ------ .pre-commit-config.yaml | 16 +++ README.md | 2 +- requirements.txt | 59 ++--------- settings.js.example | 2 +- src/BingImageGen.py | 184 ----------------------------------- src/askgpt.py | 13 ++- src/bard.py | 142 --------------------------- src/bot.py | 12 ++- src/chatgpt_bing.py | 11 +-- src/flowise.py | 16 ++- src/log.py | 6 +- src/main.py | 3 +- src/pandora_api.py | 5 +- src/send_image.py | 10 +- src/send_message.py | 6 +- 17 files changed, 79 insertions(+), 441 deletions(-) delete mode 100644 .github/workflows/pylint.yml create mode 100644 .pre-commit-config.yaml delete mode 100644 src/BingImageGen.py delete mode 100644 src/bard.py diff --git a/.env.example b/.env.example index 85ae41e..8d347cd 100644 --- a/.env.example +++ b/.env.example @@ -17,4 +17,4 @@ FLOWISE_API_URL="http://localhost:3000/api/v1/prediction/xxxx" # Optional FLOWISE_API_KEY="xxxxxxxxxxxxxxxxxxxxxxx" # Optional PANDORA_API_ENDPOINT="http://pandora:8008" # Optional, for !talk, !goon command PANDORA_API_MODEL="text-davinci-002-render-sha-mobile" # Optional -TEMPERATURE="0.8" # Optional \ No newline at end of file +TEMPERATURE="0.8" # Optional diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml deleted file mode 100644 index 3af25c3..0000000 --- a/.github/workflows/pylint.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: Pylint - -on: - push: - paths: - - 'src/**' - -jobs: - build: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.10", "3.11"] - steps: - - uses: actions/checkout@v3 - - name: Install libolm-dev - run: | - sudo apt install -y libolm-dev - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - cache: 'pip' - - name: Install dependencies - run: | - pip install -U pip setuptools wheel - pip install -r requirements.txt - pip install pylint - - name: Analysing the code with pylint - run: | - pylint $(git ls-files '*.py') --errors-only diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..d811573 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,16 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - repo: https://github.com/psf/black + rev: 23.9.1 + hooks: + - id: black + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.0.289 + hooks: + - id: ruff + args: [--fix, --exit-non-zero-on-fix] diff --git a/README.md b/README.md index 4ca09ee..b591b59 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ This is a simple Matrix bot that support using OpenAI API, Langchain to generate ## Feature -1. Support official openai api and self host models([LocalAI](https://github.com/go-skynet/LocalAI)) +1. Support official openai api and self host models([LocalAI](https://github.com/go-skynet/LocalAI)) 2. Support E2E Encrypted Room 3. Colorful code blocks 4. Langchain([Flowise](https://github.com/FlowiseAI/Flowise)) diff --git a/requirements.txt b/requirements.txt index 250e033..e884258 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,51 +1,8 @@ -aiofiles==23.1.0 -aiohttp==3.8.4 -aiohttp-socks==0.7.1 -aiosignal==1.3.1 -anyio==3.6.2 -async-timeout==4.0.2 -atomicwrites==1.4.1 -attrs==22.2.0 -blobfile==2.0.1 -cachetools==4.2.4 -certifi==2022.12.7 -cffi==1.15.1 -charset-normalizer==3.1.0 -cryptography==41.0.0 -filelock==3.11.0 -frozenlist==1.3.3 -future==0.18.3 -h11==0.14.0 -h2==4.1.0 -hpack==4.0.0 -httpcore==0.16.3 -httpx==0.23.3 -hyperframe==6.0.1 -idna==3.4 -jsonschema==4.17.3 -Logbook==1.5.3 -lxml==4.9.2 -Markdown==3.4.3 -matrix-nio[e2e]==0.20.2 -multidict==6.0.4 -peewee==3.16.0 -Pillow==9.5.0 -pycparser==2.21 -pycryptodome==3.17 -pycryptodomex==3.17 -pyrsistent==0.19.3 -python-cryptography-fernet-wrapper==1.0.4 -python-magic==0.4.27 -python-olm==3.1.3 -python-socks==2.2.0 -regex==2023.3.23 -requests==2.31.0 -rfc3986==1.5.0 -six==1.16.0 -sniffio==1.3.0 -tiktoken==0.3.3 -toml==0.10.2 -unpaddedbase64==2.1.0 -urllib3==1.26.15 -wcwidth==0.2.6 -yarl==1.8.2 +aiofiles +aiohttp +Markdown +matrix-nio[e2e] +Pillow +tiktoken +tenacity +python-magic diff --git a/settings.js.example b/settings.js.example index 321880f..57ec272 100644 --- a/settings.js.example +++ b/settings.js.example @@ -98,4 +98,4 @@ export default { // (Optional) Possible options: "chatgpt", "bing". // clientToUse: 'bing', }, -}; \ No newline at end of file +}; diff --git a/src/BingImageGen.py b/src/BingImageGen.py deleted file mode 100644 index 21371fc..0000000 --- a/src/BingImageGen.py +++ /dev/null @@ -1,184 +0,0 @@ -""" -Code derived from: -https://github.com/acheong08/EdgeGPT/blob/f940cecd24a4818015a8b42a2443dd97c3c2a8f4/src/ImageGen.py -""" - -from log import getlogger -from uuid import uuid4 -import os -import contextlib -import aiohttp -import asyncio -import random -import requests -import regex - -logger = getlogger() - -BING_URL = "https://www.bing.com" -# Generate random IP between range 13.104.0.0/14 -FORWARDED_IP = ( - f"13.{random.randint(104, 107)}.{random.randint(0, 255)}.{random.randint(0, 255)}" -) -HEADERS = { - "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", # noqa: E501 - "accept-language": "en-US,en;q=0.9", - "cache-control": "max-age=0", - "content-type": "application/x-www-form-urlencoded", - "referrer": "https://www.bing.com/images/create/", - "origin": "https://www.bing.com", - "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36 Edg/110.0.1587.63", # noqa: E501 - "x-forwarded-for": FORWARDED_IP, -} - - -class ImageGenAsync: - """ - Image generation by Microsoft Bing - Parameters: - auth_cookie: str - """ - - def __init__(self, auth_cookie: str, quiet: bool = True) -> None: - self.session = aiohttp.ClientSession( - headers=HEADERS, - cookies={"_U": auth_cookie}, - ) - self.quiet = quiet - - async def __aenter__(self): - return self - - async def __aexit__(self, *excinfo) -> None: - await self.session.close() - - def __del__(self): - try: - loop = asyncio.get_running_loop() - except RuntimeError: - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - loop.run_until_complete(self._close()) - - async def _close(self): - await self.session.close() - - async def get_images(self, prompt: str) -> list: - """ - Fetches image links from Bing - Parameters: - prompt: str - """ - if not self.quiet: - print("Sending request...") - url_encoded_prompt = requests.utils.quote(prompt) - # https://www.bing.com/images/create?q=&rt=3&FORM=GENCRE - url = f"{BING_URL}/images/create?q={url_encoded_prompt}&rt=4&FORM=GENCRE" - async with self.session.post(url, allow_redirects=False) as response: - content = await response.text() - if "this prompt has been blocked" in content.lower(): - raise Exception( - "Your prompt has been blocked by Bing. Try to change any bad words and try again.", # noqa: E501 - ) - if response.status != 302: - # if rt4 fails, try rt3 - url = ( - f"{BING_URL}/images/create?q={url_encoded_prompt}&rt=3&FORM=GENCRE" - ) - async with self.session.post( - url, - allow_redirects=False, - timeout=200, - ) as response3: - if response3.status != 302: - print(f"ERROR: {response3.text}") - raise Exception("Redirect failed") - response = response3 - # Get redirect URL - redirect_url = response.headers["Location"].replace("&nfy=1", "") - request_id = redirect_url.split("id=")[-1] - await self.session.get(f"{BING_URL}{redirect_url}") - # https://www.bing.com/images/create/async/results/{ID}?q={PROMPT} - polling_url = f"{BING_URL}/images/create/async/results/{request_id}?q={url_encoded_prompt}" # noqa: E501 - # Poll for results - if not self.quiet: - print("Waiting for results...") - while True: - if not self.quiet: - print(".", end="", flush=True) - # By default, timeout is 300s, change as needed - response = await self.session.get(polling_url) - if response.status != 200: - raise Exception("Could not get results") - content = await response.text() - if content and content.find("errorMessage") == -1: - break - - await asyncio.sleep(1) - continue - # Use regex to search for src="" - image_links = regex.findall(r'src="([^"]+)"', content) - # Remove size limit - normal_image_links = [link.split("?w=")[0] for link in image_links] - # Remove duplicates - normal_image_links = list(set(normal_image_links)) - - # Bad images - bad_images = [ - "https://r.bing.com/rp/in-2zU3AJUdkgFe7ZKv19yPBHVs.png", - "https://r.bing.com/rp/TX9QuO3WzcCJz1uaaSwQAz39Kb0.jpg", - ] - for im in normal_image_links: - if im in bad_images: - raise Exception("Bad images") - # No images - if not normal_image_links: - raise Exception("No images") - return normal_image_links - - async def save_images( - self, links: list, output_dir: str, output_four_images: bool - ) -> list: - """ - Saves images to output directory - """ - with contextlib.suppress(FileExistsError): - os.mkdir(output_dir) - - image_path_list = [] - - if output_four_images: - for link in links: - image_name = str(uuid4()) - image_path = os.path.join(output_dir, f"{image_name}.jpeg") - try: - async with self.session.get( - link, raise_for_status=True - ) as response: - with open(image_path, "wb") as output_file: - async for chunk in response.content.iter_chunked(8192): - output_file.write(chunk) - image_path_list.append(image_path) - except aiohttp.client_exceptions.InvalidURL as url_exception: - raise Exception( - "Inappropriate contents found in the generated images. Please try again or try another prompt." - ) from url_exception # noqa: E501 - else: - image_name = str(uuid4()) - if links: - link = links.pop() - try: - async with self.session.get( - link, raise_for_status=True - ) as response: - image_path = os.path.join(output_dir, f"{image_name}.jpeg") - with open(image_path, "wb") as output_file: - async for chunk in response.content.iter_chunked(8192): - output_file.write(chunk) - image_path_list.append(image_path) - except aiohttp.client_exceptions.InvalidURL as url_exception: - raise Exception( - "Inappropriate contents found in the generated images. Please try again or try another prompt." - ) from url_exception # noqa: E501 - - return image_path_list diff --git a/src/askgpt.py b/src/askgpt.py index bd7c22f..d3c37ca 100644 --- a/src/askgpt.py +++ b/src/askgpt.py @@ -1,6 +1,6 @@ -import aiohttp -import asyncio import json + +import aiohttp from log import getlogger logger = getlogger() @@ -10,7 +10,9 @@ class askGPT: def __init__(self, session: aiohttp.ClientSession): self.session = session - async def oneTimeAsk(self, prompt: str, api_endpoint: str, headers: dict, temperature: float = 0.8) -> str: + async def oneTimeAsk( + self, prompt: str, api_endpoint: str, headers: dict, temperature: float = 0.8 + ) -> str: jsons = { "model": "gpt-3.5-turbo", "messages": [ @@ -25,7 +27,10 @@ class askGPT: while max_try > 0: try: async with self.session.post( - url=api_endpoint, json=jsons, headers=headers, timeout=120 + url=api_endpoint, + json=jsons, + headers=headers, + timeout=120, ) as response: status_code = response.status if not status_code == 200: diff --git a/src/bard.py b/src/bard.py deleted file mode 100644 index a71d6a4..0000000 --- a/src/bard.py +++ /dev/null @@ -1,142 +0,0 @@ -""" -Code derived from: https://github.com/acheong08/Bard/blob/main/src/Bard.py -""" - -import random -import string -import re -import json -import httpx - - -class Bardbot: - """ - A class to interact with Google Bard. - Parameters - session_id: str - The __Secure-1PSID cookie. - timeout: int - Request timeout in seconds. - session: requests.Session - Requests session object. - """ - - __slots__ = [ - "headers", - "_reqid", - "SNlM0e", - "conversation_id", - "response_id", - "choice_id", - "session_id", - "session", - "timeout", - ] - - def __init__( - self, - session_id: str, - timeout: int = 20, - ): - headers = { - "Host": "bard.google.com", - "X-Same-Domain": "1", - "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36", - "Content-Type": "application/x-www-form-urlencoded;charset=UTF-8", - "Origin": "https://bard.google.com", - "Referer": "https://bard.google.com/", - } - self._reqid = int("".join(random.choices(string.digits, k=4))) - self.conversation_id = "" - self.response_id = "" - self.choice_id = "" - self.session_id = session_id - self.session = httpx.AsyncClient() - self.session.headers = headers - self.session.cookies.set("__Secure-1PSID", session_id) - self.timeout = timeout - - @classmethod - async def create( - cls, - session_id: str, - timeout: int = 20, - ) -> "Bardbot": - instance = cls(session_id, timeout) - instance.SNlM0e = await instance.__get_snlm0e() - return instance - - async def __get_snlm0e(self): - # Find "SNlM0e":"" - if not self.session_id or self.session_id[-1] != ".": - raise Exception( - "__Secure-1PSID value must end with a single dot. Enter correct __Secure-1PSID value.", - ) - resp = await self.session.get( - "https://bard.google.com/", - timeout=10, - ) - if resp.status_code != 200: - raise Exception( - f"Response code not 200. Response Status is {resp.status_code}", - ) - SNlM0e = re.search(r"SNlM0e\":\"(.*?)\"", resp.text) - if not SNlM0e: - raise Exception( - "SNlM0e value not found in response. Check __Secure-1PSID value.", - ) - return SNlM0e.group(1) - - async def ask(self, message: str) -> dict: - """ - Send a message to Google Bard and return the response. - :param message: The message to send to Google Bard. - :return: A dict containing the response from Google Bard. - """ - # url params - params = { - "bl": "boq_assistant-bard-web-server_20230523.13_p0", - "_reqid": str(self._reqid), - "rt": "c", - } - - # message arr -> data["f.req"]. Message is double json stringified - message_struct = [ - [message], - None, - [self.conversation_id, self.response_id, self.choice_id], - ] - data = { - "f.req": json.dumps([None, json.dumps(message_struct)]), - "at": self.SNlM0e, - } - resp = await self.session.post( - "https://bard.google.com/_/BardChatUi/data/assistant.lamda.BardFrontendService/StreamGenerate", - params=params, - data=data, - timeout=self.timeout, - ) - chat_data = json.loads(resp.content.splitlines()[3])[0][2] - if not chat_data: - return {"content": f"Google Bard encountered an error: {resp.content}."} - json_chat_data = json.loads(chat_data) - images = set() - if len(json_chat_data) >= 3: - if len(json_chat_data[4][0]) >= 4: - if json_chat_data[4][0][4]: - for img in json_chat_data[4][0][4]: - images.add(img[0][0][0]) - results = { - "content": json_chat_data[0][0], - "conversation_id": json_chat_data[1][0], - "response_id": json_chat_data[1][1], - "factualityQueries": json_chat_data[3], - "textQuery": json_chat_data[2][0] if json_chat_data[2] is not None else "", - "choices": [{"id": i[0], "content": i[1]} for i in json_chat_data[4]], - "images": images, - } - self.conversation_id = results["conversation_id"] - self.response_id = results["response_id"] - self.choice_id = results["choices"][0]["id"] - self._reqid += 100000 - return results diff --git a/src/bot.py b/src/bot.py index 04a62fa..ca57f6f 100644 --- a/src/bot.py +++ b/src/bot.py @@ -822,7 +822,7 @@ class Bot: ) except TimeoutError: await send_room_message(self.client, room_id, reply_message="TimeoutError") - except Exception as e: + except Exception: await send_room_message( self.client, room_id, @@ -838,9 +838,13 @@ class Bot: await self.client.room_typing(room_id) if self.flowise_api_key is not None: headers = {"Authorization": f"Bearer {self.flowise_api_key}"} - response = await flowise_query(self.flowise_api_url, prompt, self.session, headers) + response = await flowise_query( + self.flowise_api_url, prompt, self.session, headers + ) else: - response = await flowise_query(self.flowise_api_url, prompt, self.session) + response = await flowise_query( + self.flowise_api_url, prompt, self.session + ) await send_room_message( self.client, room_id, @@ -850,7 +854,7 @@ class Bot: user_message=raw_user_message, markdown_formatted=self.markdown_formatted, ) - except Exception as e: + except Exception: await send_room_message( self.client, room_id, diff --git a/src/chatgpt_bing.py b/src/chatgpt_bing.py index b148821..3feb879 100644 --- a/src/chatgpt_bing.py +++ b/src/chatgpt_bing.py @@ -1,5 +1,4 @@ import aiohttp -import asyncio from log import getlogger logger = getlogger() @@ -42,8 +41,8 @@ async def test_chatgpt(): { "clientOptions": { "clientToUse": "chatgpt", - } - } + }, + }, ) resp = await gptbot.queryChatGPT(payload) content = resp["response"] @@ -63,12 +62,12 @@ async def test_bing(): { "clientOptions": { "clientToUse": "bing", - } - } + }, + }, ) resp = await gptbot.queryBing(payload) content = "".join( - [body["text"] for body in resp["details"]["adaptiveCards"][0]["body"]] + [body["text"] for body in resp["details"]["adaptiveCards"][0]["body"]], ) payload["conversationSignature"] = resp["conversationSignature"] payload["conversationId"] = resp["conversationId"] diff --git a/src/flowise.py b/src/flowise.py index 65b2c12..500dbf6 100644 --- a/src/flowise.py +++ b/src/flowise.py @@ -1,7 +1,9 @@ import aiohttp -# need refactor: flowise_api does not support context converstaion, temporarily set it aside -async def flowise_query(api_url: str, prompt: str, session: aiohttp.ClientSession, headers: dict = None) -> str: + +async def flowise_query( + api_url: str, prompt: str, session: aiohttp.ClientSession, headers: dict = None +) -> str: """ Sends a query to the Flowise API and returns the response. @@ -16,19 +18,25 @@ async def flowise_query(api_url: str, prompt: str, session: aiohttp.ClientSessio """ if headers: response = await session.post( - api_url, json={"question": prompt}, headers=headers + api_url, + json={"question": prompt}, + headers=headers, ) else: response = await session.post(api_url, json={"question": prompt}) return await response.json() + async def test(): session = aiohttp.ClientSession() - api_url = "http://127.0.0.1:3000/api/v1/prediction/683f9ea8-e670-4d51-b657-0886eab9cea1" + api_url = ( + "http://127.0.0.1:3000/api/v1/prediction/683f9ea8-e670-4d51-b657-0886eab9cea1" + ) prompt = "What is the capital of France?" response = await flowise_query(api_url, prompt, session) print(response) + if __name__ == "__main__": import asyncio diff --git a/src/log.py b/src/log.py index db5f708..5d4976a 100644 --- a/src/log.py +++ b/src/log.py @@ -1,6 +1,6 @@ import logging -from pathlib import Path import os +from pathlib import Path log_path = Path(os.path.dirname(__file__)).parent / "bot.log" @@ -20,10 +20,10 @@ def getlogger(): # create formatters warn_format = logging.Formatter( - "%(asctime)s - %(funcName)s - %(levelname)s - %(message)s" + "%(asctime)s - %(funcName)s - %(levelname)s - %(message)s", ) error_format = logging.Formatter( - "%(asctime)s - %(name)s - %(funcName)s - %(levelname)s - %(message)s" + "%(asctime)s - %(name)s - %(funcName)s - %(levelname)s - %(message)s", ) info_format = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s") diff --git a/src/main.py b/src/main.py index 86853ef..28940ce 100644 --- a/src/main.py +++ b/src/main.py @@ -2,6 +2,7 @@ import asyncio import json import os from pathlib import Path + from bot import Bot from log import getlogger @@ -12,7 +13,7 @@ async def main(): need_import_keys = False config_path = Path(os.path.dirname(__file__)).parent / "config.json" if os.path.isfile(config_path): - fp = open(config_path, "r", encoding="utf8") + fp = open(config_path, encoding="utf8") config = json.load(fp) matrix_bot = Bot( diff --git a/src/pandora_api.py b/src/pandora_api.py index 71fd299..4b4d1c5 100644 --- a/src/pandora_api.py +++ b/src/pandora_api.py @@ -1,7 +1,8 @@ # API wrapper for https://github.com/pengzhile/pandora/blob/master/doc/HTTP-API.md -import uuid -import aiohttp import asyncio +import uuid + +import aiohttp class Pandora: diff --git a/src/send_image.py b/src/send_image.py index c70fd69..5529f2c 100644 --- a/src/send_image.py +++ b/src/send_image.py @@ -3,11 +3,13 @@ code derived from: https://matrix-nio.readthedocs.io/en/latest/examples.html#sending-an-image """ import os + import aiofiles.os import magic -from PIL import Image -from nio import AsyncClient, UploadResponse from log import getlogger +from nio import AsyncClient +from nio import UploadResponse +from PIL import Image logger = getlogger() @@ -31,13 +33,13 @@ async def send_room_image(client: AsyncClient, room_id: str, image: str): filesize=file_stat.st_size, ) if not isinstance(resp, UploadResponse): - logger.warning(f"Failed to generate image. Failure response: {resp}") + logger.warning(f"Failed to upload image. Failure response: {resp}") await client.room_send( room_id, message_type="m.room.message", content={ "msgtype": "m.text", - "body": f"Failed to generate image. Failure response: {resp}", + "body": f"Failed to upload image. Failure response: {resp}", }, ignore_unverified_devices=True, ) diff --git a/src/send_message.py b/src/send_message.py index bddda24..946360b 100644 --- a/src/send_message.py +++ b/src/send_message.py @@ -1,7 +1,8 @@ -from nio import AsyncClient import re + import markdown from log import getlogger +from nio import AsyncClient logger = getlogger() @@ -28,7 +29,8 @@ async def send_room_message( "body": reply_message, "format": "org.matrix.custom.html", "formatted_body": markdown.markdown( - reply_message, extensions=["nl2br", "tables", "fenced_code"] + reply_message, + extensions=["nl2br", "tables", "fenced_code"], ), } else: