Introduce pre-commit-hooks

.env.example

@@ -17,4 +17,4 @@ FLOWISE_API_URL="http://localhost:3000/api/v1/prediction/xxxx" # Optional
 FLOWISE_API_KEY="xxxxxxxxxxxxxxxxxxxxxxx" # Optional
 PANDORA_API_ENDPOINT="http://pandora:8008" # Optional, for !talk, !goon command
 PANDORA_API_MODEL="text-davinci-002-render-sha-mobile" # Optional
-TEMPERATURE="0.8" # Optional
+TEMPERATURE="0.8" # Optional

.github/workflows/pylint.yml

@@ -1,31 +0,0 @@
-name: Pylint
-
-on:
-  push:
-    paths:
-      - 'src/**'
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.10", "3.11"]
-    steps:
-    - uses: actions/checkout@v3
-    - name: Install libolm-dev
-      run: |
-        sudo apt install -y libolm-dev
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python-version }}
-        cache: 'pip'
-    - name: Install dependencies
-      run: |
-        pip install -U pip setuptools wheel
-        pip install -r requirements.txt
-        pip install pylint
-    - name: Analysing the code with pylint
-      run: |
-        pylint $(git ls-files '*.py') --errors-only

.pre-commit-config.yaml

@@ -0,0 +1,16 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+  - repo: https://github.com/psf/black
+    rev: 23.9.1
+    hooks:
+      - id: black
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.0.289
+    hooks:
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix]

README.md

@@ -5,7 +5,7 @@ This is a simple Matrix bot that support using OpenAI API, Langchain to generate
 
 ## Feature
 
-1. Support official openai api and self host models([LocalAI](https://github.com/go-skynet/LocalAI))   
+1. Support official openai api and self host models([LocalAI](https://github.com/go-skynet/LocalAI))
 2. Support E2E Encrypted Room
 3. Colorful code blocks
 4. Langchain([Flowise](https://github.com/FlowiseAI/Flowise))

requirements.txt

@@ -1,51 +1,8 @@
-aiofiles==23.1.0
+aiofiles
-aiohttp==3.8.4
+aiohttp
-aiohttp-socks==0.7.1
+Markdown
-aiosignal==1.3.1
+matrix-nio[e2e]
-anyio==3.6.2
+Pillow
-async-timeout==4.0.2
+tiktoken
-atomicwrites==1.4.1
+tenacity
-attrs==22.2.0
+python-magic
-blobfile==2.0.1
-cachetools==4.2.4
-certifi==2022.12.7
-cffi==1.15.1
-charset-normalizer==3.1.0
-cryptography==41.0.0
-filelock==3.11.0
-frozenlist==1.3.3
-future==0.18.3
-h11==0.14.0
-h2==4.1.0
-hpack==4.0.0
-httpcore==0.16.3
-httpx==0.23.3
-hyperframe==6.0.1
-idna==3.4
-jsonschema==4.17.3
-Logbook==1.5.3
-lxml==4.9.2
-Markdown==3.4.3
-matrix-nio[e2e]==0.20.2
-multidict==6.0.4
-peewee==3.16.0
-Pillow==9.5.0
-pycparser==2.21
-pycryptodome==3.17
-pycryptodomex==3.17
-pyrsistent==0.19.3
-python-cryptography-fernet-wrapper==1.0.4
-python-magic==0.4.27
-python-olm==3.1.3
-python-socks==2.2.0
-regex==2023.3.23
-requests==2.31.0
-rfc3986==1.5.0
-six==1.16.0
-sniffio==1.3.0
-tiktoken==0.3.3
-toml==0.10.2
-unpaddedbase64==2.1.0
-urllib3==1.26.15
-wcwidth==0.2.6
-yarl==1.8.2

settings.js.example

@@ -98,4 +98,4 @@ export default {
         // (Optional) Possible options: "chatgpt", "bing".
         // clientToUse: 'bing',
     },
-};
+};

src/BingImageGen.py

@@ -1,184 +0,0 @@
-"""
-Code derived from:
-https://github.com/acheong08/EdgeGPT/blob/f940cecd24a4818015a8b42a2443dd97c3c2a8f4/src/ImageGen.py
-"""
-
-from log import getlogger
-from uuid import uuid4
-import os
-import contextlib
-import aiohttp
-import asyncio
-import random
-import requests
-import regex
-
-logger = getlogger()
-
-BING_URL = "https://www.bing.com"
-# Generate random IP between range 13.104.0.0/14
-FORWARDED_IP = (
-    f"13.{random.randint(104, 107)}.{random.randint(0, 255)}.{random.randint(0, 255)}"
-)
-HEADERS = {
-    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",  # noqa: E501
-    "accept-language": "en-US,en;q=0.9",
-    "cache-control": "max-age=0",
-    "content-type": "application/x-www-form-urlencoded",
-    "referrer": "https://www.bing.com/images/create/",
-    "origin": "https://www.bing.com",
-    "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36 Edg/110.0.1587.63",  # noqa: E501
-    "x-forwarded-for": FORWARDED_IP,
-}
-
-
-class ImageGenAsync:
-    """
-    Image generation by Microsoft Bing
-    Parameters:
-        auth_cookie: str
-    """
-
-    def __init__(self, auth_cookie: str, quiet: bool = True) -> None:
-        self.session = aiohttp.ClientSession(
-            headers=HEADERS,
-            cookies={"_U": auth_cookie},
-        )
-        self.quiet = quiet
-
-    async def __aenter__(self):
-        return self
-
-    async def __aexit__(self, *excinfo) -> None:
-        await self.session.close()
-
-    def __del__(self):
-        try:
-            loop = asyncio.get_running_loop()
-        except RuntimeError:
-            loop = asyncio.new_event_loop()
-            asyncio.set_event_loop(loop)
-        loop.run_until_complete(self._close())
-
-    async def _close(self):
-        await self.session.close()
-
-    async def get_images(self, prompt: str) -> list:
-        """
-        Fetches image links from Bing
-        Parameters:
-            prompt: str
-        """
-        if not self.quiet:
-            print("Sending request...")
-        url_encoded_prompt = requests.utils.quote(prompt)
-        # https://www.bing.com/images/create?q=<PROMPT>&rt=3&FORM=GENCRE
-        url = f"{BING_URL}/images/create?q={url_encoded_prompt}&rt=4&FORM=GENCRE"
-        async with self.session.post(url, allow_redirects=False) as response:
-            content = await response.text()
-            if "this prompt has been blocked" in content.lower():
-                raise Exception(
-                    "Your prompt has been blocked by Bing. Try to change any bad words and try again.",  # noqa: E501
-                )
-            if response.status != 302:
-                # if rt4 fails, try rt3
-                url = (
-                    f"{BING_URL}/images/create?q={url_encoded_prompt}&rt=3&FORM=GENCRE"
-                )
-                async with self.session.post(
-                    url,
-                    allow_redirects=False,
-                    timeout=200,
-                ) as response3:
-                    if response3.status != 302:
-                        print(f"ERROR: {response3.text}")
-                        raise Exception("Redirect failed")
-                    response = response3
-        # Get redirect URL
-        redirect_url = response.headers["Location"].replace("&nfy=1", "")
-        request_id = redirect_url.split("id=")[-1]
-        await self.session.get(f"{BING_URL}{redirect_url}")
-        # https://www.bing.com/images/create/async/results/{ID}?q={PROMPT}
-        polling_url = f"{BING_URL}/images/create/async/results/{request_id}?q={url_encoded_prompt}"  # noqa: E501
-        # Poll for results
-        if not self.quiet:
-            print("Waiting for results...")
-        while True:
-            if not self.quiet:
-                print(".", end="", flush=True)
-            # By default, timeout is 300s, change as needed
-            response = await self.session.get(polling_url)
-            if response.status != 200:
-                raise Exception("Could not get results")
-            content = await response.text()
-            if content and content.find("errorMessage") == -1:
-                break
-
-            await asyncio.sleep(1)
-            continue
-        # Use regex to search for src=""
-        image_links = regex.findall(r'src="([^"]+)"', content)
-        # Remove size limit
-        normal_image_links = [link.split("?w=")[0] for link in image_links]
-        # Remove duplicates
-        normal_image_links = list(set(normal_image_links))
-
-        # Bad images
-        bad_images = [
-            "https://r.bing.com/rp/in-2zU3AJUdkgFe7ZKv19yPBHVs.png",
-            "https://r.bing.com/rp/TX9QuO3WzcCJz1uaaSwQAz39Kb0.jpg",
-        ]
-        for im in normal_image_links:
-            if im in bad_images:
-                raise Exception("Bad images")
-        # No images
-        if not normal_image_links:
-            raise Exception("No images")
-        return normal_image_links
-
-    async def save_images(
-        self, links: list, output_dir: str, output_four_images: bool
-    ) -> list:
-        """
-        Saves images to output directory
-        """
-        with contextlib.suppress(FileExistsError):
-            os.mkdir(output_dir)
-
-        image_path_list = []
-
-        if output_four_images:
-            for link in links:
-                image_name = str(uuid4())
-                image_path = os.path.join(output_dir, f"{image_name}.jpeg")
-                try:
-                    async with self.session.get(
-                        link, raise_for_status=True
-                    ) as response:
-                        with open(image_path, "wb") as output_file:
-                            async for chunk in response.content.iter_chunked(8192):
-                                output_file.write(chunk)
-                    image_path_list.append(image_path)
-                except aiohttp.client_exceptions.InvalidURL as url_exception:
-                    raise Exception(
-                        "Inappropriate contents found in the generated images. Please try again or try another prompt."
-                    ) from url_exception  # noqa: E501
-        else:
-            image_name = str(uuid4())
-            if links:
-                link = links.pop()
-                try:
-                    async with self.session.get(
-                        link, raise_for_status=True
-                    ) as response:
-                        image_path = os.path.join(output_dir, f"{image_name}.jpeg")
-                        with open(image_path, "wb") as output_file:
-                            async for chunk in response.content.iter_chunked(8192):
-                                output_file.write(chunk)
-                        image_path_list.append(image_path)
-                except aiohttp.client_exceptions.InvalidURL as url_exception:
-                    raise Exception(
-                        "Inappropriate contents found in the generated images. Please try again or try another prompt."
-                    ) from url_exception  # noqa: E501
-
-        return image_path_list

src/askgpt.py

@@ -1,6 +1,6 @@
-import aiohttp
-import asyncio
 import json
+
+import aiohttp
 from log import getlogger
 
 logger = getlogger()
@@ -10,7 +10,9 @@ class askGPT:
     def __init__(self, session: aiohttp.ClientSession):
         self.session = session
 
-    async def oneTimeAsk(self, prompt: str, api_endpoint: str, headers: dict, temperature: float = 0.8) -> str:
+    async def oneTimeAsk(
+        self, prompt: str, api_endpoint: str, headers: dict, temperature: float = 0.8
+    ) -> str:
         jsons = {
             "model": "gpt-3.5-turbo",
             "messages": [
@@ -25,7 +27,10 @@ class askGPT:
         while max_try > 0:
             try:
                 async with self.session.post(
-                    url=api_endpoint, json=jsons, headers=headers, timeout=120
+                    url=api_endpoint,
+                    json=jsons,
+                    headers=headers,
+                    timeout=120,
                 ) as response:
                     status_code = response.status
                     if not status_code == 200:

src/bard.py

@@ -1,142 +0,0 @@
-"""
-Code derived from: https://github.com/acheong08/Bard/blob/main/src/Bard.py
-"""
-
-import random
-import string
-import re
-import json
-import httpx
-
-
-class Bardbot:
-    """
-    A class to interact with Google Bard.
-    Parameters
-        session_id: str
-            The __Secure-1PSID cookie.
-        timeout: int
-            Request timeout in seconds.
-        session: requests.Session
-            Requests session object.
-    """
-
-    __slots__ = [
-        "headers",
-        "_reqid",
-        "SNlM0e",
-        "conversation_id",
-        "response_id",
-        "choice_id",
-        "session_id",
-        "session",
-        "timeout",
-    ]
-
-    def __init__(
-        self,
-        session_id: str,
-        timeout: int = 20,
-    ):
-        headers = {
-            "Host": "bard.google.com",
-            "X-Same-Domain": "1",
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36",
-            "Content-Type": "application/x-www-form-urlencoded;charset=UTF-8",
-            "Origin": "https://bard.google.com",
-            "Referer": "https://bard.google.com/",
-        }
-        self._reqid = int("".join(random.choices(string.digits, k=4)))
-        self.conversation_id = ""
-        self.response_id = ""
-        self.choice_id = ""
-        self.session_id = session_id
-        self.session = httpx.AsyncClient()
-        self.session.headers = headers
-        self.session.cookies.set("__Secure-1PSID", session_id)
-        self.timeout = timeout
-
-    @classmethod
-    async def create(
-        cls,
-        session_id: str,
-        timeout: int = 20,
-    ) -> "Bardbot":
-        instance = cls(session_id, timeout)
-        instance.SNlM0e = await instance.__get_snlm0e()
-        return instance
-
-    async def __get_snlm0e(self):
-        # Find "SNlM0e":"<ID>"
-        if not self.session_id or self.session_id[-1] != ".":
-            raise Exception(
-                "__Secure-1PSID value must end with a single dot. Enter correct __Secure-1PSID value.",
-            )
-        resp = await self.session.get(
-            "https://bard.google.com/",
-            timeout=10,
-        )
-        if resp.status_code != 200:
-            raise Exception(
-                f"Response code not 200. Response Status is {resp.status_code}",
-            )
-        SNlM0e = re.search(r"SNlM0e\":\"(.*?)\"", resp.text)
-        if not SNlM0e:
-            raise Exception(
-                "SNlM0e value not found in response. Check __Secure-1PSID value.",
-            )
-        return SNlM0e.group(1)
-
-    async def ask(self, message: str) -> dict:
-        """
-        Send a message to Google Bard and return the response.
-        :param message: The message to send to Google Bard.
-        :return: A dict containing the response from Google Bard.
-        """
-        # url params
-        params = {
-            "bl": "boq_assistant-bard-web-server_20230523.13_p0",
-            "_reqid": str(self._reqid),
-            "rt": "c",
-        }
-
-        # message arr -> data["f.req"]. Message is double json stringified
-        message_struct = [
-            [message],
-            None,
-            [self.conversation_id, self.response_id, self.choice_id],
-        ]
-        data = {
-            "f.req": json.dumps([None, json.dumps(message_struct)]),
-            "at": self.SNlM0e,
-        }
-        resp = await self.session.post(
-            "https://bard.google.com/_/BardChatUi/data/assistant.lamda.BardFrontendService/StreamGenerate",
-            params=params,
-            data=data,
-            timeout=self.timeout,
-        )
-        chat_data = json.loads(resp.content.splitlines()[3])[0][2]
-        if not chat_data:
-            return {"content": f"Google Bard encountered an error: {resp.content}."}
-        json_chat_data = json.loads(chat_data)
-        images = set()
-        if len(json_chat_data) >= 3:
-            if len(json_chat_data[4][0]) >= 4:
-                if json_chat_data[4][0][4]:
-                    for img in json_chat_data[4][0][4]:
-                        images.add(img[0][0][0])
-        results = {
-            "content": json_chat_data[0][0],
-            "conversation_id": json_chat_data[1][0],
-            "response_id": json_chat_data[1][1],
-            "factualityQueries": json_chat_data[3],
-            "textQuery": json_chat_data[2][0] if json_chat_data[2] is not None else "",
-            "choices": [{"id": i[0], "content": i[1]} for i in json_chat_data[4]],
-            "images": images,
-        }
-        self.conversation_id = results["conversation_id"]
-        self.response_id = results["response_id"]
-        self.choice_id = results["choices"][0]["id"]
-        self._reqid += 100000
-        return results

src/bot.py

@@ -822,7 +822,7 @@ class Bot:
             )
         except TimeoutError:
             await send_room_message(self.client, room_id, reply_message="TimeoutError")
-        except Exception as e:
+        except Exception:
             await send_room_message(
                 self.client,
                 room_id,
@@ -838,9 +838,13 @@ class Bot:
             await self.client.room_typing(room_id)
             if self.flowise_api_key is not None:
                 headers = {"Authorization": f"Bearer {self.flowise_api_key}"}
-                response = await flowise_query(self.flowise_api_url, prompt, self.session, headers)
+                response = await flowise_query(
+                    self.flowise_api_url, prompt, self.session, headers
+                )
             else:
-                response = await flowise_query(self.flowise_api_url, prompt, self.session)
+                response = await flowise_query(
+                    self.flowise_api_url, prompt, self.session
+                )
             await send_room_message(
                 self.client,
                 room_id,
@@ -850,7 +854,7 @@ class Bot:
                 user_message=raw_user_message,
                 markdown_formatted=self.markdown_formatted,
             )
-        except Exception as e:
+        except Exception:
             await send_room_message(
                 self.client,
                 room_id,

src/chatgpt_bing.py

@@ -1,5 +1,4 @@
 import aiohttp
-import asyncio
 from log import getlogger
 
 logger = getlogger()
@@ -42,8 +41,8 @@ async def test_chatgpt():
             {
                 "clientOptions": {
                     "clientToUse": "chatgpt",
-                }
+                },
-            }
+            },
         )
         resp = await gptbot.queryChatGPT(payload)
         content = resp["response"]
@@ -63,12 +62,12 @@ async def test_bing():
             {
                 "clientOptions": {
                     "clientToUse": "bing",
-                }
+                },
-            }
+            },
         )
         resp = await gptbot.queryBing(payload)
         content = "".join(
-            [body["text"] for body in resp["details"]["adaptiveCards"][0]["body"]]
+            [body["text"] for body in resp["details"]["adaptiveCards"][0]["body"]],
         )
         payload["conversationSignature"] = resp["conversationSignature"]
         payload["conversationId"] = resp["conversationId"]

src/flowise.py

@@ -1,7 +1,9 @@
 import aiohttp
-# need refactor: flowise_api does not support context converstaion, temporarily set it aside
 
-async def flowise_query(api_url: str, prompt: str, session: aiohttp.ClientSession, headers: dict = None) -> str:
+
+async def flowise_query(
+    api_url: str, prompt: str, session: aiohttp.ClientSession, headers: dict = None
+) -> str:
     """
     Sends a query to the Flowise API and returns the response.
 
@@ -16,19 +18,25 @@ async def flowise_query(api_url: str, prompt: str, session: aiohttp.ClientSessio
     """
     if headers:
         response = await session.post(
-            api_url, json={"question": prompt}, headers=headers
+            api_url,
+            json={"question": prompt},
+            headers=headers,
         )
     else:
         response = await session.post(api_url, json={"question": prompt})
     return await response.json()
 
+
 async def test():
     session = aiohttp.ClientSession()
-    api_url = "http://127.0.0.1:3000/api/v1/prediction/683f9ea8-e670-4d51-b657-0886eab9cea1"
+    api_url = (
+        "http://127.0.0.1:3000/api/v1/prediction/683f9ea8-e670-4d51-b657-0886eab9cea1"
+    )
     prompt = "What is the capital of France?"
     response = await flowise_query(api_url, prompt, session)
     print(response)
 
+
 if __name__ == "__main__":
     import asyncio
 

src/log.py

@@ -1,6 +1,6 @@
 import logging
-from pathlib import Path
 import os
+from pathlib import Path
 
 log_path = Path(os.path.dirname(__file__)).parent / "bot.log"
 
@@ -20,10 +20,10 @@ def getlogger():
 
         # create formatters
         warn_format = logging.Formatter(
-            "%(asctime)s - %(funcName)s - %(levelname)s - %(message)s"
+            "%(asctime)s - %(funcName)s - %(levelname)s - %(message)s",
         )
         error_format = logging.Formatter(
-            "%(asctime)s - %(name)s - %(funcName)s - %(levelname)s - %(message)s"
+            "%(asctime)s - %(name)s - %(funcName)s - %(levelname)s - %(message)s",
         )
         info_format = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
 

src/main.py

@@ -2,6 +2,7 @@ import asyncio
 import json
 import os
 from pathlib import Path
+
 from bot import Bot
 from log import getlogger
 
@@ -12,7 +13,7 @@ async def main():
     need_import_keys = False
     config_path = Path(os.path.dirname(__file__)).parent / "config.json"
     if os.path.isfile(config_path):
-        fp = open(config_path, "r", encoding="utf8")
+        fp = open(config_path, encoding="utf8")
         config = json.load(fp)
 
         matrix_bot = Bot(

src/pandora_api.py

@@ -1,7 +1,8 @@
 # API wrapper for https://github.com/pengzhile/pandora/blob/master/doc/HTTP-API.md
-import uuid
-import aiohttp
 import asyncio
+import uuid
+
+import aiohttp
 
 
 class Pandora:

src/send_image.py

@@ -3,11 +3,13 @@ code derived from:
 https://matrix-nio.readthedocs.io/en/latest/examples.html#sending-an-image
 """
 import os
+
 import aiofiles.os
 import magic
-from PIL import Image
-from nio import AsyncClient, UploadResponse
 from log import getlogger
+from nio import AsyncClient
+from nio import UploadResponse
+from PIL import Image
 
 logger = getlogger()
 
@@ -31,13 +33,13 @@ async def send_room_image(client: AsyncClient, room_id: str, image: str):
             filesize=file_stat.st_size,
         )
     if not isinstance(resp, UploadResponse):
-        logger.warning(f"Failed to generate image. Failure response: {resp}")
+        logger.warning(f"Failed to upload image. Failure response: {resp}")
         await client.room_send(
             room_id,
             message_type="m.room.message",
             content={
                 "msgtype": "m.text",
-                "body": f"Failed to generate image. Failure response: {resp}",
+                "body": f"Failed to upload image. Failure response: {resp}",
             },
             ignore_unverified_devices=True,
         )

src/send_message.py

@@ -1,7 +1,8 @@
-from nio import AsyncClient
 import re
+
 import markdown
 from log import getlogger
+from nio import AsyncClient
 
 logger = getlogger()
 
@@ -28,7 +29,8 @@ async def send_room_message(
                     "body": reply_message,
                     "format": "org.matrix.custom.html",
                     "formatted_body": markdown.markdown(
-                        reply_message, extensions=["nl2br", "tables", "fenced_code"]
+                        reply_message,
+                        extensions=["nl2br", "tables", "fenced_code"],
                     ),
                 }
             else: