From 70d8a8aa0c5fa9f58c5ed5f8f87624721be0674e Mon Sep 17 00:00:00 2001 From: hibobmaster Date: Thu, 20 Apr 2023 00:45:28 +0800 Subject: [PATCH] Release the code --- .dockerignore | 23 ++ .env.example | 8 + .github/workflows/docker-release.yml | 76 ++++ .github/workflows/pylint.yml | 30 ++ .gitignore | 10 + Dockerfile | 30 ++ README.md | 101 ++++- bot.py | 579 +++++++++++++++++++++++++++ compose.yaml | 24 ++ config.json.example | 10 + log.py | 34 ++ requirements.txt | 51 +++ send_message.py | 27 ++ 13 files changed, 1001 insertions(+), 2 deletions(-) create mode 100644 .dockerignore create mode 100644 .env.example create mode 100644 .github/workflows/docker-release.yml create mode 100644 .github/workflows/pylint.yml create mode 100644 Dockerfile create mode 100644 bot.py create mode 100644 compose.yaml create mode 100644 config.json.example create mode 100644 log.py create mode 100644 requirements.txt create mode 100644 send_message.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..d83621c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,23 @@ +.gitignore +*.md +Dockerfile +Dockerfile-dev +.dockerignore +config.json +config.json.sample +db +bot.log +venv +.venv +*.yaml +*.yml +.git +.idea +__pycache__ +.env +.env.example +.github +settings.js +models +output +element-keys.txt \ No newline at end of file diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..485966f --- /dev/null +++ b/.env.example @@ -0,0 +1,8 @@ +HOMESERVER="https://xxxxx.xxxx.xxxx" +USER_ID="@xxxx:matrix.org" +PASSWORD="xxxxxxxxxx" +DEVICE_ID="GMIAZSVFF" +ROOM_ID="!xxxxxxxx:xxx.xxx.xxx" +IMPORT_KEYS_PATH="element-keys.txt" +IMPORT_KEYS_PASSWORD="xxxxxxxxxx" +MODEL_SIZE="base" \ No newline at end of file diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml new file mode 100644 index 0000000..29456cb --- /dev/null +++ b/.github/workflows/docker-release.yml @@ -0,0 +1,76 @@ +name: Publish Docker image + +on: + workflow_dispatch: + release: + types: [published] + +jobs: + push_to_registry: + name: Push Docker image to registry + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Check out the repo + uses: actions/checkout@v3 + + - name: Log in to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: hibobmaster/matrix-stt-bot + tags: | + type=raw,value=latest + type=ref,event=tag + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build and push Docker image(dockerhub) + uses: docker/build-push-action@v4 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Docker metadata(ghcr) + id: meta2 + uses: docker/metadata-action@v4 + with: + images: ghcr.io/hibobmaster/matrix-stt-bot + tags: | + type=raw,value=latest + type=sha,format=long + + - name: Build and push Docker image(ghcr) + uses: docker/build-push-action@v4 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta2.outputs.tags }} + labels: ${{ steps.meta2.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml new file mode 100644 index 0000000..a16a46a --- /dev/null +++ b/.github/workflows/pylint.yml @@ -0,0 +1,30 @@ +name: Pylint + +on: [push, pull_request] + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + strategy: + matrix: + python-version: ["3.10"] + steps: + - uses: actions/checkout@v3 + - name: Install libolm-dev + run: | + sudo apt install -y libolm-dev + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Install dependencies + run: | + pip install -U pip setuptools wheel + pip install -r requirements.txt + pip install pylint + - name: Analysing the code with pylint + run: | + pylint $(git ls-files '*.py') --errors-only diff --git a/.gitignore b/.gitignore index b6e4761..d8eaa00 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,16 @@ share/python-wheels/ *.egg MANIFEST +# Custom file/path +config.json +.env +db +models +element-keys.txt +output +Dockerfile-dev +compose-dev.yaml + # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..07f90a3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,30 @@ +FROM python:3.10-bullseye as base + +FROM base as pybuilder + +RUN set -eux; \ + apt update; \ + apt install -y --no-install-recommends \ + libolm-dev \ + ; \ + rm -rf /var/lib/apt/lists/* + +COPY requirements.txt /requirements.txt +RUN pip install -U pip setuptools wheel && pip install --user -r /requirements.txt && rm /requirements.txt + + +FROM base as runner +RUN set -eux; \ + apt update; \ + apt install -y --no-install-recommends \ + libolm-dev \ + ; \ + rm -rf /var/lib/apt/lists/* + +COPY --from=pybuilder /root/.local /usr/local +COPY . /app + + +FROM runner +WORKDIR /app +CMD ["python", "bot.py"] diff --git a/README.md b/README.md index 501a5e1..55e7731 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,99 @@ -# matrix-stt-bot -A simple matrix bot transcribes your voice to text message +# Introduction +This is a simple matrix bot that transcribes your voice to text message using faster-whisper, a reimplementation of OpenAI's Whisper model using CTranslate2. + +## Feature + +1. Liberate your hands: support automatically speech to text transcribtion +2. Support E2EE Room +3. Self host your service without privacy problem + +## Installation and Setup + +1. Edit `config.json` or `.env` with proper values +2. Edit `compose.yaml` +3. Launch the container + +Here is a guide to make bot works on E2E encrypted room. + +For explainations and complete parameter list see: https://github.com/hibobmaster/matrix-stt-bot/wiki + +1. Create `config.json` + +Tips: set a non-exist `room_id` at the first time to prevent bot handling historical message which may mess your room up. +```json +{ + "homeserver": "https://matrix.org", + "user_id": "@xxxx:matrix.org", + "password": "xxxxxxxxxx", + "device_id": "GMIAZSVFF", + "room_id": "!xxxxxxxx:xxx.xxx.xxx", + "model_size": "base", + "import_keys_path": "element-keys.txt", + "import_keys_password": "xxxxxxxxxxxx" +} +``` +2. Create `compose.yaml` + +```yaml +services: + app: + image: ghcr.io/hibobmaster/matrix-stt-bot:latest + container_name: matrix-stt-bot + restart: always + # build: + # context: . + # dockerfile: ./Dockerfile + # env_file: + # - .env + volumes: + # use env file or config.json + - ./config.json:/app/config.json + # use touch to create an empty file stt_db, for persist database only + - ./stt_db:/app/db + # import_keys path + - ./element-keys.txt:/app/element-keys.txt + # store whisper models that program will download + - ./models:/app/models + networks: + - matrix_network + +networks: + matrix_network: +``` +Get your E2E room keys here: +![e2e-room-keys](https://i.imgur.com/WTKlXob.jpg) +Notice: If you deploy [matrix_chatgpt_bot](https://github.com/hibobmaster/matrix_chatgpt_bot) along with this project, remember do not use the same database name. + +3. Launch for the first time +```sh +docker compose up +``` +You will get notice: `INFO - start import_keys process` + +After `INFO - import_keys success, please remove import_keys configuration!!!` + +Wait a second, to see if `stt_db` has finished syncing_progress (The space occupied is about 100kb and above?) + +Then `Ctrl+C` stop the container + +4. Edit `config.json` again + +Remove `import_keys_path` and `import_keys_password` options + +Set a correct `room_id` or remove it if you hope the bot to work in the rooms it is in. + +5. Finally + +Launch the container +```sh +docker compose up -d +``` + +## Demo +![demo1](https://i.imgur.com/vntImys.png) +![demo2](https://i.imgur.com/VkOOVZA.png) + +## Thanks +1. https://github.com/guillaumekln/faster-whisper +2. https://github.com/poljar/matrix-nio +3. https://github.com/8go/matrix-commander \ No newline at end of file diff --git a/bot.py b/bot.py new file mode 100644 index 0000000..4927ab0 --- /dev/null +++ b/bot.py @@ -0,0 +1,579 @@ +import os +import sys +import traceback +from typing import Union, Optional +import aiofiles +import asyncio +import uuid +import json +from nio import (AsyncClient, AsyncClientConfig, InviteMemberEvent, JoinError, + KeyVerificationCancel, KeyVerificationEvent, DownloadError, + KeyVerificationKey, KeyVerificationMac, KeyVerificationStart, + LocalProtocolError, LoginResponse, MatrixRoom, MegolmEvent, + RoomMessageAudio, RoomEncryptedAudio, ToDeviceError, crypto, + EncryptionError) +from nio.store.database import SqliteStore + +from faster_whisper import WhisperModel + +from log import getlogger +from send_message import send_room_message + +from unsync import unsync + +logger = getlogger() + + +class Bot: + def __init__( + self, + homeserver: str, + user_id: str, + device_id: str, + room_id: Union[str, None] = None, + password: Union[str, None] = None, + access_token: Union[str, None] = None, + import_keys_path: Optional[str] = None, + import_keys_password: Optional[str] = None, + model_size: str = "tiny", + device: str = "cpu", + compute_type: str = "int8", + cpu_threads: int = 0, + num_workers: int = 1, + download_root: str = "models", + ): + if (homeserver is None or user_id is None + or device_id is None): + logger.warning("homeserver && user_id && device_id is required") + sys.exit(1) + + if (password is None and access_token is None): + logger.warning("password or access_toekn is required") + sys.exit(1) + + self.homeserver = homeserver + self.user_id = user_id + self.password = password + self.access_token = access_token + self.device_id = device_id + self.room_id = room_id + self.import_keys_path = import_keys_path + self.import_keys_password = import_keys_password + self.model_size = model_size + self.device = device + self.compute_type = compute_type + self.cpu_threads = cpu_threads + self.num_workers = num_workers + self.download_root = download_root + + if model_size is None: + self.model_size = "tiny" + + if device is None: + self.device = "cpu" + + if compute_type is None: + self.compute_type = "int8" + + if cpu_threads is None: + self.cpu_threads = 0 + + if num_workers is None: + self.num_workers = 1 + + if download_root is None: + self.download_root = "models" + if not os.path.exists("models"): + os.mkdir("models") + + # initialize AsyncClient object + self.store_path = os.getcwd() + self.config = AsyncClientConfig(store=SqliteStore, + store_name="db", + store_sync_tokens=True, + encryption_enabled=True, + ) + self.client = AsyncClient(homeserver=self.homeserver, user=self.user_id, device_id=self.device_id, + config=self.config, store_path=self.store_path,) + + if self.access_token is not None: + self.client.access_token = self.access_token + + # setup event callbacks + self.client.add_event_callback( + self.message_callback, (RoomMessageAudio, RoomEncryptedAudio, )) + self.client.add_event_callback( + self.decryption_failure, (MegolmEvent, )) + self.client.add_event_callback( + self.invite_callback, (InviteMemberEvent, )) + self.client.add_to_device_callback( + self.to_device_callback, (KeyVerificationEvent, )) + + # intialize whisper model + if not os.path.exists(os.path.join(self.download_root, "model.txt")): + # that means we have not downloaded the model yet + logger.info("Model downloading") + self.model_size_or_path = self.model_size + with open(os.path.join(self.download_root, "model.txt"), "w") as f: + f.write(self.model_size) + else: + # model exists + f = open(os.path.join(self.download_root, "model.txt"), "r") + old_model = f.read() + if old_model != self.model_size: + # model size changed + self.model_size_or_path = self.model_size + f.close() + with open(os.path.join(self.download_root, "model.txt"), "w") as f: + f.write(self.model_size) + f.close() + else: + # use existed model + self.model_size_or_path = self.download_root + + self.model = WhisperModel( + model_size_or_path=self.model_size_or_path, + device=self.device, + compute_type=self.compute_type, + cpu_threads=self.cpu_threads, + num_workers=self.num_workers, + download_root=self.download_root,) + + def __del__(self): + try: + loop = asyncio.get_running_loop() + except RuntimeError as e: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + loop.run_until_complete(self._close()) + + async def _close(self): + await self.client.close() + logger.info("Bot stopped!") + + # message_callback event + async def message_callback(self, room: MatrixRoom, + event: Union[RoomMessageAudio, RoomEncryptedAudio]) -> None: + if self.room_id is None: + room_id = room.room_id + else: + # if event room id does not match the room id in config, return + if room.room_id != self.room_id: + return + room_id = self.room_id + + # reply event_id + reply_to_event_id = event.event_id + + # sender_id + sender_id = event.sender + + # construct filename + if not os.path.exists("output"): + os.mkdir("output") + ext = os.path.splitext(event.body)[-1] + filename = os.path.join("output", str(uuid.uuid4()) + ext) + + try: + if isinstance(event, RoomMessageAudio): # for audio event + mxc = event.url # audio mxc + # download unencrypted audio file + resp = await self.download_mxc(mxc=mxc) + if isinstance(resp, DownloadError): + logger.error("Download of media file failed") + else: + media_data = resp.body + + async with aiofiles.open(filename, "wb") as f: + await f.write(media_data) + await f.close() + + if isinstance(event, RoomEncryptedAudio): # for encrypted audio event + mxc = event.url # audio mxc + # download encrypted audio file + resp = await self.download_mxc(mxc=mxc) + if isinstance(resp, DownloadError): + logger.error("Download of media file failed") + else: + media_data = resp.body + async with aiofiles.open(filename, "wb") as f: + await f.write( + crypto.attachments.decrypt_attachment( + media_data, + event.source["content"]["file"]["key"][ + "k" + ], + event.source["content"]["file"]["hashes"][ + "sha256" + ], + event.source["content"]["file"]["iv"], + ) + ) + await f.close() + except Exception as e: + logger.error(e, exc_info=True) + + # use whisper to transribe audio to text + try: + await self.client.room_typing(room_id) + message = self.transcribe(filename) + await send_room_message( + client=self.client, + room_id=room_id, + reply_message=message, + sender_id=sender_id, + reply_to_event_id=reply_to_event_id, + ) + + except Exception as e: + logger.error(e) + + # remove audio file + logger.info("audio file removed") + os.remove(filename) + + # message_callback decryption_failure event + async def decryption_failure(self, room: MatrixRoom, event: MegolmEvent) -> None: + if not isinstance(event, MegolmEvent): + return + + logger.error( + f"Failed to decrypt message: {event.event_id} from {event.sender} in {room.room_id}\n" + + "Please make sure the bot current session is verified" + ) + + # invite_callback event + async def invite_callback(self, room: MatrixRoom, event: InviteMemberEvent) -> None: + """Handle an incoming invite event. + If an invite is received, then join the room specified in the invite. + code copied from: https://github.com/8go/matrix-eno-bot/blob/ad037e02bd2960941109e9526c1033dd157bb212/callbacks.py#L104 + """ + logger.debug(f"Got invite to {room.room_id} from {event.sender}.") + # Attempt to join 3 times before giving up + for attempt in range(3): + result = await self.client.join(room.room_id) + if type(result) == JoinError: + logger.error( + f"Error joining room {room.room_id} (attempt %d): %s", + attempt, result.message, + ) + else: + break + else: + logger.error("Unable to join room: %s", room.room_id) + + # Successfully joined room + logger.info(f"Joined {room.room_id}") + + # to_device_callback event + async def to_device_callback(self, event: KeyVerificationEvent) -> None: + """Handle events sent to device. + + Specifically this will perform Emoji verification. + It will accept an incoming Emoji verification requests + and follow the verification protocol. + code copied from: https://github.com/8go/matrix-eno-bot/blob/ad037e02bd2960941109e9526c1033dd157bb212/callbacks.py#L127 + """ + try: + client = self.client + logger.debug( + f"Device Event of type {type(event)} received in " + "to_device_cb().") + + if isinstance(event, KeyVerificationStart): # first step + """ first step: receive KeyVerificationStart + KeyVerificationStart( + source={'content': + {'method': 'm.sas.v1', + 'from_device': 'DEVICEIDXY', + 'key_agreement_protocols': + ['curve25519-hkdf-sha256', 'curve25519'], + 'hashes': ['sha256'], + 'message_authentication_codes': + ['hkdf-hmac-sha256', 'hmac-sha256'], + 'short_authentication_string': + ['decimal', 'emoji'], + 'transaction_id': 'SomeTxId' + }, + 'type': 'm.key.verification.start', + 'sender': '@user2:example.org' + }, + sender='@user2:example.org', + transaction_id='SomeTxId', + from_device='DEVICEIDXY', + method='m.sas.v1', + key_agreement_protocols=[ + 'curve25519-hkdf-sha256', 'curve25519'], + hashes=['sha256'], + message_authentication_codes=[ + 'hkdf-hmac-sha256', 'hmac-sha256'], + short_authentication_string=['decimal', 'emoji']) + """ + + if "emoji" not in event.short_authentication_string: + estr = ("Other device does not support emoji verification " + f"{event.short_authentication_string}. Aborting.") + print(estr) + logger.info(estr) + return + resp = await client.accept_key_verification( + event.transaction_id) + if isinstance(resp, ToDeviceError): + estr = f"accept_key_verification() failed with {resp}" + print(estr) + logger.info(estr) + + sas = client.key_verifications[event.transaction_id] + + todevice_msg = sas.share_key() + resp = await client.to_device(todevice_msg) + if isinstance(resp, ToDeviceError): + estr = f"to_device() failed with {resp}" + print(estr) + logger.info(estr) + + elif isinstance(event, KeyVerificationCancel): # anytime + """ at any time: receive KeyVerificationCancel + KeyVerificationCancel(source={ + 'content': {'code': 'm.mismatched_sas', + 'reason': 'Mismatched authentication string', + 'transaction_id': 'SomeTxId'}, + 'type': 'm.key.verification.cancel', + 'sender': '@user2:example.org'}, + sender='@user2:example.org', + transaction_id='SomeTxId', + code='m.mismatched_sas', + reason='Mismatched short authentication string') + """ + + # There is no need to issue a + # client.cancel_key_verification(tx_id, reject=False) + # here. The SAS flow is already cancelled. + # We only need to inform the user. + estr = (f"Verification has been cancelled by {event.sender} " + f"for reason \"{event.reason}\".") + print(estr) + logger.info(estr) + + elif isinstance(event, KeyVerificationKey): # second step + """ Second step is to receive KeyVerificationKey + KeyVerificationKey( + source={'content': { + 'key': 'SomeCryptoKey', + 'transaction_id': 'SomeTxId'}, + 'type': 'm.key.verification.key', + 'sender': '@user2:example.org' + }, + sender='@user2:example.org', + transaction_id='SomeTxId', + key='SomeCryptoKey') + """ + sas = client.key_verifications[event.transaction_id] + + print(f"{sas.get_emoji()}") + # don't log the emojis + + # The bot process must run in forground with a screen and + # keyboard so that user can accept/reject via keyboard. + # For emoji verification bot must not run as service or + # in background. + # yn = input("Do the emojis match? (Y/N) (C for Cancel) ") + # automatic match, so we use y + yn = "y" + if yn.lower() == "y": + estr = ("Match! The verification for this " + "device will be accepted.") + print(estr) + logger.info(estr) + resp = await client.confirm_short_auth_string( + event.transaction_id) + if isinstance(resp, ToDeviceError): + estr = ("confirm_short_auth_string() " + f"failed with {resp}") + print(estr) + logger.info(estr) + elif yn.lower() == "n": # no, don't match, reject + estr = ("No match! Device will NOT be verified " + "by rejecting verification.") + print(estr) + logger.info(estr) + resp = await client.cancel_key_verification( + event.transaction_id, reject=True) + if isinstance(resp, ToDeviceError): + estr = (f"cancel_key_verification failed with {resp}") + print(estr) + logger.info(estr) + else: # C or anything for cancel + estr = ("Cancelled by user! Verification will be " + "cancelled.") + print(estr) + logger.info(estr) + resp = await client.cancel_key_verification( + event.transaction_id, reject=False) + if isinstance(resp, ToDeviceError): + estr = (f"cancel_key_verification failed with {resp}") + print(estr) + logger.info(estr) + + elif isinstance(event, KeyVerificationMac): # third step + """ Third step is to receive KeyVerificationMac + KeyVerificationMac( + source={'content': { + 'mac': {'ed25519:DEVICEIDXY': 'SomeKey1', + 'ed25519:SomeKey2': 'SomeKey3'}, + 'keys': 'SomeCryptoKey4', + 'transaction_id': 'SomeTxId'}, + 'type': 'm.key.verification.mac', + 'sender': '@user2:example.org'}, + sender='@user2:example.org', + transaction_id='SomeTxId', + mac={'ed25519:DEVICEIDXY': 'SomeKey1', + 'ed25519:SomeKey2': 'SomeKey3'}, + keys='SomeCryptoKey4') + """ + sas = client.key_verifications[event.transaction_id] + try: + todevice_msg = sas.get_mac() + except LocalProtocolError as e: + # e.g. it might have been cancelled by ourselves + estr = (f"Cancelled or protocol error: Reason: {e}.\n" + f"Verification with {event.sender} not concluded. " + "Try again?") + print(estr) + logger.info(estr) + else: + resp = await client.to_device(todevice_msg) + if isinstance(resp, ToDeviceError): + estr = f"to_device failed with {resp}" + print(estr) + logger.info(estr) + estr = (f"sas.we_started_it = {sas.we_started_it}\n" + f"sas.sas_accepted = {sas.sas_accepted}\n" + f"sas.canceled = {sas.canceled}\n" + f"sas.timed_out = {sas.timed_out}\n" + f"sas.verified = {sas.verified}\n" + f"sas.verified_devices = {sas.verified_devices}\n") + print(estr) + logger.info(estr) + estr = ("Emoji verification was successful!\n" + "Initiate another Emoji verification from " + "another device or room if desired. " + "Or if done verifying, hit Control-C to stop the " + "bot in order to restart it as a service or to " + "run it in the background.") + print(estr) + logger.info(estr) + else: + estr = (f"Received unexpected event type {type(event)}. " + f"Event is {event}. Event will be ignored.") + print(estr) + logger.info(estr) + except BaseException: + estr = traceback.format_exc() + print(estr) + logger.info(estr) + + # bot login + + async def login(self) -> None: + if self.access_token is not None: + logger.info("Login via access_token") + else: + logger.info("Login via password") + try: + resp = await self.client.login(password=self.password) + if not isinstance(resp, LoginResponse): + logger.error("Login Failed") + print(f"Login Failed: {resp}") + sys.exit(1) + except Exception as e: + logger.error(f"Error: {e}", exc_info=True) + + # sync messages in the room + async def sync_forever(self, timeout=30000, full_state=True) -> None: + await self.client.sync_forever(timeout=timeout, full_state=full_state) + + # download mxc + async def download_mxc(self, mxc: str, filename: Optional[str] = None): + response = await self.client.download(mxc=mxc, filename=filename) + logger.info(f"download_mxc response: {response}") + return response + + # import keys + async def import_keys(self): + resp = await self.client.import_keys( + self.import_keys_path, + self.import_keys_password + ) + if isinstance(resp, EncryptionError): + logger.error(f"import_keys failed with {resp}") + else: + logger.info(f"import_keys success, please remove import_keys configuration!!!") + + # whisper function + def transcribe(self, filename: str) -> str: + logger.info("Start transcribe!") + segments, _ = self.model.transcribe(filename, vad_filter=True) + message = "" + for segment in segments: + message += segment.text + + return message + + +async def main(): + need_import_keys = False + if os.path.exists("config.json"): + fp = open("config.json", "r", encoding="utf-8") + config = json.load(fp) + + bot = Bot( + homeserver=config.get('homeserver'), + user_id=config.get('user_id'), + password=config.get('password'), + device_id=config.get('device_id'), + room_id=config.get('room_id'), + access_token=config.get('access_token'), + import_keys_path=config.get('import_keys_path'), + import_keys_password=config.get('import_keys_password'), + model_size=config.get('model_size'), + device=config.get('device'), + compute_type=config.get('compute_type'), + cpu_threads=config.get('cpu_threads'), + num_workers=config.get('num_workers'), + download_root=config.get('download_root'), + ) + + if config.get('import_keys_path') and config.get('import_keys_password') is not None: + need_import_keys = True + + else: + bot = Bot( + homeserver=os.environ.get('HOMESERVER'), + user_id=os.environ.get('USER_ID'), + password=os.environ.get('PASSWORD'), + device_id=os.environ.get("DEVICE_ID"), + room_id=os.environ.get("ROOM_ID"), + access_token=os.environ.get("ACCESS_TOKEN"), + import_keys_path=os.environ.get("IMPORT_KEYS_PATH"), + import_keys_password=os.environ.get("IMPORT_KEYS_PASSWORD"), + model_size=os.environ.get("MODEL_SIZE"), + device=os.environ.get("DEVICE"), + compute_type=os.environ.get("COMPUTE_TYPE"), + cpu_threads=os.environ.get("CPU_THREADS"), + num_workers=os.environ.get("NUM_WORKERS"), + download_root=os.environ.get("DOWNLOAD_ROOT"), + ) + if os.environ.get("IMPORT_KEYS_PATH") and os.environ.get("IMPORT_KEYS_PASSWORD") is not None: + need_import_keys = True + + await bot.login() + if need_import_keys: + logger.info("start import_keys process, this may take a while...") + await bot.import_keys() + + await bot.sync_forever() + +if __name__ == '__main__': + logger.info("Bot started!") + asyncio.run(main()) diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..d0e38ac --- /dev/null +++ b/compose.yaml @@ -0,0 +1,24 @@ +services: + app: + image: ghcr.io/hibobmaster/matrix-stt-bot:latest + container_name: matrix-stt-bot + restart: always + # build: + # context: . + # dockerfile: ./Dockerfile + # env_file: + # - .env + volumes: + # use env file or config.json + - ./config.json:/app/config.json + # use touch to create an empty file stt_db, for persist database only(first time only!!!) + - ./stt_db:/app/db + # import_keys path + # - ./element-keys.txt:/app/element-keys.txt + # store whisper models that program will download + - ./models:/app/models + networks: + - matrix_network + +networks: + matrix_network: diff --git a/config.json.example b/config.json.example new file mode 100644 index 0000000..9d0b009 --- /dev/null +++ b/config.json.example @@ -0,0 +1,10 @@ +{ + "homeserver": "https://xxxxx.xxxx.xxxx", + "user_id": "@xxxx:matrix.org", + "password": "xxxxxxxxxx", + "device_id": "GMIAZSVFF", + "room_id": "!xxxxxxxx:xxx.xxx.xxx", + "model_size": "base", + "import_keys_path": "element-keys.txt", + "import_keys_password": "xxxxxxxxxxxx" +} \ No newline at end of file diff --git a/log.py b/log.py new file mode 100644 index 0000000..c3f96d9 --- /dev/null +++ b/log.py @@ -0,0 +1,34 @@ +import logging + + +def getlogger(): + # create a custom logger if no log handler + logger = logging.getLogger(__name__) + if not logger.hasHandlers(): + logger.setLevel(logging.INFO) + # create handlers + warn_handler = logging.StreamHandler() + info_handler = logging.StreamHandler() + error_handler = logging.FileHandler('bot.log', mode='a') + warn_handler.setLevel(logging.WARNING) + error_handler.setLevel(logging.ERROR) + info_handler.setLevel(logging.INFO) + + # create formatters + warn_format = logging.Formatter( + '%(asctime)s - %(funcName)s - %(levelname)s - %(message)s') + error_format = logging.Formatter( + '%(asctime)s - %(name)s - %(funcName)s - %(levelname)s - %(message)s') + info_format = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') + + # set formatter + warn_handler.setFormatter(warn_format) + error_handler.setFormatter(error_format) + info_handler.setFormatter(info_format) + + # add handlers to logger + logger.addHandler(warn_handler) + logger.addHandler(error_handler) + logger.addHandler(info_handler) + + return logger diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..d69fd9a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,51 @@ +aiofiles==23.1.0 +aiohttp==3.8.4 +aiohttp-socks==0.7.1 +aiosignal==1.3.1 +async-timeout==4.0.2 +atomicwrites==1.4.1 +attrs==23.1.0 +av==10.0.0 +cachetools==4.2.4 +certifi==2022.12.7 +cffi==1.15.1 +charset-normalizer==3.1.0 +coloredlogs==15.0.1 +ctranslate2==3.12.0 +faster-whisper @ git+https://github.com/guillaumekln/faster-whisper.git@3adcc12d0f91369446a624e33185c555facc8ed2 +filelock==3.12.0 +flatbuffers==23.3.3 +frozenlist==1.3.3 +future==0.18.3 +h11==0.14.0 +h2==4.1.0 +hpack==4.0.0 +huggingface-hub==0.13.4 +humanfriendly==10.0 +hyperframe==6.0.1 +idna==3.4 +jsonschema==4.17.3 +Logbook==1.5.3 +matrix-nio[e2e]==0.20.2 +mpmath==1.3.0 +multidict==6.0.4 +numpy==1.24.2 +onnxruntime==1.14.1 +packaging==23.1 +peewee==3.16.1 +protobuf==4.22.3 +pycparser==2.21 +pycryptodome==3.17 +pyrsistent==0.19.3 +python-olm==3.1.3 +python-socks==2.2.0 +PyYAML==6.0 +requests==2.28.2 +sympy==1.11.1 +tokenizers==0.13.3 +tqdm==4.65.0 +typing_extensions==4.5.0 +unpaddedbase64==2.1.0 +unsync==1.4.0 +urllib3==1.26.15 +yarl==1.8.2 diff --git a/send_message.py b/send_message.py new file mode 100644 index 0000000..b99263e --- /dev/null +++ b/send_message.py @@ -0,0 +1,27 @@ +from nio import AsyncClient + +async def send_room_message(client: AsyncClient, + room_id: str, + reply_message: str, + sender_id: str = '', + reply_to_event_id: str = '', + ) -> None: + NORMAL_BODY = content = {"msgtype": "m.text", "body": reply_message, } + if reply_to_event_id == '': + content = NORMAL_BODY + else: + body = r'> <' + sender_id + r'> sent an audio file.\n\n' + reply_message + format = r'org.matrix.custom.html' + formatted_body = r'
In reply to ' + sender_id \ + + r'
sent an audio file.
' + reply_message + + content = {"msgtype": "m.text", "body": body, "format": format, "formatted_body": formatted_body, + "m.relates_to": {"m.in_reply_to": {"event_id": reply_to_event_id}}, } + await client.room_send( + room_id, + message_type="m.room.message", + content=content, + ignore_unverified_devices=True, + ) + await client.room_typing(room_id, typing_state=False)