Alpha/InterviewAI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve question detection, capture stealth, latency; stop tracking model binaries

Browse Source 
- Detection: broaden request-starters, handle contractions, allow 1-word '?'
- Audio: PTT pre-roll (no clipped first word), drop Whisper silence hallucinations,
  greedy decoding + domain initial_prompt for faster/cleaner transcription
- AI: cap spoken-answer tokens so replies return at conversational speed
- Overlay: answers persist (no auto-hide); wire Ctrl+Shift+H show/hide toggle
- Screen capture: freeze-frame at hotkey press (immune to focus-blur lockouts),
  hide selector from screen-share (NSWindowSharingNone), higher capture resolution
- Stop tracking models/ and *.zip (large binaries; add to .gitignore)

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Charles Wambua
2026-06-26 14:36:06 +03:00
parent 1dd5b757be
commit dba78a2766
19 changed files with 1767 additions and 154102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

102
.claude/hooks/post-tool-call.py Normal file
View File

@@ -0,0 +1,102 @@
import hashlib
import json
import os
import sys
import tempfile
from datetime import datetime, timezone
from http.client import HTTPConnection, HTTPException
from pathlib import Path
import traceback
from contextlib import closing
from typing import Optional
import argparse
WEBSERVER_HOST = "localhost"
WEBSERVER_ENDPOINT = "/api/provenance/call"
PORT_FILE_SUFFIX = "-provenance-port.txt"
class ProvenanceHookError(RuntimeError):
pass
def http_request(method, host, port, location, *, body: Optional[bytes] = None, headers={}, timeout=None, wait_for_response=False) -> bytes:
with closing(HTTPConnection(host, port, timeout=timeout)) as connection:
connection.request(method, location, body=body, headers=headers)
if wait_for_response:
response = connection.getresponse()
responseText = response.read()
def get_server_port():
claude_root = os.getenv("CLAUDE_PROJECT_DIR")
path_hash = hashlib.md5(claude_root.encode('utf-8')).hexdigest()
port_file = Path(tempfile.gettempdir()) / (path_hash + PORT_FILE_SUFFIX)
return int(port_file.read_text("utf-8").strip())
def send_diff_to_webserver(file_path, timestamp_ms, wait_for_response):
try:
port = get_server_port()
except FileNotFoundError as e:
raise ProvenanceHookError(
f"Could not determine API port: {e.filename} does not exist") from e
except Exception as e:
raise ProvenanceHookError("Could not determine API port") from e
url = f"http://{WEBSERVER_HOST}:{port}{WEBSERVER_ENDPOINT}"
try:
payload = {"file_path": file_path, "timestamp": timestamp_ms}
return http_request(
"POST",
WEBSERVER_HOST,
port=port,
location=WEBSERVER_ENDPOINT,
body=json.dumps(payload, ensure_ascii=False).encode("utf-8"),
headers={'Content-Type': 'application/json'},
timeout=0.5,
wait_for_response=wait_for_response
)
except (HTTPException, OSError, ConnectionError) as e:
raise ProvenanceHookError(
f"Network error while sending diff to {url}") from e
except Exception as e:
raise ProvenanceHookError(
f"Unknown error while sending diff to {url}") from e
def extract_file_path(tool_name, tool_input):
if tool_name in ["Write", "Edit", "MultiEdit"]:
return tool_input.get('file_path', 'unknown')
if tool_name == "NotebookEdit":
return tool_input.get('notebook_path', 'unknown')
return 'unknown'
def excepthook(type, value, traceback_):
traceback.print_exception(type, value, traceback_, file=sys.stderr)
sys.exit(1)
def main():
data = json.load(sys.stdin)
tool_name = data.get('tool_name', 'unknown')
p = argparse.ArgumentParser()
p.add_argument("--wait_for_response", default=False)
args = p.parse_args()
modification_tools = [
"Write", "Edit", "MultiEdit", "NotebookEdit"
]
if tool_name in modification_tools:
tool_input = data.get('tool_input', {})
file_path = extract_file_path(tool_name, tool_input)
if file_path:
timestamp_ms = int(datetime.now(timezone.utc).timestamp() * 1000)
send_diff_to_webserver(file_path, timestamp_ms, args.wait_for_response)
if __name__ == "__main__":
sys.excepthook = excepthook
sys.exit(main())

3
.gitignore vendored
View File

@@ -102,3 +102,6 @@ dmypy.json
# pyre type checker # pyre type checker
.pyre/models/whisper/model.bin .pyre/models/whisper/model.bin
# Corporate CA bundle (machine-specific, may be sensitive)
certs/

53
CLAUDE.md Normal file
View File

@@ -0,0 +1,53 @@
# CLAUDE.md
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
## What this is
A real-time, fully on-device AI copilot for meetings (macOS-focused). **Push-to-talk** (hold Right Option) gates audio capture; it transcribes locally with **faster-whisper** (`distil-large-v3`), detects questions, and answers via a quantized **Qwen2.5-VL-7B** vision model through **llama.cpp**. The same vision model also reads **on-screen questions**: press a hotkey, drag a box, and it answers what's in the region. Answers appear in a Qt overlay hidden from screen capture. No audio or text leaves the machine.
## Commands
```bash
./run.sh # macOS/Linux: activates .venv, sets CA bundle, installs PortAudio, runs main.py
python main.py # direct run (expects deps + models already present)
python setup.py # first-time: pip install -r requirements.txt + audio device check
run.bat # Windows launcher
```
There is **no test suite, linter, or build step**. To test the AI loop without speaking, run the app and type a question into the terminal + Enter (a background thread in `main.py` feeds terminal input straight to answer generation).
`run.sh` sets `SSL_CERT_FILE`/`REQUESTS_CA_BUNDLE`/`CURL_CA_BUNDLE` to `certs/corp_ca_bundle.pem` and `HF_HUB_DISABLE_XET=1` — this is intentional for running behind an SSL-inspecting corporate proxy. Preserve it.
## Architecture
`main.py` (`MeetingAssistant`) is the orchestrator. It wires together the components and owns **question detection** (the `_is_question` heuristic and the `_INTERROGATIVES`/`_AUX_FRONT`/`_REQUEST_STARTERS`/`_EMBEDDED_MARKERS`/`_FACT_PATTERNS`/`_NOISE_PHRASES` tables). Detection is pure regex/keyword logic — no model — and deliberately catches plain-statement questions ("I was wondering about the budget"), tag questions, math, and fact fragments.
Data flow:
1. **`src/hotkeys.py`** (`HotkeyManager`) — a pynput global listener. Holding the push-to-talk key (`audio.ptt_key`, default `alt_r` = Right Option) calls `audio_listener.arm()`/`disarm()`; the screen chord (`screen.capture_key`, default `ctrl+shift+space`) calls `overlay.request_capture()`. Callbacks run on pynput's thread, so anything touching Qt must be marshaled (see below). **Requires macOS Accessibility permission.**
2. **`src/audio_listener.py`** — one `sd.RawInputStream` per source ("microphone" / "system"). In **push-to-talk mode** (default, `audio.capture_mode`), stream callbacks buffer raw PCM *only between `arm()` and `disarm()`*; on disarm the held clip per source is queued for transcription. (The original always-on `StreamSegmenter` VAD path still exists and is used only in `continuous` mode.) A single shared `WhisperModel` (config-driven `audio.whisper_model`, int8 CPU) transcribes; calls back into `MeetingAssistant.on_audio_transcript(text, timestamp, source, speaker)`.
3. **`src/diarizer.py`** (`SpeakerDiarizer`) — the "system" loopback stream is one *mixed* channel of all remote participants, so per-utterance it computes an ECAPA-TDNN voice embedding (SpeechBrain, `models/ecapa`) and online-clusters by cosine similarity into "Person 1/2/…". Best-effort; degrades to a single "Them" if unavailable.
4. **`src/context_manager.py`** (`ContextManager`) — rolling speaker-labeled transcript (deque of last 20 utterances) used as LLM context.
5. **`src/ai_engine.py`** (`AIEngine`) — `classify()` buckets a spoken question into `math | factual | open`. **Math is solved with a safe AST evaluator (`try_solve_math`), never the LLM.** Other kinds go to llama.cpp via `create_chat_completion` (**streamed**, so a newer question interrupts an in-flight one via `interrupt_event`; all generation is serialized through `_gen_lock` since llama.cpp isn't thread-safe). The model is loaded with a `Qwen25VLChatHandler` + `mmproj` when `ai.mmproj` is present (`self.vision = True`). **`answer_from_image(path)`** sends a screenshot (base64 data-URI) + instruction as a multimodal chat message — this is the screen-question path.
6. **`src/region_capture.py`** (`RegionSelector` + `grab_region`) — fullscreen dim `QWidget` with a `QRubberBand`; on drag-release it maps to global coords, grabs the region via `mss` (scaling logical→physical by comparing mss size to the Qt screen size, so it's Retina-correct), writes a temp PNG, and fires `on_done(path)`. **Must be created on the Qt main thread.**
7. **`src/overlay.py`** (`InvisibleOverlay`) — PyQt5 always-on-top frameless window. Worker threads must update it only via the `_Bridge` Qt signals (`show_answer`/`show_status`/`capture_request`), never directly. `request_capture()` emits `capture_request`, whose slot runs `self.on_capture` (set by `main.py`) on the main thread — this is how the off-thread hotkey safely launches the selector. `_exclude_from_screen_capture()` sets macOS `NSWindowSharingNone` so screen-recorders/Zoom can't see it.
Threading model: Qt event loop on the main thread; the pynput hotkey listener, audio capture, transcription, terminal input, and each answer run on their own threads. Anything touching Qt from off-thread goes through the `_Bridge` signals. Answer generation is serialized/interruptible via `answer_lock` (main.py) + `interrupt_event` + `_gen_lock` (ai_engine). The region selector and screen answer are driven from `main.py._handle_screen_capture` → `_on_region_captured` → `_screen_answer_worker`.
### `answer_mode` (config `ai.answer_mode`)
- `auto_obvious` (default) — answer math/factual directly; show a *suggested reply* for open-ended questions.
- `auto_all` — generate a committed answer for everything.
- `suggest_only` — always a draft, never committed.
`answer_sources` (config `audio.answer_sources`) controls which speaker sources actually trigger answering vs. just adding context.
## Config & assets
- **`config.yaml`** is the live config (loaded by `main.py`). `config/settings.py` is unused legacy — ignore it.
- Models under `models/` (large binaries): `models/ecapa/` (SpeechBrain diarizer), and the **Qwen2.5-VL** vision model + `mmproj` (downloaded by `setup.py` from `ggml-org/Qwen2.5-VL-7B-Instruct-GGUF`). The Whisper STT model auto-downloads by name via faster-whisper on first run (the old committed `models/whisper/` base.en dir is now only a fallback). The old `models/Qwen2.5-7B-Instruct-Q4_K_M.gguf` (text-only) is superseded by the VL model.
- **`src/screen_scanner.py` is dead code** — superseded by `src/region_capture.py`. The OCR screen-scan was always disabled (no-op stub); screen questions now go through the draw-a-box vision path instead.
- **16 GB RAM is the binding constraint.** VL model (~4.7 GB) + mmproj (~1.4 GB) + Whisper + ECAPA + Qt ≈ 12–13 GB. If memory is tight, set `audio.whisper_model: medium.en`.
## Platform notes
macOS is the primary target. Capturing other participants requires a virtual loopback device (BlackHole) routed through a Multi-Output Device — see README "Hearing other participants". The screen-capture-hiding and loopback auto-detection (`loopback_keywords`) are core to the product, not incidental.

113
README.md Normal file
View File

@@ -0,0 +1,113 @@
# Meeting Assistant
A real-time, on-device AI copilot for meetings. You **hold a key to talk**, it
detects questions (even ones phrased as plain statements), and shows answers or
suggested replies in an overlay that is hidden from screen-capture. It can also
**read a question off your screen** when you drag a box around it.
Everything runs locally: **faster-whisper** (`distil-large-v3` by default) for
speech-to-text and a quantized **Qwen2.5-VL-7B** vision model (via `llama.cpp`)
that answers both spoken and on-screen questions. No audio or text leaves your machine.
## What it does
- **Push-to-talk — no noise.** Audio is only captured while you **hold Right
Option (⌥)**; nothing is transcribed otherwise. Captures your microphone *and*
the other participants' audio (system output via a loopback device), tagged by
speaker (`You` / `Them`).
- **Reads on-screen questions.** Press **Ctrl+Shift+Space**, drag a box over a
question (e.g. on a shared slide), and the vision model reads and answers it —
including multiple-choice, code, and math.
- **Detects real questions** — interrogatives, yes/no questions, requests
("explain the rollout plan"), embedded/plain-sentence questions
("I was wondering about the budget"), tag questions ("…, right?"), math, and
fact-shaped fragments ("difference between TCP and UDP").
- **Auto-fills the obvious.** Arithmetic is solved instantly (no LLM); short
factual lookups get a direct answer. Open-ended questions get a concise
**suggested reply** you can read out, using the live meeting transcript as context.
## Hearing other participants (one-time macOS setup)
Your microphone only captures *you*. To also capture what the other participants
say, the assistant reads your system audio output through a virtual loopback
device. **BlackHole** is already detected on this machine.
The catch: if you send audio *only* to BlackHole, you won't hear it yourself. So
create a **Multi-Output Device** that plays to both your speakers/headphones and
BlackHole at once:
1. Open **Audio MIDI Setup** (Applications → Utilities).
2. Click **+** (bottom-left) → **Create Multi-Output Device**.
3. Check both **BlackHole 2ch** and your normal output (e.g. *MacBook Pro Speakers*).
4. In **System Settings → Sound → Output**, select that Multi-Output Device.
5. In your meeting app (Zoom/Teams/Meet), make sure the speaker/output is the
Multi-Output Device (or the system default).
Now meeting audio reaches both your ears and the assistant. Your microphone stays
selected as the meeting's *input*.
> Don't have BlackHole? Install with `brew install blackhole-2ch`, then re-run.
## Configuration (`config.yaml`)
```yaml
audio:
source: "both" # "microphone", "system", or "both"
capture_mode: "push_to_talk" # push_to_talk (hold key) or continuous
ptt_key: "alt_r" # push-to-talk key (Right Option). e.g. cmd_r, f8, ctrl_r
whisper_model: "distil-large-v3" # or large-v3, medium.en, or a local dir
mic_device: null # null = auto. Or a device index / name substring.
system_device: null # null = auto-detect loopback (BlackHole). Or index / name.
answer_sources: # which speakers trigger an answer
- "system" # other participants
- "microphone" # your own voice (handy for testing)
screen:
capture_key: "ctrl+shift+space" # press, then drag a box over a question
ai:
model: "Qwen2.5-VL-7B-Instruct-Q4_K_M.gguf"
mmproj: "mmproj-Qwen2.5-VL-7B-Instruct-f16.gguf" # required for screen reading
answer_mode: "auto_obvious" # auto_obvious | auto_all | suggest_only
user_name: "you"
```
> **Tight on 16 GB RAM?** The vision model + `distil-large-v3` fit, but if memory
> gets tight set `whisper_model: medium.en`.
- `auto_obvious` — answer obvious questions (math/factual) directly; show a
*suggested reply* for open-ended ones.
- `auto_all` — generate a full answer for every detected question.
- `suggest_only` — never commit; always show a draft.
## Run
```bash
./run.sh # macOS / Linux
# or
python main.py
```
First run downloads the models (run `python setup.py` once, ~6 GB for the vision
model; `distil-large-v3` auto-downloads on first launch).
On first launch grant three macOS permissions (System Settings → Privacy & Security):
- **Microphone** — to capture audio.
- **Screen Recording** — so the overlay can hide *itself* from capture, and so the
draw-a-box screen grab works.
- **Accessibility** — so the global push-to-talk and screen-grab hotkeys are seen.
You can also type a question in the terminal + Enter to test the AI directly.
## How it works
| File | Role |
|------|------|
| `src/audio_listener.py` | Captures each source. Push-to-talk buffers audio only while armed; transcribes the held clip with one shared Whisper model, tags by source. |
| `src/hotkeys.py` | Global hotkeys (pynput): hold-to-talk + screen-grab chord. |
| `main.py` | Question detection + orchestration. |
| `src/ai_engine.py` | Math fast-path, classification, and Qwen2.5-VL answering for both spoken and on-screen (`answer_from_image`) questions. |
| `src/region_capture.py` | Draw-a-box fullscreen selector + region screenshot. |
| `src/context_manager.py` | Rolling speaker-labeled meeting transcript. |
| `src/overlay.py` | Always-on-top overlay, hidden from screen capture. |
```

264
main.py
View File

@@ -17,18 +17,49 @@ from src.audio_listener import AudioListener
from src.ai_engine import AIEngine from src.ai_engine import AIEngine
from src.overlay import InvisibleOverlay from src.overlay import InvisibleOverlay
from src.context_manager import ContextManager from src.context_manager import ContextManager
from src.hotkeys import HotkeyManager
from src.region_capture import RegionSelector
from src.utils import setup_logging, get_platform from src.utils import setup_logging, get_platform
# Question words that indicate the speaker wants an answer import re
_QUESTION_WORDS = (
"what", "why", "how", "when", "where", "who", "which", # Interrogatives — a sentence starting with one of these is almost always a question
"can you", "could you", "would you", "do you", "did you", _INTERROGATIVES = (
"is there", "are there", "is it", "are you", "what", "why", "how", "when", "where", "who", "whom", "whose", "which",
"tell me", "explain", "describe", "define",
"your name", "help me",
) )
# Phrases to skip even if they contain question words (noise) # Auxiliary/modal verbs that, when fronted, signal a yes/no question
_AUX_FRONT = (
"is", "are", "am", "was", "were", "do", "does", "did", "can", "could",
"would", "will", "shall", "should", "may", "might", "has", "have", "had",
)
# Imperative/request starters that expect an answer
_REQUEST_STARTERS = (
"tell me", "tell us", "explain", "describe", "define", "show me", "give me",
"give us", "give an overview", "give me an overview", "overview of",
"walk me through", "walk us through", "run me through", "run through",
"take me through", "talk me through", "go through", "go over", "break down",
"break it down", "lay out", "list", "summarize", "summarise", "compare",
"contrast", "outline", "clarify", "elaborate", "help me", "remind me",
)
# Embedded / plain-sentence question markers ("I was wondering what the timeline is")
_EMBEDDED_MARKERS = (
"i was wondering", "i wonder", "i'd like to know", "i would like to know",
"do you know", "any idea", "any thoughts", "curious", "wondering if",
"wondering what", "wondering how", "wondering whether", "let me know",
"your thoughts on", "what's your take", "thoughts on", "would love to know",
"not sure", "can you clarify", "question for you", "question is",
)
# Fact-shaped lookups phrased as fragments ("capital of France", "python decorators")
_FACT_PATTERNS = (
"capital of", "definition of", "meaning of", "difference between",
"how many", "what year", "abbreviation for", "acronym for",
)
# Phrases to skip even if they contain question words (noise / code / UI text)
_NOISE_PHRASES = ( _NOISE_PHRASES = (
"timer cannot", "qobject", "qml", "pyside", "pyqt", "timer cannot", "qobject", "qml", "pyside", "pyqt",
"def ", "class ", "import ", "return ", "self.", "def ", "class ", "import ", "return ", "self.",
@@ -57,83 +88,141 @@ class MeetingAssistant:
# Audio listener with callback # Audio listener with callback
self.audio_listener = AudioListener(self.config, self.on_audio_transcript) self.audio_listener = AudioListener(self.config, self.on_audio_transcript)
# Which speaker sources should trigger an answer
self.answer_sources = set(
self.config.get("audio", {}).get("answer_sources", ["system", "microphone"])
)
# Global hotkeys: hold push-to-talk to capture audio, chord to grab screen
audio_cfg = self.config.get("audio", {})
screen_cfg = self.config.get("screen", {})
hotkeys_cfg = self.config.get("hotkeys", {})
self.hotkeys = HotkeyManager(
on_arm=self.audio_listener.arm,
on_disarm=self.audio_listener.disarm,
on_capture=self.overlay.request_capture, # marshals onto Qt thread
on_toggle=self.overlay.request_toggle, # marshals onto Qt thread
ptt_key=audio_cfg.get("ptt_key", "alt_r"),
capture_key=screen_cfg.get("capture_key", "ctrl+shift+space"),
toggle_key=hotkeys_cfg.get("toggle_overlay", "ctrl+shift+h"),
)
# Screen-capture handler runs on the Qt main thread (via the overlay bridge)
self.overlay.on_capture = self._handle_screen_capture
self._selector = None # keep a ref so the selector isn't GC'd mid-use
# State management # State management
self.current_answer = None self.current_answer = None
self.answering = False self.answering = False
self.answer_lock = threading.Lock() self.answer_lock = threading.Lock()
self._last_question = "" self._last_question = ""
ptt = audio_cfg.get("ptt_key", "alt_r")
cap = screen_cfg.get("capture_key", "ctrl+shift+space")
mode = audio_cfg.get("capture_mode", "push_to_talk")
print("✅ Meeting Assistant Ready!") print("✅ Meeting Assistant Ready!")
print("==================================================") print("==================================================")
print("🎤 Listening for questions via microphone") if mode == "push_to_talk":
print(f"🎙️ Hold [{ptt}] to capture audio, release to answer")
else:
print("🎤 Listening continuously for questions")
print(f"📸 Press [{cap}] then drag a box over an on-screen question")
print(" Answers appear in the overlay (top-right)") print(" Answers appear in the overlay (top-right)")
print(" Type a question here + Enter to test AI") print(" Type a question here + Enter to test AI")
print(" Ctrl+C to stop") print(" Ctrl+C to stop")
print("==================================================") print("==================================================")
def on_audio_transcript(self, text, timestamp): def on_audio_transcript(self, text, timestamp, source="microphone", speaker=None):
"""Audio callback — runs in the audio listener thread""" """Audio callback — runs in the transcription thread"""
if not text or len(text.strip()) < 2: if not text or len(text.strip()) < 2:
return return
self.context_manager.add_audio_context(text, timestamp) speaker = speaker or ("You" if source == "microphone" else "Them")
self.context_manager.add_audio_context(text, timestamp, source, speaker)
if source not in self.answer_sources:
print(f" Context [{speaker}] (not an answer source): {text}")
return
if self._is_question(text): if self._is_question(text):
print(f"\n❓ Question detected: {text}") print(f"\n❓ Question from {speaker}: {text}")
self._generate_answer(text) self._generate_answer(text, source, speaker)
else: else:
print(f" Context (not a question): {text}") print(f" Context [{speaker}] (not a question): {text}")
def _is_question(self, text): def _is_question(self, text):
"""Enhanced question detection for faster responses""" """Detect questions, including ones phrased as plain statements.
if len(text.split()) < 2: # Reduced from 3 for shorter questions
return False
Combines surface cues ( '?'), interrogatives, fronted auxiliaries,
requests/imperatives, embedded markers ('I was wondering...'),
tag questions ('..., right?'), math, and fact-shaped fragments.
"""
text_lower = text.lower().strip() text_lower = text.lower().strip()
if not text_lower:
return False
# Skip obvious noise / code text # Skip obvious noise / code text
if any(p in text_lower for p in _NOISE_PHRASES): if any(p in text_lower for p in _NOISE_PHRASES):
return False return False
# Ends with "?" → always a question # Explicit question mark → always a question (even one word: "Why?")
if text.rstrip().endswith("?"): if text.rstrip().endswith("?"):
return True return True
# Check for math expressions (2+2, two plus two)
import re
math_patterns = [
r'\d+\s*[\+\-\*\/]\s*\d+', # 2+2, 5-3
r'\d+\s*plus\s*\d+', # 2 plus 2
r'\d+\s*minus\s*\d+', # 5 minus 3
r'\d+\s*times\s*\d+', # 4 times 5
r'\d+\s*divided by\s*\d+', # 10 divided by 2
]
for pattern in math_patterns:
if re.search(pattern, text_lower):
return True
# Check for short implicit questions (2-5 words without filler)
words = text_lower.split() words = text_lower.split()
if 2 <= len(words) <= 5: if len(words) < 2:
filler = {'um', 'uh', 'like', 'so', 'well', 'actually', 'basically'} return False
content_words = [w for w in words if w not in filler]
if len(content_words) >= 2: stripped = text_lower.rstrip(".!? ")
# "two plus two", "capital france", "python example"
# Tag questions: "..., right?", "..., correct?", "isn't it"
if re.search(r"\b(right|correct|yeah|okay|ok|no)\s*$", stripped) and len(words) >= 4:
if re.search(r",\s*(right|correct|okay|ok)\b", text_lower):
return True return True
# Starts with or contains a question word # Math expressions ("2+2", "5 plus 3", "10 divided by 2")
if any(text_lower.startswith(w) or f" {w} " in text_lower for w in _QUESTION_WORDS): math_patterns = (
r"\d+\s*[\+\-\*\/]\s*\d+",
r"\d+\s*(plus|minus|times|divided by|multiplied by)\s*\d+",
)
if any(re.search(p, text_lower) for p in math_patterns):
return True return True
# Check for command starters first = words[0]
command_starters = ['tell me', 'explain', 'describe', 'show me', 'give me', 'find', 'search'] # normalize contractions so "what's"/"who's"/"isn't"/"don't" still match
for starter in command_starters: first_base = re.sub(r"(n't|'s|'re|'ll|'d|'ve|'m)$", "", first)
if text_lower.startswith(starter):
return True # Starts with an interrogative ("what is the deadline", "what's the plan")
if first in _INTERROGATIVES or first_base in _INTERROGATIVES:
return True
# Fronted auxiliary/modal forming a yes/no question ("are we shipping friday",
# "isn't that due friday", "don't we need sign-off")
if first in _AUX_FRONT or first_base in _AUX_FRONT:
return True
# Request / imperative starters ("explain the rollout plan")
if any(stripped.startswith(s) for s in _REQUEST_STARTERS):
return True
# Embedded / plain-sentence questions ("I was wondering about the budget")
if any(m in text_lower for m in _EMBEDDED_MARKERS):
return True
# Interrogative appearing after a lead-in ("so what about the budget")
if any(f" {w} " in f" {text_lower} " for w in _INTERROGATIVES):
# avoid matching relative clauses like "the plan that we have" → require
# the interrogative within the first few words
for w in _INTERROGATIVES:
if w in words[:4]:
return True
# Fact-shaped fragments / lookups ("difference between TCP and UDP")
if any(p in text_lower for p in _FACT_PATTERNS):
return True
return False return False
def _generate_answer(self, question): def _generate_answer(self, question, source="microphone", speaker=None):
"""Generate answer with interruption support""" """Generate answer with interruption support"""
# Debounce: skip exact repeat within same run # Debounce: skip exact repeat within same run
if question.lower() == self._last_question.lower(): if question.lower() == self._last_question.lower():
@@ -148,23 +237,31 @@ class MeetingAssistant:
self.ai_engine.interrupt() self.ai_engine.interrupt()
# Start answer generation in background thread # Start answer generation in background thread
threading.Thread(target=self._answer_worker, args=(question,), daemon=True).start() threading.Thread(target=self._answer_worker,
args=(question, source, speaker), daemon=True).start()
def _answer_worker(self, question): def _answer_worker(self, question, source="microphone", speaker=None):
"""Worker thread for answer generation""" """Worker thread for answer generation"""
with self.answer_lock: with self.answer_lock:
self.answering = True self.answering = True
try: try:
# Get context and generate answer
context = self.context_manager.get_context() context = self.context_manager.get_context()
answer = self.ai_engine.answer_question(question, context) result = self.ai_engine.answer_question(question, context, source)
# Show answer in overlay if not result or not result.get("text"):
self.overlay.show_answer(answer, question) return # interrupted or empty
self.logger.info(f"Q: {question}")
answer = result["text"]
suggested = result.get("suggested", False)
asked_by = speaker or ("You" if source == "microphone" else "Them")
shown_q = f"({asked_by}) {question}"
self.overlay.show_answer(answer, shown_q, suggested)
self.logger.info(f"Q [{asked_by}]: {question}")
self.logger.info(f"A: {answer}") self.logger.info(f"A: {answer}")
print(f"\n💡 Answer: {answer}\n") label = "💬 Suggested" if suggested else "💡 Answer"
print(f"\n{label}: {answer}\n")
except Exception as e: except Exception as e:
print(f"❌ Error generating answer: {e}") print(f"❌ Error generating answer: {e}")
@@ -172,6 +269,51 @@ class MeetingAssistant:
with self.answer_lock: with self.answer_lock:
self.answering = False self.answering = False
# ------------------------------------------------------------------
# Screen questions — draw a box, read it, answer it
# ------------------------------------------------------------------
def _handle_screen_capture(self):
"""Runs on the Qt main thread. Show the draw-a-box selector."""
if self._selector is not None:
return # a selection is already in progress
print("\n📸 Draw a box around the question...")
self._selector = RegionSelector(on_done=self._on_region_captured)
self._selector.show_selector()
def _on_region_captured(self, image_path):
"""Called when the user finishes (or cancels) the selection."""
self._selector = None
if not image_path:
print(" Screen capture cancelled.")
return
self.overlay.show_status("📸 Reading the question on screen...")
threading.Thread(target=self._screen_answer_worker,
args=(image_path,), daemon=True).start()
def _screen_answer_worker(self, image_path):
with self.answer_lock:
if self.answering and hasattr(self.ai_engine, "interrupt"):
self.ai_engine.interrupt()
self.answering = True
try:
answer = self.ai_engine.answer_from_image(image_path)
if not answer:
return
self.overlay.show_answer(answer, "Screen question", suggested=False)
self.logger.info(f"Q [screen]: {image_path}")
self.logger.info(f"A: {answer}")
print(f"\n💡 Screen answer: {answer}\n")
except Exception as e:
print(f"❌ Error answering screen question: {e}")
finally:
with self.answer_lock:
self.answering = False
try:
os.remove(image_path)
except Exception:
pass
# ------------------------------------------------------------------ # ------------------------------------------------------------------
# Manual terminal input for testing # Manual terminal input for testing
# ------------------------------------------------------------------ # ------------------------------------------------------------------
@@ -195,6 +337,7 @@ class MeetingAssistant:
print("\n🎤 Starting audio listener...") print("\n🎤 Starting audio listener...")
self.overlay.start() self.overlay.start()
self.audio_listener.start() self.audio_listener.start()
self.hotkeys.start()
# Terminal input runs in a background daemon thread # Terminal input runs in a background daemon thread
t = threading.Thread(target=self._terminal_input_loop, daemon=True) t = threading.Thread(target=self._terminal_input_loop, daemon=True)
@@ -202,18 +345,25 @@ class MeetingAssistant:
try: try:
if self.overlay.app: if self.overlay.app:
sys.exit(self.overlay.app.exec_()) self.overlay.app.exec_()
except KeyboardInterrupt: except KeyboardInterrupt:
self.shutdown() pass
self.shutdown()
def shutdown(self): def shutdown(self):
print("\n🛑 Shutting down...") print("\n🛑 Shutting down...")
if hasattr(self, 'hotkeys'):
self.hotkeys.stop()
if hasattr(self, 'audio_listener'): if hasattr(self, 'audio_listener'):
self.audio_listener.stop() self.audio_listener.stop()
if hasattr(self, 'overlay'): if hasattr(self, 'overlay'):
self.overlay.stop() self.overlay.stop()
print("👋 Goodbye!") print("👋 Goodbye!")
sys.exit(0) # Hard-exit: skips Python finalizers so llama.cpp's buggy Metal teardown
# (GGML_ASSERT in ggml_metal_device_free at __cxa_finalize) can't crash us.
logging.shutdown()
sys.stdout.flush()
os._exit(0)
def main(): def main():

42
models/whisper/README.md
View File

@@ -1,42 +0,0 @@
---
language:
- en
tags:
- audio
- automatic-speech-recognition
license: mit
library_name: ctranslate2
---
# Whisper base.en model for CTranslate2
This repository contains the conversion of [openai/whisper-base.en](https://huggingface.co/openai/whisper-base.en) to the [CTranslate2](https://github.com/OpenNMT/CTranslate2) model format.
This model can be used in CTranslate2 or projects based on CTranslate2 such as [faster-whisper](https://github.com/systran/faster-whisper).
## Example
```python
from faster_whisper import WhisperModel
model = WhisperModel("base.en")
segments, info = model.transcribe("audio.mp3")
for segment in segments:
print("[%.2fs -> %.2fs] %s" % (segment.start, segment.end, segment.text))
```
## Conversion details
The original model was converted with the following command:
```
ct2-transformers-converter --model openai/whisper-base.en --output_dir faster-whisper-base.en \
--copy_files tokenizer.json --quantization float16
```
Note that the model weights are saved in FP16. This type can be changed when the model is loaded using the [`compute_type` option in CTranslate2](https://opennmt.net/CTranslate2/quantization.html).
## More information
**For more information about the original model, see its [model card](https://huggingface.co/openai/whisper-base.en).**

220
models/whisper/config.json
View File

@@ -1,220 +0,0 @@
{
"alignment_heads": [
[
3,
3
],
[
4,
7
],
[
5,
1
],
[
5,
5
],
[
5,
7
]
],
"lang_ids": [
50259,
50260,
50261,
50262,
50263,
50264,
50265,
50266,
50267,
50268,
50269,
50270,
50271,
50272,
50273,
50274,
50275,
50276,
50277,
50278,
50279,
50280,
50281,
50282,
50283,
50284,
50285,
50286,
50287,
50288,
50289,
50290,
50291,
50292,
50293,
50294,
50295,
50296,
50297,
50298,
50299,
50300,
50301,
50302,
50303,
50304,
50305,
50306,
50307,
50308,
50309,
50310,
50311,
50312,
50313,
50314,
50315,
50316,
50317,
50318,
50319,
50320,
50321,
50322,
50323,
50324,
50325,
50326,
50327,
50328,
50329,
50330,
50331,
50332,
50333,
50334,
50335,
50336,
50337,
50338,
50339,
50340,
50341,
50342,
50343,
50344,
50345,
50346,
50347,
50348,
50349,
50350,
50351,
50352,
50353,
50354,
50355,
50356
],
"suppress_ids": [
1,
2,
7,
8,
9,
10,
14,
25,
26,
27,
28,
29,
31,
58,
59,
60,
61,
62,
63,
90,
91,
92,
93,
357,
366,
438,
532,
685,
705,
796,
930,
1058,
1220,
1267,
1279,
1303,
1343,
1377,
1391,
1635,
1782,
1875,
2162,
2361,
2488,
3467,
4008,
4211,
4600,
4808,
5299,
5855,
6329,
7203,
9609,
9959,
10563,
10786,
11420,
11709,
11907,
13163,
13697,
13700,
14808,
15306,
16410,
16791,
17992,
19203,
19510,
20724,
22305,
22935,
27007,
30109,
30420,
33409,
34949,
40283,
40493,
40549,
47282,
49146,
50257,
50357,
50358,
50359,
50360,
50361
],
"suppress_ids_begin": [
220,
50256
]
}

101342
models/whisper/tokenizer.json
View File

File diff suppressed because it is too large Load Diff

51864
models/whisper/vocabulary.txt
View File

File diff suppressed because it is too large Load Diff

1
requirements.txt
View File

@@ -27,6 +27,7 @@ pywin32; platform_system == "Windows"
# Utilities # Utilities
pyautogui pyautogui
keyboard keyboard
pynput
pyyaml pyyaml
numpy numpy
scipy scipy

10
run.sh → run.sh
View File

@@ -6,6 +6,16 @@ cd "$SCRIPT_DIR"
echo "🎙️ Starting Meeting Assistant..." echo "🎙️ Starting Meeting Assistant..."
# Use the corporate-aware CA bundle (system + corporate + public CAs) so model
# downloads / revalidation work behind SSL-inspecting proxies. See README.
if [ -f "certs/corp_ca_bundle.pem" ]; then
export SSL_CERT_FILE="$SCRIPT_DIR/certs/corp_ca_bundle.pem"
export REQUESTS_CA_BUNDLE="$SSL_CERT_FILE"
export CURL_CA_BUNDLE="$SSL_CERT_FILE"
fi
# hf-xet has been flaky behind proxies; plain HTTPS transfers are more reliable.
export HF_HUB_DISABLE_XET=1
# Activate virtual environment # Activate virtual environment
if [ -d ".venv" ]; then if [ -d ".venv" ]; then
source .venv/bin/activate source .venv/bin/activate

47
setup.py
View File

@@ -17,32 +17,37 @@ def install_dependencies():
def download_models(): def download_models():
"""Download AI models""" """Download the Qwen2.5-VL vision brain (model + mmproj) into models/.
print("🤖 Downloading AI models...")
# Create models directory The Whisper STT model (distil-large-v3 by default) auto-downloads via
faster-whisper on first run, so it isn't fetched here.
"""
print("🤖 Downloading AI models (Qwen2.5-VL-7B + vision projector)...")
os.makedirs("models", exist_ok=True) os.makedirs("models", exist_ok=True)
# Download TinyLlama (small, fast, works on CPU) repo = "ggml-org/Qwen2.5-VL-7B-Instruct-GGUF"
model_urls = { files = [
"tinyllama-1.1b.Q4_K_M.gguf": "https://huggingface.co/TheBloke/TinyLlama-1.1B-GGUF/resolve/main/tinyllama-1.1b.Q4_K_M.gguf" "Qwen2.5-VL-7B-Instruct-Q4_K_M.gguf", # ~4.7 GB
} "mmproj-Qwen2.5-VL-7B-Instruct-f16.gguf", # ~1.4 GB vision projector
]
import urllib.request try:
from huggingface_hub import hf_hub_download
except ImportError:
print("❌ huggingface_hub not installed. Run: pip install -r requirements.txt")
return
for model_name, url in model_urls.items(): dest = os.path.abspath("models")
model_path = os.path.join("models", model_name) for fn in files:
if os.path.exists(os.path.join(dest, fn)):
if not os.path.exists(model_path): print(f"✅ {fn} already exists")
print(f"Downloading {model_name}...") continue
try: print(f"Downloading {fn} (this is large; behind a proxy it may be slow)...")
urllib.request.urlretrieve(url, model_path) try:
print(f"✅ Downloaded {model_name}") hf_hub_download(repo_id=repo, filename=fn, local_dir=dest)
except Exception as e: print(f"✅ Downloaded {fn}")
print(f"❌ Failed to download {model_name}: {e}") except Exception as e:
print("⚠️ Will run in fallback mode without local LLM") print(f"❌ Failed to download {fn}: {e}")
else:
print(f"✅ {model_name} already exists")
def setup_audio(): def setup_audio():

418
src/ai_engine.py
View File

@@ -1,232 +1,322 @@
import os import os
import re import re
import ast
import operator
from pathlib import Path from pathlib import Path
from collections import deque from collections import deque
from datetime import datetime from datetime import datetime
from rapidfuzz import fuzz
import threading import threading
# ----------------------------------------------------------------------
# Safe arithmetic evaluator for "obvious" math questions
# ----------------------------------------------------------------------
_OPS = {
ast.Add: operator.add, ast.Sub: operator.sub, ast.Mult: operator.mul,
ast.Div: operator.truediv, ast.Pow: operator.pow, ast.Mod: operator.mod,
ast.USub: operator.neg, ast.UAdd: operator.pos,
}
_WORD_MATH = [
(r"\bplus\b", "+"), (r"\bminus\b", "-"),
(r"\btimes\b", "*"), (r"\bmultiplied by\b", "*"),
(r"\bdivided by\b", "/"), (r"\bover\b", "/"),
(r"\bto the power of\b", "**"), (r"\bsquared\b", "**2"),
(r"\bpercent of\b", "/100*"),
]
def _eval_node(node):
if isinstance(node, ast.Constant) and isinstance(node.value, (int, float)):
return node.value
if isinstance(node, ast.BinOp) and type(node.op) in _OPS:
return _OPS[type(node.op)](_eval_node(node.left), _eval_node(node.right))
if isinstance(node, ast.UnaryOp) and type(node.op) in _OPS:
return _OPS[type(node.op)](_eval_node(node.operand))
raise ValueError("unsupported expression")
def try_solve_math(question):
"""Return a string answer for a simple arithmetic question, else None."""
q = question.lower().strip().rstrip("?.! ")
for prefix in ("what is", "what's", "whats", "how much is", "calculate", "compute", "solve"):
if q.startswith(prefix):
q = q[len(prefix):].strip()
for pat, rep in _WORD_MATH:
q = re.sub(pat, rep, q)
# keep only arithmetic characters
if not re.search(r"\d", q):
return None
expr = re.sub(r"[^0-9+\-*/().% ]", "", q).strip()
if not expr or not re.search(r"[+\-*/]", expr):
return None
try:
tree = ast.parse(expr, mode="eval")
result = _eval_node(tree.body)
if isinstance(result, float) and result.is_integer():
result = int(result)
elif isinstance(result, float):
result = round(result, 6)
pretty = re.sub(r"\s+", " ", re.sub(r"([+\-*/])", r" \1 ", expr)).strip()
return f"{pretty} = {result}"
except Exception:
return None
class AIEngine: class AIEngine:
def __init__(self, config): def __init__(self, config):
self.config = config self.config = config
ai_cfg = config.get("ai", {}) if config else {}
self.answer_mode = ai_cfg.get("answer_mode", "auto_obvious")
self.user_name = ai_cfg.get("user_name", "you")
self.max_tokens = int(ai_cfg.get("max_tokens", 160))
self.temperature = float(ai_cfg.get("temperature", 0.3))
self.n_ctx = int(ai_cfg.get("context_length", 4096))
self.model = None self.model = None
self.vision = False # True once a vision (VL) model is loaded
self.memory = deque(maxlen=8) self.memory = deque(maxlen=8)
self.fast_mode = True
self.current_generation = None
self.interrupt_event = threading.Event() self.interrupt_event = threading.Event()
# llama.cpp is not thread-safe; serialize all generations through one lock
self._gen_lock = threading.Lock()
self.load_model() self.load_model()
def load_model(self): def load_model(self):
"""Load Qwen2.5-VL (vision) when an mmproj is configured, else a plain
text GGUF. The VL model answers both spoken questions and screenshots."""
print("🤖 Loading AI model...") print("🤖 Loading AI model...")
models_dir = Path(__file__).parent.parent / "models"
ai_cfg = self.config.get("ai", {})
model_path = models_dir / ai_cfg.get("model", "Qwen2.5-VL-7B-Instruct-Q4_K_M.gguf")
model_path = ( mmproj_name = ai_cfg.get("mmproj", "mmproj-Qwen2.5-VL-7B-Instruct-f16.gguf")
Path(__file__).parent.parent mmproj_path = models_dir / mmproj_name if mmproj_name else None
/ "models"
/ "Qwen2.5-7B-Instruct-Q4_K_M.gguf"
)
from llama_cpp import Llama from llama_cpp import Llama
chat_handler = None
if mmproj_path and mmproj_path.exists():
try:
from llama_cpp.llama_chat_format import Qwen25VLChatHandler
chat_handler = Qwen25VLChatHandler(
clip_model_path=str(mmproj_path), verbose=False
)
self.vision = True
print(f" 👁️ Vision projector: {mmproj_path.name}")
except Exception as e:
print(f"⚠️ Could not load vision projector ({e}); text-only mode.")
self.model = Llama( self.model = Llama(
model_path=str(model_path), model_path=str(model_path),
chat_handler=chat_handler,
# PERFORMANCE - MAXIMUM SPEED n_gpu_layers=-1, # offload to Metal/GPU where available
n_gpu_layers=-1, n_ctx=self.n_ctx,
n_ctx=2048, # Reduced from 4096 for speed n_batch=512,
n_batch=512, # Reduced for faster prompt processing
n_threads=max(os.cpu_count() - 1, 1), n_threads=max(os.cpu_count() - 1, 1),
# SPEED OPTIMIZATIONS
offload_kqv=True, offload_kqv=True,
flash_attn=True, flash_attn=True,
use_mmap=True, use_mmap=True,
use_mlock=False, use_mlock=False,
seed=42,
# FASTER INFERENCE verbose=False,
n_parts=-1,
seed=42, # Deterministic for speed
f16_kv=True, # Use half-precision for KV cache
# STABILITY
verbose=False
) )
print(f"✅ AI ready ({'vision+text' if self.vision else 'text-only'})")
print("✅ AI ready (fast mode enabled)") # ------------------------------------------------------------------
# Public API
# ------------------------------------------------------------------
def answer_question(self, question, context, is_partial=False): def classify(self, question):
"""Answer a question with interruption support""" """math | factual | open — used to decide auto-answer vs suggest."""
q = question.lower().strip()
if try_solve_math(question):
return "math"
# short, fact-shaped lookups -> "obvious"
factual_markers = (
"capital of", "definition of", "define ", "meaning of", "what is the",
"what's the", "who is", "who was", "when did", "when was", "how many",
"what year", "convert ", "how do you spell", "abbreviation",
)
if any(m in q for m in factual_markers) and len(q.split()) <= 12:
return "factual"
return "open"
# Check for interrupt def answer_question(self, question, context, source=None):
"""Returns dict: {text, kind, suggested}. text is None if interrupted."""
if self.interrupt_event.is_set(): if self.interrupt_event.is_set():
self.interrupt_event.clear() self.interrupt_event.clear()
return None return None
question = self._normalize_question(question) question = self._normalize_question(question)
kind = self.classify(question)
# For partial questions, answer faster with fewer tokens # decide whether this is a committed answer or a suggestion
if is_partial: if self.answer_mode == "suggest_only":
response = self._generate_fast(question, context) suggested = True
elif self.answer_mode == "auto_all":
suggested = False
else: # auto_obvious
suggested = kind == "open"
# fast path: solve arithmetic directly, no LLM needed
if kind == "math":
text = try_solve_math(question)
else: else:
response = self._generate(question, context) text = self._generate(question, context, kind, suggested)
if response: # Only store if not interrupted if text is None:
self.memory.append({ return None
"question": question,
"response": response,
"time": datetime.now()
})
return response self.memory.append({"question": question, "response": text, "time": datetime.now()})
return {"text": text, "kind": kind, "suggested": suggested}
def interrupt(self): def interrupt(self):
"""Interrupt current generation"""
self.interrupt_event.set() self.interrupt_event.set()
print("🛑 Generation interrupted")
# ------------------------------------------------------------------
# Internals
# ------------------------------------------------------------------
def _normalize_question(self, question): def _normalize_question(self, question):
question = question.strip() return question.strip()
fixes = { def _system_prompt(self, kind, suggested):
"jav": "java", if suggested:
"py": "python", return (
"js": "javascript", f"You are a real-time meeting copilot for {self.user_name}. "
"api": "API" "Someone in the meeting asked a question. Draft a clear, professional "
} f"reply that {self.user_name} could say out loud. Be concise (under ~60 "
"words), specific, and natural. Do not add labels or preamble."
words = question.split() )
normalized = [] if kind == "factual":
return (
for word in words: "You are a fast, accurate assistant. Give the direct factual answer in "
lowered = word.lower() "one short sentence. No preamble, no hedging."
if lowered in fixes: )
normalized.append(fixes[lowered]) return (
else: f"You are a real-time meeting copilot for {self.user_name}. "
normalized.append(word) "Answer the latest question accurately and concisely (under ~70 words). "
"Use the meeting transcript only for context. Be direct."
question = " ".join(normalized)
# Fix cut-off endings
if question.endswith("between"):
if self.memory:
last = self.memory[-1]["question"]
question += f" and {last}"
return question
def _generate_fast(self, question, context):
"""Ultra-fast generation for partial/interruptible responses"""
# Check for interrupt before generation
if self.interrupt_event.is_set():
self.interrupt_event.clear()
return None
# Simplified prompt for speed
system_prompt = "You are a fast assistant. Answer in 1-2 short sentences max."
user_prompt = f"Q: {question}\nA:"
prompt = (
f"<|im_start|>system\n{system_prompt}<|im_end|>\n"
f"<|im_start|>user\n{user_prompt}<|im_end|>\n"
f"<|im_start|>assistant\n"
) )
try: def _user_prompt(self, question, context):
output = self.model( transcript = ""
prompt, if context:
max_tokens=30, # Very short for fast responses transcript = (context.get("audio", "") or "")[-1800:].strip()
temperature=0.1,
top_p=0.9,
repeat_penalty=1.0,
stop=["<|im_end|>", "\n", ".", "!", "?"],
echo=False
)
# Check for interrupt during generation memory_block = ""
if self.interrupt_event.is_set():
self.interrupt_event.clear()
return None
text = output["choices"][0]["text"]
return self._clean(text)
except Exception as e:
print(f"⚠️ Fast generation error: {e}")
return None
def _generate(self, question, context):
"""Standard generation for complete questions"""
# Check for interrupt before generation
if self.interrupt_event.is_set():
self.interrupt_event.clear()
return None
recent_audio = context.get("audio", "")[-1500:] if context else ""
recent_screen = context.get("screen", "")[-700:] if context else ""
memory_context = ""
if self.memory: if self.memory:
recent = list(self.memory)[-2:] for item in list(self.memory)[-2:]:
for item in recent: memory_block += f"Earlier Q: {item['question']}\nEarlier A: {item['response']}\n"
memory_context += f"Q: {item['question']}\nA: {item['response']}\n"
# Ultra-concise system prompt for speed parts = []
system_prompt = """ if transcript:
You are a realtime assistant. Answer immediately and concisely. parts.append(f"[Meeting transcript so far]\n{transcript}\n")
Keep answers under 40 words. Be direct. No explanations unless asked. if memory_block:
""" parts.append(memory_block)
parts.append(f"[Question]\n{question}")
return "\n".join(parts)
user_prompt = f""" def _stream_chat(self, messages, max_tokens):
Q: {question} """Run a chat completion, streaming so a new question can interrupt the
A:""" current one mid-generation. Returns cleaned text, or None if interrupted."""
if self.interrupt_event.is_set():
self.interrupt_event.clear()
return None
prompt = ( chunks = []
f"<|im_start|>system\n{system_prompt}<|im_end|>\n" with self._gen_lock:
f"<|im_start|>user\n{user_prompt}<|im_end|>\n" try:
f"<|im_start|>assistant\n" stream = self.model.create_chat_completion(
) messages=messages,
max_tokens=max_tokens,
temperature=self.temperature,
top_p=0.9,
repeat_penalty=1.1,
stream=True,
)
for part in stream:
if self.interrupt_event.is_set():
self.interrupt_event.clear()
return None
delta = part["choices"][0].get("delta", {})
piece = delta.get("content")
if piece:
chunks.append(piece)
except Exception as e:
print(f"⚠️ Generation error: {e}")
return "I couldn't process that."
return self._clean("".join(chunks))
def _generate(self, question, context, kind, suggested):
messages = [
{"role": "system", "content": self._system_prompt(kind, suggested)},
{"role": "user", "content": self._user_prompt(question, context)},
]
# Spoken answers are conversational — keep them short so they come back
# fast. The prompts already target ~60-70 words; cap tokens to match
# rather than spend seconds generating up to self.max_tokens (~350).
if kind == "factual":
max_tokens = 40
elif suggested:
max_tokens = min(self.max_tokens, 110)
else:
max_tokens = min(self.max_tokens, 160)
return self._stream_chat(messages, max_tokens)
# ------------------------------------------------------------------
# Screen questions — read a screenshot and answer it directly
# ------------------------------------------------------------------
def answer_from_image(self, image_path, hint=None):
"""Read the question(s) in a screenshot and answer. Returns the answer
text, or None if interrupted / unavailable."""
if not self.vision:
return ("Screen reading needs the vision model (Qwen2.5-VL + mmproj). "
"It isn't loaded.")
import base64
try: try:
output = self.model( with open(image_path, "rb") as f:
prompt, b64 = base64.b64encode(f.read()).decode("ascii")
max_tokens=60, # Reduced for speed
temperature=0.1, # Lower for faster, more deterministic
top_p=0.9,
repeat_penalty=1.0, # Disabled for speed
stop=["<|im_end|>", "\n", "."],
echo=False,
frequency_penalty=0.0,
presence_penalty=0.0
)
# Check for interrupt during generation
if self.interrupt_event.is_set():
self.interrupt_event.clear()
return None
text = output["choices"][0]["text"]
return self._clean(text)
except Exception as e: except Exception as e:
print(f"⚠️ Generation error: {e}") print(f"⚠️ Could not read screenshot {image_path}: {e}")
return "I couldn't process that." return None
data_uri = f"data:image/png;base64,{b64}"
instruction = (
"Read the question(s) shown in this screenshot and answer them directly "
"and correctly. If it is multiple-choice, state the correct option and a "
"one-line reason. If it is a coding or math problem, give the solution. "
"Be concise. No preamble."
)
if hint:
instruction += f" Extra context from {self.user_name}: {hint}"
messages = [
{"role": "system",
"content": f"You are an expert assistant helping {self.user_name} answer "
"a question shown on screen during a meeting."},
{"role": "user", "content": [
{"type": "text", "text": instruction},
{"type": "image_url", "image_url": {"url": data_uri}},
]},
]
text = self._stream_chat(messages, max_tokens=self.max_tokens)
if text:
self.memory.append({"question": "[screen question]", "response": text,
"time": datetime.now()})
return text
def _clean(self, text): def _clean(self, text):
text = re.sub(r"<\|.*?\|>", "", text)
text = re.sub(r"<.*?>", "", text) text = re.sub(r"<.*?>", "", text)
text = re.sub(r"\s+", " ", text).strip() text = re.sub(r"\s+", " ", text).strip()
if not text: if not text:
return "No response generated." return "No response generated."
# Ensure it ends with punctuation
if text[-1] not in ".!?": if text[-1] not in ".!?":
text += "." text += "."
if text[0].islower():
# Capitalize first letter
if text and text[0].islower():
text = text[0].upper() + text[1:] text = text[0].upper() + text[1:]
return text return text

852
src/audio_listener.py
View File

@@ -19,377 +19,561 @@ from faster_whisper import WhisperModel
os.environ["HF_HUB_DISABLE_SSL_VERIFY"] = "1" os.environ["HF_HUB_DISABLE_SSL_VERIFY"] = "1"
# ============================================================ # ============================================================
# 🔧 TUNING - OPTIMIZED FOR SPEED # 🔧 TUNING - OPTIMIZED FOR SPEED + ACCURACY
# ============================================================ # ============================================================
MIN_SPEECH_SECONDS = 0.8 MIN_SPEECH_SECONDS = 0.6
SILENCE_SECONDS = 0.3 SILENCE_SECONDS = 0.4
MAX_SPEECH_SECONDS = 8.0 MAX_SPEECH_SECONDS = 12.0
SPEECH_THRESHOLD_MULTIPLIER = 1.3 SPEECH_THRESHOLD_MULTIPLIER = 1.3
MIN_THRESHOLD_GAP = 400 MIN_THRESHOLD_GAP = 350
CONSECUTIVE_SPEECH_TO_START = 4 CONSECUTIVE_SPEECH_TO_START = 3
# ============================================================ # ============================================================
# Phrases Whisper hallucinates on silence / near-silent clips (a accidental
# key tap, a breath). Dropping them stops phantom questions/context.
_HALLUCINATIONS = {
"", ".", "you", "you.", "thank you", "thank you.", "thank you very much.",
"thanks for watching", "thanks for watching.", "thanks for watching!",
"please subscribe.", "subscribe.", "bye.", "bye bye.", "bye-bye.",
"okay.", "ok.", "so.", "uh.", "um.", "hmm.", "yeah.", ".....",
}
class StreamSegmenter:
"""Per-stream voice-activity segmenter.
One instance owns the speech state-machine for a single audio source
(e.g. the microphone, or the system/loopback device). It turns a stream
of 30 ms frames into complete utterances and pushes the raw PCM of each
finished utterance onto a shared transcription queue, tagged with its
source so downstream code knows who spoke.
"""
def __init__(self, source, sample_rate, frame_size, frame_duration_ms,
vad, out_queue):
self.source = source # "microphone" | "system"
self.sample_rate = sample_rate
self.frame_size = frame_size
self.frame_duration_ms = frame_duration_ms
self.vad = vad
self.out_queue = out_queue
# thresholds (set during calibration)
self.noise_floor = 0.0
self.speech_threshold = 2000.0
self.silence_threshold = 1500.0
# rolling pre-speech buffer so we don't clip the start of words
self._pre_buffer = bytearray()
self._pre_frames = 0
self._MAX_PRE = 10
# speech state
self.is_speaking = False
self.current_audio = bytearray()
self.speech_frames = 0
self.silence_frames = 0
self.consecutive_speech = 0
self.speech_rms_values = []
self.peak_rms = 0
self.speech_start_time = 0.0
def calibrate(self, rms_values):
if rms_values:
self.noise_floor = float(np.median(rms_values))
else:
self.noise_floor = 1500.0
self.speech_threshold = max(
self.noise_floor * SPEECH_THRESHOLD_MULTIPLIER,
self.noise_floor + MIN_THRESHOLD_GAP,
)
self.silence_threshold = self.noise_floor * 1.15
def process_frame(self, audio_bytes, rms, now):
"""Feed one frame. Emits a finished utterance to the queue when ready."""
# maintain pre-speech ring buffer
self._pre_buffer.extend(audio_bytes)
self._pre_frames += 1
if self._pre_frames > self._MAX_PRE:
excess = self._pre_frames - self._MAX_PRE
self._pre_buffer = self._pre_buffer[excess * self.frame_size:]
self._pre_frames = self._MAX_PRE
is_voice = False
if rms >= self.speech_threshold:
try:
is_voice = self.vad.is_speech(audio_bytes, self.sample_rate)
except Exception:
is_voice = True
if not self.is_speaking:
if is_voice:
self.consecutive_speech += 1
if self.consecutive_speech >= CONSECUTIVE_SPEECH_TO_START:
self.is_speaking = True
self.speech_start_time = now
self.current_audio = bytearray(self._pre_buffer)
self.speech_frames = self._pre_frames
self.silence_frames = 0
self.speech_rms_values = [rms]
self.peak_rms = rms
else:
self.consecutive_speech = 0
return
# --- currently speaking ---
self.current_audio.extend(audio_bytes)
self.speech_frames += 1
self.speech_rms_values.append(rms)
if rms > self.peak_rms:
self.peak_rms = rms
if len(self.speech_rms_values) > 15:
speech_median = np.median(self.speech_rms_values)
dynamic_silence = max(self.silence_threshold, speech_median * 0.6)
else:
dynamic_silence = self.silence_threshold
if rms < dynamic_silence and not is_voice:
self.silence_frames += 1
elif is_voice:
self.silence_frames = 0
speech_duration = now - self.speech_start_time
silence_duration = (self.silence_frames * self.frame_duration_ms) / 1000.0
if speech_duration >= MIN_SPEECH_SECONDS and silence_duration >= SILENCE_SECONDS:
self._emit()
elif speech_duration >= MAX_SPEECH_SECONDS:
self._emit()
def flush_if_idle(self, now):
"""Called when the input queue goes quiet — close a dangling utterance."""
if not self.is_speaking:
return
elapsed = now - self.speech_start_time
silence_duration = (self.silence_frames * self.frame_duration_ms) / 1000.0
if elapsed >= MIN_SPEECH_SECONDS and silence_duration >= SILENCE_SECONDS:
self._emit()
def _emit(self):
audio = bytes(self.current_audio)
self._reset()
# ignore clips that are too short to be meaningful
if len(audio) / (2 * self.sample_rate) >= 0.3:
self.out_queue.put((self.source, audio))
def _reset(self):
self.is_speaking = False
self.current_audio = bytearray()
self.speech_frames = 0
self.silence_frames = 0
self.consecutive_speech = 0
self.peak_rms = 0
self.speech_rms_values = []
self._pre_buffer = bytearray()
self._pre_frames = 0
class AudioListener: class AudioListener:
"""Captures one or more audio sources, segments speech per source, and
transcribes finished utterances with a single shared Whisper model.
The callback is invoked as callback(text, timestamp, source) where source
is "microphone" (you) or "system" (other meeting participants)."""
def __init__(self, config, callback): def __init__(self, config, callback):
self.config = config self.config = config
self.callback = callback # Expects callback(text, timestamp) self.callback = callback
self.running = False self.running = False
audio_cfg = config.get("audio", {}) if config else {}
self.source_mode = audio_cfg.get("source", "both")
self.loopback_keywords = [
k.lower() for k in audio_cfg.get("loopback_keywords", ["blackhole"])
]
# Capture mode: "push_to_talk" (only buffer audio while armed) or
# "continuous" (always-on VAD segmentation, the original behavior).
self.capture_mode = audio_cfg.get("capture_mode", "push_to_talk")
self._armed = False
self._ptt_buffers = {} # source -> bytearray, while armed
self._ptt_lock = threading.Lock()
# rolling pre-roll captured *before* the key is pressed, so the first
# word isn't clipped (otherwise "Run me through" → "Runs"/"Water"/...).
self._PREROLL_FRAMES = 12 # ~360 ms at 30 ms frames
self._preroll = {} # source -> deque of recent raw frames
# optional speaker diarization for the mixed "system" stream
diar_cfg = audio_cfg.get("diarization", {}) or {}
self.diarizer = None
if diar_cfg.get("enabled", True):
try:
from src.diarizer import SpeakerDiarizer
self.diarizer = SpeakerDiarizer(
model_dir=diar_cfg.get("model_dir", "models/ecapa"),
similarity_threshold=diar_cfg.get("similarity_threshold", 0.40),
max_speakers=diar_cfg.get("max_speakers", 10),
)
except Exception as e:
print(f"⚠️ Could not init diarizer: {e}")
self.sample_rate = 16000 self.sample_rate = 16000
self.frame_duration_ms = 30 self.frame_duration_ms = 30
self.frame_size = int(self.sample_rate * self.frame_duration_ms / 1000) self.frame_size = int(self.sample_rate * self.frame_duration_ms / 1000)
self.vad = webrtcvad.Vad(2) self.vad = webrtcvad.Vad(2)
self.audio_queue = queue.Queue() # frames coming in from every stream: (source, audio_bytes, rms)
self.current_audio = bytearray() self.frame_queue = queue.Queue()
self.history = deque(maxlen=20) # finished utterances awaiting transcription: (source, audio_bytes)
self.utterance_queue = queue.Queue()
# Speech state machine self.segmenters = {} # source -> StreamSegmenter
self.is_speaking = False self.streams = [] # open sd.RawInputStream objects
self.speech_frames = 0 self.last_emit = {} # source -> last transcript text (de-dup)
self.silence_frames = 0
self.consecutive_speech = 0
self.consecutive_silence = 0
self.total_frames_in_utterance = 0
self.noise_floor = 0 # Whisper model is config-driven. Default to distil-large-v3: near
self.speech_threshold = 2000 # large-v3 accuracy but much faster — and push-to-talk makes the
self.silence_threshold = 1500 # transcription latency a non-issue. Auto-downloads/caches by name
# (honors SSL_CERT_FILE / HF_HUB_DISABLE_XET set by run.sh).
whisper_name = audio_cfg.get("whisper_model", "distil-large-v3")
cpu_threads = max(os.cpu_count() - 1, 1)
# A light domain prompt biases Whisper toward plausible vocabulary so
# short conversational clips garble less ("SOLID principles", not "solid
# transports"). Override per meeting via audio.transcription_prompt.
self._initial_prompt = audio_cfg.get(
"transcription_prompt",
"A professional meeting conversation. Topics may include software "
"engineering, SOLID principles, APIs, databases, architecture, "
"deadlines, budgets, and roadmaps.",
)
print(f"🔄 Loading Whisper model '{whisper_name}'...")
self.model = None
for attempt in (whisper_name, "models/whisper", "base"):
try:
local_only = attempt == "models/whisper"
self.model = WhisperModel(
attempt,
device="cpu",
compute_type="int8",
local_files_only=local_only,
cpu_threads=cpu_threads,
)
if attempt != whisper_name:
print(f"✅ Whisper model ready (fell back to '{attempt}')")
else:
print(f"✅ Whisper model ready ('{whisper_name}')")
break
except Exception as e:
print(f"⚠️ Could not load Whisper '{attempt}': {e}")
if self.model is None:
raise RuntimeError("Could not load any Whisper model")
self.recent_rms = deque(maxlen=30) # ------------------------------------------------------------------
self.speech_rms_values = [] # Device selection
self.peak_rms = 0 # ------------------------------------------------------------------
self.speech_start_time = 0
self.last_transcription_time = 0 def _resolve_device(self, spec):
self.min_transcription_interval = 0.3 """Resolve a config value (None / int / name substring) to a device index."""
devices = sd.query_devices()
if isinstance(spec, int):
return spec
if isinstance(spec, str):
for i, dev in enumerate(devices):
if spec.lower() in dev["name"].lower() and dev["max_input_channels"] > 0:
return i
return None
self.calibrated = False def _is_loopback(self, name):
name = name.lower()
return any(k in name for k in self.loopback_keywords)
print("🔄 Loading Whisper model...") def _find_mic_device(self):
try: cfg = self.config.get("audio", {}).get("mic_device")
self.model = WhisperModel( resolved = self._resolve_device(cfg)
"models/whisper", if resolved is not None:
device="cpu", return resolved
compute_type="int8", devices = sd.query_devices()
local_files_only=True, default_input = sd.default.device[0]
cpu_threads=max(os.cpu_count() - 1, 1) # prefer the system default input, unless it's a loopback device
) if (default_input is not None and 0 <= default_input < len(devices)
print("✅ Whisper model ready") and devices[default_input]["max_input_channels"] > 0
except Exception as e: and not self._is_loopback(devices[default_input]["name"])):
print(f"⚠️ Could not load Whisper from models/whisper, trying default: {e}") return default_input
self.model = WhisperModel( for i, dev in enumerate(devices):
"base", if dev["max_input_channels"] > 0 and not self._is_loopback(dev["name"]):
device="cpu", return i
compute_type="int8", return None
cpu_threads=max(os.cpu_count() - 1, 1)
) def _find_system_device(self):
print("✅ Whisper model ready (using default 'base' model)") cfg = self.config.get("audio", {}).get("system_device")
resolved = self._resolve_device(cfg)
if resolved is not None:
return resolved
for i, dev in enumerate(sd.query_devices()):
if dev["max_input_channels"] > 0 and self._is_loopback(dev["name"]):
return i
return None
def _planned_sources(self):
"""Return {source: device_index} based on config + detected hardware."""
plan = {}
mode = self.source_mode
if mode in ("microphone", "both"):
mic = self._find_mic_device()
if mic is not None:
plan["microphone"] = mic
if mode in ("system", "both"):
sysdev = self._find_system_device()
if sysdev is not None:
plan["system"] = sysdev
return plan
# ------------------------------------------------------------------
# Startup
# ------------------------------------------------------------------
def start(self): def start(self):
self.running = True self.running = True
print("\n🎤 Available audio devices:\n") print("\n🎤 Available audio devices:\n")
devices = sd.query_devices() for i, dev in enumerate(sd.query_devices()):
for i, dev in enumerate(devices): print(f" [{i}] {dev['name']} (in: {dev['max_input_channels']}, "
print(f" [{i}] {dev['name']} (in: {dev['max_input_channels']}, out: {dev['max_output_channels']})") f"out: {dev['max_output_channels']})")
input_device = self._select_input_device() plan = self._planned_sources()
device_name = sd.query_devices(input_device)['name']
print(f"\n🎤 Using input device: {device_name}\n")
print("🔧 Calibrating background noise (3 seconds)...") if not plan:
print(" Please stay COMPLETELY silent...") print("\n❌ No usable audio input devices found for the configured source.")
return
try: if "system" not in plan and self.source_mode in ("system", "both"):
calibration_audio = sd.rec( print("\n⚠️ No system/loopback device detected (e.g. BlackHole).")
int(3.0 * self.sample_rate), print(" You will only capture the microphone, not other participants.")
samplerate=self.sample_rate, print(" See README → 'Hearing other participants' for one-time setup.")
channels=1,
dtype="int16",
device=input_device
)
sd.wait()
cal_np = np.frombuffer(calibration_audio.tobytes(), dtype=np.int16).astype(np.float32) summary = ", ".join(
f"{s}=[{d}] {sd.query_devices(d)['name']}" for s, d in plan.items()
chunk_size = self.frame_size
rms_values = []
for i in range(0, len(cal_np) - chunk_size, chunk_size):
chunk = cal_np[i:i + chunk_size]
rms = np.sqrt(np.mean(chunk ** 2))
rms_values.append(rms)
self.noise_floor = np.median(rms_values) if rms_values else 1500
self.speech_threshold = max(
self.noise_floor * SPEECH_THRESHOLD_MULTIPLIER,
self.noise_floor + MIN_THRESHOLD_GAP
)
self.silence_threshold = self.noise_floor * 1.15
print(f"📊 Noise floor: {self.noise_floor:.0f} RMS")
print(f"📊 Speech threshold: {self.speech_threshold:.0f} RMS")
print(f"📊 Silence threshold: {self.silence_threshold:.0f} RMS")
self.calibrated = True
except Exception as e:
print(f"⚠️ Calibration failed: {e}")
self.noise_floor = 1500
self.speech_threshold = 2200
self.silence_threshold = 1700
self.calibrated = True
print(f"\n📋 Settings (FAST MODE):")
print(f" Min speech: {MIN_SPEECH_SECONDS}s | Max: {MAX_SPEECH_SECONDS}s")
print(f" Silence to end: {SILENCE_SECONDS}s")
print(f" Consecutive speech to start: {CONSECUTIVE_SPEECH_TO_START} frames")
self.stream = sd.RawInputStream(
samplerate=self.sample_rate,
blocksize=self.frame_size,
dtype="int16",
channels=1,
device=input_device,
callback=self._audio_callback
) )
self.stream.start() print(f"\n🎧 Capturing: {summary}\n")
threading.Thread(target=self._processing_loop, daemon=True).start() ptt = self.capture_mode == "push_to_talk"
print(f"\n🎤 Listening... Speak now!") # In continuous mode each source gets a calibrated VAD segmenter.
print(" Press Ctrl+C to stop\n") # In push-to-talk mode we buffer the held window instead, so neither
# calibration nor the always-on segmenter is needed.
if not ptt:
for source, device in plan.items():
seg = StreamSegmenter(source, self.sample_rate, self.frame_size,
self.frame_duration_ms, self.vad, self.utterance_queue)
self._calibrate(seg, device)
self.segmenters[source] = seg
# open one input stream per source
for source, device in plan.items():
stream = sd.RawInputStream(
samplerate=self.sample_rate,
blocksize=self.frame_size,
dtype="int16",
channels=1,
device=device,
callback=self._make_callback(source),
)
stream.start()
self.streams.append(stream)
if not ptt:
threading.Thread(target=self._frame_loop, daemon=True).start()
threading.Thread(target=self._transcribe_loop, daemon=True).start()
if ptt:
print("🎤 Ready. Hold the push-to-talk key to capture; release to answer.\n")
else:
print("🎤 Listening... Speak now! (Ctrl+C to stop)\n")
# ------------------------------------------------------------------
# Push-to-talk: only capture audio while armed
# ------------------------------------------------------------------
def arm(self):
"""Begin buffering audio from every source (push-to-talk key down)."""
if self.capture_mode != "push_to_talk" or self._armed:
return
with self._ptt_lock:
# seed each source with its pre-roll so the leading word survives
self._ptt_buffers = {
src: bytearray(b"".join(frames))
for src, frames in self._preroll.items()
}
self._armed = True
print("\n🎙️ Listening (key held)...")
def disarm(self):
"""Stop buffering and queue what was captured for transcription (key up)."""
if self.capture_mode != "push_to_talk" or not self._armed:
return
self._armed = False
with self._ptt_lock:
buffers = self._ptt_buffers
self._ptt_buffers = {}
queued = False
for source, buf in buffers.items():
audio = bytes(buf)
if len(audio) / (2 * self.sample_rate) >= 0.3: # ignore < 0.3s blips
self.utterance_queue.put((source, audio))
queued = True
print("⏳ Transcribing..." if queued else " (too short — nothing captured)")
def _calibrate(self, segmenter, device):
dev_name = sd.query_devices(device)["name"]
print(f"🔧 Calibrating noise floor for '{dev_name}' (2s)...")
try:
cal = sd.rec(int(2.0 * self.sample_rate), samplerate=self.sample_rate,
channels=1, dtype="int16", device=device)
sd.wait()
cal_np = np.frombuffer(cal.tobytes(), dtype=np.int16).astype(np.float32)
rms_values = []
for i in range(0, len(cal_np) - self.frame_size, self.frame_size):
chunk = cal_np[i:i + self.frame_size]
rms_values.append(np.sqrt(np.mean(chunk ** 2)))
segmenter.calibrate(rms_values)
print(f" 📊 {segmenter.source}: noise={segmenter.noise_floor:.0f} "
f"speech>={segmenter.speech_threshold:.0f}")
except Exception as e:
print(f"⚠️ Calibration failed for {segmenter.source}: {e}")
segmenter.calibrate([])
def _make_callback(self, source):
def _cb(indata, frames, time_info, status):
if status:
# underruns are noisy and harmless; skip logging them
pass
audio_bytes = bytes(indata)
# push-to-talk: only retain audio while the key is held
if self.capture_mode == "push_to_talk":
if self._armed:
with self._ptt_lock:
self._ptt_buffers.setdefault(source, bytearray()).extend(audio_bytes)
else:
# keep a short rolling pre-roll so the first word isn't clipped
pr = self._preroll.get(source)
if pr is None:
pr = self._preroll[source] = deque(maxlen=self._PREROLL_FRAMES)
pr.append(audio_bytes)
return
# continuous mode: feed the VAD segmenter via the frame queue
samples = np.frombuffer(audio_bytes, dtype=np.int16).astype(np.float32)
rms = int(np.sqrt(np.mean(samples ** 2))) if samples.size else 0
self.frame_queue.put((source, audio_bytes, rms))
return _cb
def _select_input_device(self): def _select_input_device(self):
devices = sd.query_devices() # kept for backward compatibility
default_input = sd.default.device[0] return self._find_mic_device() or 0
if default_input is not None and default_input < len(devices):
dev = devices[default_input]
if dev['max_input_channels'] > 0:
return default_input
for i, dev in enumerate(devices):
if dev['max_input_channels'] > 0:
return i
return 0
def stop(self): def stop(self):
self.running = False self.running = False
if hasattr(self, 'stream'): for stream in self.streams:
self.stream.stop() try:
self.stream.close() stream.stop()
stream.close()
except Exception:
pass
print("🛑 Stopped") print("🛑 Stopped")
def _audio_callback(self, indata, frames, time_info, status): # ------------------------------------------------------------------
if status: # Processing
print(f"⚠️ Audio status: {status}") # ------------------------------------------------------------------
audio_bytes = bytes(indata)
rms = int(np.sqrt(np.mean(
np.frombuffer(audio_bytes, dtype=np.int16).astype(np.float32) ** 2
)))
self.audio_queue.put((audio_bytes, rms))
def _processing_loop(self):
pre_speech_buffer = bytearray()
pre_speech_frames = 0
MAX_PRE_SPEECH = 10
def _frame_loop(self):
while self.running: while self.running:
try: try:
audio_bytes, rms = self.audio_queue.get(timeout=0.5) source, audio_bytes, rms = self.frame_queue.get(timeout=0.5)
self.recent_rms.append(rms) seg = self.segmenters.get(source)
if seg:
pre_speech_buffer.extend(audio_bytes) seg.process_frame(audio_bytes, rms, time.time())
pre_speech_frames += 1
if pre_speech_frames > MAX_PRE_SPEECH:
excess = pre_speech_frames - MAX_PRE_SPEECH
bytes_to_trim = excess * self.frame_size
pre_speech_buffer = pre_speech_buffer[bytes_to_trim:]
pre_speech_frames = MAX_PRE_SPEECH
is_voice_frame = False
if rms >= self.speech_threshold:
try:
is_voice_frame = self.vad.is_speech(audio_bytes, self.sample_rate)
except:
is_voice_frame = True
if not self.is_speaking:
if is_voice_frame:
self.consecutive_speech += 1
self.consecutive_silence = 0
if self.consecutive_speech >= CONSECUTIVE_SPEECH_TO_START:
self.is_speaking = True
self.speech_start_time = time.time()
self.current_audio = bytearray(pre_speech_buffer)
self.speech_frames = pre_speech_frames
self.silence_frames = 0
self.speech_rms_values = list(self.recent_rms)[-pre_speech_frames:]
self.peak_rms = max(self.speech_rms_values) if self.speech_rms_values else rms
print(f"\n🔴 SPEAKING (peak: {self.peak_rms})")
else:
self.consecutive_speech = 0
self.consecutive_silence += 1
else:
self.current_audio.extend(audio_bytes)
self.speech_frames += 1
self.speech_rms_values.append(rms)
if rms > self.peak_rms:
self.peak_rms = rms
if len(self.speech_rms_values) > 15:
speech_median = np.median(self.speech_rms_values)
dynamic_silence = max(self.silence_threshold, speech_median * 0.6)
else:
dynamic_silence = self.silence_threshold
if rms < dynamic_silence and not is_voice_frame:
self.silence_frames += 1
else:
if is_voice_frame:
self.silence_frames = 0
if self.is_speaking:
bar_len = max(0, min(int((rms - self.noise_floor) / 60), 35))
bar = "█" * bar_len
sil = f" 🔇{self.silence_frames}" if self.silence_frames > 0 else ""
print(f"\r🎙️ {rms:5d} |{bar}{sil} ", end="")
elif self.consecutive_speech > 0:
print(f"\r👂 {rms:5d} | detecting... {self.consecutive_speech}/{CONSECUTIVE_SPEECH_TO_START} ",
end="")
if self.is_speaking:
speech_duration = time.time() - self.speech_start_time
silence_duration = (self.silence_frames * self.frame_duration_ms) / 1000
if (speech_duration >= MIN_SPEECH_SECONDS and silence_duration >= SILENCE_SECONDS):
print(f"\n✅ End ({speech_duration:.1f}s, peak: {self.peak_rms})")
self._safe_transcribe()
self._reset_speech_state()
pre_speech_buffer = bytearray()
pre_speech_frames = 0
elif speech_duration >= MAX_SPEECH_SECONDS:
print(f"\n⏰ Max ({speech_duration:.1f}s, peak: {self.peak_rms})")
self._safe_transcribe()
self._reset_speech_state()
pre_speech_buffer = bytearray()
pre_speech_frames = 0
except queue.Empty: except queue.Empty:
if self.is_speaking: now = time.time()
elapsed = time.time() - self.speech_start_time for seg in self.segmenters.values():
silence_duration = (self.silence_frames * self.frame_duration_ms) / 1000 seg.flush_if_idle(now)
if elapsed >= MIN_SPEECH_SECONDS and silence_duration >= SILENCE_SECONDS:
print(f"\n✅ Final ({elapsed:.1f}s)")
self._safe_transcribe()
self._reset_speech_state()
continue
except Exception as e: except Exception as e:
print(f"\n❌ Error: {e}") print(f"\n❌ Frame error: {e}")
def _transcribe_loop(self):
while self.running:
try:
source, audio = self.utterance_queue.get(timeout=0.5)
except queue.Empty:
continue continue
try:
self._transcribe(source, audio)
except Exception as e:
print(f"❌ Transcription error: {e}")
def _reset_speech_state(self): def _transcribe(self, source, audio):
self.is_speaking = False audio_np = np.frombuffer(audio, dtype=np.int16).astype(np.float32) / 32768.0
self.current_audio = bytearray()
self.speech_frames = 0
self.silence_frames = 0
self.consecutive_speech = 0
self.peak_rms = 0
self.speech_rms_values = []
def _safe_transcribe(self): segments, _info = self.model.transcribe(
current_time = time.time() audio_np,
language="en",
beam_size=1, # greedy: distil-whisper is tuned for it and
best_of=1, # it's markedly faster for conversational clips
temperature=0.0,
condition_on_previous_text=False,
initial_prompt=self._initial_prompt,
compression_ratio_threshold=1.8,
no_speech_threshold=0.6,
log_prob_threshold=-1.0,
word_timestamps=False,
vad_filter=True,
)
if current_time - self.last_transcription_time < self.min_transcription_interval: text = self._clean_text(" ".join(s.text.strip() for s in segments if s.text.strip()))
if not text:
return return
# drop Whisper's silence hallucinations ("Thank you.", "Bye.", "you")
self.last_transcription_time = current_time if text.lower().strip() in _HALLUCINATIONS:
audio_duration = len(self.current_audio) / (2 * self.sample_rate)
if audio_duration < 0.3:
print(" (Too short)")
return return
if self._is_repetitive(text):
return
# de-dup back-to-back identical transcripts on the same source
if text == self.last_emit.get(source):
return
self.last_emit[source] = text
self._transcribe() # resolve who spoke
if source == "system":
speaker = self.diarizer.identify(audio_np) if self.diarizer else "Them"
else:
speaker = "You"
def _transcribe(self): icon = "🎙️" if source == "microphone" else "🔊"
try: print(f"\n{icon} {speaker}: \"{text}\"")
audio_np = np.frombuffer(self.current_audio, dtype=np.int16).astype(np.float32) / 32768.0
segments, info = self.model.transcribe( if self.callback:
audio_np, self.callback(text, datetime.now(), source, speaker)
language="en",
beam_size=3,
best_of=3,
temperature=[0.0, 0.2],
condition_on_previous_text=False,
compression_ratio_threshold=1.8,
no_speech_threshold=0.6,
log_prob_threshold=-1.0,
word_timestamps=False,
vad_filter=True,
)
text_parts = [] # ------------------------------------------------------------------
for segment in segments: # Text helpers
text = segment.text.strip() # ------------------------------------------------------------------
if text:
text_parts.append(text)
full_text = " ".join(text_parts)
full_text = self._clean_text(full_text)
if not full_text:
print(" (No text)")
return
if self._is_repetitive(full_text):
print(f" 🚫 Repetitive")
return
print(f"\n🎤 \"{full_text}\"")
self.history.append({
"text": full_text,
"timestamp": datetime.now()
})
is_command = self._is_command(full_text)
tag = "🎯 Command" if is_command else "📝 Speech"
print(f"{tag}: {full_text}")
# Callback with just text and timestamp (original format)
if self.callback:
self.callback(full_text, datetime.now())
except Exception as e:
print(f"❌ Transcription error: {e}")
def _clean_text(self, text): def _clean_text(self, text):
text = re.sub(r'\s+', ' ', text).strip() text = re.sub(r"\s+", " ", text).strip()
if not text: if not text:
return "" return ""
text = re.sub(r'\b(\w+)( \1\b){2,}', r'\1', text, flags=re.IGNORECASE) text = re.sub(r"\b(\w+)( \1\b){2,}", r"\1", text, flags=re.IGNORECASE)
text = ''.join(char for char in text if char.isprintable() or char in ' \t\n\r') text = "".join(c for c in text if c.isprintable() or c in " \t\n\r")
if text: if len(text) > 1:
text = text[0].upper() + text[1:] if len(text) > 1 else text.upper() text = text[0].upper() + text[1:]
return text.strip() return text.strip()
def _is_repetitive(self, text): def _is_repetitive(self, text):
words = text.lower().split() words = text.lower().split()
if len(words) < 4: if len(words) < 4:
return False return False
unique_ratio = len(set(words)) / len(words) if len(set(words)) / len(words) < 0.3 and len(words) > 8:
if unique_ratio < 0.3 and len(words) > 8:
return True return True
mid = len(words) // 2 mid = len(words) // 2
if mid > 0: if mid > 0:
@@ -398,59 +582,3 @@ class AudioListener:
if first == second and len(first) > 15: if first == second and len(first) > 15:
return True return True
return False return False
def _is_command(self, text):
"""Aggressive command/question detection"""
if not text or len(text) < 2:
return False
lowered = text.lower().strip()
# Always command if ends with ?
if lowered.endswith('?'):
return True
# Math expressions
math_patterns = [
r'\d+\s*[\+\-\*\/]\s*\d+',
r'\d+\s*plus\s*\d+',
r'\d+\s*minus\s*\d+',
r'\d+\s*times\s*\d+',
r'\d+\s*divided by\s*\d+',
]
import re
for pattern in math_patterns:
if re.search(pattern, lowered):
return True
# Question starters
starters = {
'what', 'why', 'how', 'when', 'where', 'who', 'which',
'can', 'could', 'would', 'will', 'do', 'does', 'did',
'is', 'are', 'was', 'were', 'should', 'shall', 'may',
'tell', 'explain', 'describe', 'show', 'give'
}
words = lowered.split()
if words and words[0] in starters:
return True
# Question phrases
question_phrases = [
'what is', 'what are', 'how do', 'how to', 'can you',
'tell me', 'explain', 'describe', 'show me'
]
for phrase in question_phrases:
if phrase in lowered:
return True
# Short statements (2-6 words) - treat as commands/questions
if 2 <= len(words) <= 6:
filler = {'um', 'uh', 'like', 'you', 'know', 'so', 'well'}
content_words = [w for w in words if w not in filler]
if len(content_words) >= 2:
if any(op in lowered for op in ['plus', 'minus', 'times']):
return True
if len(content_words) <= 4:
return True
return False

48
src/context_manager.py
View File

@@ -1,49 +1,53 @@
from collections import deque from collections import deque
import json
class ContextManager: class ContextManager:
def __init__(self, config): def __init__(self, config):
self.config = config self.config = config
self.audio_context = deque(maxlen=10) # Last 10 audio chunks self.audio_context = deque(maxlen=20) # last 20 utterances (both sources)
self.screen_context = deque(maxlen=5) # Last 5 screen captures self.screen_context = deque(maxlen=5)
self.last_text = "" self.last_text = ""
def add_audio_context(self, text, timestamp): def add_audio_context(self, text, timestamp, source="microphone", speaker=None):
"""Add audio transcript to context""" """Add a transcribed utterance, tagged with who spoke."""
self.audio_context.append({ self.audio_context.append({
'text': text, "text": text,
'timestamp': timestamp.isoformat(), "timestamp": timestamp.isoformat(),
'type': 'audio' "source": source,
"speaker": speaker or self._speaker(source),
"type": "audio",
}) })
self.last_text = text self.last_text = text
def add_screen_context(self, text, timestamp, region): def add_screen_context(self, text, timestamp, region):
"""Add screen text to context"""
self.screen_context.append({ self.screen_context.append({
'text': text, "text": text,
'timestamp': timestamp.isoformat(), "timestamp": timestamp.isoformat(),
'region': region, "region": region,
'type': 'screen' "type": "screen",
}) })
self.last_text = text self.last_text = text
def _speaker(self, source):
return "Them" if source == "system" else "You"
def get_context(self): def get_context(self):
"""Get current context for LLM""" """Context for the LLM, with speaker labels so it can follow the meeting."""
context = { transcript = "\n".join(
'audio': ' '.join([item['text'] for item in self.audio_context]), f"{item.get('speaker') or self._speaker(item.get('source'))}: {item['text']}"
'screen': ' '.join([item['text'] for item in self.screen_context]), for item in self.audio_context
'recent_audio': list(self.audio_context)[-3:], )
'recent_screen': list(self.screen_context)[-2:] return {
"audio": transcript,
"screen": " ".join(item["text"] for item in self.screen_context),
"recent_audio": list(self.audio_context)[-3:],
"recent_screen": list(self.screen_context)[-2:],
} }
return context
def get_last_text(self): def get_last_text(self):
"""Get last detected text"""
return self.last_text return self.last_text
def clear(self): def clear(self):
"""Clear all context"""
self.audio_context.clear() self.audio_context.clear()
self.screen_context.clear() self.screen_context.clear()
self.last_text = "" self.last_text = ""

96
src/diarizer.py Normal file
View File

@@ -0,0 +1,96 @@
"""Lightweight online speaker diarization.
The OS only gives us a single *mixed* stream of all remote participants (via
the loopback device), so we can't get per-person audio channels. Instead, for
each finished utterance we compute a voice fingerprint (ECAPA-TDNN speaker
embedding) and cluster it online: each new utterance is matched to the most
similar known speaker, or starts a new one ("Person 1", "Person 2", ...).
This is best-effort: very short utterances, people talking over each other, or
very similar voices will reduce accuracy. It is intended to *separate* the
conversation, not to identify people by name.
"""
import os
import warnings
import numpy as np
class SpeakerDiarizer:
def __init__(self, model_dir="models/ecapa", similarity_threshold=0.40,
max_speakers=10, min_seconds=0.7, device="cpu"):
self.similarity_threshold = float(similarity_threshold)
self.max_speakers = int(max_speakers)
self.min_samples = int(min_seconds * 16000)
self.available = False
self._model = None
# speaker registry: list of dicts {centroid: np.ndarray, count: int, label: str}
self.speakers = []
try:
warnings.filterwarnings("ignore")
from speechbrain.inference.speaker import EncoderClassifier
self._model = EncoderClassifier.from_hparams(
source=model_dir, savedir=model_dir, run_opts={"device": device}
)
self.available = True
print("✅ Speaker diarization ready (ECAPA voice fingerprints)")
except Exception as e:
print(f"⚠️ Diarization unavailable, falling back to single 'Them': {e}")
def _embed(self, audio_np):
import torch
with warnings.catch_warnings():
warnings.simplefilter("ignore")
wav = torch.from_numpy(audio_np.astype("float32")).unsqueeze(0)
emb = self._model.encode_batch(wav).squeeze().detach().cpu().numpy()
norm = np.linalg.norm(emb)
return emb / norm if norm > 0 else emb
def identify(self, audio_np):
"""Return a speaker label for this utterance's audio (16k mono float32)."""
if not self.available:
return "Them"
# too short to fingerprint reliably -> attribute to most recent speaker
if audio_np.shape[0] < self.min_samples:
return self.speakers[-1]["label"] if self.speakers else "Person 1"
try:
emb = self._embed(audio_np)
except Exception:
return "Them"
if not self.speakers:
return self._add_speaker(emb)
sims = [float(np.dot(emb, s["centroid"])) for s in self.speakers]
best = int(np.argmax(sims))
if sims[best] >= self.similarity_threshold:
self._update_speaker(best, emb)
return self.speakers[best]["label"]
if len(self.speakers) < self.max_speakers:
return self._add_speaker(emb)
# registry full: attach to nearest existing speaker
self._update_speaker(best, emb)
return self.speakers[best]["label"]
def _add_speaker(self, emb):
label = f"Person {len(self.speakers) + 1}"
self.speakers.append({"centroid": emb, "count": 1, "label": label})
return label
def _update_speaker(self, idx, emb):
s = self.speakers[idx]
# running mean of embeddings, renormalised to the unit sphere
c = (s["centroid"] * s["count"] + emb) / (s["count"] + 1)
norm = np.linalg.norm(c)
s["centroid"] = c / norm if norm > 0 else c
s["count"] += 1
def reset(self):
self.speakers = []

174
src/hotkeys.py Normal file
View File

@@ -0,0 +1,174 @@
"""Global hotkeys via pynput.
Two jobs:
* Push-to-talk: hold a key (default Right Option / ``alt_r``) to capture audio;
release to stop. Fires ``on_arm`` on key-down and ``on_disarm`` on key-up.
* Screen grab: a chord (default ``ctrl+shift+space``) fires ``on_capture`` once
per press to trigger the draw-a-box screenshot flow.
Callbacks run on pynput's listener thread, so they must be cheap and thread-safe.
Anything touching the Qt UI should marshal onto the main thread (see overlay's
signal bridge) — this module only invokes the callbacks it is given.
macOS requires a one-time **Accessibility** permission grant for global keys
(System Settings → Privacy & Security → Accessibility).
"""
try:
from pynput import keyboard
_PYNPUT_OK = True
except Exception as _e: # pragma: no cover - import guard
keyboard = None
_PYNPUT_OK = False
_IMPORT_ERROR = _e
def _build_maps():
named = {
"space": keyboard.Key.space, "enter": keyboard.Key.enter,
"tab": keyboard.Key.tab, "esc": keyboard.Key.esc,
"alt": keyboard.Key.alt, "alt_l": keyboard.Key.alt_l, "alt_r": keyboard.Key.alt_r,
"ctrl": keyboard.Key.ctrl, "ctrl_l": keyboard.Key.ctrl_l, "ctrl_r": keyboard.Key.ctrl_r,
"shift": keyboard.Key.shift, "shift_l": keyboard.Key.shift_l, "shift_r": keyboard.Key.shift_r,
"cmd": keyboard.Key.cmd, "cmd_l": keyboard.Key.cmd_l, "cmd_r": keyboard.Key.cmd_r,
}
for i in range(1, 21):
fk = getattr(keyboard.Key, f"f{i}", None)
if fk is not None:
named[f"f{i}"] = fk
# collapse left/right modifier variants to a single token for chord matching
canon = {}
for k in ("ctrl", "ctrl_l", "ctrl_r"):
canon[named[k]] = "ctrl"
for k in ("shift", "shift_l", "shift_r"):
canon[named[k]] = "shift"
for k in ("alt", "alt_l", "alt_r"):
canon[named[k]] = "alt"
for k in ("cmd", "cmd_l", "cmd_r"):
canon[named[k]] = "cmd"
return named, canon
class HotkeyManager:
def __init__(self, on_arm=None, on_disarm=None, on_capture=None,
ptt_key="alt_r", capture_key="ctrl+shift+space",
on_toggle=None, toggle_key=""):
self.on_arm = on_arm
self.on_disarm = on_disarm
self.on_capture = on_capture
self.on_toggle = on_toggle
self.ptt_key_name = (ptt_key or "alt_r").strip().lower()
self.capture_key_name = (capture_key or "").strip().lower()
self.toggle_key_name = (toggle_key or "").strip().lower()
self._listener = None
self._ptt_key = None
self._capture_tokens = set()
self._toggle_tokens = set()
self._named = {}
self._canon = {}
self._down = set() # canonical tokens currently held
self._armed = False # ptt key currently down
self._capture_fired = False
self._toggle_fired = False
# ------------------------------------------------------------------
def _resolve(self, name):
name = name.strip().lower()
if name in self._named:
return self._named[name]
if len(name) == 1:
return keyboard.KeyCode.from_char(name)
return None
def _canon_of(self, key):
if key in self._canon:
return self._canon[key]
if isinstance(key, keyboard.Key):
return key.name
if isinstance(key, keyboard.KeyCode) and key.char:
return key.char.lower()
return None
def _safe(self, cb):
if cb is None:
return
try:
cb()
except Exception as e:
print(f"⚠️ Hotkey callback error: {e}")
# ------------------------------------------------------------------
def start(self):
if not _PYNPUT_OK:
print(f"⚠️ Hotkeys unavailable (pynput import failed: {_IMPORT_ERROR}).")
print(" Install with: pip install pynput")
return False
self._named, self._canon = _build_maps()
self._ptt_key = self._resolve(self.ptt_key_name)
self._capture_tokens = {
t.strip() for t in self.capture_key_name.split("+") if t.strip()
}
self._toggle_tokens = {
t.strip() for t in self.toggle_key_name.split("+") if t.strip()
}
self._listener = keyboard.Listener(
on_press=self._on_press, on_release=self._on_release
)
self._listener.daemon = True
self._listener.start()
print(f"⌨️ Push-to-talk: hold [{self.ptt_key_name}] to capture audio.")
if self._capture_tokens:
print(f"⌨️ Screen grab: press [{self.capture_key_name}] then drag a box.")
if self._toggle_tokens:
print(f"⌨️ Show/hide overlay: press [{self.toggle_key_name}].")
print(" (macOS: grant Accessibility permission if keys don't respond.)")
return True
def stop(self):
if self._listener:
self._listener.stop()
self._listener = None
# ------------------------------------------------------------------
def _on_press(self, key):
token = self._canon_of(key)
if token:
self._down.add(token)
if self._ptt_key is not None and key == self._ptt_key and not self._armed:
self._armed = True
self._safe(self.on_arm)
if (self._capture_tokens and not self._capture_fired
and self._capture_tokens.issubset(self._down)):
self._capture_fired = True
self._safe(self.on_capture)
if (self._toggle_tokens and not self._toggle_fired
and self._toggle_tokens.issubset(self._down)):
self._toggle_fired = True
self._safe(self.on_toggle)
def _on_release(self, key):
if self._ptt_key is not None and key == self._ptt_key and self._armed:
self._armed = False
self._safe(self.on_disarm)
token = self._canon_of(key)
if token:
self._down.discard(token)
if self._capture_tokens and not self._capture_tokens.issubset(self._down):
self._capture_fired = False
if self._toggle_tokens and not self._toggle_tokens.issubset(self._down):
self._toggle_fired = False

35
src/overlay.py
View File

@@ -14,9 +14,11 @@ from PyQt5.QtGui import QFont, QTextCursor, QColor
class _Bridge(QObject): class _Bridge(QObject):
"""Signal bridge so audio/AI threads can safely update the Qt UI.""" """Signal bridge so audio/AI threads can safely update the Qt UI."""
answer_ready = pyqtSignal(str, str) # answer, question answer_ready = pyqtSignal(str, str, bool) # answer, question, suggested
status_ready = pyqtSignal(str) status_ready = pyqtSignal(str)
hide_now = pyqtSignal() hide_now = pyqtSignal()
capture_request = pyqtSignal() # trigger draw-a-box screen grab
toggle_request = pyqtSignal() # show/hide the overlay window
class InvisibleOverlay: class InvisibleOverlay:
@@ -29,6 +31,7 @@ class InvisibleOverlay:
self.header_label = None self.header_label = None
self.hide_timer = None self.hide_timer = None
self._bridge = None self._bridge = None
self.on_capture = None # main.py sets the screen-capture handler
# ------------------------------------------------------------------ # ------------------------------------------------------------------
# Startup # Startup
@@ -44,6 +47,8 @@ class InvisibleOverlay:
self._bridge.answer_ready.connect(self._on_answer) self._bridge.answer_ready.connect(self._on_answer)
self._bridge.status_ready.connect(self._on_status) self._bridge.status_ready.connect(self._on_status)
self._bridge.hide_now.connect(self._auto_hide) self._bridge.hide_now.connect(self._auto_hide)
self._bridge.capture_request.connect(self._on_capture_request)
self._bridge.toggle_request.connect(self.toggle_visibility)
self.hide_timer = QTimer() self.hide_timer = QTimer()
self.hide_timer.setSingleShot(True) self.hide_timer.setSingleShot(True)
@@ -127,14 +132,24 @@ class InvisibleOverlay:
# Public API — safe to call from any thread # Public API — safe to call from any thread
# ------------------------------------------------------------------ # ------------------------------------------------------------------
def show_answer(self, answer, question=None): def show_answer(self, answer, question=None, suggested=False):
if self._bridge: if self._bridge:
self._bridge.answer_ready.emit(answer or "", question or "") self._bridge.answer_ready.emit(answer or "", question or "", bool(suggested))
def show_status(self, status): def show_status(self, status):
if self._bridge: if self._bridge:
self._bridge.status_ready.emit(status or "") self._bridge.status_ready.emit(status or "")
def request_capture(self):
"""Thread-safe: ask the Qt main thread to start the draw-a-box capture."""
if self._bridge:
self._bridge.capture_request.emit()
def request_toggle(self):
"""Thread-safe: ask the Qt main thread to show/hide the overlay."""
if self._bridge:
self._bridge.toggle_request.emit()
def toggle_visibility(self): def toggle_visibility(self):
if self.window: if self.window:
if self.window.isVisible(): if self.window.isVisible():
@@ -152,12 +167,13 @@ class InvisibleOverlay:
# Private slots — always run on the main Qt thread # Private slots — always run on the main Qt thread
# ------------------------------------------------------------------ # ------------------------------------------------------------------
def _on_answer(self, answer, question): def _on_answer(self, answer, question, suggested):
lines = [] lines = []
if question: if question:
lines.append(f"Q: {question}") lines.append(f"Q: {question}")
lines.append("") lines.append("")
lines.append(f"A: {answer}") label = "💬 Suggested reply" if suggested else "A"
lines.append(f"{label}: {answer}")
self.text_widget.setPlainText("\n".join(lines)) self.text_widget.setPlainText("\n".join(lines))
cursor = self.text_widget.textCursor() cursor = self.text_widget.textCursor()
@@ -166,13 +182,20 @@ class InvisibleOverlay:
self.window.show() self.window.show()
self.window.raise_() self.window.raise_()
self.hide_timer.start(12000) # Keep the answer up until the next answer replaces it (or it's hidden
# via the toggle hotkey) — don't auto-hide while you're still reading.
self.hide_timer.stop()
def _on_status(self, status): def _on_status(self, status):
self.text_widget.setPlainText(status) self.text_widget.setPlainText(status)
self.window.show() self.window.show()
self.hide_timer.start(4000) self.hide_timer.start(4000)
def _on_capture_request(self):
"""Runs on the Qt main thread — hand off to the registered handler."""
if self.on_capture:
self.on_capture()
def _auto_hide(self): def _auto_hide(self):
if self.window: if self.window:
self.window.hide() self.window.hide()

181
src/region_capture.py Normal file
View File

@@ -0,0 +1,181 @@
"""Draw-a-box screen capture.
A fullscreen translucent overlay lets the user drag a rectangle over an on-screen
question (e.g. a shared slide). The selected region is grabbed with ``mss`` and
written to a temp PNG, which the vision model then reads and answers.
Must be created and shown on the **Qt main thread** (QWidget rule). The pynput
hotkey runs off-thread, so route the trigger through a Qt signal first (see
``main.py`` / the overlay bridge).
"""
import os
import platform
import tempfile
from PyQt5.QtWidgets import QWidget, QRubberBand, QApplication
from PyQt5.QtCore import Qt, QRect, QSize
def capture_fullscreen():
"""Grab the whole primary screen *right now* and return (PIL.Image, (w, h)).
Captured at the OS-compositor level via ``mss``, so it bypasses page-level
copy/right-click/screenshot blocking (common on quiz/exam pages). Grab this
the instant the hotkey fires — before the selector can steal focus — so a
page that blanks its content on blur is captured while still visible.
(True DRM/HDCP-protected video still renders black; that's unfixable here.)
"""
import mss
from PIL import Image
with mss.mss() as sct:
mon = sct.monitors[1] # primary physical monitor
raw = sct.grab(mon)
img = Image.frombytes("RGB", raw.size, raw.bgra, "raw", "BGRX")
return img, (raw.width, raw.height)
def crop_region(frame, phys_size, global_rect, screen_geom, max_side=1600):
"""Crop a pre-captured full-screen ``frame`` to the selected region → PNG path.
Scales logical → physical pixels by comparing the captured size against the
Qt screen size, so it is correct on Retina/HiDPI without hardcoding DPR.
"""
from PIL import Image
raw_w, raw_h = phys_size
sx = raw_w / max(screen_geom.width(), 1)
sy = raw_h / max(screen_geom.height(), 1)
left = max(0, int((global_rect.x() - screen_geom.x()) * sx))
top = max(0, int((global_rect.y() - screen_geom.y()) * sy))
right = min(raw_w, int((global_rect.x() + global_rect.width() - screen_geom.x()) * sx))
bottom = min(raw_h, int((global_rect.y() + global_rect.height() - screen_geom.y()) * sy))
if right <= left or bottom <= top:
raise ValueError("empty crop region")
img = frame.crop((left, top, right, bottom))
# Keep text legible for the vision model. Small selections are upscaled so
# dense exam/quiz text reads cleanly; huge ones are capped to limit tokens.
longest = max(img.width, img.height)
if longest > max_side:
scale = max_side / longest
elif longest < 700: # tiny crop → upscale so text is readable
scale = min(700 / max(longest, 1), 3.0)
else:
scale = 1.0
if scale != 1.0:
img = img.resize((max(1, int(img.width * scale)),
max(1, int(img.height * scale))), Image.LANCZOS)
fd, path = tempfile.mkstemp(prefix="meeting_q_", suffix=".png")
os.close(fd)
img.save(path)
return path
def grab_region(global_rect, screen_geom, max_side=1600):
"""Live grab + crop of a region (fallback when freeze-frame is unavailable)."""
frame, phys = capture_fullscreen()
return crop_region(frame, phys, global_rect, screen_geom, max_side)
def _exclude_from_capture(widget):
"""macOS: set NSWindowSharingNone so the selector is invisible to Zoom/
screen-record/screen-share — others never see the dim veil or the box."""
if platform.system() != "Darwin":
return
try:
import objc
from ctypes import c_void_p
nsview = objc.objc_object(c_void_p=int(widget.winId()))
nsview.window().setSharingType_(0) # NSWindowSharingNone
except Exception as e:
print(f"⚠️ Selector capture-exclusion unavailable: {e}")
class RegionSelector(QWidget):
"""Fullscreen dim overlay with a rubber-band selection. Calls ``on_done(path)``
with the captured PNG path, or ``None`` if cancelled/too small."""
def __init__(self, on_done):
super().__init__()
self.on_done = on_done
self._origin = None
self._screen = QApplication.primaryScreen()
geo = self._screen.geometry()
# Freeze-frame FIRST, while the page under the cursor still has focus and
# its content is visible — before this window appears and can steal it.
try:
self._frame, self._phys = capture_fullscreen()
except Exception as ex:
print(f"⚠️ Could not capture screen frame: {ex}")
self._frame, self._phys = None, (geo.width(), geo.height())
self.setWindowFlags(
Qt.FramelessWindowHint | Qt.WindowStaysOnTopHint | Qt.Tool
)
# Don't activate / pull keyboard focus on show, so a focus-sensitive
# page (proctored exam, etc.) doesn't blank or flag on appearance.
self.setAttribute(Qt.WA_ShowWithoutActivating, True)
self.setGeometry(geo)
self.setWindowOpacity(0.25) # dim veil over the screen
self.setStyleSheet("background-color: #000000;")
self.setCursor(Qt.CrossCursor)
self._rubber = QRubberBand(QRubberBand.Rectangle, self)
def show_selector(self):
self.show()
self.raise_()
# NB: no activateWindow() — keep focus on the page being captured.
_exclude_from_capture(self) # invisible to screen share
# --- mouse / key handling ---------------------------------------
def keyPressEvent(self, e):
if e.key() == Qt.Key_Escape:
self._finish(None)
def mousePressEvent(self, e):
self._origin = e.pos()
self._rubber.setGeometry(QRect(self._origin, QSize()))
self._rubber.show()
def mouseMoveEvent(self, e):
if self._origin is not None:
self._rubber.setGeometry(QRect(self._origin, e.pos()).normalized())
def mouseReleaseEvent(self, e):
if self._origin is None:
return self._finish(None)
rect_local = QRect(self._origin, e.pos()).normalized()
self._rubber.hide()
top_left = self.mapToGlobal(rect_local.topLeft())
grect = QRect(top_left, rect_local.size())
if grect.width() < 5 or grect.height() < 5:
return self._finish(None)
self.hide()
# Crop from the frame captured at open time — no fresh grab, so the
# result is immune to any content that blanked after we took focus.
path = None
try:
if self._frame is not None:
path = crop_region(self._frame, self._phys, grect,
self._screen.geometry())
else:
path = grab_region(grect, self._screen.geometry())
except Exception as ex:
print(f"⚠️ Screen crop failed: {ex}")
self._finish(path)
def _finish(self, path):
try:
self.close()
except Exception:
pass
cb, self.on_done = self.on_done, None
if cb:
cb(path)