Improve question detection, capture stealth, latency; stop tracking model binaries
- Detection: broaden request-starters, handle contractions, allow 1-word '?' - Audio: PTT pre-roll (no clipped first word), drop Whisper silence hallucinations, greedy decoding + domain initial_prompt for faster/cleaner transcription - AI: cap spoken-answer tokens so replies return at conversational speed - Overlay: answers persist (no auto-hide); wire Ctrl+Shift+H show/hide toggle - Screen capture: freeze-frame at hotkey press (immune to focus-blur lockouts), hide selector from screen-share (NSWindowSharingNone), higher capture resolution - Stop tracking models/ and *.zip (large binaries; add to .gitignore) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Charles Wambua
264
main.py
264
main.py
@@ -17,18 +17,49 @@ from src.audio_listener import AudioListener
|
||||
from src.ai_engine import AIEngine
|
||||
from src.overlay import InvisibleOverlay
|
||||
from src.context_manager import ContextManager
|
||||
from src.hotkeys import HotkeyManager
|
||||
from src.region_capture import RegionSelector
|
||||
from src.utils import setup_logging, get_platform
|
||||
|
||||
# Question words that indicate the speaker wants an answer
|
||||
_QUESTION_WORDS = (
|
||||
"what", "why", "how", "when", "where", "who", "which",
|
||||
"can you", "could you", "would you", "do you", "did you",
|
||||
"is there", "are there", "is it", "are you",
|
||||
"tell me", "explain", "describe", "define",
|
||||
"your name", "help me",
|
||||
import re
|
||||
|
||||
# Interrogatives — a sentence starting with one of these is almost always a question
|
||||
_INTERROGATIVES = (
|
||||
"what", "why", "how", "when", "where", "who", "whom", "whose", "which",
|
||||
)
|
||||
|
||||
# Phrases to skip even if they contain question words (noise)
|
||||
# Auxiliary/modal verbs that, when fronted, signal a yes/no question
|
||||
_AUX_FRONT = (
|
||||
"is", "are", "am", "was", "were", "do", "does", "did", "can", "could",
|
||||
"would", "will", "shall", "should", "may", "might", "has", "have", "had",
|
||||
)
|
||||
|
||||
# Imperative/request starters that expect an answer
|
||||
_REQUEST_STARTERS = (
|
||||
"tell me", "tell us", "explain", "describe", "define", "show me", "give me",
|
||||
"give us", "give an overview", "give me an overview", "overview of",
|
||||
"walk me through", "walk us through", "run me through", "run through",
|
||||
"take me through", "talk me through", "go through", "go over", "break down",
|
||||
"break it down", "lay out", "list", "summarize", "summarise", "compare",
|
||||
"contrast", "outline", "clarify", "elaborate", "help me", "remind me",
|
||||
)
|
||||
|
||||
# Embedded / plain-sentence question markers ("I was wondering what the timeline is")
|
||||
_EMBEDDED_MARKERS = (
|
||||
"i was wondering", "i wonder", "i'd like to know", "i would like to know",
|
||||
"do you know", "any idea", "any thoughts", "curious", "wondering if",
|
||||
"wondering what", "wondering how", "wondering whether", "let me know",
|
||||
"your thoughts on", "what's your take", "thoughts on", "would love to know",
|
||||
"not sure", "can you clarify", "question for you", "question is",
|
||||
)
|
||||
|
||||
# Fact-shaped lookups phrased as fragments ("capital of France", "python decorators")
|
||||
_FACT_PATTERNS = (
|
||||
"capital of", "definition of", "meaning of", "difference between",
|
||||
"how many", "what year", "abbreviation for", "acronym for",
|
||||
)
|
||||
|
||||
# Phrases to skip even if they contain question words (noise / code / UI text)
|
||||
_NOISE_PHRASES = (
|
||||
"timer cannot", "qobject", "qml", "pyside", "pyqt",
|
||||
"def ", "class ", "import ", "return ", "self.",
|
||||
@@ -57,83 +88,141 @@ class MeetingAssistant:
|
||||
# Audio listener with callback
|
||||
self.audio_listener = AudioListener(self.config, self.on_audio_transcript)
|
||||
|
||||
# Which speaker sources should trigger an answer
|
||||
self.answer_sources = set(
|
||||
self.config.get("audio", {}).get("answer_sources", ["system", "microphone"])
|
||||
)
|
||||
|
||||
# Global hotkeys: hold push-to-talk to capture audio, chord to grab screen
|
||||
audio_cfg = self.config.get("audio", {})
|
||||
screen_cfg = self.config.get("screen", {})
|
||||
hotkeys_cfg = self.config.get("hotkeys", {})
|
||||
self.hotkeys = HotkeyManager(
|
||||
on_arm=self.audio_listener.arm,
|
||||
on_disarm=self.audio_listener.disarm,
|
||||
on_capture=self.overlay.request_capture, # marshals onto Qt thread
|
||||
on_toggle=self.overlay.request_toggle, # marshals onto Qt thread
|
||||
ptt_key=audio_cfg.get("ptt_key", "alt_r"),
|
||||
capture_key=screen_cfg.get("capture_key", "ctrl+shift+space"),
|
||||
toggle_key=hotkeys_cfg.get("toggle_overlay", "ctrl+shift+h"),
|
||||
)
|
||||
# Screen-capture handler runs on the Qt main thread (via the overlay bridge)
|
||||
self.overlay.on_capture = self._handle_screen_capture
|
||||
self._selector = None # keep a ref so the selector isn't GC'd mid-use
|
||||
|
||||
# State management
|
||||
self.current_answer = None
|
||||
self.answering = False
|
||||
self.answer_lock = threading.Lock()
|
||||
self._last_question = ""
|
||||
|
||||
ptt = audio_cfg.get("ptt_key", "alt_r")
|
||||
cap = screen_cfg.get("capture_key", "ctrl+shift+space")
|
||||
mode = audio_cfg.get("capture_mode", "push_to_talk")
|
||||
print("✅ Meeting Assistant Ready!")
|
||||
print("==================================================")
|
||||
print("🎤 Listening for questions via microphone")
|
||||
if mode == "push_to_talk":
|
||||
print(f"🎙️ Hold [{ptt}] to capture audio, release to answer")
|
||||
else:
|
||||
print("🎤 Listening continuously for questions")
|
||||
print(f"📸 Press [{cap}] then drag a box over an on-screen question")
|
||||
print(" Answers appear in the overlay (top-right)")
|
||||
print(" Type a question here + Enter to test AI")
|
||||
print(" Ctrl+C to stop")
|
||||
print("==================================================")
|
||||
|
||||
def on_audio_transcript(self, text, timestamp):
|
||||
"""Audio callback — runs in the audio listener thread"""
|
||||
def on_audio_transcript(self, text, timestamp, source="microphone", speaker=None):
|
||||
"""Audio callback — runs in the transcription thread"""
|
||||
if not text or len(text.strip()) < 2:
|
||||
return
|
||||
|
||||
self.context_manager.add_audio_context(text, timestamp)
|
||||
speaker = speaker or ("You" if source == "microphone" else "Them")
|
||||
self.context_manager.add_audio_context(text, timestamp, source, speaker)
|
||||
|
||||
if source not in self.answer_sources:
|
||||
print(f" Context [{speaker}] (not an answer source): {text}")
|
||||
return
|
||||
|
||||
if self._is_question(text):
|
||||
print(f"\n❓ Question detected: {text}")
|
||||
self._generate_answer(text)
|
||||
print(f"\n❓ Question from {speaker}: {text}")
|
||||
self._generate_answer(text, source, speaker)
|
||||
else:
|
||||
print(f" Context (not a question): {text}")
|
||||
print(f" Context [{speaker}] (not a question): {text}")
|
||||
|
||||
def _is_question(self, text):
|
||||
"""Enhanced question detection for faster responses"""
|
||||
if len(text.split()) < 2: # Reduced from 3 for shorter questions
|
||||
return False
|
||||
"""Detect questions, including ones phrased as plain statements.
|
||||
|
||||
Combines surface cues ( '?'), interrogatives, fronted auxiliaries,
|
||||
requests/imperatives, embedded markers ('I was wondering...'),
|
||||
tag questions ('..., right?'), math, and fact-shaped fragments.
|
||||
"""
|
||||
text_lower = text.lower().strip()
|
||||
if not text_lower:
|
||||
return False
|
||||
|
||||
# Skip obvious noise / code text
|
||||
if any(p in text_lower for p in _NOISE_PHRASES):
|
||||
return False
|
||||
|
||||
# Ends with "?" → always a question
|
||||
# Explicit question mark → always a question (even one word: "Why?")
|
||||
if text.rstrip().endswith("?"):
|
||||
return True
|
||||
|
||||
# Check for math expressions (2+2, two plus two)
|
||||
import re
|
||||
math_patterns = [
|
||||
r'\d+\s*[\+\-\*\/]\s*\d+', # 2+2, 5-3
|
||||
r'\d+\s*plus\s*\d+', # 2 plus 2
|
||||
r'\d+\s*minus\s*\d+', # 5 minus 3
|
||||
r'\d+\s*times\s*\d+', # 4 times 5
|
||||
r'\d+\s*divided by\s*\d+', # 10 divided by 2
|
||||
]
|
||||
for pattern in math_patterns:
|
||||
if re.search(pattern, text_lower):
|
||||
return True
|
||||
|
||||
# Check for short implicit questions (2-5 words without filler)
|
||||
words = text_lower.split()
|
||||
if 2 <= len(words) <= 5:
|
||||
filler = {'um', 'uh', 'like', 'so', 'well', 'actually', 'basically'}
|
||||
content_words = [w for w in words if w not in filler]
|
||||
if len(content_words) >= 2:
|
||||
# "two plus two", "capital france", "python example"
|
||||
if len(words) < 2:
|
||||
return False
|
||||
|
||||
stripped = text_lower.rstrip(".!? ")
|
||||
|
||||
# Tag questions: "..., right?", "..., correct?", "isn't it"
|
||||
if re.search(r"\b(right|correct|yeah|okay|ok|no)\s*$", stripped) and len(words) >= 4:
|
||||
if re.search(r",\s*(right|correct|okay|ok)\b", text_lower):
|
||||
return True
|
||||
|
||||
# Starts with or contains a question word
|
||||
if any(text_lower.startswith(w) or f" {w} " in text_lower for w in _QUESTION_WORDS):
|
||||
# Math expressions ("2+2", "5 plus 3", "10 divided by 2")
|
||||
math_patterns = (
|
||||
r"\d+\s*[\+\-\*\/]\s*\d+",
|
||||
r"\d+\s*(plus|minus|times|divided by|multiplied by)\s*\d+",
|
||||
)
|
||||
if any(re.search(p, text_lower) for p in math_patterns):
|
||||
return True
|
||||
|
||||
# Check for command starters
|
||||
command_starters = ['tell me', 'explain', 'describe', 'show me', 'give me', 'find', 'search']
|
||||
for starter in command_starters:
|
||||
if text_lower.startswith(starter):
|
||||
return True
|
||||
first = words[0]
|
||||
# normalize contractions so "what's"/"who's"/"isn't"/"don't" still match
|
||||
first_base = re.sub(r"(n't|'s|'re|'ll|'d|'ve|'m)$", "", first)
|
||||
|
||||
# Starts with an interrogative ("what is the deadline", "what's the plan")
|
||||
if first in _INTERROGATIVES or first_base in _INTERROGATIVES:
|
||||
return True
|
||||
|
||||
# Fronted auxiliary/modal forming a yes/no question ("are we shipping friday",
|
||||
# "isn't that due friday", "don't we need sign-off")
|
||||
if first in _AUX_FRONT or first_base in _AUX_FRONT:
|
||||
return True
|
||||
|
||||
# Request / imperative starters ("explain the rollout plan")
|
||||
if any(stripped.startswith(s) for s in _REQUEST_STARTERS):
|
||||
return True
|
||||
|
||||
# Embedded / plain-sentence questions ("I was wondering about the budget")
|
||||
if any(m in text_lower for m in _EMBEDDED_MARKERS):
|
||||
return True
|
||||
|
||||
# Interrogative appearing after a lead-in ("so what about the budget")
|
||||
if any(f" {w} " in f" {text_lower} " for w in _INTERROGATIVES):
|
||||
# avoid matching relative clauses like "the plan that we have" → require
|
||||
# the interrogative within the first few words
|
||||
for w in _INTERROGATIVES:
|
||||
if w in words[:4]:
|
||||
return True
|
||||
|
||||
# Fact-shaped fragments / lookups ("difference between TCP and UDP")
|
||||
if any(p in text_lower for p in _FACT_PATTERNS):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def _generate_answer(self, question):
|
||||
def _generate_answer(self, question, source="microphone", speaker=None):
|
||||
"""Generate answer with interruption support"""
|
||||
# Debounce: skip exact repeat within same run
|
||||
if question.lower() == self._last_question.lower():
|
||||
@@ -148,23 +237,31 @@ class MeetingAssistant:
|
||||
self.ai_engine.interrupt()
|
||||
|
||||
# Start answer generation in background thread
|
||||
threading.Thread(target=self._answer_worker, args=(question,), daemon=True).start()
|
||||
threading.Thread(target=self._answer_worker,
|
||||
args=(question, source, speaker), daemon=True).start()
|
||||
|
||||
def _answer_worker(self, question):
|
||||
def _answer_worker(self, question, source="microphone", speaker=None):
|
||||
"""Worker thread for answer generation"""
|
||||
with self.answer_lock:
|
||||
self.answering = True
|
||||
|
||||
try:
|
||||
# Get context and generate answer
|
||||
context = self.context_manager.get_context()
|
||||
answer = self.ai_engine.answer_question(question, context)
|
||||
result = self.ai_engine.answer_question(question, context, source)
|
||||
|
||||
# Show answer in overlay
|
||||
self.overlay.show_answer(answer, question)
|
||||
self.logger.info(f"Q: {question}")
|
||||
if not result or not result.get("text"):
|
||||
return # interrupted or empty
|
||||
|
||||
answer = result["text"]
|
||||
suggested = result.get("suggested", False)
|
||||
|
||||
asked_by = speaker or ("You" if source == "microphone" else "Them")
|
||||
shown_q = f"({asked_by}) {question}"
|
||||
self.overlay.show_answer(answer, shown_q, suggested)
|
||||
self.logger.info(f"Q [{asked_by}]: {question}")
|
||||
self.logger.info(f"A: {answer}")
|
||||
print(f"\n💡 Answer: {answer}\n")
|
||||
label = "💬 Suggested" if suggested else "💡 Answer"
|
||||
print(f"\n{label}: {answer}\n")
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Error generating answer: {e}")
|
||||
@@ -172,6 +269,51 @@ class MeetingAssistant:
|
||||
with self.answer_lock:
|
||||
self.answering = False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Screen questions — draw a box, read it, answer it
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _handle_screen_capture(self):
|
||||
"""Runs on the Qt main thread. Show the draw-a-box selector."""
|
||||
if self._selector is not None:
|
||||
return # a selection is already in progress
|
||||
print("\n📸 Draw a box around the question...")
|
||||
self._selector = RegionSelector(on_done=self._on_region_captured)
|
||||
self._selector.show_selector()
|
||||
|
||||
def _on_region_captured(self, image_path):
|
||||
"""Called when the user finishes (or cancels) the selection."""
|
||||
self._selector = None
|
||||
if not image_path:
|
||||
print(" Screen capture cancelled.")
|
||||
return
|
||||
self.overlay.show_status("📸 Reading the question on screen...")
|
||||
threading.Thread(target=self._screen_answer_worker,
|
||||
args=(image_path,), daemon=True).start()
|
||||
|
||||
def _screen_answer_worker(self, image_path):
|
||||
with self.answer_lock:
|
||||
if self.answering and hasattr(self.ai_engine, "interrupt"):
|
||||
self.ai_engine.interrupt()
|
||||
self.answering = True
|
||||
try:
|
||||
answer = self.ai_engine.answer_from_image(image_path)
|
||||
if not answer:
|
||||
return
|
||||
self.overlay.show_answer(answer, "Screen question", suggested=False)
|
||||
self.logger.info(f"Q [screen]: {image_path}")
|
||||
self.logger.info(f"A: {answer}")
|
||||
print(f"\n💡 Screen answer: {answer}\n")
|
||||
except Exception as e:
|
||||
print(f"❌ Error answering screen question: {e}")
|
||||
finally:
|
||||
with self.answer_lock:
|
||||
self.answering = False
|
||||
try:
|
||||
os.remove(image_path)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Manual terminal input for testing
|
||||
# ------------------------------------------------------------------
|
||||
@@ -195,6 +337,7 @@ class MeetingAssistant:
|
||||
print("\n🎤 Starting audio listener...")
|
||||
self.overlay.start()
|
||||
self.audio_listener.start()
|
||||
self.hotkeys.start()
|
||||
|
||||
# Terminal input runs in a background daemon thread
|
||||
t = threading.Thread(target=self._terminal_input_loop, daemon=True)
|
||||
@@ -202,18 +345,25 @@ class MeetingAssistant:
|
||||
|
||||
try:
|
||||
if self.overlay.app:
|
||||
sys.exit(self.overlay.app.exec_())
|
||||
self.overlay.app.exec_()
|
||||
except KeyboardInterrupt:
|
||||
self.shutdown()
|
||||
pass
|
||||
self.shutdown()
|
||||
|
||||
def shutdown(self):
|
||||
print("\n🛑 Shutting down...")
|
||||
if hasattr(self, 'hotkeys'):
|
||||
self.hotkeys.stop()
|
||||
if hasattr(self, 'audio_listener'):
|
||||
self.audio_listener.stop()
|
||||
if hasattr(self, 'overlay'):
|
||||
self.overlay.stop()
|
||||
print("👋 Goodbye!")
|
||||
sys.exit(0)
|
||||
# Hard-exit: skips Python finalizers so llama.cpp's buggy Metal teardown
|
||||
# (GGML_ASSERT in ggml_metal_device_free at __cxa_finalize) can't crash us.
|
||||
logging.shutdown()
|
||||
sys.stdout.flush()
|
||||
os._exit(0)
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
Reference in New Issue
Block a user