425 lines
17 KiB
Python
425 lines
17 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Meeting Assistant - Real-time AI Copilot for Meetings
|
|
"""
|
|
|
|
import sys
|
|
import os
|
|
import yaml
|
|
import threading
|
|
import logging
|
|
from pathlib import Path
|
|
from datetime import datetime
|
|
|
|
sys.path.insert(0, str(Path(__file__).parent))
|
|
|
|
from src.audio_listener import AudioListener
|
|
from src.ai_engine import AIEngine
|
|
from src.overlay import InvisibleOverlay
|
|
from src.context_manager import ContextManager
|
|
from src.hotkeys import HotkeyManager
|
|
from src.region_capture import RegionSelector
|
|
from src.utils import setup_logging, get_platform
|
|
|
|
import re
|
|
|
|
# Interrogatives — a sentence starting with one of these is almost always a question
|
|
_INTERROGATIVES = (
|
|
"what", "why", "how", "when", "where", "who", "whom", "whose", "which",
|
|
)
|
|
|
|
# Auxiliary/modal verbs that, when fronted, signal a yes/no question
|
|
_AUX_FRONT = (
|
|
"is", "are", "am", "was", "were", "do", "does", "did", "can", "could",
|
|
"would", "will", "shall", "should", "may", "might", "has", "have", "had",
|
|
)
|
|
|
|
# Imperative/request starters that expect an answer
|
|
_REQUEST_STARTERS = (
|
|
"tell me", "tell us", "explain", "describe", "define", "show me", "give me",
|
|
"give us", "give an overview", "give me an overview", "overview of",
|
|
"walk me through", "walk us through", "run me through", "run through",
|
|
"take me through", "talk me through", "go through", "go over", "break down",
|
|
"break it down", "lay out", "list", "summarize", "summarise", "compare",
|
|
"contrast", "outline", "clarify", "elaborate", "help me", "remind me",
|
|
)
|
|
|
|
# Embedded / plain-sentence question markers ("I was wondering what the timeline is")
|
|
_EMBEDDED_MARKERS = (
|
|
"i was wondering", "i wonder", "i'd like to know", "i would like to know",
|
|
"do you know", "any idea", "any thoughts", "curious", "wondering if",
|
|
"wondering what", "wondering how", "wondering whether", "let me know",
|
|
"your thoughts on", "what's your take", "thoughts on", "would love to know",
|
|
"not sure", "can you clarify", "question for you", "question is",
|
|
)
|
|
|
|
# Fact-shaped lookups phrased as fragments ("capital of France", "python decorators")
|
|
_FACT_PATTERNS = (
|
|
"capital of", "definition of", "meaning of", "difference between",
|
|
"how many", "what year", "abbreviation for", "acronym for",
|
|
)
|
|
|
|
# Phrases to skip even if they contain question words (noise / code / UI text)
|
|
_NOISE_PHRASES = (
|
|
"timer cannot", "qobject", "qml", "pyside", "pyqt",
|
|
"def ", "class ", "import ", "return ", "self.",
|
|
)
|
|
|
|
|
|
class MeetingAssistant:
|
|
def __init__(self):
|
|
"""Initialize all components"""
|
|
print("🚀 Starting Meeting Assistant...")
|
|
print(f"💻 Platform: {get_platform()}")
|
|
|
|
# Load config
|
|
config_path = Path(__file__).parent / "config.yaml"
|
|
with open(config_path, 'r') as f:
|
|
self.config = yaml.safe_load(f)
|
|
|
|
# Setup logging
|
|
self.logger = setup_logging()
|
|
|
|
# Initialize components
|
|
self.ai_engine = AIEngine(self.config)
|
|
self.context_manager = ContextManager(self.config)
|
|
self.overlay = InvisibleOverlay(self.config)
|
|
|
|
# Audio listener with callback
|
|
self.audio_listener = AudioListener(self.config, self.on_audio_transcript)
|
|
|
|
# Which speaker sources should trigger an answer
|
|
self.answer_sources = set(
|
|
self.config.get("audio", {}).get("answer_sources", ["system", "microphone"])
|
|
)
|
|
|
|
# Global hotkeys: hold push-to-talk to capture audio, chord to grab screen
|
|
audio_cfg = self.config.get("audio", {})
|
|
screen_cfg = self.config.get("screen", {})
|
|
hotkeys_cfg = self.config.get("hotkeys", {})
|
|
self.hotkeys = HotkeyManager(
|
|
on_arm=self.audio_listener.arm,
|
|
on_disarm=self.audio_listener.disarm,
|
|
on_capture=self.overlay.request_capture, # marshals onto Qt thread
|
|
on_toggle=self.overlay.request_toggle, # marshals onto Qt thread
|
|
on_summary=self._handle_summary, # spawns its own worker thread
|
|
on_listen=self._handle_toggle_listen, # switch auto-listen / push-to-talk
|
|
ptt_key=audio_cfg.get("ptt_key", "alt_r"),
|
|
capture_key=screen_cfg.get("capture_key", "ctrl+shift+space"),
|
|
toggle_key=hotkeys_cfg.get("toggle_overlay", "ctrl+shift+h"),
|
|
summary_key=hotkeys_cfg.get("meeting_summary", "ctrl+shift+s"),
|
|
listen_key=hotkeys_cfg.get("toggle_listening", "ctrl+shift+m"),
|
|
)
|
|
# Screen-capture handler runs on the Qt main thread (via the overlay bridge)
|
|
self.overlay.on_capture = self._handle_screen_capture
|
|
self._selector = None # keep a ref so the selector isn't GC'd mid-use
|
|
|
|
# State management
|
|
self.current_answer = None
|
|
self.answering = False
|
|
self.answer_lock = threading.Lock()
|
|
self._last_question = ""
|
|
|
|
ptt = audio_cfg.get("ptt_key", "alt_r")
|
|
cap = screen_cfg.get("capture_key", "ctrl+shift+space")
|
|
mode = audio_cfg.get("capture_mode", "push_to_talk")
|
|
print("✅ Meeting Assistant Ready!")
|
|
print("==================================================")
|
|
listen_key = hotkeys_cfg.get("toggle_listening", "ctrl+shift+m")
|
|
if mode == "push_to_talk":
|
|
print(f"🎙️ Hold [{ptt}] to capture audio, release to answer")
|
|
else:
|
|
print("🎤 Auto-listening (no key needed)")
|
|
print(f"🔁 Press [{listen_key}] to toggle auto-listen ⇄ push-to-talk")
|
|
print(f"📸 Press [{cap}] then drag a box over an on-screen question")
|
|
summ = hotkeys_cfg.get("meeting_summary", "ctrl+shift+s")
|
|
print(f"📝 Press [{summ}] for a summary of the meeting so far")
|
|
print(" Answers appear in the overlay (top-right)")
|
|
print(" Type a question here + Enter to test AI")
|
|
print(" Ctrl+C to stop")
|
|
print("==================================================")
|
|
|
|
def on_audio_transcript(self, text, timestamp, source="microphone", speaker=None):
|
|
"""Audio callback — runs in the transcription thread"""
|
|
if not text or len(text.strip()) < 2:
|
|
return
|
|
|
|
speaker = speaker or ("You" if source == "microphone" else "Them")
|
|
self.context_manager.add_audio_context(text, timestamp, source, speaker)
|
|
|
|
if source not in self.answer_sources:
|
|
print(f" Context [{speaker}] (not an answer source): {text}")
|
|
return
|
|
|
|
if self._is_question(text):
|
|
print(f"\n❓ Question from {speaker}: {text}")
|
|
self._generate_answer(text, source, speaker)
|
|
else:
|
|
print(f" Context [{speaker}] (not a question): {text}")
|
|
|
|
def _is_question(self, text):
|
|
"""Detect questions, including ones phrased as plain statements.
|
|
|
|
Combines surface cues ( '?'), interrogatives, fronted auxiliaries,
|
|
requests/imperatives, embedded markers ('I was wondering...'),
|
|
tag questions ('..., right?'), math, and fact-shaped fragments.
|
|
"""
|
|
text_lower = text.lower().strip()
|
|
if not text_lower:
|
|
return False
|
|
|
|
# Skip obvious noise / code text
|
|
if any(p in text_lower for p in _NOISE_PHRASES):
|
|
return False
|
|
|
|
# Explicit question mark → always a question (even one word: "Why?")
|
|
if text.rstrip().endswith("?"):
|
|
return True
|
|
|
|
words = text_lower.split()
|
|
if len(words) < 2:
|
|
return False
|
|
|
|
stripped = text_lower.rstrip(".!? ")
|
|
|
|
# Tag questions: "..., right?", "..., correct?", "isn't it"
|
|
if re.search(r"\b(right|correct|yeah|okay|ok|no)\s*$", stripped) and len(words) >= 4:
|
|
if re.search(r",\s*(right|correct|okay|ok)\b", text_lower):
|
|
return True
|
|
|
|
# Math expressions ("2+2", "5 plus 3", "10 divided by 2")
|
|
math_patterns = (
|
|
r"\d+\s*[\+\-\*\/]\s*\d+",
|
|
r"\d+\s*(plus|minus|times|divided by|multiplied by)\s*\d+",
|
|
)
|
|
if any(re.search(p, text_lower) for p in math_patterns):
|
|
return True
|
|
|
|
first = words[0]
|
|
# normalize contractions so "what's"/"who's"/"isn't"/"don't" still match
|
|
first_base = re.sub(r"(n't|'s|'re|'ll|'d|'ve|'m)$", "", first)
|
|
|
|
# Starts with an interrogative ("what is the deadline", "what's the plan")
|
|
if first in _INTERROGATIVES or first_base in _INTERROGATIVES:
|
|
return True
|
|
|
|
# Fronted auxiliary/modal forming a yes/no question ("are we shipping friday",
|
|
# "isn't that due friday", "don't we need sign-off")
|
|
if first in _AUX_FRONT or first_base in _AUX_FRONT:
|
|
return True
|
|
|
|
# Request / imperative starters ("explain the rollout plan")
|
|
if any(stripped.startswith(s) for s in _REQUEST_STARTERS):
|
|
return True
|
|
|
|
# Embedded / plain-sentence questions ("I was wondering about the budget")
|
|
if any(m in text_lower for m in _EMBEDDED_MARKERS):
|
|
return True
|
|
|
|
# Interrogative appearing after a lead-in ("so what about the budget")
|
|
if any(f" {w} " in f" {text_lower} " for w in _INTERROGATIVES):
|
|
# avoid matching relative clauses like "the plan that we have" → require
|
|
# the interrogative within the first few words
|
|
for w in _INTERROGATIVES:
|
|
if w in words[:4]:
|
|
return True
|
|
|
|
# Fact-shaped fragments / lookups ("difference between TCP and UDP")
|
|
if any(p in text_lower for p in _FACT_PATTERNS):
|
|
return True
|
|
|
|
return False
|
|
|
|
def _generate_answer(self, question, source="microphone", speaker=None):
|
|
"""Generate answer with interruption support"""
|
|
# Debounce: skip exact repeat within same run
|
|
if question.lower() == self._last_question.lower():
|
|
return
|
|
self._last_question = question
|
|
|
|
# Check if we should interrupt current answer
|
|
with self.answer_lock:
|
|
if self.answering:
|
|
print("🔄 Interrupting previous answer...")
|
|
if hasattr(self.ai_engine, 'interrupt'):
|
|
self.ai_engine.interrupt()
|
|
|
|
# Start answer generation in background thread
|
|
threading.Thread(target=self._answer_worker,
|
|
args=(question, source, speaker), daemon=True).start()
|
|
|
|
def _answer_worker(self, question, source="microphone", speaker=None):
|
|
"""Worker thread for answer generation"""
|
|
with self.answer_lock:
|
|
self.answering = True
|
|
|
|
try:
|
|
context = self.context_manager.get_context()
|
|
result = self.ai_engine.answer_question(question, context, source)
|
|
|
|
if not result or not result.get("text"):
|
|
return # interrupted or empty
|
|
|
|
answer = result["text"]
|
|
suggested = result.get("suggested", False)
|
|
|
|
asked_by = speaker or ("You" if source == "microphone" else "Them")
|
|
shown_q = f"({asked_by}) {question}"
|
|
self.overlay.show_answer(answer, shown_q, suggested)
|
|
self.logger.info(f"Q [{asked_by}]: {question}")
|
|
self.logger.info(f"A: {answer}")
|
|
label = "💬 Suggested" if suggested else "💡 Answer"
|
|
print(f"\n{label}: {answer}\n")
|
|
|
|
except Exception as e:
|
|
print(f"❌ Error generating answer: {e}")
|
|
finally:
|
|
with self.answer_lock:
|
|
self.answering = False
|
|
|
|
# ------------------------------------------------------------------
|
|
# Screen questions — draw a box, read it, answer it
|
|
# ------------------------------------------------------------------
|
|
|
|
def _handle_screen_capture(self):
|
|
"""Runs on the Qt main thread. Show the draw-a-box selector."""
|
|
if self._selector is not None:
|
|
return # a selection is already in progress
|
|
print("\n📸 Draw a box around the question...")
|
|
self._selector = RegionSelector(on_done=self._on_region_captured)
|
|
self._selector.show_selector()
|
|
|
|
def _on_region_captured(self, image_path):
|
|
"""Called when the user finishes (or cancels) the selection."""
|
|
self._selector = None
|
|
if not image_path:
|
|
print(" Screen capture cancelled.")
|
|
return
|
|
self.overlay.show_status("📸 Reading the question on screen...")
|
|
threading.Thread(target=self._screen_answer_worker,
|
|
args=(image_path,), daemon=True).start()
|
|
|
|
def _screen_answer_worker(self, image_path):
|
|
with self.answer_lock:
|
|
if self.answering and hasattr(self.ai_engine, "interrupt"):
|
|
self.ai_engine.interrupt()
|
|
self.answering = True
|
|
try:
|
|
answer = self.ai_engine.answer_from_image(image_path)
|
|
if not answer:
|
|
return
|
|
self.overlay.show_answer(answer, "Screen question", suggested=False)
|
|
self.logger.info(f"Q [screen]: {image_path}")
|
|
self.logger.info(f"A: {answer}")
|
|
print(f"\n💡 Screen answer: {answer}\n")
|
|
except Exception as e:
|
|
print(f"❌ Error answering screen question: {e}")
|
|
finally:
|
|
with self.answer_lock:
|
|
self.answering = False
|
|
try:
|
|
os.remove(image_path)
|
|
except Exception:
|
|
pass
|
|
|
|
# ------------------------------------------------------------------
|
|
# End-of-meeting summary
|
|
# ------------------------------------------------------------------
|
|
|
|
def _handle_toggle_listen(self):
|
|
"""Hotkey handler: switch between auto-listen and push-to-talk."""
|
|
on = self.audio_listener.toggle_auto_listen()
|
|
if on:
|
|
self.overlay.show_status("🔊 Auto-listen ON — listening on its own")
|
|
else:
|
|
self.overlay.show_status("🎙️ Push-to-talk — hold the key to capture")
|
|
|
|
def _handle_summary(self):
|
|
"""Hotkey handler (pynput thread): summarize the meeting in the background."""
|
|
threading.Thread(target=self._summary_worker, daemon=True).start()
|
|
|
|
def _summary_worker(self):
|
|
transcript = self.context_manager.get_full_transcript()
|
|
if not transcript:
|
|
self.overlay.show_status("No meeting captured yet — nothing to summarize.")
|
|
print(" (No transcript captured yet.)")
|
|
return
|
|
with self.answer_lock:
|
|
if self.answering and hasattr(self.ai_engine, "interrupt"):
|
|
self.ai_engine.interrupt()
|
|
self.answering = True
|
|
try:
|
|
self.overlay.show_status("📝 Summarizing the meeting...")
|
|
print("\n📝 Summarizing the meeting...")
|
|
summary = self.ai_engine.summarize_meeting(transcript)
|
|
if not summary:
|
|
return
|
|
self.overlay.show_answer(summary, "Meeting summary", suggested=False)
|
|
self.logger.info("Meeting summary generated.")
|
|
self.logger.info(f"Summary: {summary}")
|
|
print(f"\n📋 Meeting Summary:\n{summary}\n")
|
|
except Exception as e:
|
|
print(f"❌ Error summarizing meeting: {e}")
|
|
finally:
|
|
with self.answer_lock:
|
|
self.answering = False
|
|
|
|
# ------------------------------------------------------------------
|
|
# Manual terminal input for testing
|
|
# ------------------------------------------------------------------
|
|
|
|
def _terminal_input_loop(self):
|
|
print("\n Type a question and press Enter to test the AI directly.")
|
|
print(" (Leave blank to skip)\n")
|
|
while True:
|
|
try:
|
|
line = input()
|
|
if line.strip():
|
|
self._generate_answer(line.strip())
|
|
except (EOFError, KeyboardInterrupt):
|
|
break
|
|
|
|
# ------------------------------------------------------------------
|
|
# Run / shutdown
|
|
# ------------------------------------------------------------------
|
|
|
|
def run(self):
|
|
print("\n🎤 Starting audio listener...")
|
|
self.overlay.start()
|
|
self.audio_listener.start()
|
|
self.hotkeys.start()
|
|
|
|
# Terminal input runs in a background daemon thread
|
|
t = threading.Thread(target=self._terminal_input_loop, daemon=True)
|
|
t.start()
|
|
|
|
try:
|
|
if self.overlay.app:
|
|
self.overlay.app.exec_()
|
|
except KeyboardInterrupt:
|
|
pass
|
|
self.shutdown()
|
|
|
|
def shutdown(self):
|
|
print("\n🛑 Shutting down...")
|
|
if hasattr(self, 'hotkeys'):
|
|
self.hotkeys.stop()
|
|
if hasattr(self, 'audio_listener'):
|
|
self.audio_listener.stop()
|
|
if hasattr(self, 'overlay'):
|
|
self.overlay.stop()
|
|
print("👋 Goodbye!")
|
|
# Hard-exit: skips Python finalizers so llama.cpp's buggy Metal teardown
|
|
# (GGML_ASSERT in ggml_metal_device_free at __cxa_finalize) can't crash us.
|
|
logging.shutdown()
|
|
sys.stdout.flush()
|
|
os._exit(0)
|
|
|
|
|
|
def main():
|
|
assistant = MeetingAssistant()
|
|
assistant.run()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |