Alpha/InterviewAI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added context match

Browse Source
This commit is contained in:
Charles Wambua
2026-06-26 15:38:09 +03:00
parent dba78a2766
commit 138580475c
6 changed files with 294 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
.gitignore vendored
View File

@@ -39,11 +39,12 @@ config/production.yaml
*.local.yaml
*.secret.yaml
# Model files (large binary files)
models/*.gguf
models/*.bin
models/whisper/model.bin
models/whisper/*.gguf
# Model files (large binaries — downloaded by setup.py / faster-whisper, not tracked)
models/
# Build artifacts / archives
*.zip
meeting_assistant.zip
# Cache directories
.cache/

52
main.py
View File

@@ -102,9 +102,13 @@ class MeetingAssistant:
on_disarm=self.audio_listener.disarm,
on_capture=self.overlay.request_capture, # marshals onto Qt thread
on_toggle=self.overlay.request_toggle, # marshals onto Qt thread
on_summary=self._handle_summary, # spawns its own worker thread
on_listen=self._handle_toggle_listen, # switch auto-listen / push-to-talk
ptt_key=audio_cfg.get("ptt_key", "alt_r"),
capture_key=screen_cfg.get("capture_key", "ctrl+shift+space"),
toggle_key=hotkeys_cfg.get("toggle_overlay", "ctrl+shift+h"),
summary_key=hotkeys_cfg.get("meeting_summary", "ctrl+shift+s"),
listen_key=hotkeys_cfg.get("toggle_listening", "ctrl+shift+m"),
)
# Screen-capture handler runs on the Qt main thread (via the overlay bridge)
self.overlay.on_capture = self._handle_screen_capture
@@ -121,11 +125,15 @@ class MeetingAssistant:
mode = audio_cfg.get("capture_mode", "push_to_talk")
print("✅ Meeting Assistant Ready!")
print("==================================================")
listen_key = hotkeys_cfg.get("toggle_listening", "ctrl+shift+m")
if mode == "push_to_talk":
print(f"🎙️ Hold [{ptt}] to capture audio, release to answer")
else:
print("🎤 Listening continuously for questions")
print("🎤 Auto-listening (no key needed)")
print(f"🔁 Press [{listen_key}] to toggle auto-listen ⇄ push-to-talk")
print(f"📸 Press [{cap}] then drag a box over an on-screen question")
summ = hotkeys_cfg.get("meeting_summary", "ctrl+shift+s")
print(f"📝 Press [{summ}] for a summary of the meeting so far")
print(" Answers appear in the overlay (top-right)")
print(" Type a question here + Enter to test AI")
print(" Ctrl+C to stop")
@@ -314,6 +322,48 @@ class MeetingAssistant:
except Exception:
pass
# ------------------------------------------------------------------
# End-of-meeting summary
# ------------------------------------------------------------------
def _handle_toggle_listen(self):
"""Hotkey handler: switch between auto-listen and push-to-talk."""
on = self.audio_listener.toggle_auto_listen()
if on:
self.overlay.show_status("🔊 Auto-listen ON — listening on its own")
else:
self.overlay.show_status("🎙️ Push-to-talk — hold the key to capture")
def _handle_summary(self):
"""Hotkey handler (pynput thread): summarize the meeting in the background."""
threading.Thread(target=self._summary_worker, daemon=True).start()
def _summary_worker(self):
transcript = self.context_manager.get_full_transcript()
if not transcript:
self.overlay.show_status("No meeting captured yet — nothing to summarize.")
print(" (No transcript captured yet.)")
return
with self.answer_lock:
if self.answering and hasattr(self.ai_engine, "interrupt"):
self.ai_engine.interrupt()
self.answering = True
try:
self.overlay.show_status("📝 Summarizing the meeting...")
print("\n📝 Summarizing the meeting...")
summary = self.ai_engine.summarize_meeting(transcript)
if not summary:
return
self.overlay.show_answer(summary, "Meeting summary", suggested=False)
self.logger.info("Meeting summary generated.")
self.logger.info(f"Summary: {summary}")
print(f"\n📋 Meeting Summary:\n{summary}\n")
except Exception as e:
print(f"❌ Error summarizing meeting: {e}")
finally:
with self.answer_lock:
self.answering = False
# ------------------------------------------------------------------
# Manual terminal input for testing
# ------------------------------------------------------------------

81
src/ai_engine.py
View File

@@ -190,13 +190,16 @@ class AIEngine:
)
if kind == "factual":
return (
"You are a fast, accurate assistant. Give the direct factual answer in "
"one short sentence. No preamble, no hedging."
"You are a fast, accurate assistant in an ongoing conversation. "
"Use the recent conversation to resolve follow-ups and references "
"(e.g. \"its\", \"that\", \"the second one\"). Give the direct factual "
"answer in one or two short sentences. No preamble, no hedging."
)
return (
f"You are a real-time meeting copilot for {self.user_name}. "
"Answer the latest question accurately and concisely (under ~70 words). "
"Use the meeting transcript only for context. Be direct."
f"You are a real-time meeting copilot for {self.user_name} in an ongoing "
"conversation. Use the recent conversation and meeting transcript to "
"resolve follow-ups and references (\"it\", \"that\", \"those\"). Answer the "
"current question accurately and concisely (under ~70 words). Be direct."
)
def _user_prompt(self, question, context):
@@ -204,20 +207,28 @@ class AIEngine:
if context:
transcript = (context.get("audio", "") or "")[-1800:].strip()
# Recent Q&A thread so follow-ups resolve ("state its 4 core principles"
# after "what is Java" → the model sees the Java exchange).
memory_block = ""
if self.memory:
for item in list(self.memory)[-2:]:
memory_block += f"Earlier Q: {item['question']}\nEarlier A: {item['response']}\n"
turns = []
for item in list(self.memory)[-5:]:
turns.append(f"Q: {item['question']}\nA: {item['response']}")
memory_block = "\n".join(turns)
parts = []
if transcript:
parts.append(f"[Meeting transcript so far]\n{transcript}\n")
if memory_block:
parts.append(memory_block)
parts.append(f"[Question]\n{question}")
parts.append(
"[Recent conversation — resolve references like \"it\", \"that\", "
"\"those\", \"the second one\" against these earlier turns]\n"
f"{memory_block}\n"
)
parts.append(f"[Current question]\n{question}")
return "\n".join(parts)
def _stream_chat(self, messages, max_tokens):
def _stream_chat(self, messages, max_tokens, multiline=False):
"""Run a chat completion, streaming so a new question can interrupt the
current one mid-generation. Returns cleaned text, or None if interrupted."""
if self.interrupt_event.is_set():
@@ -247,7 +258,7 @@ class AIEngine:
print(f"⚠️ Generation error: {e}")
return "I couldn't process that."
return self._clean("".join(chunks))
return self._clean("".join(chunks), multiline=multiline)
def _generate(self, question, context, kind, suggested):
messages = [
@@ -309,9 +320,55 @@ class AIEngine:
"time": datetime.now()})
return text
def _clean(self, text):
# ------------------------------------------------------------------
# End-of-meeting summary
# ------------------------------------------------------------------
def summarize_meeting(self, transcript):
"""Summarize a speaker-labeled meeting transcript into a structured recap.
Returns the summary text, or None if interrupted / nothing to summarize."""
if not transcript or not transcript.strip():
return None
system = (
f"You are a meeting note-taker for {self.user_name}. You are given a "
"speaker-labeled transcript of a call. Write a clear, faithful recap. "
"Only use what is in the transcript — never invent decisions, names, or "
"numbers. If a section has nothing, write \"None\"."
)
user = (
"Summarize this meeting under these exact headings:\n"
"Overview: 1-2 sentences on what the meeting was about.\n"
"Key points: the main things discussed (bullet list).\n"
"Decisions / agreements: what was agreed (bullet list).\n"
"Proposals: who proposed what — attribute to the speaker label "
"(e.g. \"Person 1 proposed …\").\n"
"Action items: who needs to do what next (bullet list).\n"
"Open questions: anything left unresolved.\n\n"
f"[Transcript]\n{transcript}"
)
messages = [
{"role": "system", "content": system},
{"role": "user", "content": user},
]
# Summaries are longer than live answers — allow more room and keep the
# line/section structure (don't collapse to a single run-on line).
return self._stream_chat(messages, max_tokens=max(self.max_tokens, 600),
multiline=True)
def _clean(self, text, multiline=False):
text = re.sub(r"<\|.*?\|>", "", text)
text = re.sub(r"<.*?>", "", text)
if multiline:
# Preserve line/section structure for summaries; just tidy spacing
# and drop stray markdown so it reads cleanly in the plain-text overlay.
text = text.replace("**", "")
text = re.sub(r"(?m)^\s*#{1,6}\s*", "", text) # markdown headings
text = re.sub(r"[ \t]+", " ", text)
text = re.sub(r" *\n *", "\n", text)
text = re.sub(r"\n{3,}", "\n\n", text).strip()
return text or "No response generated."
text = re.sub(r"\s+", " ", text).strip()
if not text:
return "No response generated."

141
src/audio_listener.py
View File

@@ -62,6 +62,7 @@ class StreamSegmenter:
self.noise_floor = 0.0
self.speech_threshold = 2000.0
self.silence_threshold = 1500.0
self.calibrated = False # set once live calibration has run
# rolling pre-speech buffer so we don't clip the start of words
self._pre_buffer = bytearray()
@@ -196,6 +197,13 @@ class AudioListener:
# Capture mode: "push_to_talk" (only buffer audio while armed) or
# "continuous" (always-on VAD segmentation, the original behavior).
self.capture_mode = audio_cfg.get("capture_mode", "push_to_talk")
# Auto-listen ON → continuous VAD, listens on its own (no key).
# Auto-listen OFF → push-to-talk, capture only while the key is held.
# Togglable at runtime via toggle_auto_listen(); starts from config.
self._auto_listen = self.capture_mode == "continuous"
self._reset_segs = False # frame loop drops in-progress utterances
self._calib_rms = {} # source -> rms samples for live calibration
self._warned_silent_system = False # one-time "loopback is silent" notice
self._armed = False
self._ptt_buffers = {} # source -> bytearray, while armed
self._ptt_lock = threading.Lock()
@@ -358,17 +366,14 @@ class AudioListener:
)
print(f"\n🎧 Capturing: {summary}\n")
ptt = self.capture_mode == "push_to_talk"
# In continuous mode each source gets a calibrated VAD segmenter.
# In push-to-talk mode we buffer the held window instead, so neither
# calibration nor the always-on segmenter is needed.
if not ptt:
for source, device in plan.items():
seg = StreamSegmenter(source, self.sample_rate, self.frame_size,
self.frame_duration_ms, self.vad, self.utterance_queue)
self._calibrate(seg, device)
self.segmenters[source] = seg
# Build a VAD segmenter per source so auto-listen can run at ANY time
# (the mode is togglable at runtime). They self-calibrate from the live
# audio the first time auto-listen is active, so startup stays fast and
# push-to-talk users don't pay a calibration delay.
for source, device in plan.items():
seg = StreamSegmenter(source, self.sample_rate, self.frame_size,
self.frame_duration_ms, self.vad, self.utterance_queue)
self.segmenters[source] = seg
# open one input stream per source
for source, device in plan.items():
@@ -383,14 +388,14 @@ class AudioListener:
stream.start()
self.streams.append(stream)
if not ptt:
threading.Thread(target=self._frame_loop, daemon=True).start()
# The frame loop runs always but only does work while auto-listen feeds it.
threading.Thread(target=self._frame_loop, daemon=True).start()
threading.Thread(target=self._transcribe_loop, daemon=True).start()
if ptt:
print("🎤 Ready. Hold the push-to-talk key to capture; release to answer.\n")
if self._auto_listen:
print("🎤 Auto-listening (no key needed). Toggle to push-to-talk anytime.\n")
else:
print("🎤 Listening... Speak now! (Ctrl+C to stop)\n")
print("🎤 Ready. Hold the push-to-talk key to capture; release to answer.\n")
# ------------------------------------------------------------------
# Push-to-talk: only capture audio while armed
@@ -398,7 +403,7 @@ class AudioListener:
def arm(self):
"""Begin buffering audio from every source (push-to-talk key down)."""
if self.capture_mode != "push_to_talk" or self._armed:
if self._auto_listen or self._armed: # key is ignored while auto-listening
return
with self._ptt_lock:
# seed each source with its pre-roll so the leading word survives
@@ -411,7 +416,7 @@ class AudioListener:
def disarm(self):
"""Stop buffering and queue what was captured for transcription (key up)."""
if self.capture_mode != "push_to_talk" or not self._armed:
if self._auto_listen or not self._armed:
return
self._armed = False
with self._ptt_lock:
@@ -421,10 +426,38 @@ class AudioListener:
for source, buf in buffers.items():
audio = bytes(buf)
if len(audio) / (2 * self.sample_rate) >= 0.3: # ignore < 0.3s blips
# Flag a silent loopback (meeting audio not routed into BlackHole).
if source == "system" and not self._warned_silent_system:
samples = np.frombuffer(audio, dtype=np.int16).astype(np.float32)
if samples.size and np.sqrt(np.mean(samples ** 2)) < 15:
self._warned_silent_system = True
print(" ⚠️ Captured nothing from the other participants — the "
"loopback (BlackHole) is silent.")
print(" Set your meeting app's Speaker to a Multi-Output "
"Device that includes BlackHole (README → 'Hearing other "
"participants').")
self.utterance_queue.put((source, audio))
queued = True
print("⏳ Transcribing..." if queued else " (too short — nothing captured)")
def toggle_auto_listen(self):
"""Flip between auto-listen (continuous VAD, no key) and push-to-talk.
Returns the new auto-listen state (True = auto-listening). Safe to call
from the hotkey thread."""
self._auto_listen = not self._auto_listen
if self._auto_listen:
# entering auto-listen: drop any half-held PTT capture
self._armed = False
with self._ptt_lock:
self._ptt_buffers = {}
print("\n🔊 Auto-listen ON — listening on its own (no key needed).")
else:
# leaving auto-listen: tell the frame loop to drop in-progress speech
self._reset_segs = True
print("\n🎙️ Auto-listen OFF — back to push-to-talk (hold the key).")
return self._auto_listen
def _calibrate(self, segmenter, device):
dev_name = sd.query_devices(device)["name"]
print(f"🔧 Calibrating noise floor for '{dev_name}' (2s)...")
@@ -450,22 +483,22 @@ class AudioListener:
# underruns are noisy and harmless; skip logging them
pass
audio_bytes = bytes(indata)
# push-to-talk: only retain audio while the key is held
if self.capture_mode == "push_to_talk":
if self._armed:
with self._ptt_lock:
self._ptt_buffers.setdefault(source, bytearray()).extend(audio_bytes)
else:
# keep a short rolling pre-roll so the first word isn't clipped
pr = self._preroll.get(source)
if pr is None:
pr = self._preroll[source] = deque(maxlen=self._PREROLL_FRAMES)
pr.append(audio_bytes)
# auto-listen: feed the VAD segmenter via the frame queue
if self._auto_listen:
samples = np.frombuffer(audio_bytes, dtype=np.int16).astype(np.float32)
rms = int(np.sqrt(np.mean(samples ** 2))) if samples.size else 0
self.frame_queue.put((source, audio_bytes, rms))
return
# continuous mode: feed the VAD segmenter via the frame queue
samples = np.frombuffer(audio_bytes, dtype=np.int16).astype(np.float32)
rms = int(np.sqrt(np.mean(samples ** 2))) if samples.size else 0
self.frame_queue.put((source, audio_bytes, rms))
# push-to-talk: only retain audio while the key is held
if self._armed:
with self._ptt_lock:
self._ptt_buffers.setdefault(source, bytearray()).extend(audio_bytes)
else:
# keep a short rolling pre-roll so the first word isn't clipped
pr = self._preroll.get(source)
if pr is None:
pr = self._preroll[source] = deque(maxlen=self._PREROLL_FRAMES)
pr.append(audio_bytes)
return _cb
def _select_input_device(self):
@@ -490,15 +523,51 @@ class AudioListener:
while self.running:
try:
source, audio_bytes, rms = self.frame_queue.get(timeout=0.5)
seg = self.segmenters.get(source)
if seg:
seg.process_frame(audio_bytes, rms, time.time())
except queue.Empty:
if self._reset_segs:
for seg in self.segmenters.values():
seg._reset()
self._reset_segs = False
now = time.time()
for seg in self.segmenters.values():
seg.flush_if_idle(now)
continue
except Exception as e:
print(f"\n❌ Frame error: {e}")
continue
# auto-listen was just turned off → drop any in-progress utterances
if self._reset_segs:
for seg in self.segmenters.values():
seg._reset()
self._reset_segs = False
seg = self.segmenters.get(source)
if not seg:
continue
# Self-calibrate the noise floor from the first ~1s of live audio the
# first time this source is heard, then start detecting speech.
if not seg.calibrated:
buf = self._calib_rms.setdefault(source, [])
buf.append(rms)
if len(buf) >= 33: # ~1s at 30 ms frames
median_rms = float(np.median(buf))
seg.calibrate(buf)
seg.calibrated = True
self._calib_rms.pop(source, None)
print(f" 📊 auto-listen calibrated [{source}]: "
f"speech>={seg.speech_threshold:.0f}")
# A silent loopback means the meeting audio isn't routed in.
if source == "system" and median_rms < 15:
print(" ⚠️ System/loopback device looks SILENT — the meeting "
"audio isn't reaching it.")
print(" Set your meeting app's Speaker to a Multi-Output "
"Device that includes BlackHole (see README → 'Hearing "
"other participants').")
continue
seg.process_frame(audio_bytes, rms, time.time())
def _transcribe_loop(self):

29
src/context_manager.py
View File

@@ -6,17 +6,22 @@ class ContextManager:
self.config = config
self.audio_context = deque(maxlen=20) # last 20 utterances (both sources)
self.screen_context = deque(maxlen=5)
# full speaker-labeled meeting transcript (for the end-of-call summary)
self.full_transcript = deque(maxlen=2000)
self.last_text = ""
def add_audio_context(self, text, timestamp, source="microphone", speaker=None):
"""Add a transcribed utterance, tagged with who spoke."""
self.audio_context.append({
speaker = speaker or self._speaker(source)
entry = {
"text": text,
"timestamp": timestamp.isoformat(),
"source": source,
"speaker": speaker or self._speaker(source),
"speaker": speaker,
"type": "audio",
})
}
self.audio_context.append(entry)
self.full_transcript.append(entry)
self.last_text = text
def add_screen_context(self, text, timestamp, region):
@@ -47,7 +52,25 @@ class ContextManager:
def get_last_text(self):
return self.last_text
def get_full_transcript(self, max_chars=9000):
"""Whole speaker-labeled meeting transcript for summarization.
Trimmed from the front to ``max_chars`` so it fits the model context;
returns "" if nothing has been captured yet."""
lines = [
f"{item.get('speaker') or self._speaker(item.get('source'))}: {item['text']}"
for item in self.full_transcript
]
transcript = "\n".join(lines).strip()
if len(transcript) > max_chars:
transcript = "…(earlier conversation trimmed)…\n" + transcript[-max_chars:]
return transcript
def utterance_count(self):
return len(self.full_transcript)
def clear(self):
self.audio_context.clear()
self.screen_context.clear()
self.full_transcript.clear()
self.last_text = ""

38
src/hotkeys.py
View File

@@ -53,19 +53,27 @@ def _build_maps():
class HotkeyManager:
def __init__(self, on_arm=None, on_disarm=None, on_capture=None,
ptt_key="alt_r", capture_key="ctrl+shift+space",
on_toggle=None, toggle_key=""):
on_toggle=None, toggle_key="",
on_summary=None, summary_key="",
on_listen=None, listen_key=""):
self.on_arm = on_arm
self.on_disarm = on_disarm
self.on_capture = on_capture
self.on_toggle = on_toggle
self.on_summary = on_summary
self.on_listen = on_listen
self.ptt_key_name = (ptt_key or "alt_r").strip().lower()
self.capture_key_name = (capture_key or "").strip().lower()
self.toggle_key_name = (toggle_key or "").strip().lower()
self.summary_key_name = (summary_key or "").strip().lower()
self.listen_key_name = (listen_key or "").strip().lower()
self._listener = None
self._ptt_key = None
self._capture_tokens = set()
self._toggle_tokens = set()
self._summary_tokens = set()
self._listen_tokens = set()
self._named = {}
self._canon = {}
@@ -73,6 +81,8 @@ class HotkeyManager:
self._armed = False # ptt key currently down
self._capture_fired = False
self._toggle_fired = False
self._summary_fired = False
self._listen_fired = False
# ------------------------------------------------------------------
@@ -117,6 +127,12 @@ class HotkeyManager:
self._toggle_tokens = {
t.strip() for t in self.toggle_key_name.split("+") if t.strip()
}
self._summary_tokens = {
t.strip() for t in self.summary_key_name.split("+") if t.strip()
}
self._listen_tokens = {
t.strip() for t in self.listen_key_name.split("+") if t.strip()
}
self._listener = keyboard.Listener(
on_press=self._on_press, on_release=self._on_release
@@ -129,6 +145,10 @@ class HotkeyManager:
print(f"⌨️ Screen grab: press [{self.capture_key_name}] then drag a box.")
if self._toggle_tokens:
print(f"⌨️ Show/hide overlay: press [{self.toggle_key_name}].")
if self._summary_tokens:
print(f"⌨️ Meeting summary: press [{self.summary_key_name}].")
if self._listen_tokens:
print(f"⌨️ Auto-listen on/off: press [{self.listen_key_name}].")
print(" (macOS: grant Accessibility permission if keys don't respond.)")
return True
@@ -158,6 +178,16 @@ class HotkeyManager:
self._toggle_fired = True
self._safe(self.on_toggle)
if (self._summary_tokens and not self._summary_fired
and self._summary_tokens.issubset(self._down)):
self._summary_fired = True
self._safe(self.on_summary)
if (self._listen_tokens and not self._listen_fired
and self._listen_tokens.issubset(self._down)):
self._listen_fired = True
self._safe(self.on_listen)
def _on_release(self, key):
if self._ptt_key is not None and key == self._ptt_key and self._armed:
self._armed = False
@@ -172,3 +202,9 @@ class HotkeyManager:
if self._toggle_tokens and not self._toggle_tokens.issubset(self._down):
self._toggle_fired = False
if self._summary_tokens and not self._summary_tokens.issubset(self._down):
self._summary_fired = False
if self._listen_tokens and not self._listen_tokens.issubset(self._down):
self._listen_fired = False