diff --git a/VoiceInk/Services/ParakeetTranscriptionService.swift b/VoiceInk/Services/ParakeetTranscriptionService.swift index da2d092..71b6de7 100644 --- a/VoiceInk/Services/ParakeetTranscriptionService.swift +++ b/VoiceInk/Services/ParakeetTranscriptionService.swift @@ -137,10 +137,16 @@ class ParakeetTranscriptionService: TranscriptionService { logger.notice("๐Ÿฆœ Warning: Empty transcription result for \(audioSamples.count) samples - possible vocabulary issue") } + var text = result.text + if UserDefaults.standard.object(forKey: "IsTextFormattingEnabled") as? Bool ?? true { - return WhisperTextFormatter.format(result.text) + text = WhisperTextFormatter.format(text) } - return result.text + + // Apply hallucination and filler word filtering + text = WhisperHallucinationFilter.filter(text) + + return text } private func readAudioSamples(from url: URL) throws -> [Float] { diff --git a/VoiceInk/Whisper/WhisperHallucinationFilter.swift b/VoiceInk/Whisper/WhisperHallucinationFilter.swift index f95d59d..bea80b3 100644 --- a/VoiceInk/Whisper/WhisperHallucinationFilter.swift +++ b/VoiceInk/Whisper/WhisperHallucinationFilter.swift @@ -4,49 +4,48 @@ import os struct WhisperHallucinationFilter { private static let logger = Logger(subsystem: "com.prakashjoshipax.voiceink", category: "WhisperHallucinationFilter") - // Pattern-based approach for detecting hallucinations - focusing on format indicators private static let hallucinationPatterns = [ - // Text in various types of brackets - the most reliable hallucination indicators - #"\[.*?\]"#, // [Text in square brackets] - #"\(.*?\)"#, // (Text in parentheses) - #"\{.*?\}"#, // {Text in curly braces} - #"<.*?>"#, // - - // Text with special formatting - #"\*.*?\*"#, // *Text with asterisks* - #"_.*?_"#, // _Text with underscores_ - - // Time indicators often added by Whisper - #"(?i)\d{1,2}:\d{2}(:\d{2})?\s*-\s*\d{1,2}:\d{2}(:\d{2})?"# // 00:00 - 00:00 format + #"\[.*?\]"#, // Square brackets + #"\(.*?\)"#, // Parentheses + #"\{.*?\}"# // Curly braces + ] + + private static let fillerWords = [ + "uh", "um", "uhm", "umm", "uhh", "uhhh", "er", "ah", "eh", + "hmm", "hm", "h", "m", "mmm", "mm", "mh", "ha", "ehh" ] - - /// Removes hallucinations from transcription text using pattern matching - /// - Parameter text: Original transcription text from Whisper - /// - Returns: Filtered text with hallucinations removed static func filter(_ text: String) -> String { - logger.notice("๐Ÿงน Applying pattern-based hallucination filter to transcription") - + logger.notice("๐Ÿงน Filtering hallucinations and filler words") var filteredText = text - - // Remove pattern-based hallucinations + + // Remove bracketed hallucinations for pattern in hallucinationPatterns { if let regex = try? NSRegularExpression(pattern: pattern) { let range = NSRange(filteredText.startIndex..., in: filteredText) filteredText = regex.stringByReplacingMatches(in: filteredText, options: [], range: range, withTemplate: "") } } - - // Clean up extra whitespace and newlines that might be left after removing hallucinations + + // Remove filler words + for fillerWord in fillerWords { + let pattern = "\\b\(NSRegularExpression.escapedPattern(for: fillerWord))\\b[,.]?" + if let regex = try? NSRegularExpression(pattern: pattern, options: .caseInsensitive) { + let range = NSRange(filteredText.startIndex..., in: filteredText) + filteredText = regex.stringByReplacingMatches(in: filteredText, options: [], range: range, withTemplate: "") + } + } + + // Clean whitespace filteredText = filteredText.replacingOccurrences(of: #"\s{2,}"#, with: " ", options: .regularExpression) filteredText = filteredText.trimmingCharacters(in: .whitespacesAndNewlines) - - // Add logging to track effectiveness + + // Log results if filteredText != text { - logger.notice("โœ… Removed hallucinations using pattern matching") + logger.notice("โœ… Removed hallucinations and filler words") } else { - logger.notice("โœ… No hallucinations detected with pattern matching") + logger.notice("โœ… No hallucinations or filler words found") } - + return filteredText } } \ No newline at end of file