From 829d91174a5fba55d6526e06583c89bbe19ece5d Mon Sep 17 00:00:00 2001 From: ghaddaditw <40211818-ghaddaditw@users.noreply.replit.com> Date: Sun, 8 Jun 2025 00:12:48 +0000 Subject: [PATCH] Integrate AI voice assistant throughout the app for improved user experience Adds voice control with speech recognition to App, TasksPage, and new VoiceShortcuts components using useVoiceIntegration hook. Replit-Commit-Author: Agent Replit-Commit-Session-Id: ff0be73b-afdd-4747-978b-bb8301fb0a82 Replit-Commit-Screenshot-Url: https://storage.googleapis.com/screenshot-production-us-central1/9777c70b-fc38-4831-8d6b-78dfffe041b0/e8e3b3ae-293b-4d28-8c9c-bd319f1aca2f.jpg --- client/src/App.tsx | 2 + .../components/voice/FloatingVoiceControl.tsx | 186 ++++++++++++++++++ .../src/components/voice/VoiceShortcuts.tsx | 84 ++++++++ client/src/context/VoiceProvider.tsx | 4 + client/src/pages/TasksPage.tsx | 15 +- 5 files changed, 290 insertions(+), 1 deletion(-) create mode 100644 client/src/components/voice/FloatingVoiceControl.tsx create mode 100644 client/src/components/voice/VoiceShortcuts.tsx diff --git a/client/src/App.tsx b/client/src/App.tsx index e263502..dff90b9 100644 --- a/client/src/App.tsx +++ b/client/src/App.tsx @@ -3,6 +3,7 @@ import { queryClient } from "./lib/queryClient"; import { QueryClientProvider } from "@tanstack/react-query"; import { Toaster } from "@/components/ui/toaster"; import { TooltipProvider } from "@/components/ui/tooltip"; +import { FloatingVoiceControl } from "@/components/voice/FloatingVoiceControl"; import { AuthProvider } from "@/context/AuthContext"; import { ThemeProvider } from "@/context/ThemeContext"; import { VoiceProvider } from "@/context/VoiceProvider"; @@ -50,6 +51,7 @@ function App() { + diff --git a/client/src/components/voice/FloatingVoiceControl.tsx b/client/src/components/voice/FloatingVoiceControl.tsx new file mode 100644 index 0000000..f1b4b1c --- /dev/null +++ b/client/src/components/voice/FloatingVoiceControl.tsx @@ -0,0 +1,186 @@ +import { useState, useEffect } from 'react'; +import { useVoiceIntegration } from '@/hooks/useVoiceIntegration'; +import { Button } from '@/components/ui/button'; +import { Card, CardContent } from '@/components/ui/card'; +import { Badge } from '@/components/ui/badge'; +import { Mic, MicOff, Volume2, VolumeX, Settings, Loader2 } from 'lucide-react'; +import { cn } from '@/lib/utils'; + +export function FloatingVoiceControl() { + const { + isListening, + isSupported, + isProcessing, + currentTranscript, + toggleListening, + speak, + } = useVoiceIntegration(); + + const [isExpanded, setIsExpanded] = useState(false); + const [lastCommand, setLastCommand] = useState(''); + const [showTranscript, setShowTranscript] = useState(false); + + useEffect(() => { + if (currentTranscript) { + setLastCommand(currentTranscript); + setShowTranscript(true); + const timer = setTimeout(() => setShowTranscript(false), 3000); + return () => clearTimeout(timer); + } + }, [currentTranscript]); + + if (!isSupported) { + return null; + } + + const handleVoiceToggle = () => { + toggleListening(); + if (!isListening) { + speak('Voice assistant activated. How can I help you?'); + } + }; + + const handleQuickCommand = (command: string) => { + speak(command); + }; + + return ( +
+ {/* Transcript Display */} + {showTranscript && currentTranscript && ( + + +
You said:
+
{currentTranscript}
+ + + )} + + {/* Expanded Controls */} + {isExpanded && ( + + +
+
+

Voice Assistant

+ +
+ +
+
Quick Commands:
+
+ + + + +
+
+ +
+
Try saying:
+
+
• "Create task buy groceries"
+
• "Add expense 25 for lunch"
+
• "Go to dashboard"
+
• "What's my balance?"
+
+
+ + {lastCommand && ( +
+
Last Command:
+
{lastCommand}
+
+ )} +
+ + + )} + + {/* Main Voice Button */} +
+ {isExpanded && ( + + )} + + + + +
+ + {/* Status Indicators */} +
+ {isListening && ( + + Listening + + )} + {isProcessing && ( + + Processing + + )} +
+
+ ); +} \ No newline at end of file diff --git a/client/src/components/voice/VoiceShortcuts.tsx b/client/src/components/voice/VoiceShortcuts.tsx new file mode 100644 index 0000000..9256160 --- /dev/null +++ b/client/src/components/voice/VoiceShortcuts.tsx @@ -0,0 +1,84 @@ +import { useEffect } from 'react'; +import { useVoiceIntegration } from '@/hooks/useVoiceIntegration'; +import { useLocation } from 'wouter'; + +interface VoiceShortcutsProps { + page?: string; + onVoiceCommand?: (command: string) => void; +} + +export function VoiceShortcuts({ page, onVoiceCommand }: VoiceShortcutsProps) { + const { speak } = useVoiceIntegration(); + const [location] = useLocation(); + + useEffect(() => { + const handleKeyDown = (event: KeyboardEvent) => { + // Voice activation with Ctrl+Space + if (event.ctrlKey && event.code === 'Space') { + event.preventDefault(); + if (onVoiceCommand) { + onVoiceCommand('activate'); + } + speak('Voice assistant activated. How can I help you?'); + } + + // Quick voice commands with Ctrl+Shift+[Key] + if (event.ctrlKey && event.shiftKey) { + switch (event.code) { + case 'KeyT': + event.preventDefault(); + speak('Opening tasks'); + window.location.href = '/tasks'; + break; + case 'KeyF': + event.preventDefault(); + speak('Opening finances'); + window.location.href = '/finances'; + break; + case 'KeyD': + event.preventDefault(); + speak('Opening dashboard'); + window.location.href = '/dashboard'; + break; + case 'KeyH': + event.preventDefault(); + speak('Voice shortcuts: Control space to activate voice, Control shift T for tasks, Control shift F for finances, Control shift D for dashboard'); + break; + } + } + }; + + document.addEventListener('keydown', handleKeyDown); + return () => document.removeEventListener('keydown', handleKeyDown); + }, [speak, onVoiceCommand]); + + // Page-specific voice announcements + useEffect(() => { + const announcePageContext = () => { + switch (page) { + case 'tasks': + speak('Tasks page. Say create task, complete task, or filter tasks'); + break; + case 'finances': + speak('Finances page. Say add expense, add income, or show summary'); + break; + case 'dashboard': + speak('Dashboard loaded. Say go to tasks, finances, or ask for a summary'); + break; + case 'voice': + speak('Voice commands page. Try saying help for available commands'); + break; + case 'ai': + speak('AI assistant page. Ask me anything or say tell me a joke'); + break; + } + }; + + if (page) { + const timer = setTimeout(announcePageContext, 1500); + return () => clearTimeout(timer); + } + }, [page, speak, location]); + + return null; +} \ No newline at end of file diff --git a/client/src/context/VoiceProvider.tsx b/client/src/context/VoiceProvider.tsx index 42f3bfc..a6d491f 100644 --- a/client/src/context/VoiceProvider.tsx +++ b/client/src/context/VoiceProvider.tsx @@ -373,12 +373,16 @@ export function VoiceProvider({ children }: { children: React.ReactNode }) { value={{ isListening, isSupported, + isProcessing, + currentTranscript, language, startListening, stopListening, + toggleListening, setLanguage, executeVoiceCommand, speak, + speakAndExecute, }} > {children} diff --git a/client/src/pages/TasksPage.tsx b/client/src/pages/TasksPage.tsx index 20f85c8..d888958 100644 --- a/client/src/pages/TasksPage.tsx +++ b/client/src/pages/TasksPage.tsx @@ -1,5 +1,6 @@ -import { useState, useMemo } from "react"; +import { useState, useMemo, useEffect } from "react"; import { useQuery, useMutation, useQueryClient } from "@tanstack/react-query"; +import { useVoiceIntegration } from "@/hooks/useVoiceIntegration"; import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; @@ -32,6 +33,7 @@ interface Task { export default function TasksPage() { const { toast } = useToast(); const queryClient = useQueryClient(); + const { speak } = useVoiceIntegration(); const [open, setOpen] = useState(false); const [editingTask, setEditingTask] = useState(null); const [selectedTasks, setSelectedTasks] = useState([]); @@ -53,6 +55,16 @@ export default function TasksPage() { const allTasks: Task[] = (tasksResponse as any)?.tasks || []; + // Voice announcement when page loads + useEffect(() => { + if (allTasks.length > 0) { + const timer = setTimeout(() => { + speak(`Tasks page loaded. You have ${allTasks.length} tasks. Say "create task" followed by a task name to add a new task.`); + }, 1000); + return () => clearTimeout(timer); + } + }, [allTasks.length, speak]); + // Enhanced filtering and sorting const filteredAndSortedTasks = useMemo(() => { let filtered = allTasks.filter((task) => { @@ -194,6 +206,7 @@ export default function TasksPage() { const handleCreateTask = (e: React.FormEvent) => { e.preventDefault(); createTaskMutation.mutate(newTask); + speak(`Creating task "${newTask.title}"`); }; const handleEditTask = (e: React.FormEvent) => {