Integrate AI voice assistant throughout the app for improved user experience

Adds voice control with speech recognition to App, TasksPage, and new VoiceShortcuts components using useVoiceIntegration hook. Replit-Commit-Author: Agent Replit-Commit-Session-Id: ff0be73b-afdd-4747-978b-bb8301fb0a82 Replit-Commit-Screenshot-Url: https://storage.googleapis.com/screenshot-production-us-central1/9777c70b-fc38-4831-8d6b-78dfffe041b0/e8e3b3ae-293b-4d28-8c9c-bd319f1aca2f.jpg
2025-06-08 00:12:48 +00:00
parent 7f9d54e150
commit 829d91174a
5 changed files with 290 additions and 1 deletions
@@ -3,6 +3,7 @@ import { queryClient } from "./lib/queryClient";
 import { QueryClientProvider } from "@tanstack/react-query";
 import { Toaster } from "@/components/ui/toaster";
 import { TooltipProvider } from "@/components/ui/tooltip";
+import { FloatingVoiceControl } from "@/components/voice/FloatingVoiceControl";
 import { AuthProvider } from "@/context/AuthContext";
 import { ThemeProvider } from "@/context/ThemeContext";
 import { VoiceProvider } from "@/context/VoiceProvider";
@@ -50,6 +51,7 @@ function App() {
              <TooltipProvider>
                <Toaster />
                <Router />
+                <FloatingVoiceControl />
              </TooltipProvider>
            </VoiceProvider>
          </NotificationProvider>
@@ -0,0 +1,186 @@
+import { useState, useEffect } from 'react';
+import { useVoiceIntegration } from '@/hooks/useVoiceIntegration';
+import { Button } from '@/components/ui/button';
+import { Card, CardContent } from '@/components/ui/card';
+import { Badge } from '@/components/ui/badge';
+import { Mic, MicOff, Volume2, VolumeX, Settings, Loader2 } from 'lucide-react';
+import { cn } from '@/lib/utils';
+
+export function FloatingVoiceControl() {
+  const {
+    isListening,
+    isSupported,
+    isProcessing,
+    currentTranscript,
+    toggleListening,
+    speak,
+  } = useVoiceIntegration();
+
+  const [isExpanded, setIsExpanded] = useState(false);
+  const [lastCommand, setLastCommand] = useState('');
+  const [showTranscript, setShowTranscript] = useState(false);
+
+  useEffect(() => {
+    if (currentTranscript) {
+      setLastCommand(currentTranscript);
+      setShowTranscript(true);
+      const timer = setTimeout(() => setShowTranscript(false), 3000);
+      return () => clearTimeout(timer);
+    }
+  }, [currentTranscript]);
+
+  if (!isSupported) {
+    return null;
+  }
+
+  const handleVoiceToggle = () => {
+    toggleListening();
+    if (!isListening) {
+      speak('Voice assistant activated. How can I help you?');
+    }
+  };
+
+  const handleQuickCommand = (command: string) => {
+    speak(command);
+  };
+
+  return (
+    <div className="fixed bottom-6 right-6 z-50">
+      {/* Transcript Display */}
+      {showTranscript && currentTranscript && (
+        <Card className="mb-4 max-w-xs animate-in slide-in-from-bottom-2">
+          <CardContent className="p-3">
+            <div className="text-xs text-muted-foreground mb-1">You said:</div>
+            <div className="text-sm">{currentTranscript}</div>
+          </CardContent>
+        </Card>
+      )}
+
+      {/* Expanded Controls */}
+      {isExpanded && (
+        <Card className="mb-4 w-80 animate-in slide-in-from-bottom-2">
+          <CardContent className="p-4">
+            <div className="space-y-4">
+              <div className="flex items-center justify-between">
+                <h3 className="font-semibold">Voice Assistant</h3>
+                <Button
+                  variant="ghost"
+                  size="sm"
+                  onClick={() => setIsExpanded(false)}
+                >
+                  ×
+                </Button>
+              </div>
+
+              <div className="space-y-2">
+                <div className="text-sm font-medium">Quick Commands:</div>
+                <div className="grid grid-cols-2 gap-2">
+                  <Button
+                    variant="outline"
+                    size="sm"
+                    onClick={() => handleQuickCommand('Go to dashboard')}
+                  >
+                    Dashboard
+                  </Button>
+                  <Button
+                    variant="outline"
+                    size="sm"
+                    onClick={() => handleQuickCommand('Go to tasks')}
+                  >
+                    Tasks
+                  </Button>
+                  <Button
+                    variant="outline"
+                    size="sm"
+                    onClick={() => handleQuickCommand('Go to finances')}
+                  >
+                    Finances
+                  </Button>
+                  <Button
+                    variant="outline"
+                    size="sm"
+                    onClick={() => handleQuickCommand('Tell me a joke')}
+                  >
+                    Joke
+                  </Button>
+                </div>
+              </div>
+
+              <div className="space-y-2">
+                <div className="text-sm font-medium">Try saying:</div>
+                <div className="text-xs text-muted-foreground space-y-1">
+                  <div>• "Create task buy groceries"</div>
+                  <div>• "Add expense 25 for lunch"</div>
+                  <div>• "Go to dashboard"</div>
+                  <div>• "What's my balance?"</div>
+                </div>
+              </div>
+
+              {lastCommand && (
+                <div className="space-y-2">
+                  <div className="text-sm font-medium">Last Command:</div>
+                  <div className="text-xs bg-muted p-2 rounded">{lastCommand}</div>
+                </div>
+              )}
+            </div>
+          </CardContent>
+        </Card>
+      )}
+
+      {/* Main Voice Button */}
+      <div className="flex items-center gap-2">
+        {isExpanded && (
+          <Button
+            variant="outline"
+            size="icon"
+            onClick={() => handleQuickCommand('Help')}
+          >
+            <Settings className="w-4 h-4" />
+          </Button>
+        )}
+
+        <Button
+          onClick={handleVoiceToggle}
+          disabled={isProcessing}
+          className={cn(
+            "w-14 h-14 rounded-full shadow-lg transition-all duration-200",
+            isListening 
+              ? "bg-red-500 hover:bg-red-600 animate-pulse" 
+              : "bg-primary hover:bg-primary/90",
+            isProcessing && "opacity-75"
+          )}
+        >
+          {isProcessing ? (
+            <Loader2 className="w-6 h-6 text-white animate-spin" />
+          ) : isListening ? (
+            <MicOff className="w-6 h-6 text-white" />
+          ) : (
+            <Mic className="w-6 h-6 text-white" />
+          )}
+        </Button>
+
+        <Button
+          variant="outline"
+          size="icon"
+          onClick={() => setIsExpanded(!isExpanded)}
+        >
+          <Volume2 className="w-4 h-4" />
+        </Button>
+      </div>
+
+      {/* Status Indicators */}
+      <div className="flex justify-center mt-2 gap-1">
+        {isListening && (
+          <Badge variant="destructive" className="text-xs animate-pulse">
+            Listening
+          </Badge>
+        )}
+        {isProcessing && (
+          <Badge variant="default" className="text-xs">
+            Processing
+          </Badge>
+        )}
+      </div>
+    </div>
+  );
+}
@@ -0,0 +1,84 @@
+import { useEffect } from 'react';
+import { useVoiceIntegration } from '@/hooks/useVoiceIntegration';
+import { useLocation } from 'wouter';
+
+interface VoiceShortcutsProps {
+  page?: string;
+  onVoiceCommand?: (command: string) => void;
+}
+
+export function VoiceShortcuts({ page, onVoiceCommand }: VoiceShortcutsProps) {
+  const { speak } = useVoiceIntegration();
+  const [location] = useLocation();
+
+  useEffect(() => {
+    const handleKeyDown = (event: KeyboardEvent) => {
+      // Voice activation with Ctrl+Space
+      if (event.ctrlKey && event.code === 'Space') {
+        event.preventDefault();
+        if (onVoiceCommand) {
+          onVoiceCommand('activate');
+        }
+        speak('Voice assistant activated. How can I help you?');
+      }
+
+      // Quick voice commands with Ctrl+Shift+[Key]
+      if (event.ctrlKey && event.shiftKey) {
+        switch (event.code) {
+          case 'KeyT':
+            event.preventDefault();
+            speak('Opening tasks');
+            window.location.href = '/tasks';
+            break;
+          case 'KeyF':
+            event.preventDefault();
+            speak('Opening finances');
+            window.location.href = '/finances';
+            break;
+          case 'KeyD':
+            event.preventDefault();
+            speak('Opening dashboard');
+            window.location.href = '/dashboard';
+            break;
+          case 'KeyH':
+            event.preventDefault();
+            speak('Voice shortcuts: Control space to activate voice, Control shift T for tasks, Control shift F for finances, Control shift D for dashboard');
+            break;
+        }
+      }
+    };
+
+    document.addEventListener('keydown', handleKeyDown);
+    return () => document.removeEventListener('keydown', handleKeyDown);
+  }, [speak, onVoiceCommand]);
+
+  // Page-specific voice announcements
+  useEffect(() => {
+    const announcePageContext = () => {
+      switch (page) {
+        case 'tasks':
+          speak('Tasks page. Say create task, complete task, or filter tasks');
+          break;
+        case 'finances':
+          speak('Finances page. Say add expense, add income, or show summary');
+          break;
+        case 'dashboard':
+          speak('Dashboard loaded. Say go to tasks, finances, or ask for a summary');
+          break;
+        case 'voice':
+          speak('Voice commands page. Try saying help for available commands');
+          break;
+        case 'ai':
+          speak('AI assistant page. Ask me anything or say tell me a joke');
+          break;
+      }
+    };
+
+    if (page) {
+      const timer = setTimeout(announcePageContext, 1500);
+      return () => clearTimeout(timer);
+    }
+  }, [page, speak, location]);
+
+  return null;
+}
@@ -373,12 +373,16 @@ export function VoiceProvider({ children }: { children: React.ReactNode }) {
      value={{
        isListening,
        isSupported,
+        isProcessing,
+        currentTranscript,
        language,
        startListening,
        stopListening,
+        toggleListening,
        setLanguage,
        executeVoiceCommand,
        speak,
+        speakAndExecute,
      }}
    >
      {children}
@@ -1,5 +1,6 @@
-import { useState, useMemo } from "react";
+import { useState, useMemo, useEffect } from "react";
 import { useQuery, useMutation, useQueryClient } from "@tanstack/react-query";
+import { useVoiceIntegration } from "@/hooks/useVoiceIntegration";
 import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
 import { Button } from "@/components/ui/button";
 import { Input } from "@/components/ui/input";
@@ -32,6 +33,7 @@ interface Task {
 export default function TasksPage() {
  const { toast } = useToast();
  const queryClient = useQueryClient();
+  const { speak } = useVoiceIntegration();
  const [open, setOpen] = useState(false);
  const [editingTask, setEditingTask] = useState<Task | null>(null);
  const [selectedTasks, setSelectedTasks] = useState<number[]>([]);
@@ -53,6 +55,16 @@ export default function TasksPage() {

  const allTasks: Task[] = (tasksResponse as any)?.tasks || [];

+  // Voice announcement when page loads
+  useEffect(() => {
+    if (allTasks.length > 0) {
+      const timer = setTimeout(() => {
+        speak(`Tasks page loaded. You have ${allTasks.length} tasks. Say "create task" followed by a task name to add a new task.`);
+      }, 1000);
+      return () => clearTimeout(timer);
+    }
+  }, [allTasks.length, speak]);
+
  // Enhanced filtering and sorting
  const filteredAndSortedTasks = useMemo(() => {
    let filtered = allTasks.filter((task) => {
@@ -194,6 +206,7 @@ export default function TasksPage() {
  const handleCreateTask = (e: React.FormEvent) => {
    e.preventDefault();
    createTaskMutation.mutate(newTask);
+    speak(`Creating task "${newTask.title}"`);
  };

  const handleEditTask = (e: React.FormEvent) => {