Show description
Digital Whiteboard - Hand Tracking & Voice
Digital Whiteboard - Hand Tracking & Voice
🎤
📝
💾
🗑️
✏️
🧽
🎤 Listening...
Digital Whiteboard
✏️ Pen
🧽 Eraser
🗑️ Clear
🖼️ Toggle BG
Saved: 0 notes
Brush:
5px
👋 Wave to start
Pinch near buttons to activate
Point to draw | Fist to erase
Say "save note" or "open gallery"
×
📝 Saved Notes & Speech Archive
Touch or click any note to load it back to the whiteboard
📝 Note Saved!
Digital Whiteboard - Hand Tracking & Voice
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Digital Whiteboard - Hand Tracking & Voice</title>
<script src="https://cdnjs.cloudflare.com/ajax/libs/three.js/r128/three.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/@mediapipe/camera_utils/camera_utils.js"></script>
<script src="https://cdn.jsdelivr.net/npm/@mediapipe/control_utils/control_utils.js"></script>
<script src="https://cdn.jsdelivr.net/npm/@mediapipe/drawing_utils/drawing_utils.js"></script>
<script src="https://cdn.jsdelivr.net/npm/@mediapipe/hands/hands.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/sql.js/1.8.0/sql-wasm.js"></script>
<style>
* {
margin: 0;
padding: 0;
box-sizing: border-box;
}
body {
font-family: 'Arial', sans-serif;
background: #1a1a1a;
overflow: hidden;
}
#container {
position: relative;
width: 100vw;
height: 100vh;
}
#whiteboard {
position: absolute;
top: 0;
left: 0;
background: white;
cursor: crosshair;
z-index: 1;
}
#camera-container {
position: absolute;
top: 20px;
right: 20px;
width: 320px;
height: 240px;
border: 3px solid #00ff88;
border-radius: 15px;
overflow: hidden;
z-index: 10;
background: #000;
}
#camera-feed {
width: 100%;
height: 100%;
object-fit: cover;
/* Removed transform: scaleX(-1) to show correct hand */
}
#hand-overlay {
position: absolute;
top: 0;
left: 0;
width: 100%;
height: 100%;
pointer-events: none;
z-index: 11;
}
#controls {
position: absolute;
top: 20px;
left: 20px;
background: rgba(0, 0, 0, 0.8);
padding: 15px;
border-radius: 10px;
color: white;
z-index: 10;
backdrop-filter: blur(10px);
}
#gesture-status {
position: absolute;
bottom: 20px;
right: 350px;
background: rgba(0, 0, 0, 0.8);
padding: 10px;
border-radius: 10px;
color: #00ff88;
z-index: 10;
font-weight: bold;
}
/* Virtual Buttons */
.virtual-btn {
position: absolute;
width: 80px;
height: 80px;
border-radius: 50%;
background: rgba(0, 255, 136, 0.3);
border: 3px solid #00ff88;
display: flex;
align-items: center;
justify-content: center;
font-size: 24px;
color: white;
z-index: 15;
transition: all 0.3s;
cursor: pointer;
user-select: none;
}
.virtual-btn.active {
background: rgba(0, 255, 136, 0.8);
transform: scale(1.2);
box-shadow: 0 0 20px #00ff88;
}
.virtual-btn.pinching {
background: #00ff88;
color: black;
transform: scale(0.9);
}
/* Left side buttons */
#voice-btn {
top: 280px;
left: 20px;
}
#gallery-btn {
top: 380px;
left: 20px;
}
#save-btn {
top: 480px;
left: 20px;
}
#clear-btn {
top: 580px;
left: 20px;
}
/* Right side buttons */
#tool-pen {
top: 280px;
right: 20px;
}
#tool-eraser {
top: 380px;
right: 20px;
}
/* Color picker panel */
#color-panel {
position: absolute;
bottom: 20px;
left: 20px;
background: rgba(0, 0, 0, 0.8);
padding: 20px;
border-radius: 15px;
backdrop-filter: blur(10px);
z-index: 15;
display: flex;
flex-direction: column;
gap: 10px;
}
#color-palette {
display: grid;
grid-template-columns: repeat(6, 1fr);
gap: 10px;
margin-bottom: 10px;
}
.color-btn {
width: 40px;
height: 40px;
border-radius: 50%;
border: 3px solid transparent;
cursor: pointer;
transition: all 0.3s;
position: relative;
}
.color-btn.active {
border-color: #00ff88;
transform: scale(1.2);
box-shadow: 0 0 15px rgba(0, 255, 136, 0.5);
}
.color-btn.pinching {
transform: scale(0.8);
box-shadow: 0 0 20px #fff;
}
#brush-controls {
display: flex;
align-items: center;
gap: 10px;
color: white;
}
#brush-size-display {
color: #00ff88;
font-weight: bold;
min-width: 30px;
}
.tool-btn {
background: #333;
color: white;
border: none;
padding: 8px 12px;
margin: 5px;
border-radius: 5px;
cursor: pointer;
transition: all 0.3s;
}
.tool-btn:hover {
background: #00ff88;
color: black;
}
.tool-btn.active {
background: #00ff88;
color: black;
}
input[type="file"] {
margin: 5px 0;
}
#text-display {
position: absolute;
top: 50%;
left: 50%;
transform: translate(-50%, -50%);
font-size: 48px;
font-weight: bold;
color: #333;
text-align: center;
pointer-events: none;
z-index: 5;
text-shadow: 2px 2px 4px rgba(0,0,0,0.1);
max-width: 80%;
word-wrap: break-word;
}
/* Gallery Modal Styles */
#gallery-modal {
position: fixed;
top: 0;
left: 0;
width: 100%;
height: 100%;
background: rgba(0, 0, 0, 0.95);
z-index: 100;
display: none;
overflow: auto;
}
#gallery-header {
position: sticky;
top: 0;
background: rgba(0, 0, 0, 0.9);
padding: 20px;
text-align: center;
color: white;
backdrop-filter: blur(10px);
border-bottom: 2px solid #00ff88;
}
#gallery-content {
padding: 20px;
display: grid;
grid-template-columns: repeat(auto-fill, minmax(300px, 1fr));
gap: 20px;
max-width: 1200px;
margin: 0 auto;
}
.note-card {
background: rgba(255, 255, 255, 0.1);
border-radius: 15px;
padding: 15px;
color: white;
backdrop-filter: blur(10px);
border: 2px solid transparent;
transition: all 0.3s;
cursor: pointer;
}
.note-card:hover {
border-color: #00ff88;
transform: translateY(-5px);
}
.note-preview {
width: 100%;
height: 200px;
background: white;
border-radius: 10px;
margin-bottom: 10px;
overflow: hidden;
}
.note-preview img {
width: 100%;
height: 100%;
object-fit: cover;
}
.note-info {
padding: 10px 0;
}
.note-text {
background: rgba(0, 0, 0, 0.3);
padding: 10px;
border-radius: 8px;
margin: 10px 0;
font-style: italic;
max-height: 100px;
overflow-y: auto;
}
.note-timestamp {
font-size: 12px;
color: #888;
text-align: right;
}
.note-actions {
display: flex;
gap: 10px;
margin-top: 10px;
}
.note-action-btn {
flex: 1;
padding: 8px;
border: none;
border-radius: 5px;
cursor: pointer;
transition: all 0.3s;
}
.delete-btn {
background: #ff4444;
color: white;
}
.load-btn {
background: #00ff88;
color: black;
}
.close-gallery {
position: absolute;
top: 20px;
right: 20px;
background: #ff4444;
color: white;
border: none;
padding: 10px 15px;
border-radius: 50%;
cursor: pointer;
font-size: 20px;
width: 50px;
height: 50px;
}
#save-status {
position: fixed;
top: 50%;
left: 50%;
transform: translate(-50%, -50%);
background: rgba(0, 255, 136, 0.9);
color: black;
padding: 20px 40px;
border-radius: 10px;
font-size: 24px;
font-weight: bold;
z-index: 200;
display: none;
}
#pinch-indicator {
position: absolute;
width: 20px;
height: 20px;
background: #ff0040;
border-radius: 50%;
z-index: 20;
display: none;
pointer-events: none;
}
#instructions {
position: absolute;
bottom: 20px;
right: 20px;
background: rgba(0, 0, 0, 0.8);
color: white;
padding: 10px 20px;
border-radius: 10px;
text-align: center;
z-index: 10;
font-size: 14px;
max-width: 300px;
}
#voice-indicator {
position: absolute;
top: 240px;
left: 20px;
background: rgba(0, 0, 0, 0.8);
color: white;
padding: 10px 15px;
border-radius: 10px;
z-index: 10;
display: none;
}
#voice-indicator.listening {
display: block;
animation: pulse 1s infinite;
background: rgba(0, 255, 136, 0.8);
color: black;
}
@keyframes pulse {
0% { opacity: 1; }
50% { opacity: 0.5; }
100% { opacity: 1; }
}
</style>
</head>
<body>
<div id="container">
<canvas id="whiteboard"></canvas>
<div id="text-display"></div>
<!-- Virtual Buttons -->
<div class="virtual-btn" id="voice-btn" title="Toggle Voice Recognition">🎤</div>
<div class="virtual-btn" id="gallery-btn" title="View Notes Gallery">📝</div>
<div class="virtual-btn" id="save-btn" title="Save Current Note">💾</div>
<div class="virtual-btn" id="clear-btn" title="Clear Whiteboard">🗑️</div>
<div class="virtual-btn" id="tool-pen" title="Pen Tool">✏️</div>
<div class="virtual-btn" id="tool-eraser" title="Eraser Tool">🧽</div>
<div id="pinch-indicator"></div>
<div id="voice-indicator">🎤 Listening...</div>
<div id="camera-container">
<video id="camera-feed" autoplay muted playsinline></video>
<canvas id="hand-overlay"></canvas>
</div>
<div id="controls">
<h3>Digital Whiteboard</h3>
<button class="tool-btn active" data-tool="pen">✏️ Pen</button>
<button class="tool-btn" data-tool="eraser">🧽 Eraser</button>
<button class="tool-btn" onclick="clearWhiteboard()">🗑️ Clear</button>
<br>
<input type="file" id="bg-upload" accept="image/*">
<button class="tool-btn" onclick="toggleBackground()">🖼️ Toggle BG</button>
<br>
<div style="font-size: 12px; margin-top: 5px;">
Saved: <span id="notes-count">0</span> notes
</div>
</div>
<!-- Color Picker Panel -->
<div id="color-panel">
<div id="color-palette"></div>
<div id="brush-controls">
<span>Brush:</span>
<input type="range" id="brush-size" min="2" max="30" value="5">
<span id="brush-size-display">5px</span>
</div>
</div>
<div id="gesture-status">
<div id="current-gesture">👋 Wave to start</div>
</div>
<div id="instructions">
Pinch near buttons to activate<br>
Point to draw | Fist to erase<br>
Say "save note" or "open gallery"
</div>
</div>
<!-- Gallery Modal -->
<div id="gallery-modal">
<button class="close-gallery" onclick="closeGallery()">×</button>
<div id="gallery-header">
<h2>📝 Saved Notes & Speech Archive</h2>
<p>Touch or click any note to load it back to the whiteboard</p>
</div>
<div id="gallery-content">
<!-- Notes will be loaded here -->
</div>
</div>
<!-- Save Status Notification -->
<div id="save-status">📝 Note Saved!</div>
<script>
// Global variables
let whiteboard, ctx, camera, hands;
let isDrawing = false;
let currentTool = 'pen';
let currentColor = '#000000';
let brushSize = 5;
let recognition = null;
let isListening = false;
let backgroundImage = null;
let showBackground = true;
// Hand tracking variables
let handLandmarks = null;
let gestureState = 'none';
let lastGestureTime = 0;
let fingerTip = null;
let thumbTip = null;
let isPinching = false;
let lastPinchTime = 0;
// Database variables
let db = null;
let SQL = null;
let currentSpeechText = '';
let allSpeechTexts = [];
// Virtual buttons and color buttons
let virtualButtons = [];
let colorButtons = [];
// Color palette
const colors = [
'#000000', '#FF0000', '#00FF00', '#0000FF',
'#FFFF00', '#FF00FF', '#00FFFF', '#FFA500',
'#800080', '#008000', '#800000', '#000080'
];
// Initialize everything
window.onload = function() {
initDatabase();
setupCanvas();
setupCamera();
setupHandTracking();
setupVoiceRecognition();
setupEventListeners();
setupVirtualButtons();
setupColorPalette();
};
function setupVirtualButtons() {
virtualButtons = [
{ id: 'voice-btn', action: toggleVoice, element: document.getElementById('voice-btn') },
{ id: 'gallery-btn', action: openGallery, element: document.getElementById('gallery-btn') },
{ id: 'save-btn', action: saveCurrentNote, element: document.getElementById('save-btn') },
{ id: 'clear-btn', action: clearWhiteboard, element: document.getElementById('clear-btn') },
{ id: 'tool-pen', action: () => switchTool('pen'), element: document.getElementById('tool-pen') },
{ id: 'tool-eraser', action: () => switchTool('eraser'), element: document.getElementById('tool-eraser') }
];
// Add click listeners as backup
virtualButtons.forEach(btn => {
btn.element.addEventListener('click', btn.action);
});
}
function setupColorPalette() {
const palette = document.getElementById('color-palette');
colors.forEach((color, index) => {
const colorBtn = document.createElement('div');
colorBtn.className = 'color-btn';
colorBtn.style.backgroundColor = color;
colorBtn.setAttribute('data-color', color);
if (color === currentColor) {
colorBtn.classList.add('active');
}
colorBtn.addEventListener('click', () => selectColor(color));
palette.appendChild(colorBtn);
colorButtons.push({
element: colorBtn,
color: color,
action: () => selectColor(color)
});
});
}
function selectColor(color) {
currentColor = color;
// Update active state
colorButtons.forEach(btn => {
btn.element.classList.toggle('active', btn.color === color);
});
updateGestureDisplay(`🎨 Color: ${color}`);
}
function switchTool(tool) {
currentTool = tool;
updateToolButtons();
updateVirtualButtonStates();
}
function updateVirtualButtonStates() {
// Update tool button appearances
document.getElementById('tool-pen').classList.toggle('active', currentTool === 'pen');
document.getElementById('tool-eraser').classList.toggle('active', currentTool === 'eraser');
// Update voice button appearance
document.getElementById('voice-btn').classList.toggle('active', isListening);
document.getElementById('voice-indicator').classList.toggle('listening', isListening);
}
async function initDatabase() {
try {
// Initialize SQL.js
SQL = await initSqlJs({
locateFile: file => `https://cdnjs.cloudflare.com/ajax/libs/sql.js/1.8.0/${file}`
});
// Create database
db = new SQL.Database();
// Create tables
db.run(`
CREATE TABLE IF NOT EXISTS notes (
id INTEGER PRIMARY KEY AUTOINCREMENT,
title TEXT,
speech_content TEXT,
image_data TEXT,
timestamp DATETIME DEFAULT CURRENT_TIMESTAMP
)
`);
db.run(`
CREATE TABLE IF NOT EXISTS speech_log (
id INTEGER PRIMARY KEY AUTOINCREMENT,
content TEXT,
timestamp DATETIME DEFAULT CURRENT_TIMESTAMP
)
`);
console.log('Database initialized successfully');
updateNotesCount();
} catch (error) {
console.error('Database initialization failed:', error);
}
}
function saveToSpeechLog(text) {
if (!db || !text.trim()) return;
try {
db.run('INSERT INTO speech_log (content) VALUES (?)', [text]);
allSpeechTexts.push(text);
console.log('Speech saved to log:', text);
} catch (error) {
console.error('Error saving speech:', error);
}
}
function saveCurrentNote() {
if (!db) return;
try {
// Get canvas as base64 image
const imageData = whiteboard.toDataURL();
// Combine recent speech texts
const speechContent = allSpeechTexts.slice(-5).join(' | ');
// Generate title
const timestamp = new Date().toLocaleString();
const title = speechContent ?
`${speechContent.substring(0, 30)}...` :
`Whiteboard ${timestamp}`;
// Save to database
db.run(`
INSERT INTO notes (title, speech_content, image_data)
VALUES (?, ?, ?)
`, [title, speechContent, imageData]);
// Show save confirmation
showSaveStatus();
updateNotesCount();
// Clear recent speech texts
allSpeechTexts = [];
console.log('Note saved successfully');
} catch (error) {
console.error('Error saving note:', error);
}
}
function showSaveStatus() {
const status = document.getElementById('save-status');
status.style.display = 'block';
setTimeout(() => {
status.style.display = 'none';
}, 2000);
}
function updateNotesCount() {
if (!db) return;
try {
const result = db.exec('SELECT COUNT(*) as count FROM notes');
const count = result[0] ? result[0].values[0][0] : 0;
document.getElementById('notes-count').textContent = count;
} catch (error) {
console.error('Error counting notes:', error);
}
}
function openGallery() {
if (!db) return;
try {
// Get all notes
const result = db.exec(`
SELECT id, title, speech_content, image_data, timestamp
FROM notes
ORDER BY timestamp DESC
`);
const galleryContent = document.getElementById('gallery-content');
galleryContent.innerHTML = '';
if (result.length === 0 || result[0].values.length === 0) {
galleryContent.innerHTML = `
<div style="text-align: center; color: white; grid-column: 1 / -1;">
<h3>No notes saved yet</h3>
<p>Create some drawings and speech, then save them!</p>
</div>
`;
} else {
result[0].values.forEach(note => {
const [id, title, speechContent, imageData, timestamp] = note;
const noteCard = document.createElement('div');
noteCard.className = 'note-card';
noteCard.innerHTML = `
<div class="note-preview">
<img src="${imageData}" alt="Note preview">
</div>
<div class="note-info">
<h4>${title}</h4>
${speechContent ? `<div class="note-text">"${speechContent}"</div>` : ''}
<div class="note-timestamp">${new Date(timestamp).toLocaleString()}</div>
</div>
<div class="note-actions">
<button class="note-action-btn load-btn" onclick="loadNote(${id})">📂 Load</button>
<button class="note-action-btn delete-btn" onclick="deleteNote(${id})">🗑️ Delete</button>
</div>
`;
galleryContent.appendChild(noteCard);
});
}
document.getElementById('gallery-modal').style.display = 'block';
} catch (error) {
console.error('Error loading notes:', error);
}
}
function closeGallery() {
document.getElementById('gallery-modal').style.display = 'none';
}
function loadNote(noteId) {
if (!db) return;
try {
const result = db.exec('SELECT image_data FROM notes WHERE id = ?', [noteId]);
if (result.length > 0 && result[0].values.length > 0) {
const imageData = result[0].values[0][0];
// Create new image and draw it to canvas
const img = new Image();
img.onload = function() {
clearWhiteboard();
ctx.drawImage(img, 0, 0);
};
img.src = imageData;
closeGallery();
}
} catch (error) {
console.error('Error loading note:', error);
}
}
function deleteNote(noteId) {
if (!db) return;
if (confirm('Are you sure you want to delete this note?')) {
try {
db.run('DELETE FROM notes WHERE id = ?', [noteId]);
updateNotesCount();
openGallery(); // Refresh gallery
} catch (error) {
console.error('Error deleting note:', error);
}
}
}
function setupCanvas() {
whiteboard = document.getElementById('whiteboard');
ctx = whiteboard.getContext('2d');
// Set canvas size to window size
whiteboard.width = window.innerWidth;
whiteboard.height = window.innerHeight;
// Set initial drawing style
ctx.lineWidth = brushSize;
ctx.lineCap = 'round';
ctx.strokeStyle = currentColor;
}
async function setupCamera() {
camera = document.getElementById('camera-feed');
try {
const stream = await navigator.mediaDevices.getUserMedia({
video: { width: 640, height: 480 }
});
camera.srcObject = stream;
} catch (error) {
console.error('Camera access denied:', error);
document.getElementById('gesture-status').innerHTML =
'<div style="color: #ff4444">Camera access required</div>';
}
}
function setupHandTracking() {
hands = new Hands({
locateFile: (file) => {
return `https://cdn.jsdelivr.net/npm/@mediapipe/hands/${file}`;
}
});
hands.setOptions({
maxNumHands: 1,
modelComplexity: 1,
minDetectionConfidence: 0.5,
minTrackingConfidence: 0.5
});
hands.onResults(onHandResults);
if (camera) {
const cameraInstance = new Camera(camera, {
onFrame: async () => {
await hands.send({ image: camera });
},
width: 640,
height: 480
});
cameraInstance.start();
}
}
function onHandResults(results) {
const handOverlay = document.getElementById('hand-overlay');
const overlayCtx = handOverlay.getContext('2d');
// Set overlay canvas size
handOverlay.width = 320;
handOverlay.height = 240;
// Clear previous drawings
overlayCtx.clearRect(0, 0, handOverlay.width, handOverlay.height);
if (results.multiHandLandmarks && results.multiHandLandmarks.length > 0) {
handLandmarks = results.multiHandLandmarks[0];
// Draw hand landmarks
drawConnectors(overlayCtx, handLandmarks, HAND_CONNECTIONS, {color: '#00ff88', lineWidth: 2});
drawLandmarks(overlayCtx, handLandmarks, {color: '#ff0040', lineWidth: 1});
// Process gestures and pinch detection
processGestures(handLandmarks);
detectPinchInteractions(handLandmarks);
// Get finger tip for drawing (FIXED MIRRORING)
fingerTip = getFingerTip(handLandmarks);
thumbTip = getThumbTip(handLandmarks);
// Handle drawing based on gesture
handleHandDrawing();
} else {
handLandmarks = null;
fingerTip = null;
thumbTip = null;
gestureState = 'none';
isPinching = false;
updateGestureDisplay('👋 Wave to start');
hidePinchIndicator();
}
}
function detectPinchInteractions(landmarks) {
if (!fingerTip || !thumbTip) return;
// Calculate distance between thumb and index finger
const distance = Math.sqrt(
Math.pow(fingerTip.x - thumbTip.x, 2) +
Math.pow(fingerTip.y - thumbTip.y, 2)
);
const pinchThreshold = 60; // pixels
const currentlyPinching = distance < pinchThreshold;
// Show pinch indicator
if (currentlyPinching) {
showPinchIndicator((fingerTip.x + thumbTip.x) / 2, (fingerTip.y + thumbTip.y) / 2);
} else {
hidePinchIndicator();
}
// Detect pinch activation (transition from not pinching to pinching)
if (currentlyPinching && !isPinching && Date.now() - lastPinchTime > 500) {
checkVirtualButtonInteractions(fingerTip.x, fingerTip.y);
checkColorButtonInteractions(fingerTip.x, fingerTip.y);
lastPinchTime = Date.now();
}
isPinching = currentlyPinching;
}
function checkVirtualButtonInteractions(x, y) {
virtualButtons.forEach(btn => {
const rect = btn.element.getBoundingClientRect();
const centerX = rect.left + rect.width / 2;
const centerY = rect.top + rect.height / 2;
const distance = Math.sqrt(
Math.pow(x - centerX, 2) +
Math.pow(y - centerY, 2)
);
if (distance < 80) { // Activation radius
btn.element.classList.add('pinching');
setTimeout(() => btn.element.classList.remove('pinching'), 200);
btn.action();
updateGestureDisplay(`🤏 ${btn.id}`);
}
});
}
function checkColorButtonInteractions(x, y) {
colorButtons.forEach(btn => {
const rect = btn.element.getBoundingClientRect();
const centerX = rect.left + rect.width / 2;
const centerY = rect.top + rect.height / 2;
const distance = Math.sqrt(
Math.pow(x - centerX, 2) +
Math.pow(y - centerY, 2)
);
if (distance < 40) { // Smaller radius for color buttons
btn.element.classList.add('pinching');
setTimeout(() => btn.element.classList.remove('pinching'), 200);
btn.action();
}
});
}
function showPinchIndicator(x, y) {
const indicator = document.getElementById('pinch-indicator');
indicator.style.display = 'block';
indicator.style.left = (x - 10) + 'px';
indicator.style.top = (y - 10) + 'px';
}
function hidePinchIndicator() {
document.getElementById('pinch-indicator').style.display = 'none';
}
function processGestures(landmarks) {
const now = Date.now();
if (now - lastGestureTime < 500) return; // Debounce
const fingers = getFingerStates(landmarks);
// Detect gestures (keeping original gesture detection)
if (fingers.index && !fingers.middle && !fingers.ring && !fingers.pinky && fingers.thumb) {
// Pointing - drawing mode
if (gestureState !== 'drawing') {
gestureState = 'drawing';
currentTool = 'pen';
updateToolButtons();
updateVirtualButtonStates();
updateGestureDisplay('✏️ Drawing Mode');
lastGestureTime = now;
}
} else if (!fingers.index && !fingers.middle && !fingers.ring && !fingers.pinky && !fingers.thumb) {
// Fist - eraser mode
if (gestureState !== 'eraser') {
gestureState = 'eraser';
currentTool = 'eraser';
updateToolButtons();
updateVirtualButtonStates();
updateGestureDisplay('🧽 Eraser Mode');
lastGestureTime = now;
}
}
}
function getFingerStates(landmarks) {
return {
thumb: landmarks[4].y < landmarks[3].y,
index: landmarks[8].y < landmarks[6].y,
middle: landmarks[12].y < landmarks[10].y,
ring: landmarks[16].y < landmarks[14].y,
pinky: landmarks[20].y < landmarks[18].y
};
}
function getFingerTip(landmarks) {
// FIXED: Mirror x coordinate to match camera display
const tip = landmarks[8];
return {
x: (1 - tip.x) * whiteboard.width, // Mirror x coordinate
y: tip.y * whiteboard.height
};
}
function getThumbTip(landmarks) {
// FIXED: Mirror x coordinate to match camera display
const tip = landmarks[4];
return {
x: (1 - tip.x) * whiteboard.width, // Mirror x coordinate
y: tip.y * whiteboard.height
};
}
function handleHandDrawing() {
if (!fingerTip || gestureState !== 'drawing') return;
if (!isDrawing) {
isDrawing = true;
ctx.beginPath();
ctx.moveTo(fingerTip.x, fingerTip.y);
} else {
if (currentTool === 'pen') {
ctx.globalCompositeOperation = 'source-over';
ctx.strokeStyle = currentColor;
} else if (currentTool === 'eraser') {
ctx.globalCompositeOperation = 'destination-out';
}
ctx.lineWidth = brushSize;
ctx.lineTo(fingerTip.x, fingerTip.y);
ctx.stroke();
}
}
function setupVoiceRecognition() {
if ('webkitSpeechRecognition' in window || 'SpeechRecognition' in window) {
const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition;
recognition = new SpeechRecognition();
recognition.continuous = true;
recognition.interimResults = true;
recognition.lang = 'en-US';
recognition.onstart = function() {
isListening = true;
updateVirtualButtonStates();
};
recognition.onend = function() {
isListening = false;
updateVirtualButtonStates();
};
recognition.onerror = function(event) {
console.error('Speech recognition error:', event.error);
isListening = false;
updateVirtualButtonStates();
};
recognition.onresult = function(event) {
let finalTranscript = '';
for (let i = event.resultIndex; i < event.results.length; i++) {
if (event.results[i].isFinal) {
finalTranscript += event.results[i][0].transcript;
}
}
if (finalTranscript) {
currentSpeechText = finalTranscript;
displayText(finalTranscript);
// Save to speech log
saveToSpeechLog(finalTranscript);
// Check for voice commands
processVoiceCommands(finalTranscript.toLowerCase());
}
};
}
}
function processVoiceCommands(text) {
if (text.includes('save note') || text.includes('save this') || text.includes('save drawing')) {
saveCurrentNote();
updateGestureDisplay('💾 Voice Save!');
} else if (text.includes('open notes') || text.includes('show notes') || text.includes('view notes')) {
openGallery();
updateGestureDisplay('📝 Gallery Opened!');
} else if (text.includes('clear board') || text.includes('clear screen') || text.includes('erase all')) {
clearWhiteboard();
updateGestureDisplay('🗑️ Voice Clear!');
}
}
function displayText(text) {
const textDisplay = document.getElementById('text-display');
textDisplay.textContent = text;
// Add some flair
textDisplay.style.animation = 'none';
setTimeout(() => {
textDisplay.style.animation = 'pulse 2s ease-in-out';
}, 10);
// Clear text after 5 seconds
setTimeout(() => {
textDisplay.textContent = '';
}, 5000);
}
function setupEventListeners() {
// Tool buttons
document.querySelectorAll('.tool-btn').forEach(btn => {
btn.addEventListener('click', function() {
if (this.dataset.tool) {
currentTool = this.dataset.tool;
updateToolButtons();
updateVirtualButtonStates();
}
});
});
// Brush size
document.getElementById('brush-size').addEventListener('input', function() {
brushSize = this.value;
document.getElementById('brush-size-display').textContent = this.value + 'px';
});
// Background upload
document.getElementById('bg-upload').addEventListener('change', handleBackgroundUpload);
// Mouse drawing fallback
whiteboard.addEventListener('mousedown', startDrawing);
whiteboard.addEventListener('mousemove', draw);
whiteboard.addEventListener('mouseup', stopDrawing);
whiteboard.addEventListener('mouseout', stopDrawing);
// Touch drawing
whiteboard.addEventListener('touchstart', handleTouch);
whiteboard.addEventListener('touchmove', handleTouch);
whiteboard.addEventListener('touchend', stopDrawing);
// Window resize
window.addEventListener('resize', function() {
whiteboard.width = window.innerWidth;
whiteboard.height = window.innerHeight;
redrawBackground();
});
// Gallery modal click outside to close
document.getElementById('gallery-modal').addEventListener('click', function(e) {
if (e.target === this) {
closeGallery();
}
});
}
function startDrawing(e) {
if (gestureState === 'drawing') return; // Hand tracking has priority
isDrawing = true;
const rect = whiteboard.getBoundingClientRect();
const x = e.clientX - rect.left;
const y = e.clientY - rect.top;
ctx.beginPath();
ctx.moveTo(x, y);
}
function draw(e) {
if (!isDrawing || gestureState === 'drawing') return;
const rect = whiteboard.getBoundingClientRect();
const x = e.clientX - rect.left;
const y = e.clientY - rect.top;
if (currentTool === 'pen') {
ctx.globalCompositeOperation = 'source-over';
ctx.strokeStyle = currentColor;
} else if (currentTool === 'eraser') {
ctx.globalCompositeOperation = 'destination-out';
}
ctx.lineWidth = brushSize;
ctx.lineTo(x, y);
ctx.stroke();
}
function stopDrawing() {
isDrawing = false;
}
function handleTouch(e) {
e.preventDefault();
const touch = e.touches[0];
const mouseEvent = new MouseEvent(e.type === 'touchstart' ? 'mousedown' :
e.type === 'touchmove' ? 'mousemove' : 'mouseup', {
clientX: touch.clientX,
clientY: touch.clientY
});
whiteboard.dispatchEvent(mouseEvent);
}
function updateToolButtons() {
document.querySelectorAll('.tool-btn').forEach(btn => {
btn.classList.remove('active');
if (btn.dataset.tool === currentTool) {
btn.classList.add('active');
}
});
}
function updateGestureDisplay(text) {
document.getElementById('current-gesture').textContent = text;
}
function toggleVoice() {
if (!recognition) return;
try {
if (isListening) {
recognition.stop();
} else {
// Stop any existing recognition before starting new one
if (recognition) {
recognition.abort();
}
setTimeout(() => {
recognition.start();
}, 100);
}
} catch (error) {
console.error('Voice toggle error:', error);
isListening = false;
updateVirtualButtonStates();
}
}
function clearWhiteboard() {
ctx.clearRect(0, 0, whiteboard.width, whiteboard.height);
redrawBackground();
}
function handleBackgroundUpload(e) {
const file = e.target.files[0];
if (file) {
const reader = new FileReader();
reader.onload = function(event) {
backgroundImage = new Image();
backgroundImage.onload = function() {
redrawBackground();
};
backgroundImage.src = event.target.result;
};
reader.readAsDataURL(file);
}
}
function toggleBackground() {
showBackground = !showBackground;
redrawBackground();
}
function redrawBackground() {
if (backgroundImage && showBackground) {
ctx.save();
ctx.globalCompositeOperation = 'destination-over';
ctx.drawImage(backgroundImage, 0, 0, whiteboard.width, whiteboard.height);
ctx.restore();
}
}
// Helper functions for MediaPipe drawing
function drawConnectors(ctx, landmarks, connections, style) {
ctx.strokeStyle = style.color;
ctx.lineWidth = style.lineWidth;
for (const connection of connections) {
const from = landmarks[connection[0]];
const to = landmarks[connection[1]];
ctx.beginPath();
ctx.moveTo(from.x * ctx.canvas.width, from.y * ctx.canvas.height);
ctx.lineTo(to.x * ctx.canvas.width, to.y * ctx.canvas.height);
ctx.stroke();
}
}
function drawLandmarks(ctx, landmarks, style) {
ctx.fillStyle = style.color;
ctx.strokeStyle = style.color;
ctx.lineWidth = style.lineWidth;
for (const landmark of landmarks) {
ctx.beginPath();
ctx.arc(landmark.x * ctx.canvas.width, landmark.y * ctx.canvas.height, 3, 0, 2 * Math.PI);
ctx.fill();
}
}
// MediaPipe hand connections
const HAND_CONNECTIONS = [
[0, 1], [1, 2], [2, 3], [3, 4],
[0, 5], [5, 6], [6, 7], [7, 8],
[5, 9], [9, 10], [10, 11], [11, 12],
[9, 13], [13, 14], [14, 15], [15, 16],
[13, 17], [17, 18], [18, 19], [19, 20],
[0, 17]
];
</script>
</body>
</html>