From 65830eaea3980a9e22ecc23632dd9982bac97b68 Mon Sep 17 00:00:00 2001 From: Zylan Date: Tue, 4 Feb 2025 20:52:02 +0800 Subject: [PATCH] mathpix --- app.py | 72 ++++- models/factory.py | 4 +- models/mathpix.py | 283 +++++++++++++++++ static/base.css | 146 +++++++++ static/components.css | 717 ++++++++++++++++++++++++++++++++++++++++++ static/js/core.js | 603 +++++++++++++++++++++++++++++++++++ static/js/events.js | 385 +++++++++++++++++++++++ static/js/history.js | 93 ++++++ static/js/main.js | 46 ++- static/js/settings.js | 22 +- templates/index.html | 22 ++ 11 files changed, 2368 insertions(+), 25 deletions(-) create mode 100644 models/mathpix.py create mode 100644 static/base.css create mode 100644 static/components.css create mode 100644 static/js/core.js create mode 100644 static/js/events.js create mode 100644 static/js/history.js diff --git a/app.py b/app.py index 9340c11..2c30244 100644 --- a/app.py +++ b/app.py @@ -83,7 +83,14 @@ def stream_model_response(response_generator, sid): # Stream responses for response in response_generator: - socketio.emit('claude_response', response, room=sid) + # For Mathpix responses, use text_extracted event + if isinstance(response.get('content', ''), str) and 'mathpix' in response.get('model', ''): + socketio.emit('text_extracted', { + 'content': response['content'] + }, room=sid) + else: + # For AI model responses, use claude_response event + socketio.emit('claude_response', response, room=sid) except Exception as e: error_msg = f"Streaming error: {str(e)}" @@ -119,26 +126,59 @@ def handle_screenshot_request(): def handle_text_extraction(data): try: print("Starting text extraction...") - image_data = data['image'] # Base64 encoded image - # Convert base64 to PIL Image - image_bytes = base64.b64decode(image_data) - image = Image.open(BytesIO(image_bytes)) + # Validate input data + if not data or not isinstance(data, dict): + raise ValueError("Invalid request data") + + if 'image' not in data: + raise ValueError("No image data provided") + + image_data = data['image'] + if not isinstance(image_data, str): + raise ValueError("Invalid image data format") + + settings = data.get('settings', {}) + if not isinstance(settings, dict): + raise ValueError("Invalid settings format") - # Temporarily disabled text extraction - extracted_text = "Text extraction is currently disabled" + mathpix_key = settings.get('mathpixApiKey') + if not mathpix_key: + raise ValueError("Mathpix API key is required") - # Send the extracted text back to the client - socketio.emit('text_extraction_response', { - 'success': True, - 'text': extracted_text + try: + app_id, app_key = mathpix_key.split(':') + if not app_id.strip() or not app_key.strip(): + raise ValueError() + except ValueError: + raise ValueError("Invalid Mathpix API key format. Expected format: 'app_id:app_key'") + + print("Creating Mathpix model instance...") + model = ModelFactory.create_model( + model_name='mathpix', + api_key=mathpix_key + ) + + print("Starting text extraction thread...") + extraction_thread = Thread( + target=stream_model_response, + args=(model.analyze_image(image_data), request.sid) + ) + extraction_thread.daemon = True # Make thread daemon so it doesn't block shutdown + extraction_thread.start() + + except ValueError as e: + error_msg = str(e) + print(f"Validation error: {error_msg}") + socketio.emit('text_extracted', { + 'error': error_msg }, room=request.sid) - except Exception as e: - print(f"Text extraction error: {str(e)}") - socketio.emit('text_extraction_response', { - 'success': False, - 'error': f'Text extraction error: {str(e)}' + error_msg = f"Text extraction error: {str(e)}" + print(f"Unexpected error: {error_msg}") + print(f"Error details: {type(e).__name__}") + socketio.emit('text_extracted', { + 'error': error_msg }, room=request.sid) @socketio.on('analyze_text') diff --git a/models/factory.py b/models/factory.py index 6490e2c..ac5e339 100644 --- a/models/factory.py +++ b/models/factory.py @@ -3,12 +3,14 @@ from .base import BaseModel from .claude import ClaudeModel from .gpt4o import GPT4oModel from .deepseek import DeepSeekModel +from .mathpix import MathpixModel class ModelFactory: _models: Dict[str, Type[BaseModel]] = { 'claude-3-5-sonnet-20241022': ClaudeModel, 'gpt-4o-2024-11-20': GPT4oModel, - 'deepseek-reasoner': DeepSeekModel + 'deepseek-reasoner': DeepSeekModel, + 'mathpix': MathpixModel } @classmethod diff --git a/models/mathpix.py b/models/mathpix.py new file mode 100644 index 0000000..34239e5 --- /dev/null +++ b/models/mathpix.py @@ -0,0 +1,283 @@ +from typing import Generator, Dict, Any +import json +import requests +from .base import BaseModel + +class MathpixModel(BaseModel): + """ + Mathpix OCR model for processing images containing mathematical formulas, + text, and tables. + """ + + def __init__(self, api_key: str, temperature: float = 0.7, system_prompt: str = None): + """ + Initialize the Mathpix model. + + Args: + api_key: Mathpix API key in format "app_id:app_key" + temperature: Not used for Mathpix but kept for BaseModel compatibility + system_prompt: Not used for Mathpix but kept for BaseModel compatibility + + Raises: + ValueError: If the API key format is invalid + """ + super().__init__(api_key, temperature, system_prompt) + try: + self.app_id, self.app_key = api_key.split(':') + except ValueError: + raise ValueError("Mathpix API key must be in format 'app_id:app_key'") + + self.api_url = "https://api.mathpix.com/v3/text" + self.headers = { + "app_id": self.app_id, + "app_key": self.app_key, + "Content-Type": "application/json" + } + + # Content type presets + self.presets = { + "math": { + "formats": ["latex_normal", "latex_styled", "asciimath"], + "data_options": { + "include_asciimath": True, + "include_latex": True, + "include_mathml": True + }, + "ocr_options": { + "detect_formulas": True, + "enable_math_ocr": True, + "enable_handwritten": True, + "rm_spaces": True + } + }, + "text": { + "formats": ["text"], + "data_options": { + "include_latex": False, + "include_asciimath": False + }, + "ocr_options": { + "enable_spell_check": True, + "enable_handwritten": True, + "rm_spaces": False + } + }, + "table": { + "formats": ["text", "data"], + "data_options": { + "include_latex": True + }, + "ocr_options": { + "detect_tables": True, + "enable_spell_check": True, + "rm_spaces": True + } + } + } + + # Default to math preset + self.current_preset = "math" + + def analyze_image(self, image_data: str, proxies: dict = None, content_type: str = None, + confidence_threshold: float = 0.8, max_retries: int = 3) -> Generator[dict, None, None]: + """ + Analyze an image using Mathpix OCR API. + + Args: + image_data: Base64 encoded image data + proxies: Optional proxy configuration + content_type: Type of content to analyze ('math', 'text', or 'table') + confidence_threshold: Minimum confidence score to accept (0.0 to 1.0) + max_retries: Maximum number of retry attempts for failed requests + + Yields: + dict: Response chunks with status and content + """ + if content_type and content_type in self.presets: + self.current_preset = content_type + + preset = self.presets[self.current_preset] + + try: + # Prepare request payload + payload = { + "src": f"data:image/jpeg;base64,{image_data}", + "formats": preset["formats"], + "data_options": preset["data_options"], + "ocr_options": preset["ocr_options"] + } + + # Initialize retry counter + retry_count = 0 + + while retry_count < max_retries: + try: + # Send request to Mathpix API with timeout + response = requests.post( + self.api_url, + headers=self.headers, + json=payload, + proxies=proxies, + timeout=25 # 25 second timeout + ) + + # Handle specific API error codes + if response.status_code == 429: # Rate limit exceeded + if retry_count < max_retries - 1: + retry_count += 1 + continue + else: + raise requests.exceptions.RequestException("Rate limit exceeded") + + response.raise_for_status() + result = response.json() + + # Check confidence threshold + if 'confidence' in result and result['confidence'] < confidence_threshold: + yield { + "status": "warning", + "content": f"Low confidence score: {result['confidence']:.2%}" + } + + break # Success, exit retry loop + + except (requests.exceptions.Timeout, requests.exceptions.ConnectionError): + if retry_count < max_retries - 1: + retry_count += 1 + continue + raise + + # Format the response + formatted_response = self._format_response(result) + + # Yield initial status + yield { + "status": "started", + "content": "" + } + + # Yield the formatted response + yield { + "status": "completed", + "content": formatted_response, + "model": self.get_model_identifier() + } + + except requests.exceptions.RequestException as e: + yield { + "status": "error", + "error": f"Mathpix API error: {str(e)}" + } + except Exception as e: + yield { + "status": "error", + "error": f"Error processing image: {str(e)}" + } + + def analyze_text(self, text: str, proxies: dict = None) -> Generator[dict, None, None]: + """ + Not implemented for Mathpix model as it only processes images. + """ + yield { + "status": "error", + "error": "Text analysis is not supported by Mathpix model" + } + + def get_default_system_prompt(self) -> str: + """ + Not used for Mathpix model. + """ + return "" + + def get_model_identifier(self) -> str: + """ + Return the model identifier. + """ + return "mathpix" + + def validate_api_key(self) -> bool: + """ + Validate if the API key is in the correct format (app_id:app_key). + """ + try: + app_id, app_key = self.api_key.split(':') + return bool(app_id.strip() and app_key.strip()) + except ValueError: + return False + + def _format_response(self, result: Dict[str, Any]) -> str: + """ + Format the Mathpix API response into a readable string. + + Args: + result: Raw API response from Mathpix + + Returns: + str: Formatted response string with all available formats + """ + formatted_parts = [] + + # Add confidence score if available + if 'confidence' in result: + formatted_parts.append(f"Confidence: {result['confidence']:.2%}\n") + + # Add text content + if 'text' in result: + formatted_parts.append("Text Content:") + formatted_parts.append(result['text']) + formatted_parts.append("") + + # Add LaTeX content + if 'latex_normal' in result: + formatted_parts.append("LaTeX (Normal):") + formatted_parts.append(result['latex_normal']) + formatted_parts.append("") + + if 'latex_styled' in result: + formatted_parts.append("LaTeX (Styled):") + formatted_parts.append(result['latex_styled']) + formatted_parts.append("") + + # Add data formats (ASCII math, MathML) + if 'data' in result and isinstance(result['data'], list): + for item in result['data']: + item_type = item.get('type', '') + if item_type and 'value' in item: + formatted_parts.append(f"{item_type.upper()}:") + formatted_parts.append(item['value']) + formatted_parts.append("") + + # Add table data if present + if 'tables' in result and result['tables']: + formatted_parts.append("Tables Detected:") + for i, table in enumerate(result['tables'], 1): + formatted_parts.append(f"Table {i}:") + if 'cells' in table: + # Format table as a grid + cells = table['cells'] + if cells: + max_col = max(cell.get('col', 0) for cell in cells) + 1 + max_row = max(cell.get('row', 0) for cell in cells) + 1 + grid = [['' for _ in range(max_col)] for _ in range(max_row)] + + for cell in cells: + row = cell.get('row', 0) + col = cell.get('col', 0) + text = cell.get('text', '') + grid[row][col] = text + + # Format grid as table + col_widths = [max(len(str(grid[r][c])) for r in range(max_row)) for c in range(max_col)] + for row in grid: + row_str = ' | '.join(f"{str(cell):<{width}}" for cell, width in zip(row, col_widths)) + formatted_parts.append(f"| {row_str} |") + formatted_parts.append("") + + # Add error message if present + if 'error' in result: + error_msg = result['error'] + if isinstance(error_msg, dict): + error_msg = error_msg.get('message', str(error_msg)) + formatted_parts.append(f"Error: {error_msg}") + + return "\n".join(formatted_parts).strip() diff --git a/static/base.css b/static/base.css new file mode 100644 index 0000000..a71e006 --- /dev/null +++ b/static/base.css @@ -0,0 +1,146 @@ +/* CSS Variables */ +:root { + --primary-color: #007bff; + --primary-color-rgb: 0, 123, 255; + --bg-primary: #ffffff; + --bg-secondary: #f8f9fa; + --bg-tertiary: #ffffff; + --bg-input: #ffffff; + --bg-button-secondary: #e9ecef; + --bg-toast: #333333; + --bg-toast-error: #dc3545; + --text-primary: #212529; + --text-secondary: #6c757d; + --text-toast: #ffffff; + --border-color: #dee2e6; + --status-idle: #6c757d; + --status-thinking: #ffc107; + --status-done: #28a745; + --error-color: #dc3545; + --highlight-color: rgba(0, 123, 255, 0.1); + --tooltip-bg: rgba(0, 0, 0, 0.8); + --tooltip-text: #ffffff; +} + +/* Dark Theme */ +[data-theme="dark"] { + --bg-primary: #212529; + --bg-secondary: #343a40; + --bg-tertiary: #2b3035; + --bg-input: #1a1d20; + --bg-button-secondary: #495057; + --text-primary: #f8f9fa; + --text-secondary: #adb5bd; + --border-color: #495057; + --tooltip-bg: rgba(255, 255, 255, 0.9); + --tooltip-text: #000000; +} + +/* App Container */ +.app-container { + display: flex; + flex-direction: column; + height: 100vh; + margin: 0; + background-color: var(--bg-primary); + color: var(--text-primary); +} + +/* Header Styles */ +.app-header { + display: flex; + justify-content: space-between; + align-items: center; + padding: 1rem; + background-color: var(--bg-secondary); + border-bottom: 1px solid var(--border-color); +} + +.header-left { + display: flex; + align-items: center; + gap: 1rem; +} + +.header-right { + display: flex; + gap: 0.5rem; +} + +/* Main Content */ +.app-main { + flex: 1; + display: flex; + position: relative; + overflow: hidden; +} + +.content-panel { + flex: 1; + display: flex; + flex-direction: column; + overflow: hidden; +} + +/* Utility Classes */ +.hidden { + display: none !important; +} + +/* Animations */ +@keyframes slideIn { + from { + transform: translateX(100%); + opacity: 0; + } + to { + transform: translateX(0); + opacity: 1; + } +} + +@keyframes highlight { + 0% { background-color: var(--highlight-color); } + 100% { background-color: transparent; } +} + +/* Touch Interactions */ +@media (hover: none) { + .btn-icon:active { + background-color: var(--bg-button-secondary); + } + + .section-header:active { + background-color: var(--bg-tertiary); + } +} + +/* Responsive Design */ +@media (max-width: 768px) { + html { + font-size: 16px; + height: 100%; + overflow: hidden; + } + + body { + height: 100%; + overflow: hidden; + } + + .app-header { + padding: 0.5rem; + } + + .app-header h1 { + font-size: 1.25rem; + } + + .header-right { + gap: 0.25rem; + } + + .btn-icon { + padding: 0.75rem; + } +} diff --git a/static/components.css b/static/components.css new file mode 100644 index 0000000..0543c05 --- /dev/null +++ b/static/components.css @@ -0,0 +1,717 @@ +/* Capture Section */ +.capture-section { + flex: 1; + display: flex; + flex-direction: column; + padding: 1rem; + gap: 1rem; +} + +.toolbar { + display: flex; + justify-content: space-between; + align-items: center; + gap: 1rem; +} + +.toolbar-buttons { + display: flex; + gap: 1rem; +} + +/* Image Preview */ +.image-preview { + flex: 1; + display: flex; + flex-direction: column; + gap: 1rem; + overflow: hidden; +} + +.image-container { + flex: 1; + display: flex; + justify-content: center; + align-items: center; + overflow: hidden; + background-color: var(--bg-secondary); + border-radius: 8px; + position: relative; + min-height: 200px; + max-height: 60vh; +} + +.image-container img { + max-width: 100%; + max-height: 100%; + object-fit: contain; + touch-action: pinch-zoom; + -webkit-user-select: none; + user-select: none; +} + +/* Analysis Section */ +.analysis-button { + display: flex; + flex-direction: column; + gap: 1rem; +} + +.text-editor { + display: flex; + flex-direction: column; + gap: 0.5rem; +} + +.text-editor textarea { + width: 100%; + resize: vertical; + padding: 0.5rem; + border: 1px solid var(--border-color); + border-radius: 4px; + background-color: var(--bg-secondary); + color: var(--text-primary); +} + +.text-actions { + display: flex; + justify-content: space-between; + align-items: center; +} + +/* Claude Panel */ +.claude-panel { + position: absolute; + top: 0; + right: 0; + width: 50%; + height: 100%; + background-color: var(--bg-secondary); + border-left: 1px solid var(--border-color); + display: flex; + flex-direction: column; + transition: transform 0.3s ease; +} + +.panel-header { + display: flex; + justify-content: space-between; + align-items: center; + padding: 1rem; + border-bottom: 1px solid var(--border-color); +} + +.header-title { + display: flex; + align-items: center; + gap: 0.5rem; +} + +.analysis-status { + display: flex; + align-items: center; +} + +.status-light { + width: 8px; + height: 8px; + border-radius: 50%; + background-color: var(--status-idle); +} + +.status-light.thinking { + background-color: var(--status-thinking); +} + +.status-light.done { + background-color: var(--status-done); +} + +.response-content { + flex: 1; + padding: 1rem; + overflow-y: auto; +} + +/* Settings Panel */ +.settings-panel { + position: absolute; + top: 0; + right: 0; + width: 400px; + height: 100%; + background-color: var(--bg-secondary); + border-left: 1px solid var(--border-color); + display: flex; + flex-direction: column; + transition: transform 0.3s ease; +} + +.settings-content { + flex: 1; + padding: 1rem; + overflow-y: auto; +} + +.settings-section { + margin-bottom: 1.5rem; + padding: 1rem; + border-radius: 8px; + background: var(--bg-tertiary); +} + +.section-header { + display: flex; + align-items: center; + justify-content: space-between; + padding: 1rem; + margin: -0.75rem -0.75rem 0 -0.75rem; + background: var(--bg-secondary); + border-bottom: 1px solid var(--border-color); + cursor: pointer; + user-select: none; + -webkit-tap-highlight-color: transparent; + position: relative; + z-index: 1; +} + +.section-header i { + font-size: 0.8em; + transition: transform 0.3s ease; + margin-left: 0.5rem; +} + +.settings-section:not(.expanded) .section-header i { + transform: rotate(-90deg); +} + +.section-content { + transition: all 0.3s ease; + overflow: hidden; + padding: 0 1rem; +} + +.settings-section:not(.expanded) .section-content { + max-height: 0; + padding-top: 0; + padding-bottom: 0; + opacity: 0; +} + +.settings-section.expanded .section-content { + max-height: 2000px; + padding-top: 1rem; + padding-bottom: 1rem; + opacity: 1; +} + +/* Form Elements */ +.setting-group { + margin-bottom: 1rem; +} + +.setting-group label { + display: block; + margin-bottom: 0.5rem; + display: flex; + align-items: center; + gap: 0.5rem; +} + +.setting-group label i { + color: var(--text-secondary); + cursor: help; +} + +.setting-group input[type="text"], +.setting-group input[type="password"], +.setting-group input[type="number"], +.setting-group select, +.setting-group textarea { + width: 100%; + padding: 0.5rem; + border: 1px solid var(--border-color); + border-radius: 4px; + background-color: var(--bg-input); + color: var(--text-primary); +} + +.setting-group textarea { + resize: vertical; + min-height: 100px; +} + +.input-group { + position: relative; + display: flex; + align-items: center; +} + +.input-group.highlight { + animation: highlight 0.2s ease-in-out; +} + +.input-group input { + flex: 1; + padding-right: 2.5rem; +} + +.input-group .btn-icon { + position: absolute; + right: 0.5rem; +} + +/* History Panel */ +.history-panel { + position: absolute; + top: 0; + right: 0; + width: 300px; + height: 100%; + background-color: var(--bg-secondary); + border-left: 1px solid var(--border-color); + display: flex; + flex-direction: column; + transition: transform 0.3s ease; +} + +.history-content { + flex: 1; + padding: 1rem; + overflow-y: auto; +} + +.history-item { + margin-bottom: 1rem; + background: var(--bg-tertiary); + border-radius: 8px; + overflow: hidden; + transition: all 0.3s ease; + border: 1px solid var(--border-color); +} + +.history-item-header { + padding: 1rem; + display: flex; + align-items: center; + justify-content: space-between; + cursor: pointer; + user-select: none; + -webkit-tap-highlight-color: transparent; + position: relative; + background: var(--bg-secondary); +} + +.history-image { + padding: 1rem; + border-bottom: 1px solid var(--border-color); +} + +.history-image img { + max-width: 100%; + height: auto; + border-radius: 4px; +} + +.history-empty { + height: 100%; + display: flex; + flex-direction: column; + justify-content: center; + align-items: center; + gap: 1rem; + color: var(--text-secondary); + padding: 2rem; +} + +.history-empty i { + font-size: 3rem; + opacity: 0.5; +} + +/* Crop Container */ +.crop-container { + position: fixed; + top: 0; + left: 0; + width: 100%; + height: 100%; + background-color: var(--bg-primary); + display: flex; + flex-direction: column; + z-index: 1100; +} + +.crop-wrapper { + flex: 1; + position: relative; + overflow: hidden; + background-color: var(--bg-secondary); + display: flex; + justify-content: center; + align-items: center; +} + +.crop-area { + position: relative; + width: 100%; + height: 100%; + display: flex; + justify-content: center; + align-items: center; + background-color: var(--bg-secondary); +} + +.crop-area img { + max-width: 100%; + max-height: 100%; + object-fit: contain; +} + +/* Cropper.js Customization */ +.cropper-container { + direction: ltr; + font-size: 0; + line-height: 0; + position: relative; + touch-action: none; + user-select: none; +} + +.cropper-wrap-box, +.cropper-canvas, +.cropper-drag-box, +.cropper-crop-box, +.cropper-modal { + bottom: 0; + left: 0; + position: absolute; + right: 0; + top: 0; +} + +.cropper-wrap-box { + overflow: hidden; +} + +.cropper-drag-box { + background-color: #fff; + opacity: 0; +} + +.cropper-modal { + background-color: var(--bg-primary); + opacity: 0.5; +} + +.cropper-view-box { + display: block; + height: 100%; + outline: 1px solid var(--primary-color); + outline-color: var(--primary-color); + overflow: hidden; + width: 100%; +} + +.cropper-dashed { + border: 0 dashed #eee; + display: block; + opacity: 0.5; + position: absolute; +} + +.cropper-center { + display: block; + height: 0; + left: 50%; + opacity: 0.75; + position: absolute; + top: 50%; + width: 0; +} + +.cropper-center::before, +.cropper-center::after { + background-color: #eee; + content: " "; + display: block; + position: absolute; +} + +.cropper-face { + background-color: #fff; + left: 0; + opacity: 0.1; + position: absolute; + top: 0; +} + +.cropper-line { + background-color: var(--primary-color); + display: block; + height: 100%; + opacity: 0.1; + position: absolute; + width: 100%; +} + +.cropper-point { + background-color: var(--primary-color); + height: 5px; + opacity: 0.75; + position: absolute; + width: 5px; +} + +.crop-actions { + height: 80px; + padding: 1rem; + display: flex; + justify-content: center; + gap: 1rem; + background-color: var(--bg-secondary); + border-top: 1px solid var(--border-color); +} + +.crop-actions button { + flex: 1; + max-width: 160px; + padding: 0.75rem; + font-size: 1rem; + display: flex; + align-items: center; + justify-content: center; + gap: 0.5rem; +} + +/* Toast Container */ +.toast-container { + position: fixed; + bottom: 1rem; + right: 1rem; + display: flex; + flex-direction: column; + gap: 0.5rem; + z-index: 2000; +} + +.toast { + padding: 0.75rem 1rem; + border-radius: 4px; + background-color: var(--bg-toast); + color: var(--text-toast); + box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1); + animation: slideIn 0.3s ease; +} + +.toast.error { + background-color: var(--bg-toast-error); +} + +/* Button Styles */ +.btn-primary, +.btn-secondary { + display: inline-flex; + align-items: center; + gap: 0.5rem; + padding: 0.5rem 1rem; + border: none; + border-radius: 4px; + cursor: pointer; + transition: background-color 0.2s; +} + +.btn-primary { + background-color: var(--primary-color); + color: white; +} + +.btn-secondary { + background-color: var(--bg-button-secondary); + color: var(--text-primary); + border: 1px solid var(--border-color); +} + +.btn-icon { + padding: 0.5rem; + background: none; + border: none; + color: var(--text-primary); + cursor: pointer; + transition: color 0.2s; +} + +.btn-icon:hover { + color: var(--primary-color); +} + +.button-group { + display: flex; + gap: 0.5rem; +} + +/* Tooltip styles */ +[title] { + position: relative; +} + +[title]:hover::after { + content: attr(title); + position: absolute; + bottom: 100%; + left: 50%; + transform: translateX(-50%); + padding: 0.5rem; + background: var(--tooltip-bg); + color: var(--tooltip-text); + border-radius: 4px; + font-size: 0.8em; + white-space: nowrap; + z-index: 1000; +} + +/* Responsive Design for Components */ +@media (max-width: 768px) { + .settings-content, + .history-content, + .response-content { + -webkit-overflow-scrolling: touch; + } + + .capture-section { + padding: 0.5rem; + } + + .toolbar { + flex-wrap: wrap; + } + + .button-group { + width: 100%; + } + + .button-group button { + flex: 1; + padding: 0.75rem; + font-size: 0.9rem; + } + + .image-container { + margin: 0 -0.5rem; + border-radius: 0; + background-color: var(--bg-primary); + } + + .image-container img { + width: 100%; + height: auto; + object-fit: contain; + } + + .settings-panel, + .history-panel, + .claude-panel { + width: 100%; + position: fixed; + bottom: 0; + right: 0; + height: 90vh; + transform: translateY(100%); + border-top-left-radius: 16px; + border-top-right-radius: 16px; + border-left: none; + box-shadow: 0 -2px 10px rgba(0, 0, 0, 0.1); + } + + .settings-panel:not(.hidden), + .history-panel:not(.hidden), + .claude-panel:not(.hidden) { + transform: translateY(0); + } + + .settings-content, + .history-content { + padding: 0.75rem; + } + + .settings-section { + margin-bottom: 1rem; + padding: 0.75rem; + } + + .setting-group { + margin-bottom: 0.75rem; + } + + .setting-group input[type="text"], + .setting-group input[type="password"], + .setting-group input[type="number"], + .setting-group select, + .setting-group textarea { + padding: 0.75rem; + font-size: 1rem; + } + + .crop-container { + position: fixed; + top: 0; + left: 0; + width: 100%; + height: 100%; + background-color: var(--bg-primary); + z-index: 1100; + } + + .crop-wrapper { + height: calc(100% - 80px); + overflow: hidden; + background-color: var(--bg-secondary); + } + + .crop-area { + max-width: 100%; + max-height: 100%; + } + + .crop-actions { + height: 80px; + padding: 1rem; + background-color: var(--bg-secondary); + border-top: 1px solid var(--border-color); + } + + .crop-actions button { + padding: 0.75rem 1.5rem; + font-size: 1rem; + } + + .text-editor textarea { + padding: 0.75rem; + font-size: 1rem; + min-height: 120px; + } + + .text-actions { + flex-direction: column; + gap: 0.75rem; + align-items: stretch; + } + + .text-actions button { + width: 100%; + padding: 0.75rem; + } + + .toast-container { + left: 1rem; + right: 1rem; + bottom: 1rem; + } + + .toast { + width: 100%; + padding: 1rem; + font-size: 0.9rem; + text-align: center; + } +} diff --git a/static/js/core.js b/static/js/core.js new file mode 100644 index 0000000..9fdbf84 --- /dev/null +++ b/static/js/core.js @@ -0,0 +1,603 @@ +class SnapSolver { + constructor() { + // Initialize managers first + window.uiManager = new UIManager(); + window.settingsManager = new SettingsManager(); + + this.initializeElements(); + this.initializeState(); + this.initializeConnection(); + this.setupAutoScroll(); + this.setupEventListeners(); + } + + initializeElements() { + // Capture elements + this.captureBtn = document.getElementById('captureBtn'); + this.cropBtn = document.getElementById('cropBtn'); + this.connectionStatus = document.getElementById('connectionStatus'); + this.screenshotImg = document.getElementById('screenshotImg'); + this.cropContainer = document.getElementById('cropContainer'); + this.imagePreview = document.getElementById('imagePreview'); + this.sendToClaudeBtn = document.getElementById('sendToClaude'); + this.extractTextBtn = document.getElementById('extractText'); + this.textEditor = document.getElementById('textEditor'); + this.extractedText = document.getElementById('extractedText'); + this.sendExtractedTextBtn = document.getElementById('sendExtractedText'); + this.responseContent = document.getElementById('responseContent'); + this.claudePanel = document.getElementById('claudePanel'); + this.statusLight = document.querySelector('.status-light'); + + // Verify all elements are found + const elements = [ + this.captureBtn, this.cropBtn, this.connectionStatus, this.screenshotImg, + this.cropContainer, this.imagePreview, this.sendToClaudeBtn, this.extractTextBtn, + this.textEditor, this.extractedText, this.sendExtractedTextBtn, this.responseContent, + this.claudePanel, this.statusLight + ]; + + elements.forEach((element, index) => { + if (!element) { + console.error(`Failed to initialize element at index ${index}`); + } + }); + } + + initializeState() { + this.socket = null; + this.cropper = null; + this.croppedImage = null; + this.history = JSON.parse(localStorage.getItem('snapHistory') || '[]'); + this.heartbeatInterval = null; + this.connectionCheckInterval = null; + this.isReconnecting = false; + this.lastConnectionAttempt = 0; + } + + resetConnection() { + const now = Date.now(); + const timeSinceLastAttempt = now - this.lastConnectionAttempt; + + // Prevent multiple reset attempts within 2 seconds + if (timeSinceLastAttempt < 2000) { + console.log('Skipping reset - too soon since last attempt'); + return; + } + + console.log('Resetting connection...'); + this.isReconnecting = true; + this.lastConnectionAttempt = now; + + // Clear existing intervals + if (this.heartbeatInterval) { + clearInterval(this.heartbeatInterval); + this.heartbeatInterval = null; + } + + if (this.connectionCheckInterval) { + clearInterval(this.connectionCheckInterval); + this.connectionCheckInterval = null; + } + + // Clean up existing socket + if (this.socket) { + this.socket.removeAllListeners(); + this.socket.disconnect(); + this.socket = null; + } + + // Small delay before reconnecting + setTimeout(() => { + this.initializeConnection(); + this.isReconnecting = false; + }, 100); + } + + startConnectionCheck() { + // Clear any existing interval + if (this.connectionCheckInterval) { + clearInterval(this.connectionCheckInterval); + } + + // Check connection status every 5 seconds + this.connectionCheckInterval = setInterval(() => { + if (!this.isReconnecting && (!this.socket || !this.socket.connected)) { + console.log('Connection check failed, attempting reset...'); + this.resetConnection(); + } + }, 5000); + } + + initializeCropper() { + try { + // Clean up existing cropper if any + if (this.cropper) { + this.cropper.destroy(); + this.cropper = null; + } + + // Show crop container and prepare crop area + this.cropContainer.classList.remove('hidden'); + const cropArea = document.querySelector('.crop-area'); + cropArea.innerHTML = ''; + + // Create a new image element for cropping + const cropImage = document.createElement('img'); + cropImage.src = this.screenshotImg.src; + cropArea.appendChild(cropImage); + + // Initialize Cropper.js + this.cropper = new Cropper(cropImage, { + aspectRatio: NaN, + viewMode: 1, + dragMode: 'move', + autoCropArea: 0.8, + restore: false, + modal: true, + guides: true, + highlight: true, + cropBoxMovable: true, + cropBoxResizable: true, + toggleDragModeOnDblclick: false, + ready: () => { + console.log('Cropper initialized successfully'); + }, + error: (error) => { + console.error('Cropper initialization error:', error); + window.showToast('Failed to initialize image cropper', 'error'); + } + }); + } catch (error) { + console.error('Error initializing cropper:', error); + window.showToast('Failed to initialize image cropper', 'error'); + } + } + + setupAutoScroll() { + const observer = new MutationObserver((mutations) => { + mutations.forEach((mutation) => { + if (mutation.type === 'characterData' || mutation.type === 'childList') { + this.responseContent.scrollTo({ + top: this.responseContent.scrollHeight, + behavior: 'smooth' + }); + } + }); + }); + + observer.observe(this.responseContent, { + childList: true, + characterData: true, + subtree: true + }); + } + + updateConnectionStatus(connected) { + if (!this.connectionStatus || !this.captureBtn) { + console.error('Required elements not initialized'); + return; + } + + this.connectionStatus.textContent = connected ? 'Connected' : 'Disconnected'; + this.connectionStatus.className = `status ${connected ? 'connected' : 'disconnected'}`; + + // Enable/disable capture button + if (this.captureBtn) { + this.captureBtn.disabled = !connected; + } + + if (!connected) { + // Hide UI elements when disconnected + const elements = [ + this.imagePreview, + this.cropBtn, + this.sendToClaudeBtn, + this.extractTextBtn, + this.textEditor + ]; + + elements.forEach(element => { + if (element) { + element.classList.add('hidden'); + } + }); + } + } + + updateStatusLight(status) { + if (!this.statusLight) return; + + this.statusLight.className = 'status-light'; + switch (status) { + case 'started': + case 'streaming': + this.statusLight.classList.add('processing'); + break; + case 'completed': + this.statusLight.classList.add('completed'); + break; + case 'error': + this.statusLight.classList.add('error'); + break; + default: + break; + } + } + + initializeConnection() { + // Clear any existing heartbeat interval + if (this.heartbeatInterval) { + clearInterval(this.heartbeatInterval); + this.heartbeatInterval = null; + } + + try { + // Clean up existing socket if any + if (this.socket) { + this.socket.disconnect(); + this.socket = null; + } + + console.log('Initializing socket connection...'); + this.socket = io(window.location.origin, { + reconnection: true, + reconnectionAttempts: Infinity, + reconnectionDelay: 100, // Very fast initial reconnection + reconnectionDelayMax: 1000, // Shorter max delay + timeout: 120000, + autoConnect: true, // Enable auto-connect + transports: ['websocket'], + forceNew: true, // Force a new connection on refresh + closeOnBeforeunload: false, // Prevent auto-close on page refresh + reconnectionAttempts: Infinity, // Never stop trying to reconnect + extraHeaders: { + 'X-Client-Version': '1.0' + } + }); + + // Setup heartbeat with monitoring + this.heartbeatInterval = setInterval(() => { + if (this.socket && this.socket.connected) { + const heartbeatTimeout = setTimeout(() => { + console.log('Heartbeat timeout, resetting connection...'); + if (!this.isReconnecting) { + this.resetConnection(); + } + }, 5000); // Wait 5 seconds for heartbeat response + + this.socket.emit('heartbeat'); + + // Clear timeout when heartbeat is acknowledged + this.socket.once('heartbeat_response', () => { + clearTimeout(heartbeatTimeout); + }); + } + }, 10000); + + this.socket.on('connect', () => { + console.log('Connected to server'); + this.updateConnectionStatus(true); + // Re-enable capture button on reconnection + if (this.captureBtn) { + this.captureBtn.disabled = false; + } + // Start connection check after successful connection + this.startConnectionCheck(); + }); + + this.socket.on('disconnect', (reason) => { + console.log('Disconnected from server:', reason); + this.updateConnectionStatus(false); + + // Always attempt to reconnect regardless of reason + console.log('Attempting reconnection...'); + if (!this.socket.connected && !this.isReconnecting) { + this.resetConnection(); + } + + // Clean up resources but maintain reconnection ability + if (this.heartbeatInterval) { + clearInterval(this.heartbeatInterval); + this.heartbeatInterval = null; + } + }); + + // Add reconnecting event handler + this.socket.on('reconnecting', (attemptNumber) => { + console.log(`Reconnection attempt ${attemptNumber}...`); + if (!this.isReconnecting) { + this.resetConnection(); + } + }); + + // Add reconnect_failed event handler + this.socket.on('reconnect_failed', () => { + console.log('Reconnection failed, trying again...'); + if (!this.isReconnecting) { + this.resetConnection(); + } + }); + + this.socket.on('connect_error', (error) => { + console.error('Connection error:', error); + this.updateConnectionStatus(false); + window.showToast('Connection error: ' + error.message, 'error'); + + // Enhanced exponential backoff with jitter + const attempts = this.socket.io.backoff?.attempts || 0; + const baseDelay = 1000; + const maxDelay = 10000; + const jitter = Math.random() * 1000; + const delay = Math.min(baseDelay * Math.pow(1.5, attempts) + jitter, maxDelay); + + console.log(`Scheduling reconnection attempt in ${Math.round(delay)}ms...`); + setTimeout(() => { + if (!this.socket.connected) { + console.log(`Attempting to reconnect (attempt ${attempts + 1})...`); + this.socket.connect(); + } + }, delay); + }); + + this.socket.on('heartbeat_response', () => { + console.debug('Heartbeat acknowledged'); + // Reset connection if we were in a disconnected state + if (this.connectionStatus && this.connectionStatus.textContent === 'Disconnected' && !this.isReconnecting) { + this.resetConnection(); + } + }); + + this.socket.on('error', (error) => { + console.error('Socket error:', error); + window.showToast('Socket error occurred', 'error'); + }); + + this.setupSocketEventHandlers(); + + } catch (error) { + console.error('Connection initialization error:', error); + this.updateConnectionStatus(false); + } + } + + setupSocketEventHandlers() { + if (!this.socket) { + console.error('Socket not initialized'); + return; + } + + // Screenshot response handler + this.socket.on('screenshot_response', (data) => { + if (data.success) { + this.screenshotImg.src = `data:image/png;base64,${data.image}`; + this.imagePreview.classList.remove('hidden'); + this.cropBtn.classList.remove('hidden'); + this.captureBtn.disabled = false; + this.captureBtn.innerHTML = 'Capture'; + this.sendToClaudeBtn.classList.add('hidden'); + this.extractTextBtn.classList.add('hidden'); + this.textEditor.classList.add('hidden'); + window.showToast('Screenshot captured successfully'); + } else { + window.showToast('Failed to capture screenshot: ' + data.error, 'error'); + this.captureBtn.disabled = false; + this.captureBtn.innerHTML = 'Capture'; + } + }); + + // Mathpix text extraction response handler + this.socket.on('mathpix_response', (data) => { + console.log('Received mathpix_response:', data); + this.updateStatusLight(data.status); + + switch (data.status) { + case 'started': + console.log('Text extraction started'); + this.extractedText.value = ''; + this.extractTextBtn.disabled = true; + break; + + case 'completed': + if (data.content) { + console.log('Received extracted text:', data.content); + const confidenceMatch = data.content.match(/Confidence: (\d+\.\d+)%/); + if (confidenceMatch) { + const confidence = confidenceMatch[1]; + document.getElementById('confidenceDisplay').textContent = confidence + '%'; + this.extractedText.value = data.content.replace(/Confidence: \d+\.\d+%\n\n/, ''); + } else { + this.extractedText.value = data.content; + document.getElementById('confidenceDisplay').textContent = ''; + } + this.textEditor.classList.remove('hidden'); + } + this.extractTextBtn.disabled = false; + this.extractTextBtn.innerHTML = 'Extract Text'; + window.showToast('Text extracted successfully'); + break; + + case 'error': + console.error('Text extraction error:', data.error); + const errorMessage = data.error || 'Unknown error occurred'; + window.showToast('Failed to extract text: ' + errorMessage, 'error'); + this.extractTextBtn.disabled = false; + this.extractTextBtn.innerHTML = 'Extract Text'; + break; + + default: + console.warn('Unknown mathpix response status:', data.status); + if (data.error) { + window.showToast('Text extraction failed: ' + data.error, 'error'); + this.extractTextBtn.disabled = false; + this.extractTextBtn.innerHTML = 'Extract Text'; + } + } + }); + + // AI analysis response handler + this.socket.on('claude_response', (data) => { + console.log('Received claude_response:', data); + this.updateStatusLight(data.status); + + switch (data.status) { + case 'started': + console.log('AI analysis started'); + this.responseContent.textContent = ''; + this.sendToClaudeBtn.disabled = true; + this.sendExtractedTextBtn.disabled = true; + break; + + case 'streaming': + if (data.content) { + console.log('Received AI content:', data.content); + this.responseContent.textContent += data.content; + } + break; + + case 'completed': + if (data.content) { + console.log('Received final AI content:', data.content); + this.responseContent.textContent += data.content; + } + this.sendToClaudeBtn.disabled = false; + this.sendExtractedTextBtn.disabled = false; + this.addToHistory(this.croppedImage, this.responseContent.textContent); + window.showToast('Analysis completed successfully'); + break; + + case 'error': + console.error('AI analysis error:', data.error); + const errorMessage = data.error || 'Unknown error occurred'; + this.responseContent.textContent += '\nError: ' + errorMessage; + this.sendToClaudeBtn.disabled = false; + this.sendExtractedTextBtn.disabled = false; + window.showToast('Analysis failed: ' + errorMessage, 'error'); + break; + + default: + console.warn('Unknown claude response status:', data.status); + if (data.error) { + this.responseContent.textContent += '\nError: ' + data.error; + this.sendToClaudeBtn.disabled = false; + this.sendExtractedTextBtn.disabled = false; + window.showToast('Unknown error occurred', 'error'); + } + } + }); + } + + setupEventListeners() { + // Add click handler for app title + const appTitle = document.getElementById('appTitle'); + if (appTitle) { + appTitle.addEventListener('click', () => { + this.resetInterface(); + }); + } + + // Handle page visibility changes + document.addEventListener('visibilitychange', () => { + if (document.visibilityState === 'visible') { + console.log('Page became visible, checking connection...'); + // Check connection status and reset if needed + if (this.socket && !this.socket.connected && !this.isReconnecting) { + console.log('Connection lost while page was hidden, resetting...'); + this.resetConnection(); + } + } + }); + + // Handle before unload to clean up properly + window.addEventListener('beforeunload', () => { + if (this.socket) { + console.log('Page unloading, cleaning up socket...'); + // Store connection state in sessionStorage + sessionStorage.setItem('wasConnected', 'true'); + this.socket.disconnect(); + } + }); + + // Check if we need to reconnect after a page reload + if (sessionStorage.getItem('wasConnected') === 'true') { + console.log('Page reloaded, initiating immediate reconnection...'); + sessionStorage.removeItem('wasConnected'); + // Force an immediate connection attempt + setTimeout(() => { + if (!this.socket?.connected && !this.isReconnecting) { + this.resetConnection(); + } + }, 500); + } + + this.setupCaptureEvents(); + this.setupCropEvents(); + this.setupAnalysisEvents(); + this.setupKeyboardShortcuts(); + } + + resetInterface() { + if (!this.captureBtn) return; + + // Clear all intervals + if (this.heartbeatInterval) { + clearInterval(this.heartbeatInterval); + this.heartbeatInterval = null; + } + + if (this.connectionCheckInterval) { + clearInterval(this.connectionCheckInterval); + this.connectionCheckInterval = null; + } + + // Clean up cropper if it exists + if (this.cropper) { + this.cropper.destroy(); + this.cropper = null; + } + + // Clean up socket if it exists + if (this.socket) { + this.socket.removeAllListeners(); // Remove all event listeners + this.socket.disconnect(); + this.socket = null; + } + + // Show capture button + this.captureBtn.classList.remove('hidden'); + + // Hide all panels + const panels = ['historyPanel', 'settingsPanel']; + panels.forEach(panelId => { + const panel = document.getElementById(panelId); + if (panel) panel.classList.add('hidden'); + }); + + // Reset image preview and related buttons + const elements = [ + this.imagePreview, + this.cropBtn, + this.sendToClaudeBtn, + this.extractTextBtn, + this.textEditor + ]; + + elements.forEach(element => { + if (element) element.classList.add('hidden'); + }); + + // Clear text areas + if (this.extractedText) this.extractedText.value = ''; + if (this.responseContent) this.responseContent.textContent = ''; + + const confidenceDisplay = document.getElementById('confidenceDisplay'); + if (confidenceDisplay) confidenceDisplay.textContent = ''; + + // Hide Claude panel + if (this.claudePanel) this.claudePanel.classList.add('hidden'); + } +} + +// Initialize the application +window.addEventListener('DOMContentLoaded', () => { + window.app = new SnapSolver(); +}); diff --git a/static/js/events.js b/static/js/events.js new file mode 100644 index 0000000..6d9e644 --- /dev/null +++ b/static/js/events.js @@ -0,0 +1,385 @@ +// Events handling extension for SnapSolver class +Object.assign(SnapSolver.prototype, { + setupCaptureEvents() { + if (!this.captureBtn) { + console.error('Capture button not initialized'); + return; + } + + // Capture button + this.captureBtn.addEventListener('click', async () => { + if (!this.socket) { + console.error('Socket not initialized'); + window.showToast('Connection not initialized. Please refresh the page.', 'error'); + return; + } + + if (!this.socket.connected) { + console.error('Socket not connected'); + window.showToast('Server connection lost. Attempting to reconnect...', 'error'); + this.socket.connect(); + return; + } + + try { + this.captureBtn.disabled = true; + this.captureBtn.innerHTML = 'Capturing...'; + + // Set a timeout to re-enable the button if no response is received + const timeout = setTimeout(() => { + if (this.captureBtn.disabled) { + this.captureBtn.disabled = false; + this.captureBtn.innerHTML = 'Capture'; + window.showToast('Screenshot capture timed out. Please try again.', 'error'); + } + }, 10000); + + this.socket.emit('request_screenshot', null, (error) => { + if (error) { + clearTimeout(timeout); + console.error('Screenshot error:', error); + window.showToast('Error capturing screenshot: ' + error, 'error'); + this.captureBtn.disabled = false; + this.captureBtn.innerHTML = 'Capture'; + } + }); + } catch (error) { + console.error('Capture error:', error); + window.showToast('Error requesting screenshot: ' + error.message, 'error'); + this.captureBtn.disabled = false; + this.captureBtn.innerHTML = 'Capture'; + } + }); + }, + + setupCropEvents() { + if (!this.cropBtn || !this.screenshotImg) { + console.error('Required elements not initialized'); + return; + } + + // Crop button + this.cropBtn.addEventListener('click', () => { + if (this.screenshotImg.src) { + this.initializeCropper(); + } + }); + + // Crop confirm button + const cropConfirm = document.getElementById('cropConfirm'); + if (cropConfirm) { + cropConfirm.addEventListener('click', () => { + if (this.cropper) { + try { + console.log('Starting crop operation...'); + + if (!this.cropper) { + throw new Error('Cropper not initialized'); + } + + const cropBoxData = this.cropper.getCropBoxData(); + console.log('Crop box data:', cropBoxData); + + if (!cropBoxData || typeof cropBoxData.width !== 'number' || typeof cropBoxData.height !== 'number') { + throw new Error('Invalid crop box data'); + } + + if (cropBoxData.width < 10 || cropBoxData.height < 10) { + throw new Error('Crop area is too small. Please select a larger area (minimum 10x10 pixels).'); + } + + console.log('Getting cropped canvas...'); + const canvas = this.cropper.getCroppedCanvas({ + maxWidth: 2560, + maxHeight: 1440, + fillColor: '#fff', + imageSmoothingEnabled: true, + imageSmoothingQuality: 'high', + }); + + if (!canvas) { + throw new Error('Failed to create cropped canvas'); + } + + console.log('Canvas created successfully'); + + console.log('Converting to data URL...'); + try { + this.croppedImage = canvas.toDataURL('image/png'); + console.log('Data URL conversion successful'); + } catch (dataUrlError) { + console.error('Data URL conversion error:', dataUrlError); + throw new Error('Failed to process cropped image. The image might be too large or memory insufficient.'); + } + + if (this.cropper) { + this.cropper.destroy(); + this.cropper = null; + } + + this.cropContainer.classList.add('hidden'); + const cropArea = document.querySelector('.crop-area'); + if (cropArea) cropArea.innerHTML = ''; + + this.screenshotImg.src = this.croppedImage; + this.imagePreview.classList.remove('hidden'); + this.cropBtn.classList.remove('hidden'); + this.sendToClaudeBtn.classList.remove('hidden'); + this.extractTextBtn.classList.remove('hidden'); + window.showToast('Image cropped successfully'); + } catch (error) { + console.error('Cropping error details:', { + message: error.message, + stack: error.stack, + cropperState: this.cropper ? 'initialized' : 'not initialized' + }); + window.showToast(error.message || 'Error while cropping image', 'error'); + } finally { + if (this.cropper) { + this.cropper.destroy(); + this.cropper = null; + } + } + } + }); + } + + // Crop cancel button + const cropCancel = document.getElementById('cropCancel'); + if (cropCancel) { + cropCancel.addEventListener('click', () => { + if (this.cropper) { + this.cropper.destroy(); + this.cropper = null; + } + this.cropContainer.classList.add('hidden'); + this.sendToClaudeBtn.classList.add('hidden'); + this.extractTextBtn.classList.add('hidden'); + const cropArea = document.querySelector('.crop-area'); + if (cropArea) cropArea.innerHTML = ''; + }); + } + }, + + setupAnalysisEvents() { + // Set up text extraction socket event listener once + this.socket.on('text_extracted', (data) => { + if (data.error) { + console.error('Text extraction error:', data.error); + window.showToast('Failed to extract text: ' + data.error, 'error'); + if (this.extractedText) { + this.extractedText.value = ''; + this.extractedText.disabled = false; + } + } else if (data.content) { + if (this.extractedText) { + this.extractedText.value = data.content; + this.extractedText.disabled = false; + // Scroll to make text editor visible + this.extractedText.scrollIntoView({ behavior: 'smooth', block: 'nearest' }); + } + window.showToast('Text extracted successfully'); + } + + if (this.extractTextBtn) { + this.extractTextBtn.disabled = false; + this.extractTextBtn.innerHTML = 'Extract Text'; + } + }); + + // Extract Text button + if (this.extractTextBtn) { + this.extractTextBtn.addEventListener('click', () => { + if (!this.croppedImage) { + window.showToast('Please crop the image first', 'error'); + return; + } + + const settings = window.settingsManager.getSettings(); + const mathpixAppId = settings.mathpixAppId; + const mathpixAppKey = settings.mathpixAppKey; + + if (!mathpixAppId || !mathpixAppKey) { + window.showToast('Please enter Mathpix credentials in settings', 'error'); + const settingsPanel = document.getElementById('settingsPanel'); + if (settingsPanel) settingsPanel.classList.remove('hidden'); + return; + } + + this.extractTextBtn.disabled = true; + this.extractTextBtn.innerHTML = 'Extracting...'; + + try { + // Show text editor and prepare UI + const textEditor = document.getElementById('textEditor'); + if (textEditor) { + textEditor.classList.remove('hidden'); + // Scroll to make text editor visible + textEditor.scrollIntoView({ behavior: 'smooth', block: 'nearest' }); + } + + // Clear any previous text and show loading indicator + if (this.extractedText) { + this.extractedText.value = 'Extracting text...'; + this.extractedText.disabled = true; + } + + // Set up timeout to re-enable button if no response + const timeout = setTimeout(() => { + if (this.extractTextBtn && this.extractTextBtn.disabled) { + this.extractTextBtn.disabled = false; + this.extractTextBtn.innerHTML = 'Extract Text'; + if (this.extractedText) { + this.extractedText.value = ''; + this.extractedText.disabled = false; + } + window.showToast('Text extraction timed out. Please try again.', 'error'); + } + }, 30000); // 30 second timeout + + this.socket.emit('extract_text', { + image: this.croppedImage.split(',')[1], + settings: { + mathpixApiKey: `${mathpixAppId}:${mathpixAppKey}` + } + }, (error) => { + // Clear timeout on acknowledgement + clearTimeout(timeout); + if (error) { + console.error('Text extraction error:', error); + window.showToast('Failed to start text extraction: ' + error, 'error'); + this.extractTextBtn.disabled = false; + this.extractTextBtn.innerHTML = 'Extract Text'; + if (this.extractedText) { + this.extractedText.value = ''; + this.extractedText.disabled = false; + } + } + }); + } catch (error) { + console.error('Text extraction error:', error); + window.showToast('Failed to extract text: ' + error.message, 'error'); + this.extractTextBtn.disabled = false; + this.extractTextBtn.innerHTML = 'Extract Text'; + if (this.extractedText) { + this.extractedText.value = ''; + this.extractedText.disabled = false; + } + } + }); + } + + // Send Extracted Text button + if (this.sendExtractedTextBtn && this.extractedText) { + this.sendExtractedTextBtn.addEventListener('click', () => { + const text = this.extractedText.value.trim(); + if (!text) { + window.showToast('Please enter some text', 'error'); + return; + } + + const settings = window.settingsManager.getSettings(); + const apiKey = window.settingsManager.getApiKey(); + + if (!apiKey) { + const settingsPanel = document.getElementById('settingsPanel'); + if (settingsPanel) settingsPanel.classList.remove('hidden'); + window.showToast('Please configure API key in settings', 'error'); + return; + } + + if (this.claudePanel) this.claudePanel.classList.remove('hidden'); + if (this.responseContent) this.responseContent.textContent = ''; + this.sendExtractedTextBtn.disabled = true; + + try { + this.socket.emit('analyze_text', { + text: text, + settings: { + apiKey: apiKey, + model: settings.model || 'claude-3-5-sonnet-20241022', + temperature: parseFloat(settings.temperature) || 0.7, + systemPrompt: settings.systemPrompt || 'You are an expert at analyzing questions and providing detailed solutions.', + proxyEnabled: settings.proxyEnabled || false, + proxyHost: settings.proxyHost || '127.0.0.1', + proxyPort: settings.proxyPort || '4780' + } + }); + } catch (error) { + console.error('Text analysis error:', error); + if (this.responseContent) { + this.responseContent.textContent = 'Error: Failed to send text for analysis - ' + error.message; + } + this.sendExtractedTextBtn.disabled = false; + window.showToast('Failed to send text for analysis', 'error'); + } + }); + } + + // Send to Claude button + if (this.sendToClaudeBtn) { + this.sendToClaudeBtn.addEventListener('click', () => { + if (!this.croppedImage) { + window.showToast('Please crop the image first', 'error'); + return; + } + + const settings = window.settingsManager.getSettings(); + const apiKey = window.settingsManager.getApiKey(); + + if (!apiKey) { + const settingsPanel = document.getElementById('settingsPanel'); + if (settingsPanel) settingsPanel.classList.remove('hidden'); + window.showToast('Please configure API key in settings', 'error'); + return; + } + + if (this.claudePanel) this.claudePanel.classList.remove('hidden'); + if (this.responseContent) this.responseContent.textContent = ''; + this.sendToClaudeBtn.disabled = true; + + try { + this.socket.emit('analyze_image', { + image: this.croppedImage.split(',')[1], + settings: { + apiKey: apiKey, + model: settings.model || 'claude-3-5-sonnet-20241022', + temperature: parseFloat(settings.temperature) || 0.7, + systemPrompt: settings.systemPrompt || 'You are an expert at analyzing questions and providing detailed solutions.', + proxyEnabled: settings.proxyEnabled || false, + proxyHost: settings.proxyHost || '127.0.0.1', + proxyPort: settings.proxyPort || '4780' + } + }); + } catch (error) { + console.error('Image analysis error:', error); + if (this.responseContent) { + this.responseContent.textContent = 'Error: Failed to send image for analysis - ' + error.message; + } + this.sendToClaudeBtn.disabled = false; + window.showToast('Failed to send image for analysis', 'error'); + } + }); + } + }, + + setupKeyboardShortcuts() { + // Keyboard shortcuts for capture and crop + document.addEventListener('keydown', (e) => { + if (e.ctrlKey || e.metaKey) { + switch(e.key) { + case 'c': + if (this.captureBtn && !this.captureBtn.disabled) { + this.captureBtn.click(); + } + break; + case 'x': + if (this.cropBtn && !this.cropBtn.disabled) { + this.cropBtn.click(); + } + break; + } + } + }); + } +}); diff --git a/static/js/history.js b/static/js/history.js new file mode 100644 index 0000000..11d52c3 --- /dev/null +++ b/static/js/history.js @@ -0,0 +1,93 @@ +// History management extension for SnapSolver class +Object.assign(SnapSolver.prototype, { + addToHistory(imageData, response) { + const historyItem = { + id: Date.now(), + timestamp: new Date().toISOString(), + image: imageData, + extractedText: this.extractedText.value || null, + response: response + }; + this.history.unshift(historyItem); + if (this.history.length > 10) this.history.pop(); + localStorage.setItem('snapHistory', JSON.stringify(this.history)); + window.renderHistory(); + } +}); + +// Global function for history rendering +window.renderHistory = function() { + const content = document.querySelector('.history-content'); + const history = JSON.parse(localStorage.getItem('snapHistory') || '[]'); + + if (history.length === 0) { + content.innerHTML = ` +
+ +

No history yet

+
+ `; + return; + } + + content.innerHTML = history.map(item => ` +
+
+ ${new Date(item.timestamp).toLocaleString()} + +
+ Historical screenshot +
+ `).join(''); + + // Add click handlers for history items + content.querySelectorAll('.delete-history').forEach(btn => { + btn.addEventListener('click', (e) => { + e.stopPropagation(); + const id = parseInt(btn.dataset.id); + const updatedHistory = history.filter(item => item.id !== id); + localStorage.setItem('snapHistory', JSON.stringify(updatedHistory)); + window.renderHistory(); + window.showToast('History item deleted'); + }); + }); + + content.querySelectorAll('.history-item').forEach(item => { + item.addEventListener('click', () => { + const historyItem = history.find(h => h.id === parseInt(item.dataset.id)); + if (historyItem) { + // Display the image + window.app.screenshotImg.src = historyItem.image; + window.app.imagePreview.classList.remove('hidden'); + document.getElementById('historyPanel').classList.add('hidden'); + + // Hide all action buttons in history view + window.app.cropBtn.classList.add('hidden'); + window.app.captureBtn.classList.add('hidden'); + window.app.sendToClaudeBtn.classList.add('hidden'); + window.app.extractTextBtn.classList.add('hidden'); + window.app.sendExtractedTextBtn.classList.add('hidden'); + + // Reset confidence display + document.getElementById('confidenceDisplay').textContent = ''; + + // Only show text editor if there was extracted text + if (historyItem.extractedText) { + window.app.textEditor.classList.remove('hidden'); + window.app.extractedText.value = historyItem.extractedText; + } else { + window.app.textEditor.classList.add('hidden'); + window.app.extractedText.value = ''; + } + + // Show response if it exists + if (historyItem.response) { + window.app.claudePanel.classList.remove('hidden'); + window.app.responseContent.textContent = historyItem.response; + } + } + }); + }); +}; diff --git a/static/js/main.js b/static/js/main.js index 184e7fb..bbc7e34 100644 --- a/static/js/main.js +++ b/static/js/main.js @@ -136,14 +136,24 @@ class SnapSolver { }); // Text extraction response handler - this.socket.on('text_extraction_response', (data) => { - if (data.success) { - this.extractedText.value = data.text; - this.textEditor.classList.remove('hidden'); - window.showToast('Text extracted successfully'); - } else { + this.socket.on('text_extracted', (data) => { + if (data.error) { + console.error('Text extraction error:', data.error); window.showToast('Failed to extract text: ' + data.error, 'error'); + if (this.extractedText) { + this.extractedText.value = ''; + this.extractedText.disabled = false; + } + } else if (data.content) { + if (this.extractedText) { + this.extractedText.value = data.content; + this.extractedText.disabled = false; + // Scroll to make text editor visible + this.extractedText.scrollIntoView({ behavior: 'smooth', block: 'nearest' }); + } + window.showToast('Text extracted successfully'); } + this.extractTextBtn.disabled = false; this.extractTextBtn.innerHTML = 'Extract Text'; }); @@ -405,9 +415,31 @@ class SnapSolver { this.extractTextBtn.disabled = true; this.extractTextBtn.innerHTML = 'Extracting...'; + const settings = window.settingsManager.getSettings(); + const mathpixAppId = settings.mathpixAppId; + const mathpixAppKey = settings.mathpixAppKey; + + if (!mathpixAppId || !mathpixAppKey) { + window.showToast('Please enter Mathpix credentials in settings', 'error'); + document.getElementById('settingsPanel').classList.remove('hidden'); + this.extractTextBtn.disabled = false; + this.extractTextBtn.innerHTML = 'Extract Text'; + return; + } + + // Show text editor and prepare UI + this.textEditor.classList.remove('hidden'); + if (this.extractedText) { + this.extractedText.value = 'Extracting text...'; + this.extractedText.disabled = true; + } + try { this.socket.emit('extract_text', { - image: this.croppedImage.split(',')[1] + image: this.croppedImage.split(',')[1], + settings: { + mathpixApiKey: `${mathpixAppId}:${mathpixAppKey}` + } }); } catch (error) { window.showToast('Failed to extract text: ' + error.message, 'error'); diff --git a/static/js/settings.js b/static/js/settings.js index adb5baa..bec6f46 100644 --- a/static/js/settings.js +++ b/static/js/settings.js @@ -18,6 +18,10 @@ class SettingsManager { this.proxyPortInput = document.getElementById('proxyPort'); this.proxySettings = document.getElementById('proxySettings'); + // Initialize Mathpix inputs + this.mathpixAppIdInput = document.getElementById('mathpixAppId'); + this.mathpixAppKeyInput = document.getElementById('mathpixAppKey'); + // API Key elements this.apiKeyInputs = { 'claude-3-5-sonnet-20241022': document.getElementById('claudeApiKey'), @@ -47,6 +51,14 @@ class SettingsManager { loadSettings() { const settings = JSON.parse(localStorage.getItem('aiSettings') || '{}'); + // Load Mathpix credentials + if (settings.mathpixAppId) { + this.mathpixAppIdInput.value = settings.mathpixAppId; + } + if (settings.mathpixAppKey) { + this.mathpixAppKeyInput.value = settings.mathpixAppKey; + } + // Load API keys if (settings.apiKeys) { Object.entries(this.apiKeyInputs).forEach(([model, input]) => { @@ -89,6 +101,8 @@ class SettingsManager { saveSettings() { const settings = { apiKeys: {}, + mathpixAppId: this.mathpixAppIdInput.value, + mathpixAppKey: this.mathpixAppKeyInput.value, model: this.modelSelect.value, temperature: this.temperatureInput.value, language: this.languageInput.value, @@ -129,7 +143,9 @@ class SettingsManager { systemPrompt: this.systemPromptInput.value + ` Please respond in ${this.languageInput.value}.`, proxyEnabled: this.proxyEnabledInput.checked, proxyHost: this.proxyHostInput.value, - proxyPort: this.proxyPortInput.value + proxyPort: this.proxyPortInput.value, + mathpixAppId: this.mathpixAppIdInput.value, + mathpixAppKey: this.mathpixAppKeyInput.value }; } @@ -139,6 +155,10 @@ class SettingsManager { input.addEventListener('change', () => this.saveSettings()); }); + // Save Mathpix settings on change + this.mathpixAppIdInput.addEventListener('change', () => this.saveSettings()); + this.mathpixAppKeyInput.addEventListener('change', () => this.saveSettings()); + this.modelSelect.addEventListener('change', (e) => { this.updateVisibleApiKey(e.target.value); this.saveSettings(); diff --git a/templates/index.html b/templates/index.html index a8571cf..f3f68ad 100644 --- a/templates/index.html +++ b/templates/index.html @@ -98,6 +98,28 @@
+
+

OCR Configuration

+
+ +
+ + +
+
+
+ +
+ + +
+
+
+

AI Configuration