import logging
import time
from typing import List, Dict, Any, Callable, Optional
from dataclasses import dataclass
from concurrent.futures import ThreadPoolExecutor, as_completed
import json

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)

@dataclass
class ProcessingResult:
    success: bool
    data: Any = None
    error: Optional[str] = None
    processing_time: float = 0.0

class DataProcessor:
    def __init__(self, batch_size: int = 1000, max_workers: int = 4):
        self.batch_size = batch_size
        self.max_workers = max_workers
        self.pipeline_stages: List[Callable] = []
        self.stats = {
            'processed': 0,
            'errors': 0,
            'start_time': None,
            'end_time': None
        }

    def add_stage(self, stage_func: Callable) -> 'DataProcessor':
        """Add a processing stage to the pipeline."""
        self.pipeline_stages.append(stage_func)
        return self

    def process_item(self, item: Dict[str, Any]) -> ProcessingResult:
        """Process a single item through all pipeline stages."""
        start_time = time.time()
        
        try:
            result = item.copy()
            
            # Apply each pipeline stage
            for stage in self.pipeline_stages:
                result = stage(result)
                if result is None:
                    raise ValueError(f"Stage {stage.__name__} returned None")
            
            processing_time = time.time() - start_time
            return ProcessingResult(
                success=True,
                data=result,
                processing_time=processing_time
            )
            
        except Exception as e:
            processing_time = time.time() - start_time
            error_msg = f"Error processing item: {str(e)}"
            logger.error(error_msg, exc_info=True)
            
            return ProcessingResult(
                success=False,
                error=error_msg,
                processing_time=processing_time
            )

    def process_batch(self, batch: List[Dict[str, Any]]) -> List[ProcessingResult]:
        """Process a batch of items concurrently."""
        results = []
        
        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
            # Submit all items in the batch
            future_to_item = {
                executor.submit(self.process_item, item): item 
                for item in batch
            }
            
            # Collect results as they complete
            for future in as_completed(future_to_item):
                result = future.result()
                results.append(result)
                
                # Update statistics
                if result.success:
                    self.stats['processed'] += 1
                else:
                    self.stats['errors'] += 1
        
        return results

    def process_data(self, data: List[Dict[str, Any]]) -> Dict[str, Any]:
        """Process all data in batches."""
        self.stats['start_time'] = time.time()
        self.stats['processed'] = 0
        self.stats['errors'] = 0
        
        logger.info(f"Starting processing of {len(data)} items")
        
        all_results = []
        successful_results = []
        
        # Process data in batches
        for i in range(0, len(data), self.batch_size):
            batch = data[i:i + self.batch_size]
            batch_num = i // self.batch_size + 1
            total_batches = (len(data) + self.batch_size - 1) // self.batch_size
            
            logger.info(f"Processing batch {batch_num}/{total_batches}")
            
            batch_results = self.process_batch(batch)
            all_results.extend(batch_results)
            
            # Collect successful results
            for result in batch_results:
                if result.success:
                    successful_results.append(result.data)
            
            # Log progress
            progress = (i + len(batch)) / len(data) * 100
            logger.info(f"Progress: {progress:.1f}% complete")
        
        self.stats['end_time'] = time.time()
        total_time = self.stats['end_time'] - self.stats['start_time']
        
        # Generate summary
        summary = {
            'total_items': len(data),
            'successful': self.stats['processed'],
            'errors': self.stats['errors'],
            'success_rate': self.stats['processed'] / len(data) * 100,
            'total_time': total_time,
            'items_per_second': len(data) / total_time if total_time > 0 else 0,
            'results': successful_results
        }
        
        logger.info(f"Processing complete: {summary['successful']}/{summary['total_items']} items processed successfully")
        
        return summary

# Example pipeline stages
def validate_data(item: Dict[str, Any]) -> Dict[str, Any]:
    """Validate required fields."""
    required_fields = ['id', 'name', 'email']
    
    for field in required_fields:
        if field not in item or not item[field]:
            raise ValueError(f"Missing required field: {field}")
    
    return item

def normalize_email(item: Dict[str, Any]) -> Dict[str, Any]:
    """Normalize email address."""
    if 'email' in item:
        item['email'] = item['email'].lower().strip()
    return item

def enrich_data(item: Dict[str, Any]) -> Dict[str, Any]:
    """Add computed fields."""
    item['processed_at'] = time.time()
    item['full_name'] = f"{item.get('first_name', '')} {item.get('last_name', '')}".strip()
    return item

# Usage example
if __name__ == "__main__":
    # Sample data
    sample_data = [
        {"id": 1, "name": "John Doe", "email": "JOHN@EXAMPLE.COM"},
        {"id": 2, "name": "Jane Smith", "email": "jane@example.com"},
        {"id": 3, "name": "Invalid User"},  # Missing email - will cause error
    ]
    
    # Create and configure processor
    processor = (DataProcessor(batch_size=2, max_workers=2)
                .add_stage(validate_data)
                .add_stage(normalize_email)
                .add_stage(enrich_data))
    
    # Process data
    results = processor.process_data(sample_data)
    
    # Print results
    print(json.dumps(results, indent=2, default=str))