start_servers.py 7.95 KB
#!/usr/bin/env python3
"""
Production-ready server startup script with proper error handling and monitoring.
"""

import os
import sys
import signal
import time
import subprocess
import logging
from typing import Dict, List, Optional
import multiprocessing
import threading

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
    handlers=[
        logging.StreamHandler(),
        logging.FileHandler('/tmp/search_engine_startup.log', mode='a')
    ]
)
logger = logging.getLogger(__name__)

class ServerManager:
    """Manages frontend and API server processes."""

    def __init__(self):
        self.processes: Dict[str, subprocess.Popen] = {}
        self.running = True

    def start_frontend_server(self) -> bool:
        """Start the frontend server."""
        try:
            frontend_script = os.path.join(os.path.dirname(__file__), 'frontend_server.py')

            cmd = [sys.executable, frontend_script]
            env = os.environ.copy()
            env['PYTHONUNBUFFERED'] = '1'

            process = subprocess.Popen(
                cmd,
                env=env,
                stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT,
                universal_newlines=True,
                bufsize=1
            )

            self.processes['frontend'] = process
            logger.info(f"Frontend server started with PID: {process.pid}")

            # Start monitoring thread
            threading.Thread(
                target=self._monitor_output,
                args=('frontend', process),
                daemon=True
            ).start()

            return True

        except Exception as e:
            logger.error(f"Failed to start frontend server: {e}")
            return False

    def start_api_server(self, customer: str = "customer1", es_host: str = "http://localhost:9200") -> bool:
        """Start the API server."""
        try:
            cmd = [
                sys.executable, 'main.py', 'serve',
                '--customer', customer,
                '--es-host', es_host,
                '--host', '0.0.0.0',
                '--port', '6002'
            ]

            env = os.environ.copy()
            env['PYTHONUNBUFFERED'] = '1'
            env['CUSTOMER_ID'] = customer
            env['ES_HOST'] = es_host

            process = subprocess.Popen(
                cmd,
                env=env,
                stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT,
                universal_newlines=True,
                bufsize=1
            )

            self.processes['api'] = process
            logger.info(f"API server started with PID: {process.pid}")

            # Start monitoring thread
            threading.Thread(
                target=self._monitor_output,
                args=('api', process),
                daemon=True
            ).start()

            return True

        except Exception as e:
            logger.error(f"Failed to start API server: {e}")
            return False

    def _monitor_output(self, name: str, process: subprocess.Popen):
        """Monitor process output and log appropriately."""
        try:
            for line in iter(process.stdout.readline, ''):
                if line.strip() and self.running:
                    # Filter out scanner noise for frontend server
                    if name == 'frontend':
                        noise_patterns = [
                            'code 400',
                            'Bad request version',
                            'Bad request syntax',
                            'Bad HTTP/0.9 request type'
                        ]
                        if any(pattern in line for pattern in noise_patterns):
                            continue

                    logger.info(f"[{name}] {line.strip()}")

        except Exception as e:
            if self.running:
                logger.error(f"Error monitoring {name} output: {e}")

    def check_servers(self) -> bool:
        """Check if all servers are still running."""
        all_running = True

        for name, process in self.processes.items():
            if process.poll() is not None:
                logger.error(f"{name} server has stopped with exit code: {process.returncode}")
                all_running = False

        return all_running

    def stop_all(self):
        """Stop all servers gracefully."""
        logger.info("Stopping all servers...")
        self.running = False

        for name, process in self.processes.items():
            try:
                logger.info(f"Stopping {name} server (PID: {process.pid})...")

                # Try graceful shutdown first
                process.terminate()

                # Wait up to 10 seconds for graceful shutdown
                try:
                    process.wait(timeout=10)
                    logger.info(f"{name} server stopped gracefully")
                except subprocess.TimeoutExpired:
                    # Force kill if graceful shutdown fails
                    logger.warning(f"{name} server didn't stop gracefully, forcing...")
                    process.kill()
                    process.wait()
                    logger.info(f"{name} server stopped forcefully")

            except Exception as e:
                logger.error(f"Error stopping {name} server: {e}")

        self.processes.clear()
        logger.info("All servers stopped")

def signal_handler(signum, frame):
    """Handle shutdown signals."""
    logger.info(f"Received signal {signum}, shutting down...")
    if 'manager' in globals():
        manager.stop_all()
    sys.exit(0)

def main():
    """Main function to start all servers."""
    global manager

    parser = argparse.ArgumentParser(description='Start SearchEngine servers')
    parser.add_argument('--customer', default='customer1', help='Customer ID')
    parser.add_argument('--es-host', default='http://localhost:9200', help='Elasticsearch host')
    parser.add_argument('--check-dependencies', action='store_true', help='Check dependencies before starting')
    args = parser.parse_args()

    logger.info("Starting SearchEngine servers...")
    logger.info(f"Customer: {args.customer}")
    logger.info(f"Elasticsearch: {args.es_host}")

    # Check dependencies if requested
    if args.check_dependencies:
        logger.info("Checking dependencies...")
        try:
            import slowapi
            import anyio
            logger.info("✓ All dependencies available")
        except ImportError as e:
            logger.error(f"✗ Missing dependency: {e}")
            logger.info("Please run: pip install -r requirements_server.txt")
            sys.exit(1)

    manager = ServerManager()

    # Set up signal handlers
    signal.signal(signal.SIGINT, signal_handler)
    signal.signal(signal.SIGTERM, signal_handler)

    try:
        # Start servers
        if not manager.start_api_server(args.customer, args.es_host):
            logger.error("Failed to start API server")
            sys.exit(1)

        # Wait a moment before starting frontend server
        time.sleep(2)

        if not manager.start_frontend_server():
            logger.error("Failed to start frontend server")
            manager.stop_all()
            sys.exit(1)

        logger.info("All servers started successfully!")
        logger.info("Frontend: http://localhost:6003")
        logger.info("API: http://localhost:6002")
        logger.info("API Docs: http://localhost:6002/docs")
        logger.info("Press Ctrl+C to stop all servers")

        # Monitor servers
        while manager.running:
            if not manager.check_servers():
                logger.error("One or more servers have stopped unexpectedly")
                manager.stop_all()
                sys.exit(1)

            time.sleep(5)  # Check every 5 seconds

    except KeyboardInterrupt:
        logger.info("Received interrupt signal")
    except Exception as e:
        logger.error(f"Unexpected error: {e}")
    finally:
        manager.stop_all()

if __name__ == '__main__':
    main()