Add ComfyUI output capture for crash debugging
All checks were successful
Build and Push Docker Image / build (push) Successful in 3m25s
All checks were successful
Build and Push Docker Image / build (push) Successful in 3m25s
- Add background thread to read ComfyUI stdout in real-time - Store last 200 lines in circular buffer - Echo output to RunPod logs with [ComfyUI] prefix - Include last 100 lines in error responses for debugging - Add comfyui_output field to error responses This will help diagnose why ComfyUI crashes during generation. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
54
handler.py
54
handler.py
@@ -20,6 +20,8 @@ import base64
|
|||||||
import uuid
|
import uuid
|
||||||
import subprocess
|
import subprocess
|
||||||
import signal
|
import signal
|
||||||
|
import threading
|
||||||
|
from collections import deque
|
||||||
import requests
|
import requests
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import runpod
|
import runpod
|
||||||
@@ -42,8 +44,36 @@ NODE_STEPS = "150"
|
|||||||
NODE_SPLIT_STEP = "151"
|
NODE_SPLIT_STEP = "151"
|
||||||
NODE_SAVE_VIDEO = "117"
|
NODE_SAVE_VIDEO = "117"
|
||||||
|
|
||||||
# Global ComfyUI process
|
# Global ComfyUI process and output capture
|
||||||
comfyui_process = None
|
comfyui_process = None
|
||||||
|
comfyui_output_buffer = deque(maxlen=200) # Keep last 200 lines
|
||||||
|
comfyui_output_lock = threading.Lock()
|
||||||
|
comfyui_reader_thread = None
|
||||||
|
|
||||||
|
|
||||||
|
def _read_comfyui_output():
|
||||||
|
"""Background thread to read ComfyUI stdout and store in buffer."""
|
||||||
|
global comfyui_process, comfyui_output_buffer
|
||||||
|
while comfyui_process is not None:
|
||||||
|
try:
|
||||||
|
line = comfyui_process.stdout.readline()
|
||||||
|
if line:
|
||||||
|
decoded = line.decode('utf-8', errors='replace').rstrip()
|
||||||
|
with comfyui_output_lock:
|
||||||
|
comfyui_output_buffer.append(decoded)
|
||||||
|
print(f"[ComfyUI] {decoded}") # Echo to RunPod logs
|
||||||
|
elif comfyui_process.poll() is not None:
|
||||||
|
# Process ended
|
||||||
|
break
|
||||||
|
except Exception:
|
||||||
|
break
|
||||||
|
|
||||||
|
|
||||||
|
def get_comfyui_output(last_n: int = 50) -> list:
|
||||||
|
"""Get the last N lines of ComfyUI output."""
|
||||||
|
with comfyui_output_lock:
|
||||||
|
lines = list(comfyui_output_buffer)
|
||||||
|
return lines[-last_n:] if len(lines) > last_n else lines
|
||||||
|
|
||||||
|
|
||||||
class JobLogger:
|
class JobLogger:
|
||||||
@@ -68,7 +98,7 @@ class JobLogger:
|
|||||||
|
|
||||||
def start_comfyui(logger: JobLogger = None):
|
def start_comfyui(logger: JobLogger = None):
|
||||||
"""Start ComfyUI server if not already running."""
|
"""Start ComfyUI server if not already running."""
|
||||||
global comfyui_process
|
global comfyui_process, comfyui_reader_thread, comfyui_output_buffer
|
||||||
|
|
||||||
def log(msg):
|
def log(msg):
|
||||||
if logger:
|
if logger:
|
||||||
@@ -80,6 +110,10 @@ def start_comfyui(logger: JobLogger = None):
|
|||||||
log("ComfyUI server already running")
|
log("ComfyUI server already running")
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
# Clear output buffer for fresh start
|
||||||
|
with comfyui_output_lock:
|
||||||
|
comfyui_output_buffer.clear()
|
||||||
|
|
||||||
log("Starting ComfyUI server...")
|
log("Starting ComfyUI server...")
|
||||||
|
|
||||||
comfyui_process = subprocess.Popen(
|
comfyui_process = subprocess.Popen(
|
||||||
@@ -95,6 +129,10 @@ def start_comfyui(logger: JobLogger = None):
|
|||||||
preexec_fn=os.setsid if hasattr(os, 'setsid') else None
|
preexec_fn=os.setsid if hasattr(os, 'setsid') else None
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Start background thread to capture output
|
||||||
|
comfyui_reader_thread = threading.Thread(target=_read_comfyui_output, daemon=True)
|
||||||
|
comfyui_reader_thread.start()
|
||||||
|
|
||||||
# Wait for server to be ready
|
# Wait for server to be ready
|
||||||
start_time = time.time()
|
start_time = time.time()
|
||||||
last_status_time = start_time
|
last_status_time = start_time
|
||||||
@@ -600,13 +638,21 @@ def handler(job: dict) -> dict:
|
|||||||
|
|
||||||
except TimeoutError as e:
|
except TimeoutError as e:
|
||||||
logger.log(f"ERROR: Timeout - {str(e)}")
|
logger.log(f"ERROR: Timeout - {str(e)}")
|
||||||
return {"error": str(e), "status": "timeout", "logs": logger.get_logs()}
|
comfyui_logs = get_comfyui_output(100)
|
||||||
|
logger.log(f"ComfyUI output (last 100 lines):")
|
||||||
|
for line in comfyui_logs:
|
||||||
|
logger.log(f" {line}")
|
||||||
|
return {"error": str(e), "status": "timeout", "logs": logger.get_logs(), "comfyui_output": comfyui_logs}
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
import traceback
|
import traceback
|
||||||
tb = traceback.format_exc()
|
tb = traceback.format_exc()
|
||||||
logger.log(f"ERROR: {str(e)}")
|
logger.log(f"ERROR: {str(e)}")
|
||||||
logger.log(f"Traceback:\n{tb}")
|
logger.log(f"Traceback:\n{tb}")
|
||||||
return {"error": str(e), "status": "error", "logs": logger.get_logs()}
|
comfyui_logs = get_comfyui_output(100)
|
||||||
|
logger.log(f"ComfyUI output (last 100 lines):")
|
||||||
|
for line in comfyui_logs:
|
||||||
|
logger.log(f" {line}")
|
||||||
|
return {"error": str(e), "status": "error", "logs": logger.get_logs(), "comfyui_output": comfyui_logs}
|
||||||
|
|
||||||
|
|
||||||
# RunPod serverless entry point
|
# RunPod serverless entry point
|
||||||
|
|||||||
Reference in New Issue
Block a user