Blame view

embeddings/server.py 29.1 KB
7bfb9946   tangwang   向量化模块
1
2
3
  """
  Embedding service (FastAPI).
  
ed948666   tangwang   tidy
4
5
6
  API (simple list-in, list-out; aligned by index):
  - POST /embed/text   body: ["text1", "text2", ...] -> [[...], ...]
  - POST /embed/image  body: ["url_or_path1", ...]  -> [[...], ...]
7bfb9946   tangwang   向量化模块
7
8
  """
  
0a3764c4   tangwang   优化embedding模型加载
9
  import logging
07cf5a93   tangwang   START_EMBEDDING=...
10
  import os
4747e2f4   tangwang   embedding perform...
11
  import pathlib
7bfb9946   tangwang   向量化模块
12
  import threading
efd435cf   tangwang   tei性能调优:
13
  import time
4747e2f4   tangwang   embedding perform...
14
  import uuid
efd435cf   tangwang   tei性能调优:
15
16
  from collections import deque
  from dataclasses import dataclass
4747e2f4   tangwang   embedding perform...
17
  from logging.handlers import TimedRotatingFileHandler
7bfb9946   tangwang   向量化模块
18
19
20
  from typing import Any, Dict, List, Optional
  
  import numpy as np
4747e2f4   tangwang   embedding perform...
21
22
  from fastapi import FastAPI, HTTPException, Request, Response
  from fastapi.concurrency import run_in_threadpool
7bfb9946   tangwang   向量化模块
23
  
4747e2f4   tangwang   embedding perform...
24
  from config.services_config import get_embedding_backend_config
7bfb9946   tangwang   向量化模块
25
  from embeddings.config import CONFIG
c10f90fe   tangwang   cnclip
26
  from embeddings.protocols import ImageEncoderProtocol
7bfb9946   tangwang   向量化模块
27
  
a7920e17   tangwang   项目名称和部署路径修改
28
  app = FastAPI(title="saas-search Embedding Service", version="1.0.0")
7bfb9946   tangwang   向量化模块
29
  
4747e2f4   tangwang   embedding perform...
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
  
  class _DefaultRequestIdFilter(logging.Filter):
      def filter(self, record: logging.LogRecord) -> bool:
          if not hasattr(record, "reqid"):
              record.reqid = "-1"
          return True
  
  
  def configure_embedding_logging() -> None:
      root_logger = logging.getLogger()
      if getattr(root_logger, "_embedding_logging_configured", False):
          return
  
      log_dir = pathlib.Path("logs")
      verbose_dir = log_dir / "verbose"
      log_dir.mkdir(exist_ok=True)
      verbose_dir.mkdir(parents=True, exist_ok=True)
  
      log_level = os.getenv("LOG_LEVEL", "INFO").upper()
      numeric_level = getattr(logging, log_level, logging.INFO)
      formatter = logging.Formatter(
          "%(asctime)s | reqid:%(reqid)s | %(name)s | %(levelname)s | %(message)s"
      )
      request_filter = _DefaultRequestIdFilter()
  
      root_logger.setLevel(numeric_level)
  
      file_handler = TimedRotatingFileHandler(
          filename=log_dir / "embedding_api.log",
          when="midnight",
          interval=1,
          backupCount=30,
          encoding="utf-8",
      )
      file_handler.setLevel(numeric_level)
      file_handler.setFormatter(formatter)
      file_handler.addFilter(request_filter)
      root_logger.addHandler(file_handler)
  
      error_handler = TimedRotatingFileHandler(
          filename=log_dir / "embedding_api_error.log",
          when="midnight",
          interval=1,
          backupCount=30,
          encoding="utf-8",
      )
      error_handler.setLevel(logging.ERROR)
      error_handler.setFormatter(formatter)
      error_handler.addFilter(request_filter)
      root_logger.addHandler(error_handler)
  
      verbose_logger = logging.getLogger("embedding.verbose")
      verbose_logger.setLevel(numeric_level)
      verbose_logger.handlers.clear()
      verbose_logger.propagate = False
  
      verbose_handler = TimedRotatingFileHandler(
          filename=verbose_dir / "embedding_verbose.log",
          when="midnight",
          interval=1,
          backupCount=30,
          encoding="utf-8",
      )
      verbose_handler.setLevel(numeric_level)
      verbose_handler.setFormatter(formatter)
      verbose_handler.addFilter(request_filter)
      verbose_logger.addHandler(verbose_handler)
  
      root_logger._embedding_logging_configured = True  # type: ignore[attr-defined]
  
  
  configure_embedding_logging()
  logger = logging.getLogger(__name__)
  verbose_logger = logging.getLogger("embedding.verbose")
  
0a3764c4   tangwang   优化embedding模型加载
105
  # Models are loaded at startup, not lazily
950a640e   tangwang   embeddings
106
  _text_model: Optional[Any] = None
c10f90fe   tangwang   cnclip
107
  _image_model: Optional[ImageEncoderProtocol] = None
07cf5a93   tangwang   START_EMBEDDING=...
108
  _text_backend_name: str = ""
efd435cf   tangwang   tei性能调优:
109
110
  open_text_model = os.getenv("EMBEDDING_ENABLE_TEXT_MODEL", "true").lower() in ("1", "true", "yes")
  open_image_model = os.getenv("EMBEDDING_ENABLE_IMAGE_MODEL", "true").lower() in ("1", "true", "yes")
7bfb9946   tangwang   向量化模块
111
112
113
114
  
  _text_encode_lock = threading.Lock()
  _image_encode_lock = threading.Lock()
  
4747e2f4   tangwang   embedding perform...
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
  _TEXT_MICROBATCH_WINDOW_SEC = max(
      0.0, float(os.getenv("TEXT_MICROBATCH_WINDOW_MS", "4")) / 1000.0
  )
  _TEXT_REQUEST_TIMEOUT_SEC = max(
      1.0, float(os.getenv("TEXT_REQUEST_TIMEOUT_SEC", "30"))
  )
  _TEXT_MAX_INFLIGHT = max(1, int(os.getenv("TEXT_MAX_INFLIGHT", "32")))
  _IMAGE_MAX_INFLIGHT = max(1, int(os.getenv("IMAGE_MAX_INFLIGHT", "1")))
  _OVERLOAD_STATUS_CODE = int(os.getenv("EMBEDDING_OVERLOAD_STATUS_CODE", "503"))
  _LOG_PREVIEW_COUNT = max(1, int(os.getenv("EMBEDDING_LOG_PREVIEW_COUNT", "3")))
  _LOG_TEXT_PREVIEW_CHARS = max(32, int(os.getenv("EMBEDDING_LOG_TEXT_PREVIEW_CHARS", "120")))
  _LOG_IMAGE_PREVIEW_CHARS = max(32, int(os.getenv("EMBEDDING_LOG_IMAGE_PREVIEW_CHARS", "180")))
  _VECTOR_PREVIEW_DIMS = max(1, int(os.getenv("EMBEDDING_VECTOR_PREVIEW_DIMS", "6")))
  
  
  class _InflightLimiter:
      def __init__(self, name: str, limit: int):
          self.name = name
          self.limit = max(1, int(limit))
          self._sem = threading.BoundedSemaphore(self.limit)
          self._lock = threading.Lock()
          self._active = 0
          self._rejected = 0
          self._completed = 0
          self._failed = 0
          self._max_active = 0
  
      def try_acquire(self) -> tuple[bool, int]:
          if not self._sem.acquire(blocking=False):
              with self._lock:
                  self._rejected += 1
                  active = self._active
              return False, active
          with self._lock:
              self._active += 1
              self._max_active = max(self._max_active, self._active)
              active = self._active
          return True, active
  
      def release(self, *, success: bool) -> int:
          with self._lock:
              self._active = max(0, self._active - 1)
              if success:
                  self._completed += 1
              else:
                  self._failed += 1
              active = self._active
          self._sem.release()
          return active
  
      def snapshot(self) -> Dict[str, int]:
          with self._lock:
              return {
                  "limit": self.limit,
                  "active": self._active,
                  "rejected_total": self._rejected,
                  "completed_total": self._completed,
                  "failed_total": self._failed,
                  "max_active": self._max_active,
              }
  
  
  _text_request_limiter = _InflightLimiter(name="text", limit=_TEXT_MAX_INFLIGHT)
  _image_request_limiter = _InflightLimiter(name="image", limit=_IMAGE_MAX_INFLIGHT)
  
7bfb9946   tangwang   向量化模块
180
  
efd435cf   tangwang   tei性能调优:
181
182
183
184
185
  @dataclass
  class _SingleTextTask:
      text: str
      normalize: bool
      created_at: float
4747e2f4   tangwang   embedding perform...
186
      request_id: str
efd435cf   tangwang   tei性能调优:
187
188
189
190
191
192
193
194
195
      done: threading.Event
      result: Optional[List[float]] = None
      error: Optional[Exception] = None
  
  
  _text_single_queue: "deque[_SingleTextTask]" = deque()
  _text_single_queue_cv = threading.Condition()
  _text_batch_worker: Optional[threading.Thread] = None
  _text_batch_worker_stop = False
28e57bb1   tangwang   日志体系优化
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
  
  
  def _compact_preview(text: str, max_chars: int) -> str:
      compact = " ".join((text or "").split())
      if len(compact) <= max_chars:
          return compact
      return compact[:max_chars] + "..."
  
  
  def _preview_inputs(items: List[str], max_items: int, max_chars: int) -> List[Dict[str, Any]]:
      previews: List[Dict[str, Any]] = []
      for idx, item in enumerate(items[:max_items]):
          previews.append(
              {
                  "idx": idx,
                  "len": len(item),
                  "preview": _compact_preview(item, max_chars),
              }
          )
      return previews
efd435cf   tangwang   tei性能调优:
216
217
  
  
4747e2f4   tangwang   embedding perform...
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
  def _preview_vector(vec: Optional[List[float]], max_dims: int = _VECTOR_PREVIEW_DIMS) -> List[float]:
      if not vec:
          return []
      return [round(float(v), 6) for v in vec[:max_dims]]
  
  
  def _request_log_extra(request_id: str) -> Dict[str, str]:
      return {"reqid": request_id}
  
  
  def _resolve_request_id(http_request: Request) -> str:
      header_value = http_request.headers.get("X-Request-ID")
      if header_value and header_value.strip():
          return header_value.strip()[:32]
      return str(uuid.uuid4())[:8]
  
  
  def _request_client(http_request: Request) -> str:
      client = getattr(http_request, "client", None)
      host = getattr(client, "host", None)
      return str(host or "-")
  
  
efd435cf   tangwang   tei性能调优:
241
242
  def _encode_local_st(texts: List[str], normalize_embeddings: bool) -> Any:
      with _text_encode_lock:
77516841   tangwang   tidy embeddings
243
          return _text_model.encode(
efd435cf   tangwang   tei性能调优:
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
              texts,
              batch_size=int(CONFIG.TEXT_BATCH_SIZE),
              device=CONFIG.TEXT_DEVICE,
              normalize_embeddings=normalize_embeddings,
          )
  
  
  def _start_text_batch_worker() -> None:
      global _text_batch_worker, _text_batch_worker_stop
      if _text_batch_worker is not None and _text_batch_worker.is_alive():
          return
      _text_batch_worker_stop = False
      _text_batch_worker = threading.Thread(
          target=_text_batch_worker_loop,
          name="embed-text-microbatch-worker",
          daemon=True,
      )
      _text_batch_worker.start()
      logger.info(
          "Started local_st text micro-batch worker | window_ms=%.1f max_batch=%d",
          _TEXT_MICROBATCH_WINDOW_SEC * 1000.0,
          int(CONFIG.TEXT_BATCH_SIZE),
      )
  
  
  def _stop_text_batch_worker() -> None:
      global _text_batch_worker_stop
      with _text_single_queue_cv:
          _text_batch_worker_stop = True
          _text_single_queue_cv.notify_all()
  
  
  def _text_batch_worker_loop() -> None:
      max_batch = max(1, int(CONFIG.TEXT_BATCH_SIZE))
      while True:
          with _text_single_queue_cv:
              while not _text_single_queue and not _text_batch_worker_stop:
                  _text_single_queue_cv.wait()
              if _text_batch_worker_stop:
                  return
  
              batch: List[_SingleTextTask] = [_text_single_queue.popleft()]
              deadline = time.perf_counter() + _TEXT_MICROBATCH_WINDOW_SEC
  
              while len(batch) < max_batch:
                  remaining = deadline - time.perf_counter()
                  if remaining <= 0:
                      break
                  if not _text_single_queue:
                      _text_single_queue_cv.wait(timeout=remaining)
                      continue
                  while _text_single_queue and len(batch) < max_batch:
                      batch.append(_text_single_queue.popleft())
  
          try:
4747e2f4   tangwang   embedding perform...
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
              queue_wait_ms = [(time.perf_counter() - task.created_at) * 1000.0 for task in batch]
              reqids = [task.request_id for task in batch]
              logger.info(
                  "text microbatch dispatch | size=%d queue_wait_ms_min=%.2f queue_wait_ms_max=%.2f reqids=%s preview=%s",
                  len(batch),
                  min(queue_wait_ms) if queue_wait_ms else 0.0,
                  max(queue_wait_ms) if queue_wait_ms else 0.0,
                  reqids,
                  _preview_inputs(
                      [task.text for task in batch],
                      _LOG_PREVIEW_COUNT,
                      _LOG_TEXT_PREVIEW_CHARS,
                  ),
              )
              batch_t0 = time.perf_counter()
efd435cf   tangwang   tei性能调优:
314
315
316
317
318
319
320
321
322
323
324
              embs = _encode_local_st([task.text for task in batch], normalize_embeddings=False)
              if embs is None or len(embs) != len(batch):
                  raise RuntimeError(
                      f"Text model response length mismatch in micro-batch: "
                      f"expected {len(batch)}, got {0 if embs is None else len(embs)}"
                  )
              for task, emb in zip(batch, embs):
                  vec = _as_list(emb, normalize=task.normalize)
                  if vec is None:
                      raise RuntimeError("Text model returned empty embedding in micro-batch")
                  task.result = vec
4747e2f4   tangwang   embedding perform...
325
326
327
328
329
330
331
              logger.info(
                  "text microbatch done | size=%d reqids=%s dim=%d backend_elapsed_ms=%.2f",
                  len(batch),
                  reqids,
                  len(batch[0].result) if batch and batch[0].result is not None else 0,
                  (time.perf_counter() - batch_t0) * 1000.0,
              )
efd435cf   tangwang   tei性能调优:
332
          except Exception as exc:
4747e2f4   tangwang   embedding perform...
333
334
335
336
337
338
339
              logger.error(
                  "text microbatch failed | size=%d reqids=%s error=%s",
                  len(batch),
                  [task.request_id for task in batch],
                  exc,
                  exc_info=True,
              )
efd435cf   tangwang   tei性能调优:
340
341
342
343
344
345
346
              for task in batch:
                  task.error = exc
          finally:
              for task in batch:
                  task.done.set()
  
  
4747e2f4   tangwang   embedding perform...
347
  def _encode_single_text_with_microbatch(text: str, normalize: bool, request_id: str) -> List[float]:
efd435cf   tangwang   tei性能调优:
348
349
350
351
      task = _SingleTextTask(
          text=text,
          normalize=normalize,
          created_at=time.perf_counter(),
4747e2f4   tangwang   embedding perform...
352
          request_id=request_id,
efd435cf   tangwang   tei性能调优:
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
          done=threading.Event(),
      )
      with _text_single_queue_cv:
          _text_single_queue.append(task)
          _text_single_queue_cv.notify()
  
      if not task.done.wait(timeout=_TEXT_REQUEST_TIMEOUT_SEC):
          with _text_single_queue_cv:
              try:
                  _text_single_queue.remove(task)
              except ValueError:
                  pass
          raise RuntimeError(
              f"Timed out waiting for text micro-batch worker ({_TEXT_REQUEST_TIMEOUT_SEC:.1f}s)"
          )
      if task.error is not None:
          raise task.error
      if task.result is None:
          raise RuntimeError("Text micro-batch worker returned empty result")
      return task.result
  
  
0a3764c4   tangwang   优化embedding模型加载
375
376
377
  @app.on_event("startup")
  def load_models():
      """Load models at service startup to avoid first-request latency."""
07cf5a93   tangwang   START_EMBEDDING=...
378
      global _text_model, _image_model, _text_backend_name
7bfb9946   tangwang   向量化模块
379
  
0a3764c4   tangwang   优化embedding模型加载
380
      logger.info("Loading embedding models at startup...")
7bfb9946   tangwang   向量化模块
381
  
40f1e391   tangwang   cnclip
382
383
      if open_text_model:
          try:
07cf5a93   tangwang   START_EMBEDDING=...
384
385
386
              backend_name, backend_cfg = get_embedding_backend_config()
              _text_backend_name = backend_name
              if backend_name == "tei":
77516841   tangwang   tidy embeddings
387
                  from embeddings.text_embedding_tei import TEITextModel
07cf5a93   tangwang   START_EMBEDDING=...
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
  
                  base_url = (
                      os.getenv("TEI_BASE_URL")
                      or backend_cfg.get("base_url")
                      or CONFIG.TEI_BASE_URL
                  )
                  timeout_sec = int(
                      os.getenv("TEI_TIMEOUT_SEC")
                      or backend_cfg.get("timeout_sec")
                      or CONFIG.TEI_TIMEOUT_SEC
                  )
                  logger.info("Loading text backend: tei (base_url=%s)", base_url)
                  _text_model = TEITextModel(
                      base_url=str(base_url),
                      timeout_sec=timeout_sec,
                  )
              elif backend_name == "local_st":
77516841   tangwang   tidy embeddings
405
                  from embeddings.text_embedding_sentence_transformers import Qwen3TextModel
950a640e   tangwang   embeddings
406
  
07cf5a93   tangwang   START_EMBEDDING=...
407
408
409
410
411
412
413
                  model_id = (
                      os.getenv("TEXT_MODEL_ID")
                      or backend_cfg.get("model_id")
                      or CONFIG.TEXT_MODEL_ID
                  )
                  logger.info("Loading text backend: local_st (model=%s)", model_id)
                  _text_model = Qwen3TextModel(model_id=str(model_id))
efd435cf   tangwang   tei性能调优:
414
                  _start_text_batch_worker()
07cf5a93   tangwang   START_EMBEDDING=...
415
416
417
418
419
420
              else:
                  raise ValueError(
                      f"Unsupported embedding backend: {backend_name}. "
                      "Supported: tei, local_st"
                  )
              logger.info("Text backend loaded successfully: %s", _text_backend_name)
40f1e391   tangwang   cnclip
421
          except Exception as e:
4747e2f4   tangwang   embedding perform...
422
              logger.error("Failed to load text model: %s", e, exc_info=True)
40f1e391   tangwang   cnclip
423
              raise
0a3764c4   tangwang   优化embedding模型加载
424
  
40f1e391   tangwang   cnclip
425
426
      if open_image_model:
          try:
c10f90fe   tangwang   cnclip
427
              if CONFIG.USE_CLIP_AS_SERVICE:
950a640e   tangwang   embeddings
428
429
                  from embeddings.clip_as_service_encoder import ClipAsServiceImageEncoder
  
4747e2f4   tangwang   embedding perform...
430
431
432
433
434
                  logger.info(
                      "Loading image encoder via clip-as-service: %s (configured model: %s)",
                      CONFIG.CLIP_AS_SERVICE_SERVER,
                      CONFIG.CLIP_AS_SERVICE_MODEL_NAME,
                  )
c10f90fe   tangwang   cnclip
435
436
437
438
439
440
                  _image_model = ClipAsServiceImageEncoder(
                      server=CONFIG.CLIP_AS_SERVICE_SERVER,
                      batch_size=CONFIG.IMAGE_BATCH_SIZE,
                  )
                  logger.info("Image model (clip-as-service) loaded successfully")
              else:
950a640e   tangwang   embeddings
441
442
                  from embeddings.clip_model import ClipImageModel
  
4747e2f4   tangwang   embedding perform...
443
444
445
446
447
                  logger.info(
                      "Loading local image model: %s (device: %s)",
                      CONFIG.IMAGE_MODEL_NAME,
                      CONFIG.IMAGE_DEVICE,
                  )
c10f90fe   tangwang   cnclip
448
449
450
451
452
                  _image_model = ClipImageModel(
                      model_name=CONFIG.IMAGE_MODEL_NAME,
                      device=CONFIG.IMAGE_DEVICE,
                  )
                  logger.info("Image model (local CN-CLIP) loaded successfully")
40f1e391   tangwang   cnclip
453
          except Exception as e:
ed948666   tangwang   tidy
454
455
              logger.error("Failed to load image model: %s", e, exc_info=True)
              raise
0a3764c4   tangwang   优化embedding模型加载
456
457
  
      logger.info("All embedding models loaded successfully, service ready")
7bfb9946   tangwang   向量化模块
458
459
  
  
efd435cf   tangwang   tei性能调优:
460
461
462
463
464
  @app.on_event("shutdown")
  def stop_workers() -> None:
      _stop_text_batch_worker()
  
  
200fdddf   tangwang   embed norm
465
466
467
468
469
470
471
472
  def _normalize_vector(vec: np.ndarray) -> np.ndarray:
      norm = float(np.linalg.norm(vec))
      if not np.isfinite(norm) or norm <= 0.0:
          raise RuntimeError("Embedding vector has invalid norm (must be > 0)")
      return vec / norm
  
  
  def _as_list(embedding: Optional[np.ndarray], normalize: bool = False) -> Optional[List[float]]:
7bfb9946   tangwang   向量化模块
473
474
475
476
477
478
      if embedding is None:
          return None
      if not isinstance(embedding, np.ndarray):
          embedding = np.array(embedding, dtype=np.float32)
      if embedding.ndim != 1:
          embedding = embedding.reshape(-1)
200fdddf   tangwang   embed norm
479
480
481
482
      embedding = embedding.astype(np.float32, copy=False)
      if normalize:
          embedding = _normalize_vector(embedding).astype(np.float32, copy=False)
      return embedding.tolist()
7bfb9946   tangwang   向量化模块
483
484
485
486
  
  
  @app.get("/health")
  def health() -> Dict[str, Any]:
4747e2f4   tangwang   embedding perform...
487
      """Health check endpoint. Returns status and current throttling stats."""
0a3764c4   tangwang   优化embedding模型加载
488
489
490
      return {
          "status": "ok",
          "text_model_loaded": _text_model is not None,
07cf5a93   tangwang   START_EMBEDDING=...
491
          "text_backend": _text_backend_name,
0a3764c4   tangwang   优化embedding模型加载
492
          "image_model_loaded": _image_model is not None,
4747e2f4   tangwang   embedding perform...
493
494
495
496
497
498
499
500
501
502
          "limits": {
              "text": _text_request_limiter.snapshot(),
              "image": _image_request_limiter.snapshot(),
          },
          "text_microbatch": {
              "window_ms": round(_TEXT_MICROBATCH_WINDOW_SEC * 1000.0, 3),
              "queue_depth": len(_text_single_queue),
              "worker_alive": bool(_text_batch_worker is not None and _text_batch_worker.is_alive()),
              "request_timeout_sec": _TEXT_REQUEST_TIMEOUT_SEC,
          },
0a3764c4   tangwang   优化embedding模型加载
503
      }
7bfb9946   tangwang   向量化模块
504
505
  
  
4747e2f4   tangwang   embedding perform...
506
507
508
509
510
  def _embed_text_impl(
      normalized: List[str],
      effective_normalize: bool,
      request_id: str,
  ) -> List[Optional[List[float]]]:
0a3764c4   tangwang   优化embedding模型加载
511
512
      if _text_model is None:
          raise RuntimeError("Text model not loaded")
28e57bb1   tangwang   日志体系优化
513
  
efd435cf   tangwang   tei性能调优:
514
      t0 = time.perf_counter()
54ccf28c   tangwang   tei
515
      try:
efd435cf   tangwang   tei性能调优:
516
517
          if _text_backend_name == "local_st":
              if len(normalized) == 1 and _text_batch_worker is not None:
4747e2f4   tangwang   embedding perform...
518
519
520
521
522
523
524
                  out = [
                      _encode_single_text_with_microbatch(
                          normalized[0],
                          normalize=effective_normalize,
                          request_id=request_id,
                      )
                  ]
efd435cf   tangwang   tei性能调优:
525
                  logger.info(
4747e2f4   tangwang   embedding perform...
526
                      "text backend done | backend=%s mode=microbatch-single inputs=%d normalize=%s dim=%d backend_elapsed_ms=%.2f",
efd435cf   tangwang   tei性能调优:
527
528
529
                      _text_backend_name,
                      len(normalized),
                      effective_normalize,
28e57bb1   tangwang   日志体系优化
530
                      len(out[0]) if out and out[0] is not None else 0,
4747e2f4   tangwang   embedding perform...
531
532
                      (time.perf_counter() - t0) * 1000.0,
                      extra=_request_log_extra(request_id),
efd435cf   tangwang   tei性能调优:
533
534
535
                  )
                  return out
              embs = _encode_local_st(normalized, normalize_embeddings=False)
4747e2f4   tangwang   embedding perform...
536
              mode = "direct-batch"
efd435cf   tangwang   tei性能调优:
537
          else:
77516841   tangwang   tidy embeddings
538
              embs = _text_model.encode(
54ccf28c   tangwang   tei
539
540
541
                  normalized,
                  batch_size=int(CONFIG.TEXT_BATCH_SIZE),
                  device=CONFIG.TEXT_DEVICE,
200fdddf   tangwang   embed norm
542
                  normalize_embeddings=effective_normalize,
54ccf28c   tangwang   tei
543
              )
4747e2f4   tangwang   embedding perform...
544
              mode = "backend-batch"
54ccf28c   tangwang   tei
545
      except Exception as e:
4747e2f4   tangwang   embedding perform...
546
547
548
549
550
551
552
553
          logger.error(
              "Text embedding backend failure: %s",
              e,
              exc_info=True,
              extra=_request_log_extra(request_id),
          )
          raise RuntimeError(f"Text embedding backend failure: {e}") from e
  
ed948666   tangwang   tidy
554
555
556
557
558
      if embs is None or len(embs) != len(normalized):
          raise RuntimeError(
              f"Text model response length mismatch: expected {len(normalized)}, "
              f"got {0 if embs is None else len(embs)}"
          )
4747e2f4   tangwang   embedding perform...
559
  
ed948666   tangwang   tidy
560
561
      out: List[Optional[List[float]]] = []
      for i, emb in enumerate(embs):
200fdddf   tangwang   embed norm
562
          vec = _as_list(emb, normalize=effective_normalize)
ed948666   tangwang   tidy
563
564
565
          if vec is None:
              raise RuntimeError(f"Text model returned empty embedding for index {i}")
          out.append(vec)
4747e2f4   tangwang   embedding perform...
566
  
efd435cf   tangwang   tei性能调优:
567
      logger.info(
4747e2f4   tangwang   embedding perform...
568
          "text backend done | backend=%s mode=%s inputs=%d normalize=%s dim=%d backend_elapsed_ms=%.2f",
efd435cf   tangwang   tei性能调优:
569
          _text_backend_name,
4747e2f4   tangwang   embedding perform...
570
          mode,
efd435cf   tangwang   tei性能调优:
571
572
          len(normalized),
          effective_normalize,
28e57bb1   tangwang   日志体系优化
573
          len(out[0]) if out and out[0] is not None else 0,
4747e2f4   tangwang   embedding perform...
574
575
          (time.perf_counter() - t0) * 1000.0,
          extra=_request_log_extra(request_id),
efd435cf   tangwang   tei性能调优:
576
      )
7bfb9946   tangwang   向量化模块
577
578
579
      return out
  
  
4747e2f4   tangwang   embedding perform...
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
  @app.post("/embed/text")
  async def embed_text(
      texts: List[str],
      http_request: Request,
      response: Response,
      normalize: Optional[bool] = None,
  ) -> List[Optional[List[float]]]:
      request_id = _resolve_request_id(http_request)
      response.headers["X-Request-ID"] = request_id
  
      effective_normalize = bool(CONFIG.TEXT_NORMALIZE_EMBEDDINGS) if normalize is None else bool(normalize)
      normalized: List[str] = []
      for i, t in enumerate(texts):
          if not isinstance(t, str):
              raise HTTPException(status_code=400, detail=f"Invalid text at index {i}: must be string")
          s = t.strip()
ed948666   tangwang   tidy
596
          if not s:
4747e2f4   tangwang   embedding perform...
597
598
              raise HTTPException(status_code=400, detail=f"Invalid text at index {i}: empty string")
          normalized.append(s)
c10f90fe   tangwang   cnclip
599
  
4747e2f4   tangwang   embedding perform...
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
      accepted, active = _text_request_limiter.try_acquire()
      if not accepted:
          logger.warning(
              "embed_text rejected | client=%s backend=%s inputs=%d normalize=%s active=%d limit=%d preview=%s",
              _request_client(http_request),
              _text_backend_name,
              len(normalized),
              effective_normalize,
              active,
              _TEXT_MAX_INFLIGHT,
              _preview_inputs(normalized, _LOG_PREVIEW_COUNT, _LOG_TEXT_PREVIEW_CHARS),
              extra=_request_log_extra(request_id),
          )
          raise HTTPException(
              status_code=_OVERLOAD_STATUS_CODE,
              detail=f"Text embedding service busy: active={active}, limit={_TEXT_MAX_INFLIGHT}",
          )
  
      request_started = time.perf_counter()
      success = False
      try:
          logger.info(
              "embed_text request | client=%s backend=%s inputs=%d normalize=%s active=%d limit=%d preview=%s",
              _request_client(http_request),
              _text_backend_name,
              len(normalized),
              effective_normalize,
              active,
              _TEXT_MAX_INFLIGHT,
              _preview_inputs(normalized, _LOG_PREVIEW_COUNT, _LOG_TEXT_PREVIEW_CHARS),
              extra=_request_log_extra(request_id),
          )
          verbose_logger.info(
              "embed_text detail | payload=%s normalize=%s backend=%s",
              normalized,
              effective_normalize,
              _text_backend_name,
              extra=_request_log_extra(request_id),
          )
          out = await run_in_threadpool(_embed_text_impl, normalized, effective_normalize, request_id)
          success = True
          latency_ms = (time.perf_counter() - request_started) * 1000.0
          logger.info(
              "embed_text response | backend=%s inputs=%d normalize=%s dim=%d first_vector=%s latency_ms=%.2f",
              _text_backend_name,
              len(normalized),
              effective_normalize,
              len(out[0]) if out and out[0] is not None else 0,
              _preview_vector(out[0] if out else None),
              latency_ms,
              extra=_request_log_extra(request_id),
          )
          verbose_logger.info(
              "embed_text result detail | count=%d first_vector=%s latency_ms=%.2f",
              len(out),
              out[0][: _VECTOR_PREVIEW_DIMS] if out and out[0] is not None else [],
              latency_ms,
              extra=_request_log_extra(request_id),
          )
          return out
      except HTTPException:
          raise
      except Exception as e:
          latency_ms = (time.perf_counter() - request_started) * 1000.0
          logger.error(
              "embed_text failed | backend=%s inputs=%d normalize=%s latency_ms=%.2f error=%s",
              _text_backend_name,
              len(normalized),
              effective_normalize,
              latency_ms,
              e,
              exc_info=True,
              extra=_request_log_extra(request_id),
          )
          raise HTTPException(status_code=502, detail=str(e)) from e
      finally:
          remaining = _text_request_limiter.release(success=success)
          logger.info(
              "embed_text finalize | success=%s active_after=%d",
              success,
              remaining,
              extra=_request_log_extra(request_id),
          )
  
  
  def _embed_image_impl(
      urls: List[str],
      effective_normalize: bool,
      request_id: str,
  ) -> List[Optional[List[float]]]:
      if _image_model is None:
          raise RuntimeError("Image model not loaded")
28e57bb1   tangwang   日志体系优化
692
693
  
      t0 = time.perf_counter()
7bfb9946   tangwang   向量化模块
694
      with _image_encode_lock:
200fdddf   tangwang   embed norm
695
696
697
698
699
          vectors = _image_model.encode_image_urls(
              urls,
              batch_size=CONFIG.IMAGE_BATCH_SIZE,
              normalize_embeddings=effective_normalize,
          )
ed948666   tangwang   tidy
700
701
702
703
704
      if vectors is None or len(vectors) != len(urls):
          raise RuntimeError(
              f"Image model response length mismatch: expected {len(urls)}, "
              f"got {0 if vectors is None else len(vectors)}"
          )
4747e2f4   tangwang   embedding perform...
705
  
ed948666   tangwang   tidy
706
707
      out: List[Optional[List[float]]] = []
      for i, vec in enumerate(vectors):
200fdddf   tangwang   embed norm
708
          out_vec = _as_list(vec, normalize=effective_normalize)
ed948666   tangwang   tidy
709
710
711
          if out_vec is None:
              raise RuntimeError(f"Image model returned empty embedding for index {i}")
          out.append(out_vec)
4747e2f4   tangwang   embedding perform...
712
  
28e57bb1   tangwang   日志体系优化
713
      logger.info(
4747e2f4   tangwang   embedding perform...
714
          "image backend done | inputs=%d normalize=%s dim=%d backend_elapsed_ms=%.2f",
28e57bb1   tangwang   日志体系优化
715
716
717
          len(urls),
          effective_normalize,
          len(out[0]) if out and out[0] is not None else 0,
4747e2f4   tangwang   embedding perform...
718
719
          (time.perf_counter() - t0) * 1000.0,
          extra=_request_log_extra(request_id),
28e57bb1   tangwang   日志体系优化
720
      )
7bfb9946   tangwang   向量化模块
721
      return out
4747e2f4   tangwang   embedding perform...
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
  
  
  @app.post("/embed/image")
  async def embed_image(
      images: List[str],
      http_request: Request,
      response: Response,
      normalize: Optional[bool] = None,
  ) -> List[Optional[List[float]]]:
      request_id = _resolve_request_id(http_request)
      response.headers["X-Request-ID"] = request_id
  
      effective_normalize = bool(CONFIG.IMAGE_NORMALIZE_EMBEDDINGS) if normalize is None else bool(normalize)
      urls: List[str] = []
      for i, url_or_path in enumerate(images):
          if not isinstance(url_or_path, str):
              raise HTTPException(status_code=400, detail=f"Invalid image at index {i}: must be string URL/path")
          s = url_or_path.strip()
          if not s:
              raise HTTPException(status_code=400, detail=f"Invalid image at index {i}: empty URL/path")
          urls.append(s)
  
      accepted, active = _image_request_limiter.try_acquire()
      if not accepted:
          logger.warning(
              "embed_image rejected | client=%s inputs=%d normalize=%s active=%d limit=%d preview=%s",
              _request_client(http_request),
              len(urls),
              effective_normalize,
              active,
              _IMAGE_MAX_INFLIGHT,
              _preview_inputs(urls, _LOG_PREVIEW_COUNT, _LOG_IMAGE_PREVIEW_CHARS),
              extra=_request_log_extra(request_id),
          )
          raise HTTPException(
              status_code=_OVERLOAD_STATUS_CODE,
              detail=f"Image embedding service busy: active={active}, limit={_IMAGE_MAX_INFLIGHT}",
          )
  
      request_started = time.perf_counter()
      success = False
      try:
          logger.info(
              "embed_image request | client=%s inputs=%d normalize=%s active=%d limit=%d preview=%s",
              _request_client(http_request),
              len(urls),
              effective_normalize,
              active,
              _IMAGE_MAX_INFLIGHT,
              _preview_inputs(urls, _LOG_PREVIEW_COUNT, _LOG_IMAGE_PREVIEW_CHARS),
              extra=_request_log_extra(request_id),
          )
          verbose_logger.info(
              "embed_image detail | payload=%s normalize=%s",
              urls,
              effective_normalize,
              extra=_request_log_extra(request_id),
          )
          out = await run_in_threadpool(_embed_image_impl, urls, effective_normalize, request_id)
          success = True
          latency_ms = (time.perf_counter() - request_started) * 1000.0
          logger.info(
              "embed_image response | inputs=%d normalize=%s dim=%d first_vector=%s latency_ms=%.2f",
              len(urls),
              effective_normalize,
              len(out[0]) if out and out[0] is not None else 0,
              _preview_vector(out[0] if out else None),
              latency_ms,
              extra=_request_log_extra(request_id),
          )
          verbose_logger.info(
              "embed_image result detail | count=%d first_vector=%s latency_ms=%.2f",
              len(out),
              out[0][: _VECTOR_PREVIEW_DIMS] if out and out[0] is not None else [],
              latency_ms,
              extra=_request_log_extra(request_id),
          )
          return out
      except HTTPException:
          raise
      except Exception as e:
          latency_ms = (time.perf_counter() - request_started) * 1000.0
          logger.error(
              "embed_image failed | inputs=%d normalize=%s latency_ms=%.2f error=%s",
              len(urls),
              effective_normalize,
              latency_ms,
              e,
              exc_info=True,
              extra=_request_log_extra(request_id),
          )
          raise HTTPException(status_code=502, detail=f"Image embedding backend failure: {e}") from e
      finally:
          remaining = _image_request_limiter.release(success=success)
          logger.info(
              "embed_image finalize | success=%s active_after=%d",
              success,
              remaining,
              extra=_request_log_extra(request_id),
          )