7bfb9946
tangwang
向量化模块
|
1
2
3
|
"""
Embedding service (FastAPI).
|
ed948666
tangwang
tidy
|
4
5
6
|
API (simple list-in, list-out; aligned by index):
- POST /embed/text body: ["text1", "text2", ...] -> [[...], ...]
- POST /embed/image body: ["url_or_path1", ...] -> [[...], ...]
|
7bfb9946
tangwang
向量化模块
|
7
8
|
"""
|
0a3764c4
tangwang
优化embedding模型加载
|
9
|
import logging
|
07cf5a93
tangwang
START_EMBEDDING=...
|
10
|
import os
|
4747e2f4
tangwang
embedding perform...
|
11
|
import pathlib
|
7bfb9946
tangwang
向量化模块
|
12
|
import threading
|
efd435cf
tangwang
tei性能调优:
|
13
|
import time
|
4747e2f4
tangwang
embedding perform...
|
14
|
import uuid
|
efd435cf
tangwang
tei性能调优:
|
15
16
|
from collections import deque
from dataclasses import dataclass
|
4747e2f4
tangwang
embedding perform...
|
17
|
from logging.handlers import TimedRotatingFileHandler
|
7bfb9946
tangwang
向量化模块
|
18
19
20
|
from typing import Any, Dict, List, Optional
import numpy as np
|
4747e2f4
tangwang
embedding perform...
|
21
22
|
from fastapi import FastAPI, HTTPException, Request, Response
from fastapi.concurrency import run_in_threadpool
|
7bfb9946
tangwang
向量化模块
|
23
|
|
4747e2f4
tangwang
embedding perform...
|
24
|
from config.services_config import get_embedding_backend_config
|
7bfb9946
tangwang
向量化模块
|
25
|
from embeddings.config import CONFIG
|
c10f90fe
tangwang
cnclip
|
26
|
from embeddings.protocols import ImageEncoderProtocol
|
7bfb9946
tangwang
向量化模块
|
27
|
|
a7920e17
tangwang
项目名称和部署路径修改
|
28
|
app = FastAPI(title="saas-search Embedding Service", version="1.0.0")
|
7bfb9946
tangwang
向量化模块
|
29
|
|
4747e2f4
tangwang
embedding perform...
|
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
|
class _DefaultRequestIdFilter(logging.Filter):
def filter(self, record: logging.LogRecord) -> bool:
if not hasattr(record, "reqid"):
record.reqid = "-1"
return True
def configure_embedding_logging() -> None:
root_logger = logging.getLogger()
if getattr(root_logger, "_embedding_logging_configured", False):
return
log_dir = pathlib.Path("logs")
verbose_dir = log_dir / "verbose"
log_dir.mkdir(exist_ok=True)
verbose_dir.mkdir(parents=True, exist_ok=True)
log_level = os.getenv("LOG_LEVEL", "INFO").upper()
numeric_level = getattr(logging, log_level, logging.INFO)
formatter = logging.Formatter(
"%(asctime)s | reqid:%(reqid)s | %(name)s | %(levelname)s | %(message)s"
)
request_filter = _DefaultRequestIdFilter()
root_logger.setLevel(numeric_level)
file_handler = TimedRotatingFileHandler(
filename=log_dir / "embedding_api.log",
when="midnight",
interval=1,
backupCount=30,
encoding="utf-8",
)
file_handler.setLevel(numeric_level)
file_handler.setFormatter(formatter)
file_handler.addFilter(request_filter)
root_logger.addHandler(file_handler)
error_handler = TimedRotatingFileHandler(
filename=log_dir / "embedding_api_error.log",
when="midnight",
interval=1,
backupCount=30,
encoding="utf-8",
)
error_handler.setLevel(logging.ERROR)
error_handler.setFormatter(formatter)
error_handler.addFilter(request_filter)
root_logger.addHandler(error_handler)
verbose_logger = logging.getLogger("embedding.verbose")
verbose_logger.setLevel(numeric_level)
verbose_logger.handlers.clear()
verbose_logger.propagate = False
verbose_handler = TimedRotatingFileHandler(
filename=verbose_dir / "embedding_verbose.log",
when="midnight",
interval=1,
backupCount=30,
encoding="utf-8",
)
verbose_handler.setLevel(numeric_level)
verbose_handler.setFormatter(formatter)
verbose_handler.addFilter(request_filter)
verbose_logger.addHandler(verbose_handler)
root_logger._embedding_logging_configured = True # type: ignore[attr-defined]
configure_embedding_logging()
logger = logging.getLogger(__name__)
verbose_logger = logging.getLogger("embedding.verbose")
|
0a3764c4
tangwang
优化embedding模型加载
|
105
|
# Models are loaded at startup, not lazily
|
950a640e
tangwang
embeddings
|
106
|
_text_model: Optional[Any] = None
|
c10f90fe
tangwang
cnclip
|
107
|
_image_model: Optional[ImageEncoderProtocol] = None
|
07cf5a93
tangwang
START_EMBEDDING=...
|
108
|
_text_backend_name: str = ""
|
efd435cf
tangwang
tei性能调优:
|
109
110
|
open_text_model = os.getenv("EMBEDDING_ENABLE_TEXT_MODEL", "true").lower() in ("1", "true", "yes")
open_image_model = os.getenv("EMBEDDING_ENABLE_IMAGE_MODEL", "true").lower() in ("1", "true", "yes")
|
7bfb9946
tangwang
向量化模块
|
111
112
113
114
|
_text_encode_lock = threading.Lock()
_image_encode_lock = threading.Lock()
|
4747e2f4
tangwang
embedding perform...
|
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
|
_TEXT_MICROBATCH_WINDOW_SEC = max(
0.0, float(os.getenv("TEXT_MICROBATCH_WINDOW_MS", "4")) / 1000.0
)
_TEXT_REQUEST_TIMEOUT_SEC = max(
1.0, float(os.getenv("TEXT_REQUEST_TIMEOUT_SEC", "30"))
)
_TEXT_MAX_INFLIGHT = max(1, int(os.getenv("TEXT_MAX_INFLIGHT", "32")))
_IMAGE_MAX_INFLIGHT = max(1, int(os.getenv("IMAGE_MAX_INFLIGHT", "1")))
_OVERLOAD_STATUS_CODE = int(os.getenv("EMBEDDING_OVERLOAD_STATUS_CODE", "503"))
_LOG_PREVIEW_COUNT = max(1, int(os.getenv("EMBEDDING_LOG_PREVIEW_COUNT", "3")))
_LOG_TEXT_PREVIEW_CHARS = max(32, int(os.getenv("EMBEDDING_LOG_TEXT_PREVIEW_CHARS", "120")))
_LOG_IMAGE_PREVIEW_CHARS = max(32, int(os.getenv("EMBEDDING_LOG_IMAGE_PREVIEW_CHARS", "180")))
_VECTOR_PREVIEW_DIMS = max(1, int(os.getenv("EMBEDDING_VECTOR_PREVIEW_DIMS", "6")))
class _InflightLimiter:
def __init__(self, name: str, limit: int):
self.name = name
self.limit = max(1, int(limit))
self._sem = threading.BoundedSemaphore(self.limit)
self._lock = threading.Lock()
self._active = 0
self._rejected = 0
self._completed = 0
self._failed = 0
self._max_active = 0
def try_acquire(self) -> tuple[bool, int]:
if not self._sem.acquire(blocking=False):
with self._lock:
self._rejected += 1
active = self._active
return False, active
with self._lock:
self._active += 1
self._max_active = max(self._max_active, self._active)
active = self._active
return True, active
def release(self, *, success: bool) -> int:
with self._lock:
self._active = max(0, self._active - 1)
if success:
self._completed += 1
else:
self._failed += 1
active = self._active
self._sem.release()
return active
def snapshot(self) -> Dict[str, int]:
with self._lock:
return {
"limit": self.limit,
"active": self._active,
"rejected_total": self._rejected,
"completed_total": self._completed,
"failed_total": self._failed,
"max_active": self._max_active,
}
_text_request_limiter = _InflightLimiter(name="text", limit=_TEXT_MAX_INFLIGHT)
_image_request_limiter = _InflightLimiter(name="image", limit=_IMAGE_MAX_INFLIGHT)
|
7bfb9946
tangwang
向量化模块
|
180
|
|
efd435cf
tangwang
tei性能调优:
|
181
182
183
184
185
|
@dataclass
class _SingleTextTask:
text: str
normalize: bool
created_at: float
|
4747e2f4
tangwang
embedding perform...
|
186
|
request_id: str
|
efd435cf
tangwang
tei性能调优:
|
187
188
189
190
191
192
193
194
195
|
done: threading.Event
result: Optional[List[float]] = None
error: Optional[Exception] = None
_text_single_queue: "deque[_SingleTextTask]" = deque()
_text_single_queue_cv = threading.Condition()
_text_batch_worker: Optional[threading.Thread] = None
_text_batch_worker_stop = False
|
28e57bb1
tangwang
日志体系优化
|
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
|
def _compact_preview(text: str, max_chars: int) -> str:
compact = " ".join((text or "").split())
if len(compact) <= max_chars:
return compact
return compact[:max_chars] + "..."
def _preview_inputs(items: List[str], max_items: int, max_chars: int) -> List[Dict[str, Any]]:
previews: List[Dict[str, Any]] = []
for idx, item in enumerate(items[:max_items]):
previews.append(
{
"idx": idx,
"len": len(item),
"preview": _compact_preview(item, max_chars),
}
)
return previews
|
efd435cf
tangwang
tei性能调优:
|
216
217
|
|
4747e2f4
tangwang
embedding perform...
|
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
|
def _preview_vector(vec: Optional[List[float]], max_dims: int = _VECTOR_PREVIEW_DIMS) -> List[float]:
if not vec:
return []
return [round(float(v), 6) for v in vec[:max_dims]]
def _request_log_extra(request_id: str) -> Dict[str, str]:
return {"reqid": request_id}
def _resolve_request_id(http_request: Request) -> str:
header_value = http_request.headers.get("X-Request-ID")
if header_value and header_value.strip():
return header_value.strip()[:32]
return str(uuid.uuid4())[:8]
def _request_client(http_request: Request) -> str:
client = getattr(http_request, "client", None)
host = getattr(client, "host", None)
return str(host or "-")
|
efd435cf
tangwang
tei性能调优:
|
241
242
|
def _encode_local_st(texts: List[str], normalize_embeddings: bool) -> Any:
with _text_encode_lock:
|
77516841
tangwang
tidy embeddings
|
243
|
return _text_model.encode(
|
efd435cf
tangwang
tei性能调优:
|
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
|
texts,
batch_size=int(CONFIG.TEXT_BATCH_SIZE),
device=CONFIG.TEXT_DEVICE,
normalize_embeddings=normalize_embeddings,
)
def _start_text_batch_worker() -> None:
global _text_batch_worker, _text_batch_worker_stop
if _text_batch_worker is not None and _text_batch_worker.is_alive():
return
_text_batch_worker_stop = False
_text_batch_worker = threading.Thread(
target=_text_batch_worker_loop,
name="embed-text-microbatch-worker",
daemon=True,
)
_text_batch_worker.start()
logger.info(
"Started local_st text micro-batch worker | window_ms=%.1f max_batch=%d",
_TEXT_MICROBATCH_WINDOW_SEC * 1000.0,
int(CONFIG.TEXT_BATCH_SIZE),
)
def _stop_text_batch_worker() -> None:
global _text_batch_worker_stop
with _text_single_queue_cv:
_text_batch_worker_stop = True
_text_single_queue_cv.notify_all()
def _text_batch_worker_loop() -> None:
max_batch = max(1, int(CONFIG.TEXT_BATCH_SIZE))
while True:
with _text_single_queue_cv:
while not _text_single_queue and not _text_batch_worker_stop:
_text_single_queue_cv.wait()
if _text_batch_worker_stop:
return
batch: List[_SingleTextTask] = [_text_single_queue.popleft()]
deadline = time.perf_counter() + _TEXT_MICROBATCH_WINDOW_SEC
while len(batch) < max_batch:
remaining = deadline - time.perf_counter()
if remaining <= 0:
break
if not _text_single_queue:
_text_single_queue_cv.wait(timeout=remaining)
continue
while _text_single_queue and len(batch) < max_batch:
batch.append(_text_single_queue.popleft())
try:
|
4747e2f4
tangwang
embedding perform...
|
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
|
queue_wait_ms = [(time.perf_counter() - task.created_at) * 1000.0 for task in batch]
reqids = [task.request_id for task in batch]
logger.info(
"text microbatch dispatch | size=%d queue_wait_ms_min=%.2f queue_wait_ms_max=%.2f reqids=%s preview=%s",
len(batch),
min(queue_wait_ms) if queue_wait_ms else 0.0,
max(queue_wait_ms) if queue_wait_ms else 0.0,
reqids,
_preview_inputs(
[task.text for task in batch],
_LOG_PREVIEW_COUNT,
_LOG_TEXT_PREVIEW_CHARS,
),
)
batch_t0 = time.perf_counter()
|
efd435cf
tangwang
tei性能调优:
|
314
315
316
317
318
319
320
321
322
323
324
|
embs = _encode_local_st([task.text for task in batch], normalize_embeddings=False)
if embs is None or len(embs) != len(batch):
raise RuntimeError(
f"Text model response length mismatch in micro-batch: "
f"expected {len(batch)}, got {0 if embs is None else len(embs)}"
)
for task, emb in zip(batch, embs):
vec = _as_list(emb, normalize=task.normalize)
if vec is None:
raise RuntimeError("Text model returned empty embedding in micro-batch")
task.result = vec
|
4747e2f4
tangwang
embedding perform...
|
325
326
327
328
329
330
331
|
logger.info(
"text microbatch done | size=%d reqids=%s dim=%d backend_elapsed_ms=%.2f",
len(batch),
reqids,
len(batch[0].result) if batch and batch[0].result is not None else 0,
(time.perf_counter() - batch_t0) * 1000.0,
)
|
efd435cf
tangwang
tei性能调优:
|
332
|
except Exception as exc:
|
4747e2f4
tangwang
embedding perform...
|
333
334
335
336
337
338
339
|
logger.error(
"text microbatch failed | size=%d reqids=%s error=%s",
len(batch),
[task.request_id for task in batch],
exc,
exc_info=True,
)
|
efd435cf
tangwang
tei性能调优:
|
340
341
342
343
344
345
346
|
for task in batch:
task.error = exc
finally:
for task in batch:
task.done.set()
|
4747e2f4
tangwang
embedding perform...
|
347
|
def _encode_single_text_with_microbatch(text: str, normalize: bool, request_id: str) -> List[float]:
|
efd435cf
tangwang
tei性能调优:
|
348
349
350
351
|
task = _SingleTextTask(
text=text,
normalize=normalize,
created_at=time.perf_counter(),
|
4747e2f4
tangwang
embedding perform...
|
352
|
request_id=request_id,
|
efd435cf
tangwang
tei性能调优:
|
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
|
done=threading.Event(),
)
with _text_single_queue_cv:
_text_single_queue.append(task)
_text_single_queue_cv.notify()
if not task.done.wait(timeout=_TEXT_REQUEST_TIMEOUT_SEC):
with _text_single_queue_cv:
try:
_text_single_queue.remove(task)
except ValueError:
pass
raise RuntimeError(
f"Timed out waiting for text micro-batch worker ({_TEXT_REQUEST_TIMEOUT_SEC:.1f}s)"
)
if task.error is not None:
raise task.error
if task.result is None:
raise RuntimeError("Text micro-batch worker returned empty result")
return task.result
|
0a3764c4
tangwang
优化embedding模型加载
|
375
376
377
|
@app.on_event("startup")
def load_models():
"""Load models at service startup to avoid first-request latency."""
|
07cf5a93
tangwang
START_EMBEDDING=...
|
378
|
global _text_model, _image_model, _text_backend_name
|
7bfb9946
tangwang
向量化模块
|
379
|
|
0a3764c4
tangwang
优化embedding模型加载
|
380
|
logger.info("Loading embedding models at startup...")
|
7bfb9946
tangwang
向量化模块
|
381
|
|
40f1e391
tangwang
cnclip
|
382
383
|
if open_text_model:
try:
|
07cf5a93
tangwang
START_EMBEDDING=...
|
384
385
386
|
backend_name, backend_cfg = get_embedding_backend_config()
_text_backend_name = backend_name
if backend_name == "tei":
|
77516841
tangwang
tidy embeddings
|
387
|
from embeddings.text_embedding_tei import TEITextModel
|
07cf5a93
tangwang
START_EMBEDDING=...
|
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
|
base_url = (
os.getenv("TEI_BASE_URL")
or backend_cfg.get("base_url")
or CONFIG.TEI_BASE_URL
)
timeout_sec = int(
os.getenv("TEI_TIMEOUT_SEC")
or backend_cfg.get("timeout_sec")
or CONFIG.TEI_TIMEOUT_SEC
)
logger.info("Loading text backend: tei (base_url=%s)", base_url)
_text_model = TEITextModel(
base_url=str(base_url),
timeout_sec=timeout_sec,
)
elif backend_name == "local_st":
|
77516841
tangwang
tidy embeddings
|
405
|
from embeddings.text_embedding_sentence_transformers import Qwen3TextModel
|
950a640e
tangwang
embeddings
|
406
|
|
07cf5a93
tangwang
START_EMBEDDING=...
|
407
408
409
410
411
412
413
|
model_id = (
os.getenv("TEXT_MODEL_ID")
or backend_cfg.get("model_id")
or CONFIG.TEXT_MODEL_ID
)
logger.info("Loading text backend: local_st (model=%s)", model_id)
_text_model = Qwen3TextModel(model_id=str(model_id))
|
efd435cf
tangwang
tei性能调优:
|
414
|
_start_text_batch_worker()
|
07cf5a93
tangwang
START_EMBEDDING=...
|
415
416
417
418
419
420
|
else:
raise ValueError(
f"Unsupported embedding backend: {backend_name}. "
"Supported: tei, local_st"
)
logger.info("Text backend loaded successfully: %s", _text_backend_name)
|
40f1e391
tangwang
cnclip
|
421
|
except Exception as e:
|
4747e2f4
tangwang
embedding perform...
|
422
|
logger.error("Failed to load text model: %s", e, exc_info=True)
|
40f1e391
tangwang
cnclip
|
423
|
raise
|
0a3764c4
tangwang
优化embedding模型加载
|
424
|
|
40f1e391
tangwang
cnclip
|
425
426
|
if open_image_model:
try:
|
c10f90fe
tangwang
cnclip
|
427
|
if CONFIG.USE_CLIP_AS_SERVICE:
|
950a640e
tangwang
embeddings
|
428
429
|
from embeddings.clip_as_service_encoder import ClipAsServiceImageEncoder
|
4747e2f4
tangwang
embedding perform...
|
430
431
432
433
434
|
logger.info(
"Loading image encoder via clip-as-service: %s (configured model: %s)",
CONFIG.CLIP_AS_SERVICE_SERVER,
CONFIG.CLIP_AS_SERVICE_MODEL_NAME,
)
|
c10f90fe
tangwang
cnclip
|
435
436
437
438
439
440
|
_image_model = ClipAsServiceImageEncoder(
server=CONFIG.CLIP_AS_SERVICE_SERVER,
batch_size=CONFIG.IMAGE_BATCH_SIZE,
)
logger.info("Image model (clip-as-service) loaded successfully")
else:
|
950a640e
tangwang
embeddings
|
441
442
|
from embeddings.clip_model import ClipImageModel
|
4747e2f4
tangwang
embedding perform...
|
443
444
445
446
447
|
logger.info(
"Loading local image model: %s (device: %s)",
CONFIG.IMAGE_MODEL_NAME,
CONFIG.IMAGE_DEVICE,
)
|
c10f90fe
tangwang
cnclip
|
448
449
450
451
452
|
_image_model = ClipImageModel(
model_name=CONFIG.IMAGE_MODEL_NAME,
device=CONFIG.IMAGE_DEVICE,
)
logger.info("Image model (local CN-CLIP) loaded successfully")
|
40f1e391
tangwang
cnclip
|
453
|
except Exception as e:
|
ed948666
tangwang
tidy
|
454
455
|
logger.error("Failed to load image model: %s", e, exc_info=True)
raise
|
0a3764c4
tangwang
优化embedding模型加载
|
456
457
|
logger.info("All embedding models loaded successfully, service ready")
|
7bfb9946
tangwang
向量化模块
|
458
459
|
|
efd435cf
tangwang
tei性能调优:
|
460
461
462
463
464
|
@app.on_event("shutdown")
def stop_workers() -> None:
_stop_text_batch_worker()
|
200fdddf
tangwang
embed norm
|
465
466
467
468
469
470
471
472
|
def _normalize_vector(vec: np.ndarray) -> np.ndarray:
norm = float(np.linalg.norm(vec))
if not np.isfinite(norm) or norm <= 0.0:
raise RuntimeError("Embedding vector has invalid norm (must be > 0)")
return vec / norm
def _as_list(embedding: Optional[np.ndarray], normalize: bool = False) -> Optional[List[float]]:
|
7bfb9946
tangwang
向量化模块
|
473
474
475
476
477
478
|
if embedding is None:
return None
if not isinstance(embedding, np.ndarray):
embedding = np.array(embedding, dtype=np.float32)
if embedding.ndim != 1:
embedding = embedding.reshape(-1)
|
200fdddf
tangwang
embed norm
|
479
480
481
482
|
embedding = embedding.astype(np.float32, copy=False)
if normalize:
embedding = _normalize_vector(embedding).astype(np.float32, copy=False)
return embedding.tolist()
|
7bfb9946
tangwang
向量化模块
|
483
484
485
486
|
@app.get("/health")
def health() -> Dict[str, Any]:
|
4747e2f4
tangwang
embedding perform...
|
487
|
"""Health check endpoint. Returns status and current throttling stats."""
|
0a3764c4
tangwang
优化embedding模型加载
|
488
489
490
|
return {
"status": "ok",
"text_model_loaded": _text_model is not None,
|
07cf5a93
tangwang
START_EMBEDDING=...
|
491
|
"text_backend": _text_backend_name,
|
0a3764c4
tangwang
优化embedding模型加载
|
492
|
"image_model_loaded": _image_model is not None,
|
4747e2f4
tangwang
embedding perform...
|
493
494
495
496
497
498
499
500
501
502
|
"limits": {
"text": _text_request_limiter.snapshot(),
"image": _image_request_limiter.snapshot(),
},
"text_microbatch": {
"window_ms": round(_TEXT_MICROBATCH_WINDOW_SEC * 1000.0, 3),
"queue_depth": len(_text_single_queue),
"worker_alive": bool(_text_batch_worker is not None and _text_batch_worker.is_alive()),
"request_timeout_sec": _TEXT_REQUEST_TIMEOUT_SEC,
},
|
0a3764c4
tangwang
优化embedding模型加载
|
503
|
}
|
7bfb9946
tangwang
向量化模块
|
504
505
|
|
4747e2f4
tangwang
embedding perform...
|
506
507
508
509
510
|
def _embed_text_impl(
normalized: List[str],
effective_normalize: bool,
request_id: str,
) -> List[Optional[List[float]]]:
|
0a3764c4
tangwang
优化embedding模型加载
|
511
512
|
if _text_model is None:
raise RuntimeError("Text model not loaded")
|
28e57bb1
tangwang
日志体系优化
|
513
|
|
efd435cf
tangwang
tei性能调优:
|
514
|
t0 = time.perf_counter()
|
54ccf28c
tangwang
tei
|
515
|
try:
|
efd435cf
tangwang
tei性能调优:
|
516
517
|
if _text_backend_name == "local_st":
if len(normalized) == 1 and _text_batch_worker is not None:
|
4747e2f4
tangwang
embedding perform...
|
518
519
520
521
522
523
524
|
out = [
_encode_single_text_with_microbatch(
normalized[0],
normalize=effective_normalize,
request_id=request_id,
)
]
|
efd435cf
tangwang
tei性能调优:
|
525
|
logger.info(
|
4747e2f4
tangwang
embedding perform...
|
526
|
"text backend done | backend=%s mode=microbatch-single inputs=%d normalize=%s dim=%d backend_elapsed_ms=%.2f",
|
efd435cf
tangwang
tei性能调优:
|
527
528
529
|
_text_backend_name,
len(normalized),
effective_normalize,
|
28e57bb1
tangwang
日志体系优化
|
530
|
len(out[0]) if out and out[0] is not None else 0,
|
4747e2f4
tangwang
embedding perform...
|
531
532
|
(time.perf_counter() - t0) * 1000.0,
extra=_request_log_extra(request_id),
|
efd435cf
tangwang
tei性能调优:
|
533
534
535
|
)
return out
embs = _encode_local_st(normalized, normalize_embeddings=False)
|
4747e2f4
tangwang
embedding perform...
|
536
|
mode = "direct-batch"
|
efd435cf
tangwang
tei性能调优:
|
537
|
else:
|
77516841
tangwang
tidy embeddings
|
538
|
embs = _text_model.encode(
|
54ccf28c
tangwang
tei
|
539
540
541
|
normalized,
batch_size=int(CONFIG.TEXT_BATCH_SIZE),
device=CONFIG.TEXT_DEVICE,
|
200fdddf
tangwang
embed norm
|
542
|
normalize_embeddings=effective_normalize,
|
54ccf28c
tangwang
tei
|
543
|
)
|
4747e2f4
tangwang
embedding perform...
|
544
|
mode = "backend-batch"
|
54ccf28c
tangwang
tei
|
545
|
except Exception as e:
|
4747e2f4
tangwang
embedding perform...
|
546
547
548
549
550
551
552
553
|
logger.error(
"Text embedding backend failure: %s",
e,
exc_info=True,
extra=_request_log_extra(request_id),
)
raise RuntimeError(f"Text embedding backend failure: {e}") from e
|
ed948666
tangwang
tidy
|
554
555
556
557
558
|
if embs is None or len(embs) != len(normalized):
raise RuntimeError(
f"Text model response length mismatch: expected {len(normalized)}, "
f"got {0 if embs is None else len(embs)}"
)
|
4747e2f4
tangwang
embedding perform...
|
559
|
|
ed948666
tangwang
tidy
|
560
561
|
out: List[Optional[List[float]]] = []
for i, emb in enumerate(embs):
|
200fdddf
tangwang
embed norm
|
562
|
vec = _as_list(emb, normalize=effective_normalize)
|
ed948666
tangwang
tidy
|
563
564
565
|
if vec is None:
raise RuntimeError(f"Text model returned empty embedding for index {i}")
out.append(vec)
|
4747e2f4
tangwang
embedding perform...
|
566
|
|
efd435cf
tangwang
tei性能调优:
|
567
|
logger.info(
|
4747e2f4
tangwang
embedding perform...
|
568
|
"text backend done | backend=%s mode=%s inputs=%d normalize=%s dim=%d backend_elapsed_ms=%.2f",
|
efd435cf
tangwang
tei性能调优:
|
569
|
_text_backend_name,
|
4747e2f4
tangwang
embedding perform...
|
570
|
mode,
|
efd435cf
tangwang
tei性能调优:
|
571
572
|
len(normalized),
effective_normalize,
|
28e57bb1
tangwang
日志体系优化
|
573
|
len(out[0]) if out and out[0] is not None else 0,
|
4747e2f4
tangwang
embedding perform...
|
574
575
|
(time.perf_counter() - t0) * 1000.0,
extra=_request_log_extra(request_id),
|
efd435cf
tangwang
tei性能调优:
|
576
|
)
|
7bfb9946
tangwang
向量化模块
|
577
578
579
|
return out
|
4747e2f4
tangwang
embedding perform...
|
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
|
@app.post("/embed/text")
async def embed_text(
texts: List[str],
http_request: Request,
response: Response,
normalize: Optional[bool] = None,
) -> List[Optional[List[float]]]:
request_id = _resolve_request_id(http_request)
response.headers["X-Request-ID"] = request_id
effective_normalize = bool(CONFIG.TEXT_NORMALIZE_EMBEDDINGS) if normalize is None else bool(normalize)
normalized: List[str] = []
for i, t in enumerate(texts):
if not isinstance(t, str):
raise HTTPException(status_code=400, detail=f"Invalid text at index {i}: must be string")
s = t.strip()
|
ed948666
tangwang
tidy
|
596
|
if not s:
|
4747e2f4
tangwang
embedding perform...
|
597
598
|
raise HTTPException(status_code=400, detail=f"Invalid text at index {i}: empty string")
normalized.append(s)
|
c10f90fe
tangwang
cnclip
|
599
|
|
4747e2f4
tangwang
embedding perform...
|
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
|
accepted, active = _text_request_limiter.try_acquire()
if not accepted:
logger.warning(
"embed_text rejected | client=%s backend=%s inputs=%d normalize=%s active=%d limit=%d preview=%s",
_request_client(http_request),
_text_backend_name,
len(normalized),
effective_normalize,
active,
_TEXT_MAX_INFLIGHT,
_preview_inputs(normalized, _LOG_PREVIEW_COUNT, _LOG_TEXT_PREVIEW_CHARS),
extra=_request_log_extra(request_id),
)
raise HTTPException(
status_code=_OVERLOAD_STATUS_CODE,
detail=f"Text embedding service busy: active={active}, limit={_TEXT_MAX_INFLIGHT}",
)
request_started = time.perf_counter()
success = False
try:
logger.info(
"embed_text request | client=%s backend=%s inputs=%d normalize=%s active=%d limit=%d preview=%s",
_request_client(http_request),
_text_backend_name,
len(normalized),
effective_normalize,
active,
_TEXT_MAX_INFLIGHT,
_preview_inputs(normalized, _LOG_PREVIEW_COUNT, _LOG_TEXT_PREVIEW_CHARS),
extra=_request_log_extra(request_id),
)
verbose_logger.info(
"embed_text detail | payload=%s normalize=%s backend=%s",
normalized,
effective_normalize,
_text_backend_name,
extra=_request_log_extra(request_id),
)
out = await run_in_threadpool(_embed_text_impl, normalized, effective_normalize, request_id)
success = True
latency_ms = (time.perf_counter() - request_started) * 1000.0
logger.info(
"embed_text response | backend=%s inputs=%d normalize=%s dim=%d first_vector=%s latency_ms=%.2f",
_text_backend_name,
len(normalized),
effective_normalize,
len(out[0]) if out and out[0] is not None else 0,
_preview_vector(out[0] if out else None),
latency_ms,
extra=_request_log_extra(request_id),
)
verbose_logger.info(
"embed_text result detail | count=%d first_vector=%s latency_ms=%.2f",
len(out),
out[0][: _VECTOR_PREVIEW_DIMS] if out and out[0] is not None else [],
latency_ms,
extra=_request_log_extra(request_id),
)
return out
except HTTPException:
raise
except Exception as e:
latency_ms = (time.perf_counter() - request_started) * 1000.0
logger.error(
"embed_text failed | backend=%s inputs=%d normalize=%s latency_ms=%.2f error=%s",
_text_backend_name,
len(normalized),
effective_normalize,
latency_ms,
e,
exc_info=True,
extra=_request_log_extra(request_id),
)
raise HTTPException(status_code=502, detail=str(e)) from e
finally:
remaining = _text_request_limiter.release(success=success)
logger.info(
"embed_text finalize | success=%s active_after=%d",
success,
remaining,
extra=_request_log_extra(request_id),
)
def _embed_image_impl(
urls: List[str],
effective_normalize: bool,
request_id: str,
) -> List[Optional[List[float]]]:
if _image_model is None:
raise RuntimeError("Image model not loaded")
|
28e57bb1
tangwang
日志体系优化
|
692
693
|
t0 = time.perf_counter()
|
7bfb9946
tangwang
向量化模块
|
694
|
with _image_encode_lock:
|
200fdddf
tangwang
embed norm
|
695
696
697
698
699
|
vectors = _image_model.encode_image_urls(
urls,
batch_size=CONFIG.IMAGE_BATCH_SIZE,
normalize_embeddings=effective_normalize,
)
|
ed948666
tangwang
tidy
|
700
701
702
703
704
|
if vectors is None or len(vectors) != len(urls):
raise RuntimeError(
f"Image model response length mismatch: expected {len(urls)}, "
f"got {0 if vectors is None else len(vectors)}"
)
|
4747e2f4
tangwang
embedding perform...
|
705
|
|
ed948666
tangwang
tidy
|
706
707
|
out: List[Optional[List[float]]] = []
for i, vec in enumerate(vectors):
|
200fdddf
tangwang
embed norm
|
708
|
out_vec = _as_list(vec, normalize=effective_normalize)
|
ed948666
tangwang
tidy
|
709
710
711
|
if out_vec is None:
raise RuntimeError(f"Image model returned empty embedding for index {i}")
out.append(out_vec)
|
4747e2f4
tangwang
embedding perform...
|
712
|
|
28e57bb1
tangwang
日志体系优化
|
713
|
logger.info(
|
4747e2f4
tangwang
embedding perform...
|
714
|
"image backend done | inputs=%d normalize=%s dim=%d backend_elapsed_ms=%.2f",
|
28e57bb1
tangwang
日志体系优化
|
715
716
717
|
len(urls),
effective_normalize,
len(out[0]) if out and out[0] is not None else 0,
|
4747e2f4
tangwang
embedding perform...
|
718
719
|
(time.perf_counter() - t0) * 1000.0,
extra=_request_log_extra(request_id),
|
28e57bb1
tangwang
日志体系优化
|
720
|
)
|
7bfb9946
tangwang
向量化模块
|
721
|
return out
|
4747e2f4
tangwang
embedding perform...
|
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
|
@app.post("/embed/image")
async def embed_image(
images: List[str],
http_request: Request,
response: Response,
normalize: Optional[bool] = None,
) -> List[Optional[List[float]]]:
request_id = _resolve_request_id(http_request)
response.headers["X-Request-ID"] = request_id
effective_normalize = bool(CONFIG.IMAGE_NORMALIZE_EMBEDDINGS) if normalize is None else bool(normalize)
urls: List[str] = []
for i, url_or_path in enumerate(images):
if not isinstance(url_or_path, str):
raise HTTPException(status_code=400, detail=f"Invalid image at index {i}: must be string URL/path")
s = url_or_path.strip()
if not s:
raise HTTPException(status_code=400, detail=f"Invalid image at index {i}: empty URL/path")
urls.append(s)
accepted, active = _image_request_limiter.try_acquire()
if not accepted:
logger.warning(
"embed_image rejected | client=%s inputs=%d normalize=%s active=%d limit=%d preview=%s",
_request_client(http_request),
len(urls),
effective_normalize,
active,
_IMAGE_MAX_INFLIGHT,
_preview_inputs(urls, _LOG_PREVIEW_COUNT, _LOG_IMAGE_PREVIEW_CHARS),
extra=_request_log_extra(request_id),
)
raise HTTPException(
status_code=_OVERLOAD_STATUS_CODE,
detail=f"Image embedding service busy: active={active}, limit={_IMAGE_MAX_INFLIGHT}",
)
request_started = time.perf_counter()
success = False
try:
logger.info(
"embed_image request | client=%s inputs=%d normalize=%s active=%d limit=%d preview=%s",
_request_client(http_request),
len(urls),
effective_normalize,
active,
_IMAGE_MAX_INFLIGHT,
_preview_inputs(urls, _LOG_PREVIEW_COUNT, _LOG_IMAGE_PREVIEW_CHARS),
extra=_request_log_extra(request_id),
)
verbose_logger.info(
"embed_image detail | payload=%s normalize=%s",
urls,
effective_normalize,
extra=_request_log_extra(request_id),
)
out = await run_in_threadpool(_embed_image_impl, urls, effective_normalize, request_id)
success = True
latency_ms = (time.perf_counter() - request_started) * 1000.0
logger.info(
"embed_image response | inputs=%d normalize=%s dim=%d first_vector=%s latency_ms=%.2f",
len(urls),
effective_normalize,
len(out[0]) if out and out[0] is not None else 0,
_preview_vector(out[0] if out else None),
latency_ms,
extra=_request_log_extra(request_id),
)
verbose_logger.info(
"embed_image result detail | count=%d first_vector=%s latency_ms=%.2f",
len(out),
out[0][: _VECTOR_PREVIEW_DIMS] if out and out[0] is not None else [],
latency_ms,
extra=_request_log_extra(request_id),
)
return out
except HTTPException:
raise
except Exception as e:
latency_ms = (time.perf_counter() - request_started) * 1000.0
logger.error(
"embed_image failed | inputs=%d normalize=%s latency_ms=%.2f error=%s",
len(urls),
effective_normalize,
latency_ms,
e,
exc_info=True,
extra=_request_log_extra(request_id),
)
raise HTTPException(status_code=502, detail=f"Image embedding backend failure: {e}") from e
finally:
remaining = _image_request_limiter.release(success=success)
logger.info(
"embed_image finalize | success=%s active_after=%d",
success,
remaining,
extra=_request_log_extra(request_id),
)
|