Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2fd6272bbd | ||
|
|
470d53cad6 | ||
|
|
f2b61bae7d | ||
|
|
588f2770b7 | ||
|
|
ad743d6ab7 | ||
|
|
6145d0b005 | ||
|
|
d147fcb070 | ||
|
|
6196b93e97 | ||
|
|
b51933d625 | ||
|
|
40ff118798 | ||
|
|
d470677abe | ||
|
|
6eac3c6100 | ||
|
|
d22513730a | ||
|
|
613cece94b | ||
|
|
5cd1c12323 | ||
|
|
dcf73b25bf | ||
|
|
64a67e9c8d | ||
|
|
b6160af3c5 | ||
|
|
2ea41dfb75 | ||
|
|
cbcc419f0b | ||
|
|
687949beb2 | ||
|
|
fedaa5f5df | ||
|
|
c272c87e1f | ||
|
|
8a052ff910 | ||
|
|
d1c7a4b8c0 | ||
|
|
1a104338e8 | ||
|
|
5dd47b2d01 | ||
|
|
05c0088f1b | ||
|
|
05b48f3656 | ||
|
|
5fa1cd741e | ||
|
|
2cf95fd1c1 | ||
|
|
fb23d62fb2 | ||
|
|
eb4ecff6ed | ||
|
|
ffdf931cad | ||
|
|
5a538863b1 | ||
|
|
f6e0130035 | ||
|
|
6f882845c2 | ||
|
|
5654f83644 | ||
|
|
6d15420b08 | ||
|
|
5469eca450 | ||
|
|
94fd46c8ca | ||
|
|
57bd964d22 | ||
|
|
aac3a187d1 |
No files matched your search
+10
-1
@@ -26,8 +26,17 @@ RABBITMQ_USER=rabbitmq
|
||||
RABBITMQ_PASSWORD=rabbitmq_Gs123dA
|
||||
RABBITMQ_VHOST=/
|
||||
|
||||
# Billing queue / retry settings
|
||||
BILLING_QUEUE_NAME=billing.api.calls
|
||||
# Max retry attempts (excluding initial consumption)
|
||||
BILLING_RETRY_TIMES=3
|
||||
# Base delay seconds for exponential backoff: base * 2^retry_count
|
||||
BILLING_RETRY_BASE_DELAY_SECONDS=30
|
||||
# Max delay seconds cap
|
||||
BILLING_RETRY_MAX_DELAY_SECONDS=1800
|
||||
|
||||
# Payment config encryption key (Base64-encoded 32-byte random value for AES-GCM)
|
||||
# Quick generation: python -c "import os,base64; print(base64.b64encode(os.urandom(32)).decode())"
|
||||
PAYMENT_CONFIG_SECRET_KEY=uyyhfMJ+xpEE+IC5mPigpscekkR2ucz6Sj7sHYLhJQA=
|
||||
# Optional: key ID used to support key rotation identification
|
||||
PAYMENT_CONFIG_KEY_ID=default
|
||||
PAYMENT_CONFIG_KEY_ID=default
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
node-version: '22.16'
|
||||
- name: Pnpm install and build
|
||||
run: |
|
||||
npm install -g pnpm
|
||||
npm install -g pnpm@9
|
||||
./scripts/frontend_build.sh
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
@@ -43,4 +43,4 @@ jobs:
|
||||
QINIU_BUCKET: ${{ secrets.QINIU_BUCKET }}
|
||||
QINIU_NAME: ${{ secrets.QINIU_NAME }}
|
||||
run: cd scripts && ./docker_publish.sh ${{ secrets.DOCKER_USERNAME }} upload_qiniu
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ python-multipart
|
||||
google-auth>=2.0.0
|
||||
google-auth-oauthlib>=1.0.0
|
||||
fastapi_mcp
|
||||
mcp>=1.12.0,<2.0.0
|
||||
alipay-sdk-python>=3.0.0
|
||||
wechatpayv3[async]
|
||||
cryptography>=41.0.0
|
||||
+10
-11
@@ -3,7 +3,7 @@ import sys
|
||||
import re
|
||||
from pathlib import Path
|
||||
from dotenv import load_dotenv
|
||||
import mysql.connector
|
||||
import pymysql
|
||||
from typing import Optional, List, Tuple, Union
|
||||
|
||||
# 加载环境变量
|
||||
@@ -29,12 +29,13 @@ def get_db_connection(db_config: Optional[DBConfig] = None):
|
||||
if db_config is None:
|
||||
db_config = DBConfig()
|
||||
|
||||
return mysql.connector.connect(
|
||||
return pymysql.connect(
|
||||
host=db_config.MYSQL_HOST,
|
||||
port=db_config.MYSQL_PORT,
|
||||
user=db_config.MYSQL_USER,
|
||||
password=db_config.MYSQL_PASSWORD,
|
||||
database=db_config.MYSQL_DB
|
||||
database=db_config.MYSQL_DB,
|
||||
charset='utf8mb4'
|
||||
)
|
||||
|
||||
def get_current_version(db_config: Optional[DBConfig] = None) -> Optional[str]:
|
||||
@@ -50,19 +51,17 @@ def get_current_version(db_config: Optional[DBConfig] = None) -> Optional[str]:
|
||||
conn = get_db_connection(db_config)
|
||||
cursor = conn.cursor()
|
||||
|
||||
# 查询version配置
|
||||
cursor.execute("SELECT value FROM sys_config WHERE `key` = 'version'")
|
||||
result = cursor.fetchone()
|
||||
|
||||
cursor.close()
|
||||
conn.close()
|
||||
|
||||
if result and len(result) > 0:
|
||||
# 将元组的第一个元素转换为字符串并返回
|
||||
return str(result['value'] if isinstance(result, dict) else result[0])
|
||||
if result:
|
||||
return str(result[0])
|
||||
return None
|
||||
|
||||
except mysql.connector.Error as err:
|
||||
except pymysql.Error as err:
|
||||
# 如果表不存在或其他数据库错误,返回None
|
||||
print(f"查询版本信息失败: {err}")
|
||||
return None
|
||||
@@ -124,7 +123,7 @@ def execute_sql_file(file_path: str, db_config: Optional[DBConfig] = None) -> bo
|
||||
try:
|
||||
cursor.execute(sql)
|
||||
print(f" -> 成功")
|
||||
except mysql.connector.Error as e:
|
||||
except pymysql.Error as e:
|
||||
print(f" -> SET语句执行错误: {e}")
|
||||
# SET语句失败不影响后续执行
|
||||
continue
|
||||
@@ -144,7 +143,7 @@ def execute_sql_file(file_path: str, db_config: Optional[DBConfig] = None) -> bo
|
||||
try:
|
||||
cursor.execute(sql)
|
||||
print(f" -> 成功")
|
||||
except mysql.connector.Error as e:
|
||||
except pymysql.Error as e:
|
||||
if "Duplicate column name" in str(e):
|
||||
print(f" -> 忽略重复列错误: {e}")
|
||||
continue
|
||||
@@ -166,7 +165,7 @@ def execute_sql_file(file_path: str, db_config: Optional[DBConfig] = None) -> bo
|
||||
except FileNotFoundError:
|
||||
print(f"错误: SQL 文件不存在: {file_path}")
|
||||
return False
|
||||
except mysql.connector.Error as err:
|
||||
except pymysql.Error as err:
|
||||
print(f"数据库错误: {err}")
|
||||
if sql:
|
||||
print(f"出错的SQL语句: {sql}")
|
||||
|
||||
@@ -66,6 +66,7 @@ CREATE TABLE IF NOT EXISTS `user_apikey` (
|
||||
`expire_at` TIMESTAMP NULL DEFAULT NULL COMMENT 'Expiration timestamp',
|
||||
`created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Creation timestamp',
|
||||
`updated_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Last update timestamp',
|
||||
`is_deleted` TINYINT NOT NULL DEFAULT 0 COMMENT 'Logical deletion: 0 (normal), 1 (deleted)',
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE INDEX `uk_apikey` (`apikey`) COMMENT 'Unique API key index',
|
||||
FOREIGN KEY (`user_id`) REFERENCES `user` (`id`) ON DELETE CASCADE
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
ALTER TABLE `user_apikey` ADD COLUMN `is_deleted` TINYINT NOT NULL DEFAULT 0 COMMENT 'Logical deletion: 0 (normal), 1 (deleted)' AFTER `updated_at`;
|
||||
|
||||
INSERT INTO `sys_config` (`id`,`key`, `value`,`description`,`created_at`,`updated_at`)
|
||||
VALUES ('xpack-version','version', '1.4.0', 'Add is_deleted column to user_apikey', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
ON DUPLICATE KEY UPDATE `value` = VALUES(`value`), `description` = VALUES(`description`), `updated_at` = CURRENT_TIMESTAMP;
|
||||
@@ -5,7 +5,7 @@
|
||||
"downloads": {
|
||||
"program": {
|
||||
"xpack_mcp_market": {
|
||||
"version": "v1.4.1",
|
||||
"version": "v1.4.5",
|
||||
"docker": "xpackai/xpack-mcp-market",
|
||||
"tar": "xpack-mcp-market.tar.gz",
|
||||
"url": "https://public.eolinker.com/xpack/images/xpack-mcp-market.${VERSION}.${ARCH}.tar.gz"
|
||||
|
||||
@@ -21,9 +21,14 @@ class BillingMessageConsumer:
|
||||
|
||||
def __init__(self):
|
||||
self.queue_name = os.getenv("BILLING_QUEUE_NAME") or "billing.api.calls"
|
||||
self.retry_queue_name = f"{self.queue_name}.retry"
|
||||
self.connection = None
|
||||
self.channel = None
|
||||
self.consuming = False
|
||||
# retry times from env, default 3
|
||||
self.max_retries = int(os.getenv("BILLING_RETRY_TIMES", "3"))
|
||||
self.retry_base_delay_seconds = int(os.getenv("BILLING_RETRY_BASE_DELAY_SECONDS", "30"))
|
||||
self.retry_max_delay_seconds = int(os.getenv("BILLING_RETRY_MAX_DELAY_SECONDS", "1800"))
|
||||
self._setup_connection()
|
||||
|
||||
def _setup_connection(self):
|
||||
@@ -48,6 +53,14 @@ class BillingMessageConsumer:
|
||||
|
||||
# Declare queue
|
||||
self.channel.queue_declare(queue=self.queue_name, durable=True)
|
||||
self.channel.queue_declare(
|
||||
queue=self.retry_queue_name,
|
||||
durable=True,
|
||||
arguments={
|
||||
"x-dead-letter-exchange": "",
|
||||
"x-dead-letter-routing-key": self.queue_name,
|
||||
},
|
||||
)
|
||||
|
||||
# Set QoS, only process one message at a time
|
||||
self.channel.basic_qos(prefetch_count=1)
|
||||
@@ -124,6 +137,29 @@ class BillingMessageConsumer:
|
||||
body: Message body
|
||||
"""
|
||||
message_data = None
|
||||
settled = False
|
||||
raw_body = body.decode("utf-8", errors="replace") if isinstance(body, (bytes, bytearray)) else str(body)
|
||||
|
||||
def _safe_ack() -> None:
|
||||
nonlocal settled
|
||||
if settled:
|
||||
return
|
||||
settled = True
|
||||
try:
|
||||
channel.basic_ack(delivery_tag=method.delivery_tag)
|
||||
except Exception:
|
||||
logger.error("Failed to ack message", exc_info=True)
|
||||
|
||||
def _safe_nack(requeue: bool) -> None:
|
||||
nonlocal settled
|
||||
if settled:
|
||||
return
|
||||
settled = True
|
||||
try:
|
||||
channel.basic_nack(delivery_tag=method.delivery_tag, requeue=requeue)
|
||||
except Exception:
|
||||
logger.error("Failed to nack message", exc_info=True)
|
||||
|
||||
try:
|
||||
# Parse message
|
||||
message_data = json.loads(body)
|
||||
@@ -137,27 +173,68 @@ class BillingMessageConsumer:
|
||||
|
||||
if success:
|
||||
# Ack if processed successfully
|
||||
channel.basic_ack(delivery_tag=method.delivery_tag)
|
||||
_safe_ack()
|
||||
logger.info(f"Message processed and acknowledged: {message_data.get('user_id')}")
|
||||
else:
|
||||
# If failed, ack and drop (avoid infinite retry)
|
||||
channel.basic_ack(delivery_tag=method.delivery_tag)
|
||||
logger.error(f"Message processing failed, message dropped: {message_data.get('user_id')}")
|
||||
# Retry with limited attempts
|
||||
retry_count = self._get_retry_count(properties)
|
||||
if retry_count < self.max_retries:
|
||||
try:
|
||||
delay_ms = self._get_retry_delay_ms(retry_count)
|
||||
self._publish_delayed_retry(channel, message_data, retry_count + 1, delay_ms)
|
||||
_safe_ack()
|
||||
logger.warning(
|
||||
f"Message processing failed, scheduled retry {retry_count + 1}/{self.max_retries} after {delay_ms}ms: {message_data.get('user_id')}, tool={message_data.get('tool_name')}, "
|
||||
)
|
||||
except Exception as pub_err:
|
||||
logger.error(f"Failed to requeue message for retry: {pub_err}", exc_info=True)
|
||||
# Requeue original message
|
||||
_safe_nack(requeue=True)
|
||||
else:
|
||||
# Exceeded retries; ack and drop
|
||||
_safe_ack()
|
||||
self._log_retry_exceeded(
|
||||
reason="handler_returned_false",
|
||||
retry_count=retry_count,
|
||||
properties=properties,
|
||||
method=method,
|
||||
message_data=message_data,
|
||||
raw_body=raw_body,
|
||||
)
|
||||
|
||||
finally:
|
||||
db.close()
|
||||
try:
|
||||
db.close()
|
||||
except Exception:
|
||||
logger.error("Failed to close db session", exc_info=True)
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(f"Message format error: {str(e)}, body: {body}")
|
||||
# Format error, ack directly (do not requeue)
|
||||
channel.basic_ack(delivery_tag=method.delivery_tag)
|
||||
_safe_ack()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Exception occurred while processing message: {str(e)}", exc_info=True)
|
||||
# On exception, ack and drop (avoid infinite retry)
|
||||
try:
|
||||
channel.basic_ack(delivery_tag=method.delivery_tag)
|
||||
logger.error(f"Exception handling failed, message dropped")
|
||||
retry_count = self._get_retry_count(properties)
|
||||
if message_data is not None and retry_count < self.max_retries:
|
||||
delay_ms = self._get_retry_delay_ms(retry_count)
|
||||
self._publish_delayed_retry(channel, message_data, retry_count + 1, delay_ms, error=str(e))
|
||||
_safe_ack()
|
||||
logger.warning(
|
||||
f"Exception during processing, scheduled retry {retry_count + 1}/{self.max_retries} after {delay_ms}ms: {message_data.get('user_id')}"
|
||||
)
|
||||
else:
|
||||
_safe_ack()
|
||||
self._log_retry_exceeded(
|
||||
reason="exception",
|
||||
retry_count=retry_count,
|
||||
properties=properties,
|
||||
method=method,
|
||||
message_data=message_data,
|
||||
error=str(e),
|
||||
raw_body=raw_body,
|
||||
)
|
||||
except:
|
||||
# If even ack fails, log error but do not raise
|
||||
logger.error("Unable to acknowledge message", exc_info=True)
|
||||
@@ -173,9 +250,62 @@ class BillingMessageConsumer:
|
||||
int: Retry count
|
||||
"""
|
||||
if properties and properties.headers:
|
||||
return properties.headers.get("x-retry-count", 0)
|
||||
raw = properties.headers.get("x-retry-count", 0)
|
||||
try:
|
||||
return int(raw)
|
||||
except Exception:
|
||||
return 0
|
||||
return 0
|
||||
|
||||
def _get_retry_delay_ms(self, current_retry_count: int) -> int:
|
||||
delay_seconds = self.retry_base_delay_seconds * (2 ** current_retry_count)
|
||||
if delay_seconds > self.retry_max_delay_seconds:
|
||||
delay_seconds = self.retry_max_delay_seconds
|
||||
return int(delay_seconds * 1000)
|
||||
|
||||
def _publish_delayed_retry(self, channel, message_data: dict, next_retry_count: int, delay_ms: int, error: str | None = None) -> None:
|
||||
headers: dict[str, str | int] = {"x-retry-count": next_retry_count}
|
||||
if error:
|
||||
headers["x-last-error"] = error
|
||||
props = pika.BasicProperties(
|
||||
delivery_mode=2,
|
||||
headers=headers,
|
||||
expiration=str(delay_ms),
|
||||
)
|
||||
channel.basic_publish(exchange="", routing_key=self.retry_queue_name, body=json.dumps(message_data), properties=props)
|
||||
|
||||
def _log_retry_exceeded(self, reason: str, retry_count: int, properties, method, message_data: dict | None, error: str | None = None, raw_body: str | None = None) -> None:
|
||||
headers = {}
|
||||
if properties and properties.headers:
|
||||
try:
|
||||
headers = dict(properties.headers)
|
||||
except Exception:
|
||||
headers = {}
|
||||
event = {
|
||||
"reason": reason,
|
||||
"retry_count": retry_count,
|
||||
"max_retries": self.max_retries,
|
||||
"queue": self.queue_name,
|
||||
"routing_key": getattr(method, "routing_key", None),
|
||||
"delivery_tag": getattr(method, "delivery_tag", None),
|
||||
"headers": headers,
|
||||
"user_id": (message_data or {}).get("user_id"),
|
||||
"service_id": (message_data or {}).get("service_id"),
|
||||
"api_id": (message_data or {}).get("api_id"),
|
||||
"tool_name": (message_data or {}).get("tool_name"),
|
||||
"call_start_time": (message_data or {}).get("call_start_time"),
|
||||
"call_end_time": (message_data or {}).get("call_end_time"),
|
||||
}
|
||||
if error:
|
||||
event["error"] = error
|
||||
if raw_body is not None:
|
||||
event["raw_body"] = raw_body
|
||||
try:
|
||||
payload = json.dumps(event, ensure_ascii=False, default=str)
|
||||
except Exception:
|
||||
payload = repr(event)
|
||||
logger.error(f"ALERT billing message retries exceeded: {payload}")
|
||||
|
||||
|
||||
def start_billing_consumer():
|
||||
"""Start billing message consumer"""
|
||||
|
||||
@@ -11,17 +11,20 @@ class McpCallLogRepository:
|
||||
def __init__(self, db: Session):
|
||||
self.db = db
|
||||
|
||||
def create(self, call_log: McpCallLog) -> McpCallLog:
|
||||
def create(self, call_log: McpCallLog, commit: bool = True) -> McpCallLog:
|
||||
self.db.add(call_log)
|
||||
self.db.commit()
|
||||
self.db.refresh(call_log)
|
||||
if commit:
|
||||
self.db.commit()
|
||||
self.db.refresh(call_log)
|
||||
else:
|
||||
self.db.flush()
|
||||
return call_log
|
||||
|
||||
def get_by_id(self, log_id: str) -> Optional[McpCallLog]:
|
||||
return self.db.query(McpCallLog).filter(McpCallLog.id == log_id).first()
|
||||
|
||||
def update_status(
|
||||
self, log_id: str, status: ProcessStatus, error_msg: Optional[str] = None, wallet_history_id: Optional[str] = None
|
||||
self, log_id: str, status: ProcessStatus, error_msg: Optional[str] = None, wallet_history_id: Optional[str] = None, commit: bool = True
|
||||
) -> bool:
|
||||
log_record = self.get_by_id(log_id)
|
||||
if log_record:
|
||||
@@ -30,7 +33,10 @@ class McpCallLogRepository:
|
||||
log_record.error_msg = error_msg
|
||||
if wallet_history_id:
|
||||
log_record.wallet_history_id = wallet_history_id
|
||||
self.db.commit()
|
||||
if commit:
|
||||
self.db.commit()
|
||||
else:
|
||||
self.db.flush()
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from datetime import date, datetime, timedelta
|
||||
from typing import Optional, List, Tuple, cast
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy import func, literal_column
|
||||
from sqlalchemy import Boolean, func, literal_column
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.dialects.mysql import insert as mysql_insert
|
||||
from sqlalchemy.sql.schema import Table
|
||||
@@ -91,14 +91,12 @@ class StatsMcpServiceDateRepository:
|
||||
total = query.scalar()
|
||||
return int(total or 0)
|
||||
|
||||
def increment(self, service_id: str, stats_date: datetime, inc: int = 1) -> None:
|
||||
def increment(self, service_id: str, stats_date: datetime, inc: int = 1, commit: bool = True):
|
||||
"""Atomically increment call_count for (service_id, stats_date) under high concurrency.
|
||||
|
||||
Implementation uses MySQL's ON DUPLICATE KEY UPDATE to avoid race conditions.
|
||||
If the row does not exist, it will be inserted with call_count=inc; otherwise, call_count is increased.
|
||||
"""
|
||||
# Logger for debug purposes
|
||||
print(f"Incrementing call count for service {service_id} on {stats_date} by {inc}")
|
||||
|
||||
# Cast mapped table for typing compatibility with mysql_insert
|
||||
table: Table = cast(Table, StatsMcpServiceDate.__table__)
|
||||
@@ -114,7 +112,10 @@ class StatsMcpServiceDateRepository:
|
||||
updated_at=func.current_timestamp(),
|
||||
)
|
||||
self.db.execute(ondup)
|
||||
self.db.commit()
|
||||
if commit:
|
||||
self.db.commit()
|
||||
else:
|
||||
self.db.flush()
|
||||
|
||||
def increment_fallback(self, service_id: str, stats_date: datetime, inc: int = 1) -> None:
|
||||
"""Fallback increment strategy without dialect-specific upsert.
|
||||
|
||||
@@ -296,7 +296,7 @@ class UserWalletHistoryRepository:
|
||||
)
|
||||
return total, history
|
||||
|
||||
def add_consume_record(self, wallet_history: UserWalletHistory) -> Optional[UserWalletHistory]:
|
||||
def add_consume_record(self, wallet_history: UserWalletHistory, commit: bool = True) -> Optional[UserWalletHistory]:
|
||||
"""
|
||||
Add consume record to database.
|
||||
|
||||
@@ -310,12 +310,18 @@ class UserWalletHistoryRepository:
|
||||
# Add the wallet history record to database
|
||||
# The created_at and updated_at fields should be set by the caller
|
||||
self.db.add(wallet_history)
|
||||
self.db.commit()
|
||||
self.db.refresh(wallet_history)
|
||||
if commit:
|
||||
self.db.commit()
|
||||
self.db.refresh(wallet_history)
|
||||
else:
|
||||
self.db.flush()
|
||||
return wallet_history
|
||||
except Exception as e:
|
||||
self.logging.error(f"Failed to add consume record: {str(e)}")
|
||||
self.db.rollback()
|
||||
try:
|
||||
self.db.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
def order_list(self, payment_method: str, status: int, start: Optional[datetime] = None, end: Optional[datetime] = None) -> List[UserWalletHistory]:
|
||||
@@ -435,4 +441,4 @@ class UserWalletHistoryRepository:
|
||||
result.append({"stats_day": current_day, "count": amounts_by_day.get(current_day, 0.0)})
|
||||
current_day += timedelta(days=1)
|
||||
|
||||
return result
|
||||
return result
|
||||
@@ -32,13 +32,16 @@ class UserWalletRepository:
|
||||
"""Get user wallet with row-level lock for update"""
|
||||
return self.db.query(UserWallet).filter(UserWallet.user_id == user_id).with_for_update().first()
|
||||
|
||||
def update_balance(self, user_id: str, new_balance: float) -> bool:
|
||||
def update_balance(self, user_id: str, new_balance: float, commit: bool = True) -> bool:
|
||||
"""Update user balance"""
|
||||
wallet = self.get_by_user_id(user_id)
|
||||
if wallet:
|
||||
wallet.balance = new_balance
|
||||
wallet.updated_at = datetime.now(timezone.utc)
|
||||
self.db.commit()
|
||||
if commit:
|
||||
self.db.commit()
|
||||
else:
|
||||
self.db.flush()
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ Billing message handler service
|
||||
"""
|
||||
|
||||
import json
|
||||
from re import finditer
|
||||
import uuid
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
@@ -44,41 +45,60 @@ class BillingMessageHandler:
|
||||
Returns:
|
||||
bool: Whether processing was successful
|
||||
"""
|
||||
call_log_id: Optional[str] = None
|
||||
tx = self.db.begin()
|
||||
try:
|
||||
# Parse message
|
||||
billing_message = self._parse_message(message_data)
|
||||
if not billing_message:
|
||||
tx.rollback()
|
||||
return False
|
||||
|
||||
# Create API call log
|
||||
call_log_id = self._create_call_log(billing_message)
|
||||
if not call_log_id:
|
||||
tx.rollback()
|
||||
return False
|
||||
|
||||
# Update stats_mcp_service_date record
|
||||
# Bucket stats by hour (UTC), aligned to full hours
|
||||
|
||||
start_dt = billing_message.call_start_time
|
||||
start_utc = start_dt.replace(tzinfo=timezone.utc) if start_dt.tzinfo is None else start_dt.astimezone(timezone.utc)
|
||||
# Store as naive UTC datetime to match MySQL DATETIME behavior
|
||||
stats_date = start_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None)
|
||||
self.stats_repo.increment(billing_message.service_id, stats_date, 1)
|
||||
stats_date = start_utc.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=None)
|
||||
self.stats_repo.increment(billing_message.service_id, stats_date, 1, commit=False)
|
||||
|
||||
# Process billing logic
|
||||
if billing_message.call_success and billing_message.unit_price > 0:
|
||||
success = self._process_billing(billing_message, call_log_id)
|
||||
success = self._process_billing(billing_message, call_log_id, commit=False)
|
||||
if not success:
|
||||
# Update record status to failed
|
||||
self.call_log_repo.update_status(call_log_id, ProcessStatus.FAILED, "Billing processing failed")
|
||||
self.call_log_repo.update_status(call_log_id, ProcessStatus.FAILED, "Billing processing failed", commit=False)
|
||||
tx.rollback()
|
||||
return False
|
||||
|
||||
# Update record status to processed
|
||||
self.call_log_repo.update_status(call_log_id, ProcessStatus.PROCESSED)
|
||||
logger.info(f"Billing message processed successfully - User ID: {billing_message.user_id}, Tool: {billing_message.tool_name}")
|
||||
return True
|
||||
|
||||
if not self.call_log_repo.update_status(call_log_id, ProcessStatus.PROCESSED, commit=False):
|
||||
tx.rollback()
|
||||
return False
|
||||
tx.commit()
|
||||
logger.info(f"Billing message processed successfully - User ID: {billing_message.user_id}, Tool: {billing_message.tool_name}")
|
||||
# Update wallet cache after successful commit
|
||||
try:
|
||||
wallet = self.wallet_repo.get_by_user_id(billing_message.user_id)
|
||||
if wallet:
|
||||
self._update_wallet_cache(billing_message.user_id, Decimal(str(wallet.balance)))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to process billing message: {str(e)}", exc_info=True)
|
||||
try:
|
||||
tx.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
if call_log_id:
|
||||
try:
|
||||
self.call_log_repo.update_status(call_log_id, ProcessStatus.FAILED, "Billing processing exception")
|
||||
except Exception:
|
||||
logger.exception("Failed to update call log status after exception")
|
||||
return False
|
||||
finally:
|
||||
if tx.is_active:
|
||||
tx.rollback()
|
||||
|
||||
def _parse_message(self, message_data: dict) -> Optional[BillingMessage]:
|
||||
"""
|
||||
@@ -146,14 +166,14 @@ class BillingMessageHandler:
|
||||
updated_at=datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
created_log = self.call_log_repo.create(call_log)
|
||||
created_log = self.call_log_repo.create(call_log, commit=False)
|
||||
return created_log.id
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to create API call log record: {str(e)}")
|
||||
return None
|
||||
|
||||
def _process_billing(self, billing_message: BillingMessage, call_log_id: str) -> bool:
|
||||
def _process_billing(self, billing_message: BillingMessage, call_log_id: str, commit: bool = True) -> bool:
|
||||
"""
|
||||
Process actual billing logic
|
||||
|
||||
@@ -183,7 +203,7 @@ class BillingMessageHandler:
|
||||
|
||||
# Execute deduction
|
||||
new_balance = current_balance - amount
|
||||
success = self.wallet_repo.update_balance(user_id, float(new_balance))
|
||||
success = self.wallet_repo.update_balance(user_id, float(new_balance), commit=commit)
|
||||
if not success:
|
||||
logger.error(f"Failed to update user wallet balance - User ID: {user_id}")
|
||||
return False
|
||||
@@ -206,22 +226,15 @@ class BillingMessageHandler:
|
||||
updated_at=now,
|
||||
)
|
||||
|
||||
created_history = self.wallet_history_repo.add_consume_record(wallet_history)
|
||||
created_history = self.wallet_history_repo.add_consume_record(wallet_history, commit=commit)
|
||||
if not created_history:
|
||||
logger.error(f"Failed to create wallet history record - User ID: {user_id}")
|
||||
# Rollback balance update
|
||||
self.wallet_repo.update_balance(user_id, float(current_balance))
|
||||
return False
|
||||
|
||||
logger.info(f"Wallet history record created successfully - User ID: {user_id}, History ID: {history_id}, Created at: {now}")
|
||||
|
||||
# Update API call record's actual deduction amount and associated history record ID
|
||||
self.call_log_repo.update_status(call_log_id, ProcessStatus.PROCESSED, None, history_id)
|
||||
|
||||
|
||||
|
||||
# Update wallet balance cache in Redis
|
||||
self._update_wallet_cache(user_id, new_balance)
|
||||
self.call_log_repo.update_status(call_log_id, ProcessStatus.PROCESSED, None, history_id, commit=commit)
|
||||
|
||||
logger.info(
|
||||
f"Billing processing completed successfully - User ID: {user_id}, Amount deducted: {amount}, Balance: {current_balance} -> {new_balance}"
|
||||
|
||||
@@ -5,6 +5,7 @@ Supports both service_id (UUID) and slug_name service identifiers
|
||||
|
||||
from typing import Optional
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
from sqlalchemy import false
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import Response
|
||||
@@ -165,6 +166,10 @@ class McpController:
|
||||
|
||||
logger.info(f"Received SSE connection request - Service ID: {service_id}, Client: {client_ip}, UA: {user_agent[:50]}...")
|
||||
|
||||
# Validate Accept & Content-Type headers according to MCP Streamable HTTP rules
|
||||
accept = (req.headers.get('accept') or '').lower()
|
||||
content_type = (req.headers.get('content-type') or '').lower()
|
||||
|
||||
# Extract user ID (for billing) - must provide valid apikey
|
||||
user_info = self._extract_user_info(req)
|
||||
if not user_info:
|
||||
@@ -288,6 +293,9 @@ class McpController:
|
||||
|
||||
logger.info(f"Received StreamableHTTP request - Service ID: {service_id}, Client: {client_ip}, UA: {user_agent[:50]}...")
|
||||
|
||||
accept = (req.headers.get('accept') or '').lower()
|
||||
content_type = (req.headers.get('content-type') or '').lower()
|
||||
|
||||
# Security: validate Origin to prevent DNS rebinding (MCP spec requirement)
|
||||
origin = req.headers.get('origin')
|
||||
if origin and not self._is_origin_allowed(origin, req):
|
||||
@@ -304,9 +312,42 @@ class McpController:
|
||||
await send({'type': 'http.response.body', 'body': response_body})
|
||||
return
|
||||
|
||||
# Validate Accept & Content-Type headers according to MCP Streamable HTTP rules
|
||||
# Extract user ID (for billing) - must provide valid apikey
|
||||
user_info = self._extract_user_info(req)
|
||||
if not user_info:
|
||||
response_body = b"Missing or invalid apikey parameter"
|
||||
await send({
|
||||
'type': 'http.response.start',
|
||||
'status': 401,
|
||||
'headers': [
|
||||
[b'content-type', b'text/plain'],
|
||||
[b'content-length', str(len(response_body)).encode()],
|
||||
],
|
||||
})
|
||||
await send({'type': 'http.response.body', 'body': response_body})
|
||||
return
|
||||
user_id, apikey_id = user_info
|
||||
if not self._check_invoke_permission(user_id, service_id):
|
||||
response_body = b"Invoke permission denied"
|
||||
await send({
|
||||
'type': 'http.response.start',
|
||||
'status': 403,
|
||||
'headers': [
|
||||
[b'content-type', b'text/plain'],
|
||||
[b'content-length', str(len(response_body)).encode()],
|
||||
],
|
||||
})
|
||||
await send({'type': 'http.response.body', 'body': response_body})
|
||||
return
|
||||
|
||||
# MCP 0728 (2026-07-28) Stateless Handler for POST JSON-RPC requests
|
||||
if req.method == 'POST' and ('application/json' in content_type or req.headers.get("mcp-method") or req.headers.get("mcp-protocol-version")):
|
||||
stateless_handled = await self._handle_stateless_mcp_request(req, send, service_id, user_id, apikey_id)
|
||||
if stateless_handled:
|
||||
return
|
||||
|
||||
# Validate Accept & Content-Type headers according to legacy MCP Streamable HTTP rules
|
||||
accept = (req.headers.get('accept') or '').lower()
|
||||
content_type = (req.headers.get('content-type') or '').lower()
|
||||
if req.method == 'GET':
|
||||
if 'text/event-stream' not in accept:
|
||||
logger.warning("Missing Accept: text/event-stream for GET SSE")
|
||||
@@ -350,34 +391,6 @@ class McpController:
|
||||
})
|
||||
await send({'type': 'http.response.body', 'body': response_body})
|
||||
return
|
||||
|
||||
# Extract user ID (for billing) - must provide valid apikey
|
||||
user_info = self._extract_user_info(req)
|
||||
if not user_info:
|
||||
response_body = b"Missing or invalid apikey parameter"
|
||||
await send({
|
||||
'type': 'http.response.start',
|
||||
'status': 401,
|
||||
'headers': [
|
||||
[b'content-type', b'text/plain'],
|
||||
[b'content-length', str(len(response_body)).encode()],
|
||||
],
|
||||
})
|
||||
await send({'type': 'http.response.body', 'body': response_body})
|
||||
return
|
||||
user_id, apikey_id = user_info
|
||||
if not self._check_invoke_permission(user_id, service_id):
|
||||
response_body = b"Invoke permission denied"
|
||||
await send({
|
||||
'type': 'http.response.start',
|
||||
'status': 403,
|
||||
'headers': [
|
||||
[b'content-type', b'text/plain'],
|
||||
[b'content-length', str(len(response_body)).encode()],
|
||||
],
|
||||
})
|
||||
await send({'type': 'http.response.body', 'body': response_body})
|
||||
return
|
||||
raw_client_id = req.headers.get('x-client-instance-id') or req.headers.get('x-client-id')
|
||||
client_instance_id = (re.sub(r'[^a-zA-Z0-9._-]', '', raw_client_id)[:32]) if raw_client_id else None
|
||||
ua = user_agent or ""
|
||||
@@ -679,14 +692,22 @@ class McpController:
|
||||
|
||||
|
||||
def _extract_user_info(self, request: Request) -> Optional[tuple[str, str]]:
|
||||
"""Extract user ID and apikey ID from request by validating apikey parameter."""
|
||||
# Get apikey from URL query parameters
|
||||
"""Extract user ID and apikey ID from request by validating apikey parameter or headers."""
|
||||
# Get apikey from URL query parameters or Headers
|
||||
apikey = request.query_params.get("apikey")
|
||||
if not apikey:
|
||||
apikey = request.query_params.get("authkey")
|
||||
if not apikey:
|
||||
logger.warning("Apikey not found in URL parameters")
|
||||
return None
|
||||
if not apikey:
|
||||
auth_header = request.headers.get("authorization", "")
|
||||
if auth_header.startswith("Bearer "):
|
||||
apikey = auth_header[7:].strip()
|
||||
elif auth_header:
|
||||
apikey = auth_header.strip()
|
||||
if not apikey:
|
||||
apikey = request.headers.get("x-api-key")
|
||||
if not apikey:
|
||||
logger.warning("Apikey not found in URL parameters or Headers")
|
||||
return None
|
||||
|
||||
logger.debug(f"Validating apikey: {apikey[:10]}...") # Only log first 10 characters for debugging
|
||||
|
||||
@@ -775,3 +796,117 @@ class McpController:
|
||||
pass
|
||||
finally:
|
||||
logger.info("Session GC task stopped")
|
||||
|
||||
async def _handle_stateless_mcp_request(
|
||||
self,
|
||||
req: Request,
|
||||
send,
|
||||
service_id: str,
|
||||
user_id: str,
|
||||
apikey_id: str
|
||||
) -> bool:
|
||||
"""
|
||||
Handle MCP 0728 (2026-07-28) stateless HTTP JSON-RPC request.
|
||||
Returns True if request was handled directly, False if it should fallback to legacy session transport.
|
||||
"""
|
||||
mcp_method_hdr = req.headers.get("mcp-method")
|
||||
protocol_hdr = req.headers.get("mcp-protocol-version")
|
||||
|
||||
try:
|
||||
body_bytes = await req.body()
|
||||
if not body_bytes:
|
||||
return False
|
||||
payload = json.loads(body_bytes.decode("utf-8"))
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
if not isinstance(payload, dict):
|
||||
return False
|
||||
|
||||
method = mcp_method_hdr or payload.get("method")
|
||||
req_id = payload.get("id")
|
||||
|
||||
if not method:
|
||||
return False
|
||||
|
||||
logger.info(f"Handling MCP 0728 stateless request - Service ID: {service_id}, Method: {method}, ReqID: {req_id}")
|
||||
|
||||
result = None
|
||||
error = None
|
||||
|
||||
try:
|
||||
if method in ("server/discover", "discover"):
|
||||
result = await self.server_factory.get_server_discover(service_id)
|
||||
elif method in ("tools/list", "list_tools"):
|
||||
tools = await self.server_factory._handle_list_tools(service_id)
|
||||
result = {
|
||||
"tools": [t.model_dump(by_alias=True, exclude_none=True) for t in tools]
|
||||
}
|
||||
elif method in ("tools/call", "call_tool"):
|
||||
params = payload.get("params") or {}
|
||||
tool_name = req.headers.get("mcp-name") or params.get("name")
|
||||
arguments = params.get("arguments") or {}
|
||||
if not tool_name:
|
||||
error = {"code": -32602, "message": "Missing tool name in params or header"}
|
||||
else:
|
||||
contents, extra = await self.server_factory.execute_tool(
|
||||
service_id, tool_name, arguments, user_id, apikey_id
|
||||
)
|
||||
content_list = []
|
||||
for c in contents:
|
||||
if hasattr(c, "model_dump"):
|
||||
content_list.append(c.model_dump(by_alias=True, exclude_none=True))
|
||||
elif isinstance(c, dict):
|
||||
content_list.append(c)
|
||||
else:
|
||||
content_list.append({"type": "text", "text": str(c)})
|
||||
result = {
|
||||
"content": content_list,
|
||||
"isError": False
|
||||
}
|
||||
elif method == "initialize":
|
||||
result = {
|
||||
"protocolVersion": "2026-07-28",
|
||||
"capabilities": {
|
||||
"tools": {
|
||||
"listChanged": False,
|
||||
"ttlMs": 3600000,
|
||||
"cacheScope": "service"
|
||||
}
|
||||
},
|
||||
"serverInfo": {
|
||||
"name": f"mcp-service-{service_id}",
|
||||
"version": "1.0.0"
|
||||
}
|
||||
}
|
||||
elif method in ("notifications/initialized", "ping"):
|
||||
result = {}
|
||||
elif method in ("resources/list", "prompts/list"):
|
||||
result = {method.split("/")[0]: []}
|
||||
else:
|
||||
error = {"code": -32601, "message": f"Method not found: {method}"}
|
||||
except Exception as e:
|
||||
logger.error(f"Error handling stateless MCP method {method}: {str(e)}", exc_info=True)
|
||||
error = {"code": -32603, "message": f"Internal error: {str(e)}"}
|
||||
|
||||
response_payload = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": req_id
|
||||
}
|
||||
if error is not None:
|
||||
response_payload["error"] = error
|
||||
else:
|
||||
response_payload["result"] = result
|
||||
|
||||
resp_bytes = json.dumps(response_payload).encode("utf-8")
|
||||
await send({
|
||||
'type': 'http.response.start',
|
||||
'status': 200,
|
||||
'headers': [
|
||||
[b'content-type', b'application/json; charset=utf-8'],
|
||||
[b'content-length', str(len(resp_bytes)).encode()],
|
||||
[b'mcp-protocol-version', b'2026-07-28'],
|
||||
],
|
||||
})
|
||||
await send({'type': 'http.response.body', 'body': resp_bytes})
|
||||
return True
|
||||
@@ -199,7 +199,7 @@ def mcp_connections_stats():
|
||||
mcp_routes = [
|
||||
Route("/{service_id}", endpoint=mcp.handle_sse_connection_asgi, methods=["GET"]), # 保留原有 SSE GET
|
||||
Mount("/messages/", app=mcp.get_sse_mount_handler()), # 保留原有 SSE POST 挂载
|
||||
Route("/{service_id}/streamable-http", endpoint=mcp.handle_streamable_http_asgi, methods=["GET", "POST", "DELETE"]), # 新增 StreamableHTTP 路由
|
||||
Route("/{service_id}/streamable-http", endpoint=mcp.handle_streamable_http_asgi, methods=["GET", "POST", "DELETE"]), # StreamableHTTP 路由
|
||||
]
|
||||
|
||||
mcp_app = Starlette(routes=mcp_routes)
|
||||
|
||||
@@ -37,7 +37,7 @@ class UserApiKeyRepository:
|
||||
return cached_model
|
||||
|
||||
# Query from database if not in cache
|
||||
user_apikey = self.db.query(UserApiKey).filter(UserApiKey.apikey == apikey).first()
|
||||
user_apikey = self.db.query(UserApiKey).filter(UserApiKey.apikey == apikey, UserApiKey.is_deleted == 0).first()
|
||||
if user_apikey:
|
||||
# Cache the result using new SQLAlchemy-specific method
|
||||
CacheUtils.set_sqlalchemy_cache(cache_key, user_apikey, 300)
|
||||
|
||||
@@ -70,6 +70,43 @@ class McpServerFactory:
|
||||
logger.info("MCP server instance created successfully")
|
||||
return app
|
||||
|
||||
async def execute_tool(
|
||||
self,
|
||||
service_id: str,
|
||||
name: str,
|
||||
arguments: dict,
|
||||
user_id: str,
|
||||
apikey_id: Optional[str] = None
|
||||
) -> tuple[List[types.Content], dict]:
|
||||
"""
|
||||
Execute tool directly with billing for stateless 0728 protocol calls.
|
||||
"""
|
||||
return await self._handle_call_tool_with_billing(service_id, name, arguments, user_id, apikey_id)
|
||||
|
||||
async def get_server_discover(self, service_id: str) -> dict:
|
||||
"""
|
||||
Return server discovery metadata complying with MCP 0728 (2026-07-28) spec.
|
||||
Includes ttlMs and cacheScope for tools list caching.
|
||||
"""
|
||||
tools = await self._handle_list_tools(service_id)
|
||||
return {
|
||||
"protocolVersion": "2026-07-28",
|
||||
"capabilities": {
|
||||
"tools": {
|
||||
"listChanged": False,
|
||||
"ttlMs": 3600000, # Cache 1 hour
|
||||
"cacheScope": "service"
|
||||
},
|
||||
"resources": {},
|
||||
"prompts": {}
|
||||
},
|
||||
"serverInfo": {
|
||||
"name": f"mcp-service-{service_id}",
|
||||
"version": "1.0.0"
|
||||
},
|
||||
"tools": [tool.model_dump(by_alias=True, exclude_none=True) for tool in tools]
|
||||
}
|
||||
|
||||
async def _handle_list_tools(self, service_id: str) -> List[types.Tool]:
|
||||
"""
|
||||
Handle tools list query
|
||||
|
||||
@@ -30,7 +30,7 @@ class RabbitMQClient:
|
||||
logger.error(f"Failed to establish RabbitMQ connection: {str(e)}")
|
||||
raise
|
||||
|
||||
def publish(self, queue: str, message: str, persistent: bool = True):
|
||||
def publish(self, queue: str, message: str, persistent: bool = True, headers: dict | None = None):
|
||||
"""Publish message to queue"""
|
||||
try:
|
||||
# Ensure connection is available
|
||||
@@ -46,7 +46,9 @@ class RabbitMQClient:
|
||||
# Publish message
|
||||
properties = None
|
||||
if persistent:
|
||||
properties = pika.BasicProperties(delivery_mode=2)
|
||||
properties = pika.BasicProperties(delivery_mode=2, headers=headers or None)
|
||||
elif headers:
|
||||
properties = pika.BasicProperties(headers=headers)
|
||||
|
||||
self.channel.basic_publish(exchange="", routing_key=queue, body=message, properties=properties)
|
||||
logger.info(f"Message successfully published to queue: {queue}, message length: {len(message)} bytes")
|
||||
@@ -61,7 +63,10 @@ class RabbitMQClient:
|
||||
|
||||
# Retry sending
|
||||
self.channel.queue_declare(queue=queue, durable=True)
|
||||
properties = pika.BasicProperties(delivery_mode=2) if persistent else None
|
||||
if persistent:
|
||||
properties = pika.BasicProperties(delivery_mode=2, headers=headers or None)
|
||||
else:
|
||||
properties = pika.BasicProperties(headers=headers or None)
|
||||
self.channel.basic_publish(exchange="", routing_key=queue, body=message, properties=properties)
|
||||
logger.info(f"Message retry successful to queue {queue}")
|
||||
except Exception as retry_error:
|
||||
|
||||
Reference in new issue
Block a user