diff --git a/services/admin_service/consumers/billing_message_consumer.py b/services/admin_service/consumers/billing_message_consumer.py index 21980db..62fae6e 100644 --- a/services/admin_service/consumers/billing_message_consumer.py +++ b/services/admin_service/consumers/billing_message_consumer.py @@ -12,6 +12,7 @@ import os from services.common.config import Config from services.common.database import get_db from services.admin_service.services.billing_message_handler import BillingMessageHandler +from services.common.rabbitmq import rabbitmq_client logger = logging.getLogger(__name__) @@ -24,6 +25,8 @@ class BillingMessageConsumer: self.connection = None self.channel = None self.consuming = False + # retry times from env, default 3 + self.max_retries = int(os.getenv("BILLING_RETRY_TIMES", "3")) self._setup_connection() def _setup_connection(self): @@ -140,9 +143,21 @@ class BillingMessageConsumer: channel.basic_ack(delivery_tag=method.delivery_tag) logger.info(f"Message processed and acknowledged: {message_data.get('user_id')}") else: - # If failed, ack and drop (avoid infinite retry) - channel.basic_ack(delivery_tag=method.delivery_tag) - logger.error(f"Message processing failed, message dropped: {message_data.get('user_id')}") + # Retry with limited attempts + retry_count = self._get_retry_count(properties) + if retry_count < self.max_retries: + try: + rabbitmq_client.publish(self.queue_name, json.dumps(message_data), persistent=True, headers={"x-retry-count": retry_count + 1}) + channel.basic_ack(delivery_tag=method.delivery_tag) + logger.warning(f"Message processing failed, requeued for retry {retry_count + 1}/{self.max_retries}: {message_data.get('user_id')}") + except Exception as pub_err: + logger.error(f"Failed to requeue message for retry: {pub_err}", exc_info=True) + # Requeue original message + channel.basic_nack(delivery_tag=method.delivery_tag, requeue=True) + else: + # Exceeded retries; ack and drop + channel.basic_ack(delivery_tag=method.delivery_tag) + logger.error(f"Message processing failed, retries exceeded; message dropped: {message_data.get('user_id')}") finally: db.close() diff --git a/services/admin_service/repositories/mcp_call_log_repository.py b/services/admin_service/repositories/mcp_call_log_repository.py index 1e73ab3..ae0d429 100644 --- a/services/admin_service/repositories/mcp_call_log_repository.py +++ b/services/admin_service/repositories/mcp_call_log_repository.py @@ -11,17 +11,20 @@ class McpCallLogRepository: def __init__(self, db: Session): self.db = db - def create(self, call_log: McpCallLog) -> McpCallLog: + def create(self, call_log: McpCallLog, commit: bool = True) -> McpCallLog: self.db.add(call_log) - self.db.commit() - self.db.refresh(call_log) + if commit: + self.db.commit() + self.db.refresh(call_log) + else: + self.db.flush() return call_log def get_by_id(self, log_id: str) -> Optional[McpCallLog]: return self.db.query(McpCallLog).filter(McpCallLog.id == log_id).first() def update_status( - self, log_id: str, status: ProcessStatus, error_msg: Optional[str] = None, wallet_history_id: Optional[str] = None + self, log_id: str, status: ProcessStatus, error_msg: Optional[str] = None, wallet_history_id: Optional[str] = None, commit: bool = True ) -> bool: log_record = self.get_by_id(log_id) if log_record: @@ -30,7 +33,10 @@ class McpCallLogRepository: log_record.error_msg = error_msg if wallet_history_id: log_record.wallet_history_id = wallet_history_id - self.db.commit() + if commit: + self.db.commit() + else: + self.db.flush() return True return False diff --git a/services/admin_service/repositories/stats_mcp_service_date_repository.py b/services/admin_service/repositories/stats_mcp_service_date_repository.py index ef8157b..90ae5e9 100644 --- a/services/admin_service/repositories/stats_mcp_service_date_repository.py +++ b/services/admin_service/repositories/stats_mcp_service_date_repository.py @@ -1,7 +1,7 @@ from datetime import date, datetime, timedelta from typing import Optional, List, Tuple, cast from sqlalchemy.orm import Session -from sqlalchemy import func, literal_column +from sqlalchemy import Boolean, func, literal_column from sqlalchemy.exc import IntegrityError from sqlalchemy.dialects.mysql import insert as mysql_insert from sqlalchemy.sql.schema import Table @@ -91,14 +91,12 @@ class StatsMcpServiceDateRepository: total = query.scalar() return int(total or 0) - def increment(self, service_id: str, stats_date: datetime, inc: int = 1) -> None: + def increment(self, service_id: str, stats_date: datetime, inc: int = 1, commit: bool = True) -> bool: """Atomically increment call_count for (service_id, stats_date) under high concurrency. Implementation uses MySQL's ON DUPLICATE KEY UPDATE to avoid race conditions. If the row does not exist, it will be inserted with call_count=inc; otherwise, call_count is increased. """ - # Logger for debug purposes - print(f"Incrementing call count for service {service_id} on {stats_date} by {inc}") # Cast mapped table for typing compatibility with mysql_insert table: Table = cast(Table, StatsMcpServiceDate.__table__) @@ -114,7 +112,12 @@ class StatsMcpServiceDateRepository: updated_at=func.current_timestamp(), ) self.db.execute(ondup) - self.db.commit() + if commit: + self.db.commit() + return True + else: + self.db.flush() + return False def increment_fallback(self, service_id: str, stats_date: datetime, inc: int = 1) -> None: """Fallback increment strategy without dialect-specific upsert. diff --git a/services/admin_service/repositories/user_wallet_history_repository.py b/services/admin_service/repositories/user_wallet_history_repository.py index 3eafa92..f62ba65 100644 --- a/services/admin_service/repositories/user_wallet_history_repository.py +++ b/services/admin_service/repositories/user_wallet_history_repository.py @@ -296,7 +296,7 @@ class UserWalletHistoryRepository: ) return total, history - def add_consume_record(self, wallet_history: UserWalletHistory) -> Optional[UserWalletHistory]: + def add_consume_record(self, wallet_history: UserWalletHistory, commit: bool = True) -> Optional[UserWalletHistory]: """ Add consume record to database. @@ -310,12 +310,18 @@ class UserWalletHistoryRepository: # Add the wallet history record to database # The created_at and updated_at fields should be set by the caller self.db.add(wallet_history) - self.db.commit() - self.db.refresh(wallet_history) + if commit: + self.db.commit() + self.db.refresh(wallet_history) + else: + self.db.flush() return wallet_history except Exception as e: self.logging.error(f"Failed to add consume record: {str(e)}") - self.db.rollback() + try: + self.db.rollback() + except Exception: + pass return None def order_list(self, payment_method: str, status: int, start: Optional[datetime] = None, end: Optional[datetime] = None) -> List[UserWalletHistory]: @@ -435,4 +441,4 @@ class UserWalletHistoryRepository: result.append({"stats_day": current_day, "count": amounts_by_day.get(current_day, 0.0)}) current_day += timedelta(days=1) - return result \ No newline at end of file + return result diff --git a/services/admin_service/repositories/user_wallet_repository.py b/services/admin_service/repositories/user_wallet_repository.py index 2a0e0fb..893f8a9 100644 --- a/services/admin_service/repositories/user_wallet_repository.py +++ b/services/admin_service/repositories/user_wallet_repository.py @@ -32,13 +32,16 @@ class UserWalletRepository: """Get user wallet with row-level lock for update""" return self.db.query(UserWallet).filter(UserWallet.user_id == user_id).with_for_update().first() - def update_balance(self, user_id: str, new_balance: float) -> bool: + def update_balance(self, user_id: str, new_balance: float, commit: bool = True) -> bool: """Update user balance""" wallet = self.get_by_user_id(user_id) if wallet: wallet.balance = new_balance wallet.updated_at = datetime.now(timezone.utc) - self.db.commit() + if commit: + self.db.commit() + else: + self.db.flush() return True return False diff --git a/services/admin_service/services/billing_message_handler.py b/services/admin_service/services/billing_message_handler.py index 0c0cc0e..eda42f1 100644 --- a/services/admin_service/services/billing_message_handler.py +++ b/services/admin_service/services/billing_message_handler.py @@ -44,40 +44,57 @@ class BillingMessageHandler: Returns: bool: Whether processing was successful """ + call_log_id: Optional[str] = None + tx = self.db.begin() try: - # Parse message billing_message = self._parse_message(message_data) if not billing_message: + tx.rollback() return False - # Create API call log call_log_id = self._create_call_log(billing_message) if not call_log_id: + tx.rollback() return False - - # Update stats_mcp_service_date record - # Bucket stats by hour (UTC), aligned to full hours + start_dt = billing_message.call_start_time start_utc = start_dt.replace(tzinfo=timezone.utc) if start_dt.tzinfo is None else start_dt.astimezone(timezone.utc) - # Store as naive UTC datetime to match MySQL DATETIME behavior - stats_date = start_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None) - self.stats_repo.increment(billing_message.service_id, stats_date, 1) + stats_date = start_utc.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=None) + if not self.stats_repo.increment(billing_message.service_id, stats_date, 1, commit=False): + tx.rollback() + return False - # Process billing logic if billing_message.call_success and billing_message.unit_price > 0: - success = self._process_billing(billing_message, call_log_id) + success = self._process_billing(billing_message, call_log_id, commit=False) if not success: - # Update record status to failed - self.call_log_repo.update_status(call_log_id, ProcessStatus.FAILED, "Billing processing failed") + self.call_log_repo.update_status(call_log_id, ProcessStatus.FAILED, "Billing processing failed", commit=False) + tx.rollback() return False - - # Update record status to processed - self.call_log_repo.update_status(call_log_id, ProcessStatus.PROCESSED) - logger.info(f"Billing message processed successfully - User ID: {billing_message.user_id}, Tool: {billing_message.tool_name}") - return True + if not self.call_log_repo.update_status(call_log_id, ProcessStatus.PROCESSED, commit=False): + tx.rollback() + return False + tx.commit() + logger.info(f"Billing message processed successfully - User ID: {billing_message.user_id}, Tool: {billing_message.tool_name}") + # Update wallet cache after successful commit + try: + wallet = self.wallet_repo.get_by_user_id(billing_message.user_id) + if wallet: + self._update_wallet_cache(billing_message.user_id, Decimal(str(wallet.balance))) + except Exception: + pass + return True except Exception as e: logger.error(f"Failed to process billing message: {str(e)}", exc_info=True) + try: + tx.rollback() + except Exception: + pass + if call_log_id: + try: + self.call_log_repo.update_status(call_log_id, ProcessStatus.FAILED, "Billing processing exception") + except Exception: + logger.exception("Failed to update call log status after exception") return False def _parse_message(self, message_data: dict) -> Optional[BillingMessage]: @@ -153,7 +170,7 @@ class BillingMessageHandler: logger.error(f"Failed to create API call log record: {str(e)}") return None - def _process_billing(self, billing_message: BillingMessage, call_log_id: str) -> bool: + def _process_billing(self, billing_message: BillingMessage, call_log_id: str, commit: bool = True) -> bool: """ Process actual billing logic @@ -183,7 +200,7 @@ class BillingMessageHandler: # Execute deduction new_balance = current_balance - amount - success = self.wallet_repo.update_balance(user_id, float(new_balance)) + success = self.wallet_repo.update_balance(user_id, float(new_balance), commit=commit) if not success: logger.error(f"Failed to update user wallet balance - User ID: {user_id}") return False @@ -206,22 +223,15 @@ class BillingMessageHandler: updated_at=now, ) - created_history = self.wallet_history_repo.add_consume_record(wallet_history) + created_history = self.wallet_history_repo.add_consume_record(wallet_history, commit=commit) if not created_history: logger.error(f"Failed to create wallet history record - User ID: {user_id}") - # Rollback balance update - self.wallet_repo.update_balance(user_id, float(current_balance)) return False logger.info(f"Wallet history record created successfully - User ID: {user_id}, History ID: {history_id}, Created at: {now}") # Update API call record's actual deduction amount and associated history record ID - self.call_log_repo.update_status(call_log_id, ProcessStatus.PROCESSED, None, history_id) - - - - # Update wallet balance cache in Redis - self._update_wallet_cache(user_id, new_balance) + self.call_log_repo.update_status(call_log_id, ProcessStatus.PROCESSED, None, history_id, commit=commit) logger.info( f"Billing processing completed successfully - User ID: {user_id}, Amount deducted: {amount}, Balance: {current_balance} -> {new_balance}" diff --git a/services/common/rabbitmq.py b/services/common/rabbitmq.py index f2ec3cd..126f9e3 100644 --- a/services/common/rabbitmq.py +++ b/services/common/rabbitmq.py @@ -30,7 +30,7 @@ class RabbitMQClient: logger.error(f"Failed to establish RabbitMQ connection: {str(e)}") raise - def publish(self, queue: str, message: str, persistent: bool = True): + def publish(self, queue: str, message: str, persistent: bool = True, headers: dict | None = None): """Publish message to queue""" try: # Ensure connection is available @@ -46,7 +46,9 @@ class RabbitMQClient: # Publish message properties = None if persistent: - properties = pika.BasicProperties(delivery_mode=2) + properties = pika.BasicProperties(delivery_mode=2, headers=headers or None) + elif headers: + properties = pika.BasicProperties(headers=headers) self.channel.basic_publish(exchange="", routing_key=queue, body=message, properties=properties) logger.info(f"Message successfully published to queue: {queue}, message length: {len(message)} bytes") @@ -61,7 +63,10 @@ class RabbitMQClient: # Retry sending self.channel.queue_declare(queue=queue, durable=True) - properties = pika.BasicProperties(delivery_mode=2) if persistent else None + if persistent: + properties = pika.BasicProperties(delivery_mode=2, headers=headers or None) + else: + properties = pika.BasicProperties(headers=headers or None) self.channel.basic_publish(exchange="", routing_key=queue, body=message, properties=properties) logger.info(f"Message retry successful to queue {queue}") except Exception as retry_error: