init
This commit is contained in:
1
utils/__init__.py
Normal file
1
utils/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
# Utils package
|
||||
37
utils/admin_auth.py
Normal file
37
utils/admin_auth.py
Normal file
@@ -0,0 +1,37 @@
|
||||
from functools import wraps
|
||||
from flask import session, jsonify, redirect, url_for, request
|
||||
from models import Admin
|
||||
import pyotp
|
||||
|
||||
def admin_required(f):
|
||||
"""管理员权限装饰器"""
|
||||
@wraps(f)
|
||||
def decorated_function(*args, **kwargs):
|
||||
if 'admin_id' not in session:
|
||||
# 如果是API请求，返回JSON
|
||||
if request.path.startswith('/admin/api/'):
|
||||
return jsonify({'success': False, 'message': '请先登录'}), 401
|
||||
# 如果是页面请求，重定向到登录页
|
||||
return redirect(url_for('admin.login'))
|
||||
return f(*args, **kwargs)
|
||||
return decorated_function
|
||||
|
||||
def verify_2fa(admin: Admin, code: str) -> bool:
|
||||
"""验证2FA代码"""
|
||||
if not admin.is_2fa_enabled or not admin.totp_secret:
|
||||
return True
|
||||
|
||||
totp = pyotp.TOTP(admin.totp_secret)
|
||||
return totp.verify(code, valid_window=1)
|
||||
|
||||
def generate_2fa_secret() -> str:
|
||||
"""生成2FA密钥"""
|
||||
return pyotp.random_base32()
|
||||
|
||||
def get_2fa_qrcode_url(admin: Admin, secret: str) -> str:
|
||||
"""获取2FA二维码URL"""
|
||||
totp = pyotp.TOTP(secret)
|
||||
return totp.provisioning_uri(
|
||||
name=admin.username,
|
||||
issuer_name='短视频解析平台'
|
||||
)
|
||||
152
utils/email.py
Normal file
152
utils/email.py
Normal file
@@ -0,0 +1,152 @@
|
||||
import smtplib
|
||||
import ssl
|
||||
from email.mime.text import MIMEText
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
from email.utils import formataddr
|
||||
from email.header import Header
|
||||
from models import SMTPConfig
|
||||
import random
|
||||
|
||||
class EmailService:
|
||||
"""邮件服务类"""
|
||||
|
||||
@staticmethod
|
||||
def get_smtp_config():
|
||||
"""获取可用的SMTP配置（负载均衡）"""
|
||||
configs = SMTPConfig.query.filter_by(is_enabled=True).all()
|
||||
|
||||
if not configs:
|
||||
raise Exception("没有可用的SMTP配置")
|
||||
|
||||
# 加权随机选择
|
||||
total_weight = sum(c.weight for c in configs)
|
||||
if total_weight == 0:
|
||||
return random.choice(configs)
|
||||
|
||||
rand = random.uniform(0, total_weight)
|
||||
current = 0
|
||||
|
||||
for config in configs:
|
||||
current += config.weight
|
||||
if rand <= current:
|
||||
return config
|
||||
|
||||
return configs[-1]
|
||||
|
||||
@staticmethod
|
||||
def _do_send(smtp_config, to_email: str, subject: str, content: str, html: bool = True):
|
||||
"""实际发送邮件的内部方法"""
|
||||
server = None
|
||||
try:
|
||||
print(f"[SMTP调试] 开始发送邮件")
|
||||
print(f"[SMTP调试] 服务器: {smtp_config.host}:{smtp_config.port}")
|
||||
print(f"[SMTP调试] 用户名: {smtp_config.username}")
|
||||
print(f"[SMTP调试] 使用TLS: {smtp_config.use_tls}")
|
||||
print(f"[SMTP调试] 发件人: {smtp_config.from_email}")
|
||||
|
||||
msg = MIMEMultipart('alternative')
|
||||
from_email = smtp_config.from_email or smtp_config.username
|
||||
from_name = smtp_config.from_name or smtp_config.username
|
||||
msg['From'] = formataddr((str(Header(from_name, 'utf-8')), from_email))
|
||||
msg['To'] = to_email
|
||||
msg['Subject'] = Header(subject, 'utf-8')
|
||||
|
||||
if html:
|
||||
msg.attach(MIMEText(content, 'html', 'utf-8'))
|
||||
else:
|
||||
msg.attach(MIMEText(content, 'plain', 'utf-8'))
|
||||
|
||||
print(f"[SMTP调试] 开始连接服务器...")
|
||||
if smtp_config.port == 465:
|
||||
print(f"[SMTP调试] 使用 SSL 模式（端口 465）")
|
||||
server = smtplib.SMTP_SSL(smtp_config.host, smtp_config.port, timeout=30)
|
||||
elif smtp_config.use_tls:
|
||||
print(f"[SMTP调试] 使用 STARTTLS 模式（端口 {smtp_config.port}）")
|
||||
server = smtplib.SMTP(smtp_config.host, smtp_config.port, timeout=30)
|
||||
server.ehlo()
|
||||
context = ssl.create_default_context()
|
||||
server.starttls(context=context)
|
||||
server.ehlo()
|
||||
else:
|
||||
print(f"[SMTP调试] 使用普通 SMTP 模式（端口 {smtp_config.port}）")
|
||||
server = smtplib.SMTP(smtp_config.host, smtp_config.port, timeout=30)
|
||||
|
||||
print(f"[SMTP调试] 连接成功，开始登录...")
|
||||
server.login(smtp_config.username, smtp_config.password)
|
||||
print(f"[SMTP调试] 登录成功，开始发送邮件...")
|
||||
server.sendmail(from_email, [to_email], msg.as_string())
|
||||
print(f"[SMTP调试] 邮件发送成功！")
|
||||
|
||||
# 更新发送统计
|
||||
smtp_config.send_count += 1
|
||||
from models import db
|
||||
db.session.commit()
|
||||
|
||||
return True
|
||||
finally:
|
||||
if server:
|
||||
try:
|
||||
server.quit()
|
||||
except:
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def send_email(to_email: str, subject: str, content: str, html: bool = True):
|
||||
"""发送邮件（支持故障自动转移）"""
|
||||
configs = SMTPConfig.query.filter_by(is_enabled=True).all()
|
||||
|
||||
if not configs:
|
||||
raise Exception("没有可用的SMTP配置")
|
||||
|
||||
# 按权重排序，优先尝试高权重的配置
|
||||
configs.sort(key=lambda x: x.weight, reverse=True)
|
||||
|
||||
last_error = None
|
||||
tried_ids = []
|
||||
|
||||
for smtp_config in configs:
|
||||
tried_ids.append(smtp_config.id)
|
||||
try:
|
||||
return EmailService._do_send(smtp_config, to_email, subject, content, html)
|
||||
except Exception as e:
|
||||
last_error = str(e)
|
||||
print(f"[SMTP调试] {smtp_config.name} 发送失败: {last_error}")
|
||||
# 更新失败统计
|
||||
smtp_config.fail_count += 1
|
||||
from models import db
|
||||
db.session.commit()
|
||||
|
||||
# 如果还有其他配置，继续尝试
|
||||
remaining = len(configs) - len(tried_ids)
|
||||
if remaining > 0:
|
||||
print(f"[SMTP调试] 尝试下一个SMTP配置，剩余 {remaining} 个")
|
||||
continue
|
||||
|
||||
# 所有配置都失败了
|
||||
raise Exception(f"所有SMTP配置均发送失败，最后错误: {last_error}")
|
||||
|
||||
@staticmethod
|
||||
def send_verification_code(to_email: str, code: str, purpose: str):
|
||||
"""发送验证码邮件"""
|
||||
purpose_text = {
|
||||
'register': '注册账号',
|
||||
'reset_password': '重置密码',
|
||||
'forgot_password': '找回密码'
|
||||
}.get(purpose, '验证')
|
||||
|
||||
subject = f"【短视频解析平台】{purpose_text}验证码"
|
||||
|
||||
content = f"""
|
||||
<html>
|
||||
<body style="font-family: Arial, sans-serif; padding: 20px;">
|
||||
<h2>验证码</h2>
|
||||
<p>您正在进行<strong>{purpose_text}</strong>操作，验证码为：</p>
|
||||
<h1 style="color: #4CAF50; letter-spacing: 5px;">{code}</h1>
|
||||
<p>验证码有效期为10分钟，请勿泄露给他人。</p>
|
||||
<hr>
|
||||
<p style="color: #999; font-size: 12px;">如果这不是您的操作，请忽略此邮件。</p>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
return EmailService.send_email(to_email, subject, content, html=True)
|
||||
139
utils/health_check.py
Normal file
139
utils/health_check.py
Normal file
@@ -0,0 +1,139 @@
|
||||
from models import ParserAPI, HealthCheckConfig, HealthCheckLog
|
||||
from models import db
|
||||
from parsers.factory import ParserFactory
|
||||
from utils.email import EmailService
|
||||
from datetime import datetime
|
||||
import time
|
||||
|
||||
class HealthChecker:
|
||||
"""健康检查器"""
|
||||
|
||||
@staticmethod
|
||||
def check_api(api: ParserAPI, test_url: str) -> dict:
|
||||
"""检查单个API"""
|
||||
start_time = time.time()
|
||||
|
||||
try:
|
||||
# 创建解析器
|
||||
parser = ParserFactory.create_parser(api)
|
||||
|
||||
# 执行解析
|
||||
result = parser.parse(test_url)
|
||||
|
||||
# 计算响应时间
|
||||
response_time = int((time.time() - start_time) * 1000)
|
||||
|
||||
# 检查结果是否有效
|
||||
if result and result.get('video_url'):
|
||||
return {
|
||||
'success': True,
|
||||
'response_time': response_time,
|
||||
'error': None
|
||||
}
|
||||
else:
|
||||
return {
|
||||
'success': False,
|
||||
'response_time': response_time,
|
||||
'error': '解析结果无效'
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
response_time = int((time.time() - start_time) * 1000)
|
||||
return {
|
||||
'success': False,
|
||||
'response_time': response_time,
|
||||
'error': str(e)
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def check_platform(platform: str):
|
||||
"""检查指定平台的所有API"""
|
||||
# 获取健康检查配置
|
||||
config = HealthCheckConfig.query.filter_by(
|
||||
platform=platform,
|
||||
is_enabled=True
|
||||
).first()
|
||||
|
||||
if not config:
|
||||
return
|
||||
|
||||
# 获取该平台的所有API
|
||||
apis = ParserAPI.query.filter_by(platform=platform).all()
|
||||
|
||||
failed_apis = []
|
||||
|
||||
for api in apis:
|
||||
# 执行健康检查
|
||||
result = HealthChecker.check_api(api, config.test_url)
|
||||
|
||||
# 记录日志
|
||||
log = HealthCheckLog(
|
||||
parser_api_id=api.id,
|
||||
status='success' if result['success'] else 'failed',
|
||||
response_time=result['response_time'],
|
||||
error_message=result['error']
|
||||
)
|
||||
db.session.add(log)
|
||||
|
||||
# 更新API状态
|
||||
api.last_check_at = datetime.utcnow()
|
||||
|
||||
if result['success']:
|
||||
api.health_status = True
|
||||
api.fail_count = 0
|
||||
else:
|
||||
api.fail_count += 1
|
||||
# 连续失败3次标记为不健康
|
||||
if api.fail_count >= 3:
|
||||
api.health_status = False
|
||||
failed_apis.append({
|
||||
'name': api.name,
|
||||
'error': result['error']
|
||||
})
|
||||
|
||||
db.session.commit()
|
||||
|
||||
# 发送告警邮件
|
||||
if failed_apis and config.alert_email:
|
||||
HealthChecker.send_alert_email(platform, failed_apis, config.alert_email)
|
||||
|
||||
@staticmethod
|
||||
def send_alert_email(platform: str, failed_apis: list, alert_email: str):
|
||||
"""发送告警邮件"""
|
||||
subject = f"【短视频解析平台】{platform}接口健康检查告警"
|
||||
|
||||
content = f"""
|
||||
<html>
|
||||
<body style="font-family: Arial, sans-serif; padding: 20px;">
|
||||
<h2>接口健康检查告警</h2>
|
||||
<p>以下{platform}解析接口健康检查失败：</p>
|
||||
<ul>
|
||||
"""
|
||||
|
||||
for api in failed_apis:
|
||||
content += f"<li><strong>{api['name']}</strong>: {api['error']}</li>"
|
||||
|
||||
content += """
|
||||
</ul>
|
||||
<p>请及时检查并处理。</p>
|
||||
<hr>
|
||||
<p style="color: #999; font-size: 12px;">此邮件由系统自动发送，请勿回复。</p>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
try:
|
||||
EmailService.send_email(alert_email, subject, content, html=True)
|
||||
except Exception as e:
|
||||
print(f"发送告警邮件失败: {str(e)}")
|
||||
|
||||
@staticmethod
|
||||
def check_all_platforms():
|
||||
"""检查所有平台"""
|
||||
configs = HealthCheckConfig.query.filter_by(is_enabled=True).all()
|
||||
|
||||
for config in configs:
|
||||
try:
|
||||
HealthChecker.check_platform(config.platform)
|
||||
except Exception as e:
|
||||
print(f"检查{config.platform}平台失败: {str(e)}")
|
||||
89
utils/limiter.py
Normal file
89
utils/limiter.py
Normal file
@@ -0,0 +1,89 @@
|
||||
from datetime import date
|
||||
from models import DailyParseStat, UserGroup
|
||||
from models import db
|
||||
|
||||
class RateLimiter:
|
||||
"""限流器"""
|
||||
|
||||
@staticmethod
|
||||
def check_limit(user_id=None, ip_address=None):
|
||||
"""检查是否超过限制"""
|
||||
today = date.today()
|
||||
|
||||
if user_id:
|
||||
# 已登录用户
|
||||
stat = DailyParseStat.query.filter_by(
|
||||
user_id=user_id,
|
||||
date=today
|
||||
).first()
|
||||
|
||||
from models import User
|
||||
user = User.query.get(user_id)
|
||||
group = UserGroup.query.get(user.group_id)
|
||||
limit = group.daily_limit
|
||||
|
||||
current_count = stat.parse_count if stat else 0
|
||||
|
||||
return {
|
||||
'allowed': current_count < limit,
|
||||
'current': current_count,
|
||||
'limit': limit,
|
||||
'remaining': max(0, limit - current_count)
|
||||
}
|
||||
else:
|
||||
# 游客
|
||||
stat = DailyParseStat.query.filter_by(
|
||||
ip_address=ip_address,
|
||||
date=today
|
||||
).first()
|
||||
|
||||
from models import SiteConfig
|
||||
config = SiteConfig.query.filter_by(config_key='guest_daily_limit').first()
|
||||
limit = int(config.config_value) if config else 5
|
||||
|
||||
current_count = stat.parse_count if stat else 0
|
||||
|
||||
return {
|
||||
'allowed': current_count < limit,
|
||||
'current': current_count,
|
||||
'limit': limit,
|
||||
'remaining': max(0, limit - current_count)
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def increment_count(user_id=None, ip_address=None, success=True):
|
||||
"""增加计数"""
|
||||
today = date.today()
|
||||
|
||||
if user_id:
|
||||
stat = DailyParseStat.query.filter_by(
|
||||
user_id=user_id,
|
||||
date=today
|
||||
).first()
|
||||
|
||||
if not stat:
|
||||
stat = DailyParseStat(user_id=user_id, date=today)
|
||||
db.session.add(stat)
|
||||
|
||||
stat.parse_count = (stat.parse_count or 0) + 1
|
||||
if success:
|
||||
stat.success_count = (stat.success_count or 0) + 1
|
||||
else:
|
||||
stat.fail_count = (stat.fail_count or 0) + 1
|
||||
else:
|
||||
stat = DailyParseStat.query.filter_by(
|
||||
ip_address=ip_address,
|
||||
date=today
|
||||
).first()
|
||||
|
||||
if not stat:
|
||||
stat = DailyParseStat(ip_address=ip_address, date=today)
|
||||
db.session.add(stat)
|
||||
|
||||
stat.parse_count = (stat.parse_count or 0) + 1
|
||||
if success:
|
||||
stat.success_count = (stat.success_count or 0) + 1
|
||||
else:
|
||||
stat.fail_count = (stat.fail_count or 0) + 1
|
||||
|
||||
db.session.commit()
|
||||
141
utils/queue.py
Normal file
141
utils/queue.py
Normal file
@@ -0,0 +1,141 @@
|
||||
import json
|
||||
import time
|
||||
from datetime import datetime
|
||||
from typing import Dict, Optional
|
||||
|
||||
# 内存队列（当Redis不可用时使用）
|
||||
_memory_queue = []
|
||||
_memory_processing = {}
|
||||
_memory_results = {}
|
||||
|
||||
def get_redis_client():
|
||||
"""获取Redis客户端"""
|
||||
try:
|
||||
from app import redis_client
|
||||
return redis_client
|
||||
except:
|
||||
return None
|
||||
|
||||
class ParseQueue:
|
||||
"""解析队列管理器"""
|
||||
|
||||
QUEUE_KEY = "parse_queue"
|
||||
PROCESSING_KEY = "parse_processing"
|
||||
RESULT_KEY_PREFIX = "parse_result:"
|
||||
|
||||
@staticmethod
|
||||
def add_task(task_id: str, video_url: str, user_id: Optional[int] = None, ip_address: str = ""):
|
||||
"""添加任务到队列"""
|
||||
task = {
|
||||
'task_id': task_id,
|
||||
'video_url': video_url,
|
||||
'user_id': user_id,
|
||||
'ip_address': ip_address,
|
||||
'created_at': datetime.utcnow().isoformat(),
|
||||
'status': 'queued'
|
||||
}
|
||||
|
||||
redis_client = get_redis_client()
|
||||
if redis_client:
|
||||
redis_client.rpush(ParseQueue.QUEUE_KEY, json.dumps(task))
|
||||
else:
|
||||
# 使用内存队列
|
||||
_memory_queue.append(task)
|
||||
return task_id
|
||||
|
||||
@staticmethod
|
||||
def get_task() -> Optional[Dict]:
|
||||
"""从队列获取任务"""
|
||||
redis_client = get_redis_client()
|
||||
if redis_client:
|
||||
task_json = redis_client.lpop(ParseQueue.QUEUE_KEY)
|
||||
if task_json:
|
||||
task = json.loads(task_json)
|
||||
redis_client.hset(ParseQueue.PROCESSING_KEY, task['task_id'], json.dumps(task))
|
||||
return task
|
||||
else:
|
||||
# 使用内存队列
|
||||
if _memory_queue:
|
||||
task = _memory_queue.pop(0)
|
||||
_memory_processing[task['task_id']] = task
|
||||
return task
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def complete_task(task_id: str, result: Dict):
|
||||
"""完成任务"""
|
||||
redis_client = get_redis_client()
|
||||
if redis_client:
|
||||
redis_client.hdel(ParseQueue.PROCESSING_KEY, task_id)
|
||||
redis_client.setex(
|
||||
f"{ParseQueue.RESULT_KEY_PREFIX}{task_id}",
|
||||
3600,
|
||||
json.dumps(result)
|
||||
)
|
||||
else:
|
||||
# 使用内存
|
||||
_memory_processing.pop(task_id, None)
|
||||
_memory_results[task_id] = result
|
||||
|
||||
@staticmethod
|
||||
def get_result(task_id: str) -> Optional[Dict]:
|
||||
"""获取任务结果"""
|
||||
redis_client = get_redis_client()
|
||||
if redis_client:
|
||||
result_json = redis_client.get(f"{ParseQueue.RESULT_KEY_PREFIX}{task_id}")
|
||||
if result_json:
|
||||
return json.loads(result_json)
|
||||
else:
|
||||
# 使用内存
|
||||
return _memory_results.get(task_id)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def get_queue_length() -> int:
|
||||
"""获取队列长度"""
|
||||
redis_client = get_redis_client()
|
||||
if redis_client:
|
||||
return redis_client.llen(ParseQueue.QUEUE_KEY)
|
||||
else:
|
||||
return len(_memory_queue)
|
||||
|
||||
@staticmethod
|
||||
def get_processing_count() -> int:
|
||||
"""获取正在处理的任务数"""
|
||||
redis_client = get_redis_client()
|
||||
if redis_client:
|
||||
return redis_client.hlen(ParseQueue.PROCESSING_KEY)
|
||||
else:
|
||||
return len(_memory_processing)
|
||||
|
||||
@staticmethod
|
||||
def get_queue_status() -> Dict:
|
||||
"""获取队列状态"""
|
||||
return {
|
||||
'queued': ParseQueue.get_queue_length(),
|
||||
'processing': ParseQueue.get_processing_count()
|
||||
}
|
||||
|
||||
|
||||
class ConcurrencyController:
|
||||
"""并发控制器"""
|
||||
|
||||
@staticmethod
|
||||
def can_process() -> bool:
|
||||
"""检查是否可以处理新任务"""
|
||||
from models import SiteConfig
|
||||
config = SiteConfig.query.filter_by(config_key='max_concurrent').first()
|
||||
max_concurrent = int(config.config_value) if config else 3
|
||||
|
||||
processing_count = ParseQueue.get_processing_count()
|
||||
return processing_count < max_concurrent
|
||||
|
||||
@staticmethod
|
||||
def wait_for_slot(timeout: int = 60) -> bool:
|
||||
"""等待可用槽位"""
|
||||
start_time = time.time()
|
||||
while time.time() - start_time < timeout:
|
||||
if ConcurrencyController.can_process():
|
||||
return True
|
||||
time.sleep(0.5)
|
||||
return False
|
||||
23
utils/security.py
Normal file
23
utils/security.py
Normal file
@@ -0,0 +1,23 @@
|
||||
import bcrypt
|
||||
import random
|
||||
import string
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
"""加密密码"""
|
||||
salt = bcrypt.gensalt()
|
||||
return bcrypt.hashpw(password.encode('utf-8'), salt).decode('utf-8')
|
||||
|
||||
def verify_password(password: str, hashed: str) -> bool:
|
||||
"""验证密码"""
|
||||
return bcrypt.checkpw(password.encode('utf-8'), hashed.encode('utf-8'))
|
||||
|
||||
def generate_verification_code(length: int = 6) -> str:
|
||||
"""生成验证码"""
|
||||
return ''.join(random.choices(string.digits, k=length))
|
||||
|
||||
def get_client_ip(request):
|
||||
"""获取客户端IP"""
|
||||
if request.headers.get('X-Forwarded-For'):
|
||||
return request.headers.get('X-Forwarded-For').split(',')[0]
|
||||
return request.remote_addr
|
||||
Reference in New Issue
Block a user