367 lines
22 KiB
Python
367 lines
22 KiB
Python
"""Offline classification and extraction regression tests."""
|
||
|
||
import email
|
||
from email import policy
|
||
from email.message import EmailMessage
|
||
from email.utils import parsedate_to_datetime
|
||
from pathlib import Path
|
||
import sys
|
||
import unittest
|
||
from zoneinfo import ZoneInfo
|
||
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
sys.path.insert(0, str(ROOT / "scripts"))
|
||
import check_mail
|
||
|
||
|
||
def sample(name, folder="参考邮件"):
|
||
path = next((ROOT / folder).glob(name), None)
|
||
if path is None:
|
||
raise unittest.SkipTest("本地真实邮件样本未随仓库分发")
|
||
message = email.message_from_bytes(path.read_bytes(), policy=policy.default)
|
||
received = parsedate_to_datetime(check_mail.header_value(message, "Date"))
|
||
return check_mail.extract(message, received)
|
||
|
||
|
||
def synthetic(subject, body, rich=False):
|
||
message = EmailMessage()
|
||
message["Subject"] = subject
|
||
message.set_content(body, subtype="html" if rich else "plain")
|
||
return check_mail.extract(message, parsedate_to_datetime("Mon, 28 Sep 2026 12:00:00 +0800"))
|
||
|
||
|
||
class ExtractionTests(unittest.TestCase):
|
||
def test_four_forwarded_reference_messages(self):
|
||
expected = {
|
||
"*Account_on_Hold*": "账户验证或暂停",
|
||
"*账户已暂停*": "账户验证或暂停",
|
||
"*Abuse_Report*": "资源滥用报告",
|
||
"*存在可疑活动*": "账号可疑活动警告",
|
||
}
|
||
for pattern, kind in expected.items():
|
||
with self.subTest(pattern=pattern):
|
||
record = sample(pattern)
|
||
self.assertIsNotNone(record)
|
||
self.assertEqual(record["type"], kind)
|
||
self.assertNotEqual(record["warning"], check_mail.EMPTY)
|
||
|
||
def test_forwarded_abuse_report_keeps_explicit_account_and_resources(self):
|
||
record = sample("*Abuse_Report*")
|
||
self.assertRegex(record["account"], r"^\d{12}$")
|
||
self.assertRegex(record["resources"], r"\bi-[0-9a-f]{8,17}\b")
|
||
self.assertEqual(record["deadline"], check_mail.EMPTY)
|
||
|
||
def test_permanent_closure_confirmation_does_not_invent_body_facts(self):
|
||
record = synthetic(
|
||
"AWS Account Permanent Closure Confirmation",
|
||
"This confirms your account is permanently closed.",
|
||
)
|
||
self.assertIsNotNone(record)
|
||
self.assertEqual(record["type"], "账户永久关闭确认")
|
||
self.assertEqual(record["account"], check_mail.EMPTY)
|
||
self.assertEqual(record["warning"], check_mail.EMPTY)
|
||
self.assertEqual(record["action"], check_mail.EMPTY)
|
||
self.assertEqual(record["deadline"], check_mail.EMPTY)
|
||
|
||
def test_identity_verification_footer_is_not_ban_alert(self):
|
||
self.assertIsNone(synthetic(
|
||
"验证您的身份",
|
||
"AWS 身份验证通知。为保护您的安全，如果您发现未授权访问，请联系支持团队。",
|
||
))
|
||
|
||
def test_specific_subject_outweighs_quoted_phishing_report(self):
|
||
record = synthetic(
|
||
"AWS Account on Hold: Response Required",
|
||
"Your AWS account has been placed on hold.\n--- Forwarded history ---\n"
|
||
"An earlier AWS resource was implicated in hosting content that resembles a phishing website.",
|
||
)
|
||
self.assertEqual(record["type"], "账户验证或暂停")
|
||
|
||
def test_local_forwarded_login_code_is_filtered(self):
|
||
self.assertIsNone(sample("*FW__*.eml", "过滤邮件"))
|
||
|
||
def test_local_forwarded_access_validated_notice_is_filtered(self):
|
||
self.assertIsNone(sample("*Resources_Has_Been_Validated.eml", "过滤邮件"))
|
||
|
||
def test_local_forwarded_domain_expiration_notice_is_filtered(self):
|
||
self.assertIsNone(sample("*Domain_expiring*.eml", "过滤邮件"))
|
||
|
||
def test_domain_expiration_notices_are_filtered(self):
|
||
cases = [
|
||
("Fw:[Domain expiring] Registration for example.net expires on 2027-02-11",
|
||
"Dear AWS customer, you have disabled automatic annual renewal for this domain.", False),
|
||
("FWD: [DOMAIN EXPIRING] Registration for other.org expires on 2028-04-09",
|
||
"AWS domain registration verification: automatic renewal is disabled.", False),
|
||
("AWS notification", "Forwarded message:\nSubject: Registration for example.net "
|
||
"expires on 2027-02-11\nAutomatic renewal is disabled.", False),
|
||
("FW: AWS notification", "<div>Domain Expiring Soon</div><p>Your domain is "
|
||
"scheduled to expire. Automatic annual renewal is disabled.</p>", True),
|
||
("转发：AWS 域名到期提醒", "您的域名即将到期，自动续费已停用。", False),
|
||
("AWS 通知", "<p>域名注册将于 2027-02-11 到期</p><p>自动续费已停用。</p>", True),
|
||
]
|
||
for subject, body, rich in cases:
|
||
with self.subTest(subject=subject, rich=rich):
|
||
self.assertIsNone(synthetic(subject, body, rich))
|
||
|
||
def test_domain_renewal_business_states_are_filtered(self):
|
||
cases = [
|
||
("AWS domain renewal failed", "Automatic renewal is disabled. "
|
||
"The domain has been suspended because its registration expired."),
|
||
("AWS domain renewal reminder", "Your domain is disabled after expiration. "
|
||
"Payment verification failed; please renew its registration."),
|
||
("AWS 域名续费失败", "域名已暂停解析，域名注册已关闭。请重新验证续费付款信息。"),
|
||
("AWS domain renewal failed", "We could not charge your AWS account "
|
||
"before the domain was suspended due to registration expiry."),
|
||
("AWS 域名到期提醒", "您的 AWS 账户未扣款而域名已暂停。"),
|
||
("[Domain expiring] AWS registration notice", "If your AWS account is suspended, "
|
||
"please contact support. Automatic annual renewal is disabled."),
|
||
]
|
||
for subject, body in cases:
|
||
with self.subTest(subject=subject):
|
||
self.assertIsNone(synthetic(subject, body))
|
||
|
||
def test_domain_expiration_does_not_hide_actual_account_or_resource_risk(self):
|
||
subject = "[Domain expiring] Registration for example.net expires on 2027-02-11"
|
||
risks = [
|
||
"Your AWS account has been suspended.",
|
||
"Your AWS account has been suspended and your domain is disabled.",
|
||
"Your AWS account associated with this domain is suspended.",
|
||
"Your AWS account for this domain is suspended.",
|
||
"Your AWS account is restricted.",
|
||
"Your AWS account is disabled.",
|
||
"Your AWS account has been terminated.",
|
||
"Your AWS account is closed.",
|
||
"Your access key has been compromised. AWS security notification.",
|
||
"Your AWS resource has been implicated in hosting a phishing website.",
|
||
"We detected potentially unwanted activity in your AWS account.",
|
||
"AWS verification: Please upload your identity documents.",
|
||
"AWS: If you fail to respond, your account will be suspended.",
|
||
"您的 AWS 账户已受限。",
|
||
"您的 AWS 账户已暂停。请提交身份证明材料。",
|
||
"AWS: Please delete malicious content hosted on your resource.",
|
||
]
|
||
for body in risks:
|
||
with self.subTest(body=body):
|
||
self.assertIsNotNone(synthetic(subject, body))
|
||
for risk_subject in ("AWS Abuse Report", "AWS Suspicious Activity",
|
||
"AWS Account Permanent Closure Confirmation"):
|
||
with self.subTest(subject=risk_subject):
|
||
self.assertIsNotNone(synthetic(risk_subject, subject))
|
||
|
||
def test_generic_verification_and_expiry_notices_are_not_ban_alerts(self):
|
||
cases = [
|
||
("AWS identity verification", "Your verification session expires on 2027-02-11."),
|
||
("AWS domain security verification", "Verify your domain ownership."),
|
||
("AWS renewal verification", "Please verify your billing information."),
|
||
("AWS 域名验证", "请验证域名所有权。"),
|
||
("AWS certificate verification required", "Your SSL certificate for domain example.com "
|
||
"expires on 2026-10-17. Please renew the certificate."),
|
||
("AWS credentials verification required", "Your credentials for domain example.com "
|
||
"expire on 2026-10-17. Please rotate credentials."),
|
||
("AWS certificate verification required", "Your domain certificate expires on "
|
||
"2026-10-17. Please renew the certificate."),
|
||
("AWS certificate verification required", "Your certificate for your domain "
|
||
"example.com expires on 2026-10-17. Please renew the certificate."),
|
||
("AWS certificate verification", "Registration for certificate expires on "
|
||
"2026-10-17. Please renew the certificate."),
|
||
("AWS certificate verification", "Your certificate registration for example.com "
|
||
"expires on 2026-10-17. Please renew the certificate."),
|
||
("AWS 证书风险通知", "域名 example.com 的 SSL 证书将于 2026-10-17 到期，请更新证书。"),
|
||
("AWS 凭据风险通知", "域名 example.com 的登录凭据将于 2026-10-17 到期，请更新凭据。"),
|
||
]
|
||
for subject, body in cases:
|
||
with self.subTest(subject=subject):
|
||
self.assertIsNone(synthetic(subject, body))
|
||
|
||
def test_access_validated_notices_are_filtered(self):
|
||
cases = [
|
||
("您访问 AWS 资源的请求已得到验证", "您的 AWS 资源访问请求已完成核验，现在可以访问资源。", False),
|
||
("AWS: Your Request For Accessing AWS Resources Has Been Validated",
|
||
"The request is complete. You may access AWS resources.", False),
|
||
("Fw: 您访问 AWS 资源的请求已得到验证 / "
|
||
"Your Request For Accessing AWS Resources Has Been Validated",
|
||
"AWS 资源访问现已恢复。如果您有任何问题，请联系支持。", False),
|
||
("Fwd: AWS notification", "Forwarded message:\nSubject: Your Request For Accessing "
|
||
"AWS Resources Has Been Validated\nYou may access resources again.", False),
|
||
("FW: AWS notification", "<div>Your Request For Accessing</div><p>AWS Resources "
|
||
"Has Been Successfully Validated</p><p>You may access AWS resources.</p>", True),
|
||
("Fw: AWS 通知", "<p>您访问 AWS 资源的请求已得到验证</p><p>现在可以访问资源。</p>", True),
|
||
]
|
||
for subject, body, rich in cases:
|
||
with self.subTest(subject=subject, rich=rich):
|
||
self.assertIsNone(synthetic(subject, body, rich))
|
||
|
||
def test_success_notice_does_not_hide_actual_risk_or_required_documents(self):
|
||
success = "Your Request For Accessing AWS Resources Has Been Validated"
|
||
risks = [
|
||
"Your AWS account has been suspended.",
|
||
"Your AWS account is restricted.",
|
||
"Your AWS account is disabled.",
|
||
"Your AWS account has been terminated.",
|
||
"Your AWS account is closed.",
|
||
"Your access key has been compromised.",
|
||
"Your AWS resource has been implicated in hosting a phishing website.",
|
||
"We detected potentially unwanted activity in your AWS account.",
|
||
"Please upload your identity documents.",
|
||
"If you fail to respond, your account will be suspended.",
|
||
"您的 AWS 账户已受限。",
|
||
"您的账户已暂停。请提交身份证明材料。",
|
||
]
|
||
for body in risks:
|
||
with self.subTest(body=body):
|
||
self.assertIsNotNone(synthetic(success, body))
|
||
for subject in ("AWS Abuse Report", "AWS Suspicious Activity", "AWS Account Permanent Closure Confirmation"):
|
||
with self.subTest(subject=subject):
|
||
self.assertIsNotNone(synthetic(subject, success))
|
||
|
||
def test_other_validation_notices_remain_reported(self):
|
||
cases = [
|
||
("AWS 验证您的身份", "请上传身份证核验材料。"),
|
||
("Your Request For Accessing AWS Resources Has Not Been Validated",
|
||
"We could not validate important details. Please upload proof of identity."),
|
||
("AWS notification", "Your Request For Accessing AWS Resources Has Been Validated? "
|
||
"Your account has been placed on hold."),
|
||
]
|
||
for subject, body in cases:
|
||
with self.subTest(subject=subject):
|
||
self.assertIsNotNone(synthetic(subject, body))
|
||
|
||
def test_access_success_keeps_explicit_resource_risk_actions(self):
|
||
subject = "Your Request For Accessing AWS Resources Has Been Validated"
|
||
cases = [
|
||
"Please take action immediately to remove phishing content from your AWS resource.",
|
||
"Please delete malicious content hosted on your AWS resource.",
|
||
"You must stop abusive traffic from your AWS resource.",
|
||
"请立即删除您的 AWS 资源上的钓鱼内容。",
|
||
"必须停止 AWS 资源上的滥用行为。",
|
||
]
|
||
for body in cases:
|
||
with self.subTest(body=body):
|
||
self.assertIsNotNone(synthetic(subject, body))
|
||
|
||
def test_access_success_security_tips_are_still_filtered(self):
|
||
subject = "Your Request For Accessing AWS Resources Has Been Validated"
|
||
cases = [
|
||
"If you find phishing content, please remove phishing content from your AWS resource.",
|
||
"If you have questions, please take action to contact AWS support.",
|
||
"如果您发现钓鱼内容，请立即删除 AWS 资源上的钓鱼内容。",
|
||
"如果您有任何问题，请联系支持团队。",
|
||
"Please contact AWS support if you need help accessing resources.",
|
||
]
|
||
for body in cases:
|
||
with self.subTest(body=body):
|
||
self.assertIsNone(synthetic(subject, body))
|
||
|
||
def test_login_codes_with_security_boilerplate_are_filtered(self):
|
||
cases = [
|
||
("FW: 验证您的身份", "AWS 登录验证。您的验证码是 123456，10 分钟后过期。"
|
||
"您是否发现异常活动？如果您没有发起登录，请更换密码并启用 MFA。", False),
|
||
("Fwd: AWS Verify your identity", "Your verification code is 123456. "
|
||
"Use this code to sign in. If you notice suspicious activity or unauthorized access, "
|
||
"change your password and enable multi-factor authentication (MFA).", False),
|
||
("AWS authentication", "AWS one-time passcode: 654321. Use this code to log in. "
|
||
"It expires in 10 minutes. Verify your identity.", False),
|
||
("AWS 验证您的身份", "<div>请输入验证码</div><strong>123456</strong>"
|
||
"<p>以完成登录。如果发现异常活动，请更换密码。</p>", True),
|
||
("AWS verification code", "<p>Your security code:</p><b>123456</b>"
|
||
"<p>It expires in 10 minutes. Use this code to sign in.</p>", True),
|
||
]
|
||
for subject, body, rich in cases:
|
||
with self.subTest(subject=subject, rich=rich):
|
||
self.assertIsNone(synthetic(subject, body, rich))
|
||
|
||
def test_real_risk_with_verification_or_mfa_is_retained(self):
|
||
cases = [
|
||
("AWS Account on Hold: Verification Required", "Your account has been placed on hold. "
|
||
"Please upload your identity documents. MFA security code: 123456; use this code to sign in."),
|
||
("AWS 身份验证", "您的账户已暂停。请上传身份证核验材料。验证码 123456，可用于登录。"),
|
||
("AWS verify your identity", "Please submit proof of identity and a bank statement. "
|
||
"Your verification code is 123456. Use this code to sign in."),
|
||
("AWS Account Permanent Closure Confirmation", "Your account is permanently closed. "
|
||
"Verification code: 123456. Use this code to sign in."),
|
||
("AWS Security verification", "We have limited your ability to access AWS services. "
|
||
"Your security code is 123456. Use this code to sign in."),
|
||
("AWS Security warning", "Your access key has been compromised. "
|
||
"Verification code: 123456. Use this code to sign in."),
|
||
("AWS Account Suspended", "Verification code: 123456. Use this code to sign in."),
|
||
("AWS Account Restricted", "Verification code: 123456. Use this code to sign in."),
|
||
("AWS 账号受限", "验证码 123456，用于登录。"),
|
||
]
|
||
for subject, body in cases:
|
||
with self.subTest(subject=subject):
|
||
self.assertIsNotNone(synthetic(subject, body))
|
||
|
||
def test_conditional_security_tips_do_not_make_login_codes_into_alerts(self):
|
||
cases = [
|
||
("AWS verification code", "Your verification code is 987654. Use this code to sign in.\n"
|
||
"If you think your access key has been compromised, change your password and enable MFA."),
|
||
("AWS 验证码", "您的验证码是 987654，用于登录。\n如果您的账户已暂停，请联系支持。"),
|
||
("AWS verification code", "Your verification code is 987654. Use this code to sign in. "
|
||
"If your account is suspended, please submit identity documents to support."),
|
||
("AWS verification code", "Your verification code is 987654. Use this code to sign in. "
|
||
"If you need help, please provide identity documents to support."),
|
||
("AWS 验证码", "验证码 987654，用于登录。如果您需要帮助，请提交身份证材料给支持团队。"),
|
||
]
|
||
for subject, body in cases:
|
||
with self.subTest(subject=subject):
|
||
self.assertIsNone(synthetic(subject, body))
|
||
|
||
def test_explicit_risk_and_nonresponse_consequences_survive_conditional_tips(self):
|
||
english_code = "Your verification code is 987654. Use this code to sign in. "
|
||
cases = [
|
||
("AWS verification code", english_code +
|
||
"If you think your access key has been compromised, enable MFA. "
|
||
"Your account has been placed on hold."),
|
||
("AWS verification code", english_code +
|
||
"Your account has been placed on hold, if you think your access key has been compromised, enable MFA."),
|
||
("AWS verification code", english_code +
|
||
"If you think your access key has been compromised, enable MFA, your account has been suspended."),
|
||
("AWS 验证码", "验证码 987654，用于登录。如果您的账户已暂停，请联系支持。您的账户已受限。"),
|
||
("AWS verification code", english_code +
|
||
"If you fail to respond by 2026-10-10 your account will be suspended."),
|
||
("AWS verification code", english_code +
|
||
"If you do not upload documents by 2026-10-10, your account will be suspended."),
|
||
("AWS 验证码", "验证码 987654，用于登录。如果您未能在截止日期回复您的账号将被暂停。"),
|
||
("AWS verification code", english_code +
|
||
"Your access key has been compromised. If your account is suspended, contact support."),
|
||
("AWS verification code", english_code +
|
||
"If you need help, please provide identity documents to support, your account has been suspended."),
|
||
("AWS verification code", english_code +
|
||
"If you need help, please provide identity documents to support. Please upload proof of identity."),
|
||
]
|
||
for subject, body in cases:
|
||
with self.subTest(subject=subject, body=body):
|
||
self.assertIsNotNone(synthetic(subject, body))
|
||
|
||
def test_report_omits_missing_fields_without_reordering(self):
|
||
record = synthetic("AWS Account Permanent Closure Confirmation", "Account permanently closed.")
|
||
report = check_mail.format_report(record, ZoneInfo("Asia/Shanghai"))
|
||
self.assertEqual([line.split("：", 1)[0] for line in report.splitlines()],
|
||
["【AWS账号风控邮件】", "收件时间", "邮件主题", "风险类型"])
|
||
self.assertNotIn(check_mail.EMPTY, report)
|
||
self.assertNotIn("AWS账号ID", report)
|
||
|
||
def test_report_omits_none_empty_and_absent_fields(self):
|
||
report = check_mail.format_report({
|
||
"subject": "", "account": None, "type": "账户验证或暂停",
|
||
"warning": " ", "resources": check_mail.EMPTY, "deadline": "2026-10-10",
|
||
}, ZoneInfo("Asia/Shanghai"))
|
||
self.assertEqual(report.splitlines(), ["【AWS账号风控邮件】", "风险类型：账户验证或暂停",
|
||
"截止时间：2026-10-10"])
|
||
|
||
def test_complete_report_keeps_field_order(self):
|
||
record = synthetic("AWS Account on Hold: Response Required",
|
||
"AWS Account ID: 123456789012\nYour account has been placed on hold.\n"
|
||
"Resource i-12345678\nPlease reply before 2026-10-10.")
|
||
self.assertEqual(record["account"], "123456789012")
|
||
self.assertEqual(record["resources"], "i-12345678")
|
||
report = check_mail.format_report(record, ZoneInfo("Asia/Shanghai"))
|
||
self.assertEqual([line.split("：", 1)[0] for line in report.splitlines()],
|
||
["【AWS账号风控邮件】", "收件时间", "邮件主题", "AWS账号ID", "风险类型",
|
||
"警告信息", "涉及资源", "要求操作", "截止时间"])
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|