You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

398 lines
10 KiB

  1. package helpers
  2. import (
  3. "errors"
  4. "fmt"
  5. "io"
  6. "net/http"
  7. "os"
  8. "os/exec"
  9. "path/filepath"
  10. "runtime"
  11. "strings"
  12. "time"
  13. )
  14. const (
  15. NodeTypeMaster = "master"
  16. NodeTypeSlave = "slave"
  17. DefaultCNPort = 3100
  18. DefaultOVPort = 3101
  19. DefaultSessionSecret = "user-migration-e2e-session-secret"
  20. DefaultCNDBFileName = "cn.db"
  21. DefaultOVDBFileName = "ov.db"
  22. DefaultCNLogFileName = "cn.log"
  23. DefaultOVLogFileName = "ov.log"
  24. DefaultCNSessionName = "cn_session"
  25. DefaultOVSessionName = "ov_session"
  26. DefaultSyncAPIKey = "e2e-sync-key"
  27. DefaultRootUsername = "root"
  28. DefaultRootPassword = "Password123!"
  29. defaultNodeReadyTimout = 45 * time.Second
  30. )
  31. type NodeProcess struct {
  32. Name string
  33. Port int
  34. BaseURL string
  35. DBPath string
  36. SessionName string
  37. NodeType string
  38. LogPath string
  39. cmd *exec.Cmd
  40. logFile *os.File
  41. waitCh chan error
  42. }
  43. type Cluster struct {
  44. RepoRoot string
  45. RunDir string
  46. BinaryPath string
  47. CNDBPath string
  48. OVDBPath string
  49. CN *NodeProcess
  50. OV *NodeProcess
  51. }
  52. type Environment struct {
  53. RepoRoot string
  54. ArtifactRoot string
  55. RunDir string
  56. BinaryPath string
  57. CNDBPath string
  58. OVDBPath string
  59. Cluster *Cluster
  60. CNClient *APIClient
  61. OVClient *APIClient
  62. }
  63. func BuildBinary(repoRoot, artifactRoot string) (string, error) {
  64. binDir := filepath.Join(artifactRoot, "bin")
  65. if err := os.MkdirAll(binDir, 0o755); err != nil {
  66. return "", fmt.Errorf("create binary dir: %w", err)
  67. }
  68. binaryName := "new-api-e2e"
  69. if runtime.GOOS == "windows" {
  70. binaryName += ".exe"
  71. }
  72. binaryPath := filepath.Join(binDir, binaryName)
  73. cmd := exec.Command("go", "build", "-o", binaryPath, ".")
  74. cmd.Dir = repoRoot
  75. output, err := cmd.CombinedOutput()
  76. if err != nil {
  77. return "", fmt.Errorf("build e2e binary: %w\n%s", err, string(output))
  78. }
  79. return binaryPath, nil
  80. }
  81. func PrepareEnvironment(repoRoot, artifactRoot, binaryPath, scenarioName string) (*Environment, error) {
  82. runDir := filepath.Join(artifactRoot, "runs", sanitizeName(scenarioName))
  83. if err := os.RemoveAll(runDir); err != nil {
  84. return nil, fmt.Errorf("reset run dir: %w", err)
  85. }
  86. if err := os.MkdirAll(runDir, 0o755); err != nil {
  87. return nil, fmt.Errorf("create run dir: %w", err)
  88. }
  89. cluster := NewCluster(repoRoot, runDir, binaryPath)
  90. if err := cluster.Start(NodeTypeMaster, NodeTypeMaster); err != nil {
  91. _ = cluster.Stop()
  92. return nil, err
  93. }
  94. cnClient, err := NewAPIClient(cluster.CN.BaseURL)
  95. if err != nil {
  96. _ = cluster.Stop()
  97. return nil, err
  98. }
  99. ovClient, err := NewAPIClient(cluster.OV.BaseURL)
  100. if err != nil {
  101. _ = cluster.Stop()
  102. return nil, err
  103. }
  104. if err := SetupRoot(cnClient, DefaultRootUsername, DefaultRootPassword); err != nil {
  105. _ = cluster.Stop()
  106. return nil, fmt.Errorf("setup CN root: %w", err)
  107. }
  108. if err := SetupRoot(ovClient, DefaultRootUsername, DefaultRootPassword); err != nil {
  109. _ = cluster.Stop()
  110. return nil, fmt.Errorf("setup OV root: %w", err)
  111. }
  112. if err := cluster.Stop(); err != nil {
  113. return nil, fmt.Errorf("stop bootstrap cluster: %w", err)
  114. }
  115. if err := SeedCNData(cluster.CNDBPath); err != nil {
  116. return nil, fmt.Errorf("seed CN data: %w", err)
  117. }
  118. if err := SeedOVData(cluster.OVDBPath); err != nil {
  119. return nil, fmt.Errorf("seed OV data: %w", err)
  120. }
  121. if err := cluster.Start(NodeTypeMaster, NodeTypeSlave); err != nil {
  122. _ = cluster.Stop()
  123. return nil, err
  124. }
  125. cnClient, err = NewAPIClient(cluster.CN.BaseURL)
  126. if err != nil {
  127. _ = cluster.Stop()
  128. return nil, err
  129. }
  130. ovClient, err = NewAPIClient(cluster.OV.BaseURL)
  131. if err != nil {
  132. _ = cluster.Stop()
  133. return nil, err
  134. }
  135. if err := Login(cnClient, DefaultRootUsername, DefaultRootPassword); err != nil {
  136. _ = cluster.Stop()
  137. return nil, fmt.Errorf("login CN root: %w", err)
  138. }
  139. if err := Login(ovClient, DefaultRootUsername, DefaultRootPassword); err != nil {
  140. _ = cluster.Stop()
  141. return nil, fmt.Errorf("login OV root: %w", err)
  142. }
  143. if err := ConfigureCNRegionSync(cnClient, cluster.CN.BaseURL, cluster.OV.BaseURL, DefaultSyncAPIKey); err != nil {
  144. _ = cluster.Stop()
  145. return nil, fmt.Errorf("configure CN region sync: %w", err)
  146. }
  147. if err := ConfigureOVRegionSync(ovClient, cluster.CN.BaseURL, DefaultSyncAPIKey); err != nil {
  148. _ = cluster.Stop()
  149. return nil, fmt.Errorf("configure OV region sync: %w", err)
  150. }
  151. if err := WaitOptionApplied(cluster.OV.BaseURL, DefaultSyncAPIKey, 5, 20*time.Second); err != nil {
  152. _ = cluster.Stop()
  153. return nil, fmt.Errorf("wait OV migration API ready: %w", err)
  154. }
  155. return &Environment{
  156. RepoRoot: repoRoot,
  157. ArtifactRoot: artifactRoot,
  158. RunDir: runDir,
  159. BinaryPath: binaryPath,
  160. CNDBPath: cluster.CNDBPath,
  161. OVDBPath: cluster.OVDBPath,
  162. Cluster: cluster,
  163. CNClient: cnClient,
  164. OVClient: ovClient,
  165. }, nil
  166. }
  167. func (e *Environment) Close() error {
  168. if e == nil || e.Cluster == nil {
  169. return nil
  170. }
  171. return e.Cluster.Stop()
  172. }
  173. func NewCluster(repoRoot, runDir, binaryPath string) *Cluster {
  174. return &Cluster{
  175. RepoRoot: repoRoot,
  176. RunDir: runDir,
  177. BinaryPath: binaryPath,
  178. CNDBPath: filepath.Join(runDir, DefaultCNDBFileName),
  179. OVDBPath: filepath.Join(runDir, DefaultOVDBFileName),
  180. }
  181. }
  182. func (c *Cluster) Start(cnNodeType, ovNodeType string) error {
  183. if err := os.MkdirAll(c.RunDir, 0o755); err != nil {
  184. return fmt.Errorf("create run dir: %w", err)
  185. }
  186. cn, err := c.startNode("cn", DefaultCNPort, c.CNDBPath, DefaultCNSessionName, cnNodeType, DefaultCNLogFileName)
  187. if err != nil {
  188. return err
  189. }
  190. c.CN = cn
  191. ov, err := c.startNode("ov", DefaultOVPort, c.OVDBPath, DefaultOVSessionName, ovNodeType, DefaultOVLogFileName)
  192. if err != nil {
  193. _ = c.CN.Stop()
  194. c.CN = nil
  195. return err
  196. }
  197. c.OV = ov
  198. if err := c.CN.WaitReady(defaultNodeReadyTimout); err != nil {
  199. _ = c.Stop()
  200. return err
  201. }
  202. if err := c.OV.WaitReady(defaultNodeReadyTimout); err != nil {
  203. _ = c.Stop()
  204. return err
  205. }
  206. return nil
  207. }
  208. func (c *Cluster) Stop() error {
  209. var errs []string
  210. if c.OV != nil {
  211. if err := c.OV.Stop(); err != nil {
  212. errs = append(errs, err.Error())
  213. }
  214. c.OV = nil
  215. }
  216. if c.CN != nil {
  217. if err := c.CN.Stop(); err != nil {
  218. errs = append(errs, err.Error())
  219. }
  220. c.CN = nil
  221. }
  222. if len(errs) == 0 {
  223. return nil
  224. }
  225. return errors.New(strings.Join(errs, "; "))
  226. }
  227. func (c *Cluster) startNode(name string, port int, dbPath, sessionName, nodeType, logFileName string) (*NodeProcess, error) {
  228. logPath := filepath.Join(c.RunDir, logFileName)
  229. logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644)
  230. if err != nil {
  231. return nil, fmt.Errorf("open %s log file: %w", name, err)
  232. }
  233. systemLogDir := filepath.Join(c.RunDir, "system-logs", name)
  234. if err := os.MkdirAll(systemLogDir, 0o755); err != nil {
  235. _ = logFile.Close()
  236. return nil, fmt.Errorf("create %s system log dir: %w", name, err)
  237. }
  238. cmd := exec.Command(c.BinaryPath, "--log-dir", systemLogDir)
  239. cmd.Dir = c.RepoRoot
  240. cmd.Env = mergeEnv(os.Environ(), map[string]string{
  241. "PORT": fmt.Sprintf("%d", port),
  242. "SQLITE_PATH": dbPath,
  243. "SESSION_SECRET": DefaultSessionSecret,
  244. "SESSION_NAME": sessionName,
  245. "SESSION_SECURE": "false",
  246. "SESSION_SAMESITE": "strict",
  247. "GLOBAL_API_RATE_LIMIT_ENABLE": "false",
  248. "GLOBAL_WEB_RATE_LIMIT_ENABLE": "false",
  249. "CRITICAL_RATE_LIMIT_ENABLE": "false",
  250. "GIN_MODE": "release",
  251. "NODE_TYPE": nodeType,
  252. })
  253. cmd.Stdout = logFile
  254. cmd.Stderr = logFile
  255. waitCh := make(chan error, 1)
  256. if err := cmd.Start(); err != nil {
  257. _ = logFile.Close()
  258. return nil, fmt.Errorf("start %s node: %w", name, err)
  259. }
  260. go func() {
  261. waitCh <- cmd.Wait()
  262. }()
  263. return &NodeProcess{
  264. Name: name,
  265. Port: port,
  266. BaseURL: fmt.Sprintf("http://127.0.0.1:%d", port),
  267. DBPath: dbPath,
  268. SessionName: sessionName,
  269. NodeType: nodeType,
  270. LogPath: logPath,
  271. cmd: cmd,
  272. logFile: logFile,
  273. waitCh: waitCh,
  274. }, nil
  275. }
  276. func (n *NodeProcess) WaitReady(timeout time.Duration) error {
  277. client := &http.Client{Timeout: 1 * time.Second}
  278. deadline := time.Now().Add(timeout)
  279. for time.Now().Before(deadline) {
  280. select {
  281. case err := <-n.waitCh:
  282. if err == nil {
  283. return fmt.Errorf("%s node exited before ready; see %s", n.Name, n.LogPath)
  284. }
  285. return fmt.Errorf("%s node exited before ready: %w; see %s", n.Name, err, n.LogPath)
  286. default:
  287. }
  288. resp, err := client.Get(n.BaseURL + "/api/status")
  289. if err == nil {
  290. _, _ = io.Copy(io.Discard, resp.Body)
  291. resp.Body.Close()
  292. if resp.StatusCode == http.StatusOK {
  293. return nil
  294. }
  295. }
  296. time.Sleep(250 * time.Millisecond)
  297. }
  298. return fmt.Errorf("%s node did not become ready within %s; see %s", n.Name, timeout, n.LogPath)
  299. }
  300. func (n *NodeProcess) Stop() error {
  301. if n == nil {
  302. return nil
  303. }
  304. var errs []string
  305. killRequested := false
  306. if n.cmd != nil && n.cmd.Process != nil {
  307. if err := n.cmd.Process.Kill(); err != nil && !errors.Is(err, os.ErrProcessDone) {
  308. errs = append(errs, fmt.Sprintf("kill %s node: %v", n.Name, err))
  309. } else {
  310. killRequested = true
  311. }
  312. select {
  313. case err := <-n.waitCh:
  314. if err != nil && !killRequested && !strings.Contains(strings.ToLower(err.Error()), "signal: killed") {
  315. errs = append(errs, fmt.Sprintf("wait %s node: %v", n.Name, err))
  316. }
  317. case <-time.After(5 * time.Second):
  318. errs = append(errs, fmt.Sprintf("wait %s node timeout", n.Name))
  319. }
  320. }
  321. if n.logFile != nil {
  322. if err := n.logFile.Close(); err != nil {
  323. errs = append(errs, fmt.Sprintf("close %s log: %v", n.Name, err))
  324. }
  325. n.logFile = nil
  326. }
  327. n.cmd = nil
  328. if len(errs) == 0 {
  329. return nil
  330. }
  331. return errors.New(strings.Join(errs, "; "))
  332. }
  333. func sanitizeName(name string) string {
  334. replacer := strings.NewReplacer("/", "_", "\\", "_", ":", "_", " ", "_")
  335. return replacer.Replace(name)
  336. }
  337. func mergeEnv(base []string, overrides map[string]string) []string {
  338. filtered := make([]string, 0, len(base)+len(overrides))
  339. for _, entry := range base {
  340. parts := strings.SplitN(entry, "=", 2)
  341. if len(parts) != 2 {
  342. continue
  343. }
  344. if _, ok := overrides[parts[0]]; ok {
  345. continue
  346. }
  347. filtered = append(filtered, entry)
  348. }
  349. for key, value := range overrides {
  350. filtered = append(filtered, key+"="+value)
  351. }
  352. return filtered
  353. }