From d8ad8d6da5998c0b4f8c58f89d35b52b9cae8996 Mon Sep 17 00:00:00 2001 From: Seth Foster Date: Tue, 16 Nov 2021 12:01:13 -0500 Subject: [PATCH 1/2] make tests more stable by extending context timeout --- .golangci.yml | 1 + tests/functional/lib/mesh/climesh.go | 6 +- tests/functional/mesh/work_test.go | 238 +++++++++++++-------------- 3 files changed, 123 insertions(+), 122 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index d568777ac..b6148a599 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -66,6 +66,7 @@ linters: # - wrapcheck # TODO: Errors passed upwards should be wrapped. issues: + fix: true exclude: - "lostcancel" # TODO: Context is not canceled on multiple occasions. Needs more detailed work to be fixed. - "SA2002|thelper|testinggoroutine" # TODO: Test interface used outside of its routine, tests need to be rewritten. diff --git a/tests/functional/lib/mesh/climesh.go b/tests/functional/lib/mesh/climesh.go index 1c07d1f74..f9fe35ebe 100644 --- a/tests/functional/lib/mesh/climesh.go +++ b/tests/functional/lib/mesh/climesh.go @@ -83,11 +83,11 @@ func (n *CLINode) Start() error { nodedefPath := filepath.Join(n.dir, "nodedef.yaml") ioutil.WriteFile(nodedefPath, strData, 0o644) n.receptorCmd = exec.Command("receptor", "--config", nodedefPath) - stdout, err := os.Create(filepath.Join(n.dir, "stdout")) + stdout, err := os.OpenFile(filepath.Join(n.dir, "stdout"), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0o600) if err != nil { return err } - stderr, err := os.Create(filepath.Join(n.dir, "stderr")) + stderr, err := os.OpenFile(filepath.Join(n.dir, "stderr"), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0o600) if err != nil { return err } @@ -654,7 +654,7 @@ func (m *CLIMesh) CheckControlSockets() bool { // WaitForReady Waits for connections and routes to converge. func (m *CLIMesh) WaitForReady(ctx context.Context) error { - sleepInterval := 100 * time.Millisecond + sleepInterval := 500 * time.Millisecond if !utils.CheckUntilTimeout(ctx, sleepInterval, m.CheckControlSockets) { return errors.New("timed out while waiting for control sockets") } diff --git a/tests/functional/mesh/work_test.go b/tests/functional/mesh/work_test.go index 450314e21..0e139a998 100644 --- a/tests/functional/mesh/work_test.go +++ b/tests/functional/mesh/work_test.go @@ -195,13 +195,13 @@ func TestWork(t *testing.T) { m, err := mesh.NewCLIMeshFromYaml(data, testName) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } ctx, _ := context.WithTimeout(context.Background(), 60*time.Second) err = m.WaitForReady(ctx) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } nodes := m.Nodes() @@ -210,7 +210,7 @@ func TestWork(t *testing.T) { controller := receptorcontrol.New() err = controller.Connect(nodes[k].ControlSocket()) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } controllers[k] = controller } @@ -264,12 +264,12 @@ func TestWork(t *testing.T) { command := `{"command":"work","subcommand":"submit","worktype":"echosleepshort","tlsclient":"tlsclient","node":"node2","params":"", "ttl":"10h"}` unitID, err := controllers["node1"].WorkSubmitJSON(command) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } - ctx, _ := context.WithTimeout(context.Background(), 20*time.Second) + ctx, _ := context.WithTimeout(context.Background(), 60*time.Second) err = controllers["node1"].AssertWorkSucceeded(ctx, unitID) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } }) @@ -288,26 +288,26 @@ func TestWork(t *testing.T) { command := `{"command":"work","subcommand":"submit","worktype":"echosleepshort","tlsclient":"tlsclientwrongCN","node":"node2","params":""}` unitID, err := controllers["node1"].WorkSubmitJSON(command) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } - ctx, _ := context.WithTimeout(context.Background(), 20*time.Second) + ctx, _ := context.WithTimeout(context.Background(), 60*time.Second) err = controllers["node1"].AssertWorkFailed(ctx, unitID) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } _, err = controllers["node1"].WorkRelease(unitID) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } - ctx, _ = context.WithTimeout(context.Background(), 30*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) err = controllers["node1"].AssertWorkReleased(ctx, unitID) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } - ctx, _ = context.WithTimeout(context.Background(), 20*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) err = assertFilesReleased(ctx, nodes["node1"].Dir(), "node1", unitID) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } }) @@ -326,26 +326,26 @@ func TestWork(t *testing.T) { command := `{"command":"work","subcommand":"submit","worktype":"echosleepshort","tlsclient":"tlsclient","node":"node2","params":"","ttl":"5s"}` unitID, err := controllers["node1"].WorkSubmitJSON(command) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } - ctx, _ := context.WithTimeout(context.Background(), 20*time.Second) + ctx, _ := context.WithTimeout(context.Background(), 60*time.Second) err = controllers["node1"].AssertWorkTimedOut(ctx, unitID) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } _, err = controllers["node1"].WorkRelease(unitID) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } - ctx, _ = context.WithTimeout(context.Background(), 30*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) err = controllers["node1"].AssertWorkReleased(ctx, unitID) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } - ctx, _ = context.WithTimeout(context.Background(), 20*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) err = assertFilesReleased(ctx, nodes["node1"].Dir(), "node1", unitID) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } }) @@ -361,25 +361,25 @@ func TestWork(t *testing.T) { unitID, err := controllers["node1"].WorkSubmit("node3", "echosleeplong") if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } - ctx, _ := context.WithTimeout(context.Background(), 20*time.Second) + ctx, _ := context.WithTimeout(context.Background(), 60*time.Second) err = controllers["node1"].AssertWorkRunning(ctx, unitID) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } _, err = controllers["node1"].WorkCancel(unitID) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } - ctx, _ = context.WithTimeout(context.Background(), 20*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) err = controllers["node1"].AssertWorkCancelled(ctx, unitID) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } workStatus, err := controllers["node1"].GetWorkStatus(unitID) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } remoteUnitID := workStatus.ExtraData.(map[string]interface{})["RemoteUnitID"].(string) if remoteUnitID == "" { @@ -389,39 +389,39 @@ func TestWork(t *testing.T) { nodes["node1"].WaitForShutdown() err = nodes["node1"].Start() if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } - ctx, _ = context.WithTimeout(context.Background(), 10*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) err = m.WaitForReady(ctx) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } err = controllers["node1"].Close() if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } err = controllers["node1"].Reconnect() if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } _, err = controllers["node1"].WorkRelease(unitID) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } - ctx, _ = context.WithTimeout(context.Background(), 30*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) err = controllers["node1"].AssertWorkReleased(ctx, unitID) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } - ctx, _ = context.WithTimeout(context.Background(), 10*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) err = assertFilesReleased(ctx, nodes["node1"].Dir(), "node1", unitID) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } - ctx, _ = context.WithTimeout(context.Background(), 10*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) err = assertFilesReleased(ctx, nodes["node3"].Dir(), "node3", remoteUnitID) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } }) @@ -438,27 +438,27 @@ func TestWork(t *testing.T) { nodes["node3"].WaitForShutdown() unitID, err := controllers["node1"].WorkSubmit("node3", "echosleepshort") if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } - ctx, _ := context.WithTimeout(context.Background(), 10*time.Second) + ctx, _ := context.WithTimeout(context.Background(), 60*time.Second) err = controllers["node1"].AssertWorkPending(ctx, unitID) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } err = nodes["node3"].Start() if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } // Wait for node3 to join the mesh again ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) err = m.WaitForReady(ctx) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) err = controllers["node1"].AssertWorkSucceeded(ctx, unitID) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } }) @@ -473,21 +473,21 @@ func TestWork(t *testing.T) { unitID, err := controllers["node1"].WorkSubmit("node3", "echosleeplong") if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } - ctx, _ := context.WithTimeout(context.Background(), 20*time.Second) + ctx, _ := context.WithTimeout(context.Background(), 60*time.Second) err = controllers["node1"].AssertWorkRunning(ctx, unitID) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } - ctx, _ = context.WithTimeout(context.Background(), 20*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) err = assertStdoutFizeSize(ctx, nodes["node1"].Dir(), "node1", unitID, 1) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } err = controllers["node1"].AssertWorkResults(unitID, expectedResults[:1]) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } nodes["node2"].Shutdown() nodes["node2"].WaitForShutdown() @@ -496,21 +496,21 @@ func TestWork(t *testing.T) { ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) err = m.WaitForReady(ctx) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } - ctx, _ = context.WithTimeout(context.Background(), 30*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) err = controllers["node1"].AssertWorkSucceeded(ctx, unitID) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } - ctx, _ = context.WithTimeout(context.Background(), 30*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) err = assertStdoutFizeSize(ctx, nodes["node1"].Dir(), "node1", unitID, 10) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } err = controllers["node1"].AssertWorkResults(unitID, expectedResults) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } }) t.Run(testGroup+"/results on restarted node", func(t *testing.T) { @@ -524,33 +524,33 @@ func TestWork(t *testing.T) { unitID, err := controllers["node1"].WorkSubmit("node3", "echosleeplong") if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } - ctx, _ := context.WithTimeout(context.Background(), 20*time.Second) + ctx, _ := context.WithTimeout(context.Background(), 60*time.Second) err = controllers["node1"].AssertWorkRunning(ctx, unitID) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } nodes["node3"].Shutdown() nodes["node3"].WaitForShutdown() err = nodes["node3"].Start() if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } // Wait for node3 to join the mesh again ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) err = m.WaitForReady(ctx) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) err = controllers["node1"].AssertWorkSucceeded(ctx, unitID) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } err = controllers["node1"].AssertWorkResults(unitID, expectedResults) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } }) t.Run(testGroup+"/work submit and release to non-existent node", func(t *testing.T) { @@ -566,45 +566,45 @@ func TestWork(t *testing.T) { // node999 was never initialised unitID, err := controllers["node1"].WorkSubmit("node999", "echosleeplong") if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } // wait for 10 seconds, and check if the work is in pending state - ctx, _ := context.WithTimeout(context.Background(), 10*time.Second) + ctx, _ := context.WithTimeout(context.Background(), 60*time.Second) err = controllers["node1"].AssertWorkPending(ctx, unitID) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } nodes["node1"].Shutdown() nodes["node1"].WaitForShutdown() err = nodes["node1"].Start() if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } - ctx, _ = context.WithTimeout(context.Background(), 10*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) err = m.WaitForReady(ctx) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } err = controllers["node1"].Close() if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } err = controllers["node1"].Reconnect() if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } // release the work on node1 _, err = controllers["node1"].WorkRelease(unitID) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } - ctx, _ = context.WithTimeout(context.Background(), 30*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) err = controllers["node1"].AssertWorkReleased(ctx, unitID) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } }) t.Run(testGroup+"/reload backends while streaming work results", func(t *testing.T) { @@ -618,14 +618,14 @@ func TestWork(t *testing.T) { // submit work from node 2 to node 3 unitID, err := controllers["node2"].WorkSubmit("node3", "echosleeplong50") if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } // wait for 10 seconds, and check if the work is in pending state - ctx, _ := context.WithTimeout(context.Background(), 10*time.Second) + ctx, _ := context.WithTimeout(context.Background(), 60*time.Second) err = controllers["node2"].AssertWorkPending(ctx, unitID) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } // declare a new mesh with no connection between nodes @@ -644,14 +644,14 @@ func TestWork(t *testing.T) { // modify the existing mesh err = mesh.ModifyCLIMeshFromYaml(modifiedData, *m) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } // reload the entire mesh for k := range controllers { err = controllers[k].Reload() if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } } @@ -664,7 +664,7 @@ func TestWork(t *testing.T) { // read the work status workStatus, err := controllers["node2"].GetWorkStatus(unitID) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } // modify the mesh to have connection again @@ -690,14 +690,14 @@ func TestWork(t *testing.T) { err = mesh.ModifyCLIMeshFromYaml(withConnectionData, *m) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } // reload the entire mesh for k := range controllers { err = controllers[k].Reload() if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } } @@ -705,19 +705,19 @@ func TestWork(t *testing.T) { ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) err = m.WaitForReady(ctx) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } // ping should be successful in the mesh with connections _, err = controllers["node1"].Ping("node3") if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } - ctx, _ = context.WithTimeout(context.Background(), 15*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) err = controllers["node2"].AssertWorkSizeIncreasing(ctx, unitID, workStatus.StdoutSize) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } // it takes some time for the streaming to start again @@ -725,7 +725,7 @@ func TestWork(t *testing.T) { newWorkStatus, err := controllers["node2"].GetWorkStatus(unitID) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } if newWorkStatus.StdoutSize <= workStatus.StdoutSize { @@ -735,12 +735,12 @@ func TestWork(t *testing.T) { // cancel the work so that it doesnt run after the test ends _, err = controllers["node2"].WorkCancel(unitID) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } - ctx, _ = context.WithTimeout(context.Background(), 20*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) err = controllers["node2"].AssertWorkCancelled(ctx, unitID) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } }) } @@ -770,13 +770,13 @@ func TestRuntimeParams(t *testing.T) { m, err := mesh.NewCLIMeshFromYaml(data, t.Name()) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } ctx, _ := context.WithTimeout(context.Background(), 60*time.Second) err = m.WaitForReady(ctx) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } nodes := m.Nodes() controllers := make(map[string]*receptorcontrol.ReceptorControl) @@ -784,21 +784,21 @@ func TestRuntimeParams(t *testing.T) { controllers["node0"] = receptorcontrol.New() err = controllers["node0"].Connect(nodes["node0"].ControlSocket()) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } command := `{"command":"work","subcommand":"submit","worktype":"echo","node":"node0","params":"it worked!"}` unitID, err := controllers["node0"].WorkSubmitJSON(command) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } err = controllers["node0"].AssertWorkSucceeded(ctx, unitID) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } err = controllers["node0"].AssertWorkResults(unitID, []byte("it worked!")) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } } @@ -837,12 +837,12 @@ func TestKubeRuntimeParams(t *testing.T) { } m, err := mesh.NewCLIMeshFromYaml(data, t.Name()) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } ctx, _ := context.WithTimeout(context.Background(), 60*time.Second) err = m.WaitForReady(ctx) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } nodes := m.Nodes() controllers := make(map[string]*receptorcontrol.ReceptorControl) @@ -850,20 +850,20 @@ func TestKubeRuntimeParams(t *testing.T) { controllers["node0"] = receptorcontrol.New() err = controllers["node0"].Connect(nodes["node0"].ControlSocket()) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } command := fmt.Sprintf(`{"command": "work", "subcommand": "submit", "node": "localhost", "worktype": "echo", "secret_kube_pod": "---\napiVersion: v1\nkind: Pod\nspec:\n containers:\n - name: worker\n image: centos:8\n command:\n - bash\n args:\n - \"-c\"\n - for i in {1..5}; do echo $i;done\n", "secret_kube_config": "%s"}`, kubeconfig) unitID, err := controllers["node0"].WorkSubmitJSON(command) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } err = controllers["node0"].AssertWorkSucceeded(ctx, unitID) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } err = controllers["node0"].AssertWorkResults(unitID, []byte("1\n2\n3\n4\n5\n")) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } } @@ -891,13 +891,13 @@ func TestRuntimeParamsNotAllowed(t *testing.T) { m, err := mesh.NewCLIMeshFromYaml(data, t.Name()) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } ctx, _ := context.WithTimeout(context.Background(), 60*time.Second) err = m.WaitForReady(ctx) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } nodes := m.Nodes() controllers := make(map[string]*receptorcontrol.ReceptorControl) @@ -905,7 +905,7 @@ func TestRuntimeParamsNotAllowed(t *testing.T) { controllers["node0"] = receptorcontrol.New() err = controllers["node0"].Connect(nodes["node0"].ControlSocket()) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } command := `{"command":"work","subcommand":"submit","worktype":"echo","node":"node0","params":"it worked!"}` _, err = controllers["node0"].WorkSubmitJSON(command) @@ -941,13 +941,13 @@ func TestKubeContainerFailure(t *testing.T) { } m, err := mesh.NewCLIMeshFromYaml(data, t.Name()) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } ctx, _ := context.WithTimeout(context.Background(), 60*time.Second) err = m.WaitForReady(ctx) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } nodes := m.Nodes() controllers := make(map[string]*receptorcontrol.ReceptorControl) @@ -955,14 +955,14 @@ func TestKubeContainerFailure(t *testing.T) { controllers["node0"] = receptorcontrol.New() err = controllers["node0"].Connect(nodes["node0"].ControlSocket()) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } job := `{"command":"work","subcommand":"submit","worktype":"kubejob","node":"node0"}` unitID, err := controllers["node0"].WorkSubmitJSON(job) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } - ctx, _ = context.WithTimeout(context.Background(), 20*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) err = controllers["node0"].AssertWorkFailed(ctx, unitID) if err != nil { t.Fatal("Expected work to fail but it succeeded") @@ -1030,13 +1030,13 @@ func TestSignedWorkVerification(t *testing.T) { } m, err := mesh.NewCLIMeshFromYaml(data, t.Name()) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } ctx, _ := context.WithTimeout(context.Background(), 60*time.Second) err = m.WaitForReady(ctx) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } nodes := m.Nodes() controllers := make(map[string]*receptorcontrol.ReceptorControl) @@ -1044,18 +1044,18 @@ func TestSignedWorkVerification(t *testing.T) { controllers["node0"] = receptorcontrol.New() err = controllers["node0"].Connect(nodes["node0"].ControlSocket()) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } job := `{"command":"work","subcommand":"submit","worktype":"echo","node":"node1", "signwork":"true"}` unitID, err := controllers["node0"].WorkSubmitJSON(job) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } - ctx, _ = context.WithTimeout(context.Background(), 20*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) err = controllers["node0"].AssertWorkSucceeded(ctx, unitID) if err != nil { - t.Fatal(err) + t.Fatal(err, m.Dir()) } // node2 has the wrong public key to verify work signatures, so the work submission should fail From 384545baf8ed70396323396538404910c1373e91 Mon Sep 17 00:00:00 2001 From: Seth Foster Date: Tue, 16 Nov 2021 12:18:59 -0500 Subject: [PATCH 2/2] extend to 120 seconds for service ads --- tests/functional/mesh/work_test.go | 78 +++++++++++++++--------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/tests/functional/mesh/work_test.go b/tests/functional/mesh/work_test.go index 0e139a998..5dee503a2 100644 --- a/tests/functional/mesh/work_test.go +++ b/tests/functional/mesh/work_test.go @@ -198,7 +198,7 @@ func TestWork(t *testing.T) { t.Fatal(err, m.Dir()) } - ctx, _ := context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ := context.WithTimeout(context.Background(), 120*time.Second) err = m.WaitForReady(ctx) if err != nil { t.Fatal(err, m.Dir()) @@ -266,7 +266,7 @@ func TestWork(t *testing.T) { if err != nil { t.Fatal(err, m.Dir()) } - ctx, _ := context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ := context.WithTimeout(context.Background(), 120*time.Second) err = controllers["node1"].AssertWorkSucceeded(ctx, unitID) if err != nil { t.Fatal(err, m.Dir()) @@ -290,7 +290,7 @@ func TestWork(t *testing.T) { if err != nil { t.Fatal(err, m.Dir()) } - ctx, _ := context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ := context.WithTimeout(context.Background(), 120*time.Second) err = controllers["node1"].AssertWorkFailed(ctx, unitID) if err != nil { t.Fatal(err, m.Dir()) @@ -299,12 +299,12 @@ func TestWork(t *testing.T) { if err != nil { t.Fatal(err, m.Dir()) } - ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 120*time.Second) err = controllers["node1"].AssertWorkReleased(ctx, unitID) if err != nil { t.Fatal(err, m.Dir()) } - ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 120*time.Second) err = assertFilesReleased(ctx, nodes["node1"].Dir(), "node1", unitID) if err != nil { t.Fatal(err, m.Dir()) @@ -328,7 +328,7 @@ func TestWork(t *testing.T) { if err != nil { t.Fatal(err, m.Dir()) } - ctx, _ := context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ := context.WithTimeout(context.Background(), 120*time.Second) err = controllers["node1"].AssertWorkTimedOut(ctx, unitID) if err != nil { t.Fatal(err, m.Dir()) @@ -337,12 +337,12 @@ func TestWork(t *testing.T) { if err != nil { t.Fatal(err, m.Dir()) } - ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 120*time.Second) err = controllers["node1"].AssertWorkReleased(ctx, unitID) if err != nil { t.Fatal(err, m.Dir()) } - ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 120*time.Second) err = assertFilesReleased(ctx, nodes["node1"].Dir(), "node1", unitID) if err != nil { t.Fatal(err, m.Dir()) @@ -363,7 +363,7 @@ func TestWork(t *testing.T) { if err != nil { t.Fatal(err, m.Dir()) } - ctx, _ := context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ := context.WithTimeout(context.Background(), 120*time.Second) err = controllers["node1"].AssertWorkRunning(ctx, unitID) if err != nil { t.Fatal(err, m.Dir()) @@ -372,7 +372,7 @@ func TestWork(t *testing.T) { if err != nil { t.Fatal(err, m.Dir()) } - ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 120*time.Second) err = controllers["node1"].AssertWorkCancelled(ctx, unitID) if err != nil { t.Fatal(err, m.Dir()) @@ -391,7 +391,7 @@ func TestWork(t *testing.T) { if err != nil { t.Fatal(err, m.Dir()) } - ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 120*time.Second) err = m.WaitForReady(ctx) if err != nil { t.Fatal(err, m.Dir()) @@ -408,17 +408,17 @@ func TestWork(t *testing.T) { if err != nil { t.Fatal(err, m.Dir()) } - ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 120*time.Second) err = controllers["node1"].AssertWorkReleased(ctx, unitID) if err != nil { t.Fatal(err, m.Dir()) } - ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 120*time.Second) err = assertFilesReleased(ctx, nodes["node1"].Dir(), "node1", unitID) if err != nil { t.Fatal(err, m.Dir()) } - ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 120*time.Second) err = assertFilesReleased(ctx, nodes["node3"].Dir(), "node3", remoteUnitID) if err != nil { t.Fatal(err, m.Dir()) @@ -440,7 +440,7 @@ func TestWork(t *testing.T) { if err != nil { t.Fatal(err, m.Dir()) } - ctx, _ := context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ := context.WithTimeout(context.Background(), 120*time.Second) err = controllers["node1"].AssertWorkPending(ctx, unitID) if err != nil { t.Fatal(err, m.Dir()) @@ -450,12 +450,12 @@ func TestWork(t *testing.T) { t.Fatal(err, m.Dir()) } // Wait for node3 to join the mesh again - ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 120*time.Second) err = m.WaitForReady(ctx) if err != nil { t.Fatal(err, m.Dir()) } - ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 120*time.Second) err = controllers["node1"].AssertWorkSucceeded(ctx, unitID) if err != nil { t.Fatal(err, m.Dir()) @@ -475,12 +475,12 @@ func TestWork(t *testing.T) { if err != nil { t.Fatal(err, m.Dir()) } - ctx, _ := context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ := context.WithTimeout(context.Background(), 120*time.Second) err = controllers["node1"].AssertWorkRunning(ctx, unitID) if err != nil { t.Fatal(err, m.Dir()) } - ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 120*time.Second) err = assertStdoutFizeSize(ctx, nodes["node1"].Dir(), "node1", unitID, 1) if err != nil { t.Fatal(err, m.Dir()) @@ -493,17 +493,17 @@ func TestWork(t *testing.T) { nodes["node2"].WaitForShutdown() nodes["node2"].Start() // Wait for node2 to join the mesh again - ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 120*time.Second) err = m.WaitForReady(ctx) if err != nil { t.Fatal(err, m.Dir()) } - ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 120*time.Second) err = controllers["node1"].AssertWorkSucceeded(ctx, unitID) if err != nil { t.Fatal(err, m.Dir()) } - ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 120*time.Second) err = assertStdoutFizeSize(ctx, nodes["node1"].Dir(), "node1", unitID, 10) if err != nil { t.Fatal(err, m.Dir()) @@ -526,7 +526,7 @@ func TestWork(t *testing.T) { if err != nil { t.Fatal(err, m.Dir()) } - ctx, _ := context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ := context.WithTimeout(context.Background(), 120*time.Second) err = controllers["node1"].AssertWorkRunning(ctx, unitID) if err != nil { t.Fatal(err, m.Dir()) @@ -538,12 +538,12 @@ func TestWork(t *testing.T) { t.Fatal(err, m.Dir()) } // Wait for node3 to join the mesh again - ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 120*time.Second) err = m.WaitForReady(ctx) if err != nil { t.Fatal(err, m.Dir()) } - ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 120*time.Second) err = controllers["node1"].AssertWorkSucceeded(ctx, unitID) if err != nil { t.Fatal(err, m.Dir()) @@ -570,7 +570,7 @@ func TestWork(t *testing.T) { } // wait for 10 seconds, and check if the work is in pending state - ctx, _ := context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ := context.WithTimeout(context.Background(), 120*time.Second) err = controllers["node1"].AssertWorkPending(ctx, unitID) if err != nil { t.Fatal(err, m.Dir()) @@ -582,7 +582,7 @@ func TestWork(t *testing.T) { if err != nil { t.Fatal(err, m.Dir()) } - ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 120*time.Second) err = m.WaitForReady(ctx) if err != nil { t.Fatal(err, m.Dir()) @@ -601,7 +601,7 @@ func TestWork(t *testing.T) { if err != nil { t.Fatal(err, m.Dir()) } - ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 120*time.Second) err = controllers["node1"].AssertWorkReleased(ctx, unitID) if err != nil { t.Fatal(err, m.Dir()) @@ -622,7 +622,7 @@ func TestWork(t *testing.T) { } // wait for 10 seconds, and check if the work is in pending state - ctx, _ := context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ := context.WithTimeout(context.Background(), 120*time.Second) err = controllers["node2"].AssertWorkPending(ctx, unitID) if err != nil { t.Fatal(err, m.Dir()) @@ -702,7 +702,7 @@ func TestWork(t *testing.T) { } // wait for mesh to become ready again - ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 120*time.Second) err = m.WaitForReady(ctx) if err != nil { t.Fatal(err, m.Dir()) @@ -714,7 +714,7 @@ func TestWork(t *testing.T) { t.Fatal(err, m.Dir()) } - ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 120*time.Second) err = controllers["node2"].AssertWorkSizeIncreasing(ctx, unitID, workStatus.StdoutSize) if err != nil { t.Fatal(err, m.Dir()) @@ -737,7 +737,7 @@ func TestWork(t *testing.T) { if err != nil { t.Fatal(err, m.Dir()) } - ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 120*time.Second) err = controllers["node2"].AssertWorkCancelled(ctx, unitID) if err != nil { t.Fatal(err, m.Dir()) @@ -773,7 +773,7 @@ func TestRuntimeParams(t *testing.T) { t.Fatal(err, m.Dir()) } - ctx, _ := context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ := context.WithTimeout(context.Background(), 120*time.Second) err = m.WaitForReady(ctx) if err != nil { t.Fatal(err, m.Dir()) @@ -839,7 +839,7 @@ func TestKubeRuntimeParams(t *testing.T) { if err != nil { t.Fatal(err, m.Dir()) } - ctx, _ := context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ := context.WithTimeout(context.Background(), 120*time.Second) err = m.WaitForReady(ctx) if err != nil { t.Fatal(err, m.Dir()) @@ -894,7 +894,7 @@ func TestRuntimeParamsNotAllowed(t *testing.T) { t.Fatal(err, m.Dir()) } - ctx, _ := context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ := context.WithTimeout(context.Background(), 120*time.Second) err = m.WaitForReady(ctx) if err != nil { t.Fatal(err, m.Dir()) @@ -944,7 +944,7 @@ func TestKubeContainerFailure(t *testing.T) { t.Fatal(err, m.Dir()) } - ctx, _ := context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ := context.WithTimeout(context.Background(), 120*time.Second) err = m.WaitForReady(ctx) if err != nil { t.Fatal(err, m.Dir()) @@ -962,7 +962,7 @@ func TestKubeContainerFailure(t *testing.T) { if err != nil { t.Fatal(err, m.Dir()) } - ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 120*time.Second) err = controllers["node0"].AssertWorkFailed(ctx, unitID) if err != nil { t.Fatal("Expected work to fail but it succeeded") @@ -1033,7 +1033,7 @@ func TestSignedWorkVerification(t *testing.T) { t.Fatal(err, m.Dir()) } - ctx, _ := context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ := context.WithTimeout(context.Background(), 120*time.Second) err = m.WaitForReady(ctx) if err != nil { t.Fatal(err, m.Dir()) @@ -1052,7 +1052,7 @@ func TestSignedWorkVerification(t *testing.T) { if err != nil { t.Fatal(err, m.Dir()) } - ctx, _ = context.WithTimeout(context.Background(), 60*time.Second) + ctx, _ = context.WithTimeout(context.Background(), 120*time.Second) err = controllers["node0"].AssertWorkSucceeded(ctx, unitID) if err != nil { t.Fatal(err, m.Dir())