Basic example POC

Initial basic implementation.  Has a couple of sample tests.  Mostly
cobbled together from k8s-tests, stripped down.

.gitignore

@@ -0,0 +1 @@
+example-tests

Makefile

@@ -0,0 +1,15 @@
+all: test build
+
+verify: lint
+
+build:
+	go build ./cmd/...
+
+test:
+	go test ./...
+
+lint:
+	./hack/go-lint.sh run ./...
+
+clean:
+	rm -f example-test

cmd/example-tests/main.go

@@ -0,0 +1,72 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"sort"
+
+	"github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+	"github.com/spf13/cobra"
+)
+
+func main() {
+	root := &cobra.Command{
+		Long: "OpenShift Tests External Binary Example",
+	}
+
+	root.AddCommand(
+		newRunTestCommand(),
+		newListTestsCommand(),
+	)
+
+	gomega.RegisterFailHandler(ginkgo.Fail)
+
+	if err := func() error {
+		return root.Execute()
+	}(); err != nil {
+		if ex, ok := err.(ExitError); ok {
+			fmt.Fprintf(os.Stderr, "Ginkgo exit error %d: %v\n", ex.Code, err)
+			os.Exit(ex.Code)
+		}
+		fmt.Fprintf(os.Stderr, "error: %v\n", err)
+		os.Exit(1)
+	}
+}
+
+func newRunTestCommand() *cobra.Command {
+	testOpt := NewTestOptions(os.Stdout, os.Stderr)
+
+	cmd := &cobra.Command{
+		Use:          "run-test NAME",
+		Short:        "Run a single test by name",
+		Long:         "Execute a single test.",
+		SilenceUsage: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return testOpt.Run(args)
+		},
+	}
+	return cmd
+}
+
+func newListTestsCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:          "list",
+		Short:        "List available tests",
+		Long:         "List the available tests in this binary.",
+		SilenceUsage: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			tests := testsForSuite()
+			sort.Slice(tests, func(i, j int) bool { return tests[i].Name < tests[j].Name })
+			data, err := json.Marshal(tests)
+			if err != nil {
+				return err
+			}
+			fmt.Fprintf(os.Stdout, "%s\n", data)
+			return nil
+		},
+	}
+
+	return cmd
+}

cmd/example-tests/runtest.go

@@ -0,0 +1,139 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/onsi/ginkgo/v2"
+	"github.com/onsi/ginkgo/v2/types"
+
+	_ "github.com/openshift-eng/openshift-tests-extension/test/example"
+)
+
+// TestOptions handles running a single test.
+type TestOptions struct {
+	Out    io.Writer
+	ErrOut io.Writer
+}
+
+var _ ginkgo.GinkgoTestingT = &TestOptions{}
+
+func NewTestOptions(out io.Writer, errOut io.Writer) *TestOptions {
+	return &TestOptions{
+		Out:    out,
+		ErrOut: errOut,
+	}
+}
+
+func (opt *TestOptions) Run(args []string) error {
+	if len(args) != 1 {
+		return fmt.Errorf("only a single test name may be passed")
+	}
+
+	// Ignore the upstream suite behavior within test execution
+	ginkgo.GetSuite().ClearBeforeAndAfterSuiteNodes()
+	tests := testsForSuite()
+	var test *TestCase
+	for _, t := range tests {
+		if t.Name == args[0] {
+			test = t
+			break
+		}
+	}
+	if test == nil {
+		return fmt.Errorf("no test exists with that name: %s", args[0])
+	}
+
+	suiteConfig, reporterConfig := ginkgo.GinkgoConfiguration()
+	suiteConfig.FocusStrings = []string{fmt.Sprintf("^ %s$", regexp.QuoteMeta(test.Name))}
+
+	// These settings are matched to upstream's ginkgo configuration. See:
+	// https://github.com/kubernetes/kubernetes/blob/v1.25.0/test/e2e/framework/test_context.go#L354-L355
+	// Randomize specs as well as suites
+	suiteConfig.RandomizeAllSpecs = true
+	// https://github.com/kubernetes/kubernetes/blob/v1.25.0/hack/ginkgo-e2e.sh#L172-L173
+	suiteConfig.Timeout = 24 * time.Hour
+	reporterConfig.NoColor = true
+	reporterConfig.Verbose = true
+
+	ginkgo.SetReporterConfig(reporterConfig)
+
+	cwd, err := os.Getwd()
+	if err != nil {
+		return err
+	}
+	ginkgo.GetSuite().RunSpec(test.spec, ginkgo.Labels{}, "Kubernetes e2e suite", cwd, ginkgo.GetFailer(), ginkgo.GetWriter(), suiteConfig, reporterConfig)
+
+	var summary types.SpecReport
+	for _, report := range ginkgo.GetSuite().GetReport().SpecReports {
+		if report.NumAttempts > 0 {
+			summary = report
+		}
+	}
+
+	switch {
+	case summary.State == types.SpecStatePassed:
+		// do nothing
+	case summary.State == types.SpecStateSkipped:
+		if len(summary.Failure.Message) > 0 {
+			fmt.Fprintf(opt.ErrOut, "skip [%s:%d]: %s\n", lastFilenameSegment(summary.Failure.Location.FileName), summary.Failure.Location.LineNumber, summary.Failure.Message)
+		}
+		if len(summary.Failure.ForwardedPanic) > 0 {
+			fmt.Fprintf(opt.ErrOut, "skip [%s:%d]: %s\n", lastFilenameSegment(summary.Failure.Location.FileName), summary.Failure.Location.LineNumber, summary.Failure.ForwardedPanic)
+		}
+		return ExitError{Code: 3}
+	case summary.State == types.SpecStateFailed, summary.State == types.SpecStatePanicked, summary.State == types.SpecStateInterrupted:
+		if len(summary.Failure.ForwardedPanic) > 0 {
+			if len(summary.Failure.Location.FullStackTrace) > 0 {
+				fmt.Fprintf(opt.ErrOut, "\n%s\n", summary.Failure.Location.FullStackTrace)
+			}
+			fmt.Fprintf(opt.ErrOut, "fail [%s:%d]: Test Panicked: %s\n", lastFilenameSegment(summary.Failure.Location.FileName), summary.Failure.Location.LineNumber, summary.Failure.ForwardedPanic)
+			return ExitError{Code: 1}
+		}
+		fmt.Fprintf(opt.ErrOut, "fail [%s:%d]: %s\n", lastFilenameSegment(summary.Failure.Location.FileName), summary.Failure.Location.LineNumber, summary.Failure.Message)
+		return ExitError{Code: 1}
+	default:
+		return fmt.Errorf("unrecognized test case outcome: %#v", summary)
+	}
+	return nil
+}
+
+func (opt *TestOptions) Fail() {
+	// this function allows us to pass TestOptions as the first argument,
+	// it's empty becase we have failure check mechanism implemented above.
+}
+
+func lastFilenameSegment(filename string) string {
+	if parts := strings.Split(filename, "/vendor/"); len(parts) > 1 {
+		return parts[len(parts)-1]
+	}
+	if parts := strings.Split(filename, "/src/"); len(parts) > 1 {
+		return parts[len(parts)-1]
+	}
+	return filename
+}
+
+func testsForSuite() []*TestCase {
+	var tests []*TestCase
+
+	// Don't build the tree multiple times, it results in multiple initing of tests
+	if !ginkgo.GetSuite().InPhaseBuildTree() {
+		if err := ginkgo.GetSuite().BuildTree(); err != nil {
+			panic(err)
+		}
+	}
+
+	ginkgo.GetSuite().WalkTests(func(name string, spec types.TestSpec) {
+		testCase := &TestCase{
+			Name:      spec.Text(),
+			locations: spec.CodeLocations(),
+			spec:      spec,
+		}
+		tests = append(tests, testCase)
+	})
+	return tests
+}

cmd/example-tests/types.go

@@ -0,0 +1,69 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/onsi/ginkgo/v2/types"
+)
+
+// copied directly from github.com/openshift/origin/test/extended/util/cluster/cluster.go
+type ClusterConfiguration struct {
+	ProviderName string `json:"type"`
+
+	// These fields (and the "type" tag for ProviderName) chosen to match
+	// upstream's e2e.CloudConfig.
+	ProjectID   string
+	Region      string
+	Zone        string
+	NumNodes    int
+	MultiMaster bool
+	MultiZone   bool
+	Zones       []string
+	ConfigFile  string
+
+	// Disconnected is set for test jobs without external internet connectivity
+	Disconnected bool
+
+	// SingleReplicaTopology is set for disabling disruptive tests or tests
+	// that require high availability
+	SingleReplicaTopology bool
+
+	// NetworkPlugin is the "official" plugin name
+	NetworkPlugin string
+	// NetworkPluginMode is an optional sub-identifier for the NetworkPlugin.
+	// (Currently it is only used for OpenShiftSDN.)
+	NetworkPluginMode string `json:",omitempty"`
+
+	// HasIPv4 and HasIPv6 determine whether IPv4-specific, IPv6-specific,
+	// and dual-stack-specific tests are run
+	HasIPv4 bool
+	HasIPv6 bool
+
+	// HasSCTP determines whether SCTP connectivity tests can be run in the cluster
+	HasSCTP bool
+
+	// IsProxied determines whether we are accessing the cluster through an HTTP proxy
+	IsProxied bool
+
+	// IsIBMROKS determines whether the cluster is Managed IBM Cloud (ROKS)
+	IsIBMROKS bool
+
+	// IsNoOptionalCapabilities indicates the cluster has no optional capabilities enabled
+	HasNoOptionalCapabilities bool
+}
+
+// copied directly from github.com/openshift/origin/pkg/test/ginkgo/test.go
+type TestCase struct {
+	Name      string
+	Labels    string
+	spec      types.TestSpec
+	locations []types.CodeLocation
+}
+
+type ExitError struct {
+	Code int
+}
+
+func (e ExitError) Error() string {
+	return fmt.Sprintf("exit with code %d", e.Code)
+}

go.mod

@@ -0,0 +1,26 @@
+module github.com/openshift-eng/openshift-tests-extension
+
+go 1.22.4
+
+require (
+	github.com/onsi/ginkgo/v2 v2.20.2
+	github.com/onsi/gomega v1.30.0
+	github.com/spf13/cobra v1.8.1
+)
+
+require (
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/stretchr/testify v1.8.4 // indirect
+	golang.org/x/net v0.28.0 // indirect
+	golang.org/x/sys v0.24.0 // indirect
+	golang.org/x/text v0.17.0 // indirect
+	golang.org/x/tools v0.24.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)
+
+replace github.com/onsi/ginkgo/v2 => github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20240314173009-2cd07f4ca53d

go.sum

@@ -0,0 +1,46 @@
+github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
+github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
+github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
+github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 h1:5iH8iuqE5apketRbSFBy+X1V0o+l+8NF1avt4HWl7cA=
+github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8=
+github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
+github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20240314173009-2cd07f4ca53d h1:GDn4rF5hmB+d0tnFcPQhPy1YTooJH6U+HeYYdyjktmI=
+github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20240314173009-2cd07f4ca53d/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
+github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE=
+golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
+golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
+golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
+golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24=
+golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ=
+google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
+google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=