Add cross-version test framework (and a simple test)

.github/workflows/ci.yml

@@ -26,7 +26,7 @@ jobs:
       - name: test
         run: |
           sudo apt-get install tcl8.6 tclx
-          ./runtest --verbose --tags -slow --dump-logs --old-server-path tests/tmp/valkey-7.2.7-focal-x86_64/bin/valkey-server
+          ./runtest --verbose --tags -slow --dump-logs --other-server-path tests/tmp/valkey-7.2.7-focal-x86_64/bin/valkey-server
       - name: module api test
         run: CFLAGS='-Werror' ./runtest-moduleapi --verbose --dump-logs
       - name: validate commands.def up to date

tests/README.md

@@ -24,16 +24,17 @@ There are additional runtime options that can further adjust the test suite to
 match different external server configurations. All options are listed by
 `./runtest --help`. The following table is just a subset of the options:
 
-| Option               | Impact                                                   |
-| -------------------- | -------------------------------------------------------- |
-| `--singledb`         | Only use database 0, don't assume others are supported. |
-| `--ignore-encoding`  | Skip all checks for specific encoding.  |
-| `--ignore-digest`    | Skip key value digest validations. |
-| `--cluster-mode`     | Run in strict Valkey Cluster compatibility mode. |
-| `--large-memory`     | Enables tests that consume more than 100MB |
-| `--tls`              | Run tests with TLS. See below. |
-| `--tls-module`       | Run tests with TLS, when TLS support is built as a module. |
-| `--help`             | Displays the full set of options. |
+| Option                    | Impact                                                   |
+| ------------------------- | -------------------------------------------------------- |
+| `--singledb`              | Only use database 0, don't assume others are supported. |
+| `--ignore-encoding`       | Skip all checks for specific encoding.  |
+| `--ignore-digest`         | Skip key value digest validations. |
+| `--cluster-mode`          | Run in strict Valkey Cluster compatibility mode. |
+| `--large-memory`          | Enables tests that consume more than 100MB |
+| `--tls`                   | Run tests with TLS. See below. |
+| `--tls-module`            | Run tests with TLS, when TLS support is built as a module. |
+| `--other-server-path PATH | Run compatibility tests with an other server executable. |
+| `--help`                  | Displays the full set of options. |
 
 Running with TLS requires the following preparations:
 
@@ -88,16 +89,20 @@ The following compatibility and capability tags are currently used:
 | Tag                       | Indicates |
 | ---------------------     | --------- |
 | `external:skip`           | Not compatible with external servers. |
+| `cluster`                 | Uses cluster with multiple nodes. |
 | `cluster:skip`            | Not compatible with `--cluster-mode`. |
 | `large-memory`            | Test that requires more than 100MB |
+| `tls`                     | Uses TLS. |
 | `tls:skip`                | Not compatible with `--tls`. |
-| `needs:repl`              | Uses replication and needs to be able to `SYNC` from server. |
+| `ipv6`                    | Uses IPv6. |
+| `needs:repl`, `repl`      | Uses replication and needs to be able to `SYNC` from server. |
 | `needs:debug`             | Uses the `DEBUG` command or other debugging focused commands (like `OBJECT REFCOUNT`). |
 | `needs:pfdebug`           | Uses the `PFDEBUG` command. |
 | `needs:config-maxmemory`  | Uses `CONFIG SET` to manipulate memory limit, eviction policies, etc. |
 | `needs:config-resetstat`  | Uses `CONFIG RESETSTAT` to reset statistics. |
 | `needs:reset`             | Uses `RESET` to reset client connections. |
 | `needs:save`              | Uses `SAVE` or `BGSAVE` to create an RDB file. |
+| `needs:other-server         | Requires `--other-server-path`. |
 
 When using an external server (`--host` and `--port`), filtering using the
 `external:skip` tags is done automatically.

tests/integration/cross-version-replication.tcl

@@ -11,7 +11,7 @@ proc server_name_and_version {} {
     return "$server_name $server_version"
 }
 
-start_server {tags {"repl needs:old-server external:skip"} start-old-server 1 config "minimal.conf"} {
+start_server {tags {"repl needs:other-server external:skip"} start-other-server 1 config "minimal.conf"} {
     set primary_name_and_version [server_name_and_version]
     r set foo bar
 

tests/support/server.tcl

@@ -216,8 +216,8 @@ proc tags_acceptable {tags err_return} {
         return 0
     }
 
-    if {$::old_server_path eq {} && [lsearch $tags "needs:old-server"] >= 0} {
-        set err "Old server path not provided"
+    if {$::other_server_path eq {} && [lsearch $tags "needs:other-server"] >= 0} {
+        set err "Other server path not provided"
         return 0
     }
 
@@ -446,16 +446,16 @@ proc start_server {options {code undefined}} {
     set args {}
     set keep_persistence false
     set config_lines {}
-    set start_old_server 0
+    set start_other_server 0
 
     # Wait for the server to be ready and check for server liveness/client connectivity before starting the test.
     set wait_ready true
 
     # parse options
     foreach {option value} $options {
         switch $option {
-            "start-old-server" {
-                set start_old_server $value
+            "start-other-server" {
+                set start_other_server $value ; # boolean, 0 or 1
             }
             "config" {
                 set baseconfig $value
@@ -507,8 +507,8 @@ proc start_server {options {code undefined}} {
         return
     }
 
-    if {$start_old_server} {
-        set executable $::old_server_path
+    if {$start_other_server} {
+        set executable $::other_server_path
         if {![file executable $executable]} {
             error "File not found or not executable: $executable"
         }

tests/test_helper.tcl

@@ -69,7 +69,7 @@ set ::single_tests {}
 set ::run_solo_tests {}
 set ::skip_till ""
 set ::external 0; # If "1" this means, we are running against external instance
-set ::old_server_path {}; # Used in upgrade and inter-version tests
+set ::other_server_path {}; # Used in upgrade and inter-version tests
 set ::file ""; # If set, runs only the tests in this comma separated list
 set ::curfile ""; # Hold the filename of the current suite
 set ::accurate 0; # If true runs fuzz tests with more iterations
@@ -568,43 +568,54 @@ proc send_data_packet {fd status data {elapsed 0}} {
 
 proc print_help_screen {} {
     puts [join {
-        "--cluster          Run the cluster tests, by default cluster tests run along with all tests."
-        "--moduleapi        Run the module API tests, this option should only be used in runtest-moduleapi which will build the test module."
+        # This is for terminal output, so assume default term width of 80 columns. -----|
+        "--cluster          Run the cluster tests, by default cluster tests run along"
+        "                   with all tests."
+        "--moduleapi        Run the module API tests, this option should only be used in"
+        "                   runtest-moduleapi which will build the test module."
         "--valgrind         Run the test over valgrind."
         "--durable          suppress test crashes and keep running"
         "--stack-logging    Enable OSX leaks/malloc stack logging."
         "--accurate         Run slow randomized tests for more iterations."
         "--quiet            Don't show individual tests."
-        "--single <unit>    Just execute the specified unit (see next option). This option can be repeated."
+        "--single <unit>    Just execute the specified unit (see next option). This"
+        "                   option can be repeated."
         "--verbose          Increases verbosity."
         "--list-tests       List all the available test units."
-        "--only <test>      Just execute the specified test by test name or tests that match <test> regexp (if <test> starts with '/'). This option can be repeated."
+        "--only <test>      Just execute the specified test by test name or tests that"
+        "                   match <test> regexp (if <test> starts with '/'). This option"
+        "                   can be repeated."
         "--skip-till <unit> Skip all units until (and including) the specified one."
         "--skipunit <unit>  Skip one unit."
         "--clients <num>    Number of test clients (default 16)."
         "--timeout <sec>    Test timeout in seconds (default 20 min)."
         "--force-failure    Force the execution of a test that always fails."
         "--config <k> <v>   Extra config file argument."
-        "--skipfile <file>  Name of a file containing test names or regexp patterns (if <test> starts with '/') that should be skipped (one per line). This option can be repeated."
-        "--skiptest <test>  Test name or regexp pattern (if <test> starts with '/') to skip. This option can be repeated."
-        "--tags <tags>      Run only tests having specified tags or not having '-' prefixed tags."
+        "--skipfile <file>  Name of a file containing test names or regexp patterns (if"
+        "                   <test> starts with '/') that should be skipped (one per"
+        "                   line). This option can be repeated."
+        "--skiptest <test>  Test name or regexp pattern (if <test> starts with '/') to"
+        "                   skip. This option can be repeated."
+        "--tags <tags>      Run only tests having specified tags or not having '-'"
+        "                   prefixed tags."
         "--dont-clean       Don't delete valkey log files after the run."
         "--dont-pre-clean   Don't delete existing valkey log files before the run."
         "--no-latency       Skip latency measurements and validation by some tests."
         "--fastfail         Exit immediately once the first test fails."
         "--stop             Blocks once the first test fails."
         "--loop             Execute the specified set of tests forever."
         "--loops <count>    Execute the specified set of tests several times."
-        "--wait-server      Wait after server is started (so that you can attach a debugger)."
+        "--wait-server      Wait after server is started (so that you can attach a"
+        "                   debugger)."
         "--dump-logs        Dump server log on test failure."
         "--io-threads       Run tests with IO threads."
         "--tls              Run tests in TLS mode."
         "--tls-module       Run tests in TLS mode with Valkey module."
         "--host <addr>      Run tests against an external host."
         "--port <port>      TCP port to use against external host."
-        "--old-server-path <path>"
-        "                   Path to another version of valkey-server, used for inter-version"
-        "                   compatibility testing."
+        "--other-server-path <path>"
+        "                   Path to another version of valkey-server, used for inter-"
+        "                   version compatibility testing."
         "--baseport <port>  Initial port number for spawned valkey servers."
         "--portcount <num>  Port range for spawned valkey servers."
         "--singledb         Use a single database, avoid SELECT."
@@ -679,8 +690,8 @@ for {set j 0} {$j < [llength $argv]} {incr j} {
     } elseif {$opt eq {--port}} {
         set ::port $arg
         incr j
-    } elseif {$opt eq {--old-server-path}} {
-        set ::old_server_path $arg
+    } elseif {$opt eq {--other-server-path}} {
+        set ::other_server_path $arg
         incr j
     } elseif {$opt eq {--baseport}} {
         set ::baseport $arg