From 2954d6ee71c1b7a3209e63ff0c32966e0c073c79 Mon Sep 17 00:00:00 2001 From: s3w3nofficial Date: Sun, 22 Dec 2024 05:05:03 +0100 Subject: [PATCH] Implement basic version of DUMP and RESTORE redis commands --- libs/common/Crc64.cs | 82 ++++++ libs/common/RedisLengthEncodingUtils.cs | 78 ++++++ libs/host/Configuration/Options.cs | 7 +- libs/resources/RespCommandsDocs.json | 44 ++++ libs/resources/RespCommandsInfo.json | 50 ++++ libs/server/API/GarnetApi.cs | 4 +- libs/server/API/GarnetWatchApi.cs | 2 +- libs/server/Resp/BasicCommands.cs | 1 - libs/server/Resp/CmdStrings.cs | 1 + libs/server/Resp/KeyAdminCommands.cs | 176 +++++++++++++ libs/server/Resp/Parser/RespCommand.cs | 4 + libs/server/Resp/RespServerSession.cs | 2 + libs/server/Servers/ServerOptions.cs | 5 + test/Garnet.test/RespTests.cs | 286 +++++++++++++++++++++ website/docs/commands/api-compatibility.md | 8 +- website/docs/commands/generic-commands.md | 46 ++++ 16 files changed, 787 insertions(+), 9 deletions(-) create mode 100644 libs/common/Crc64.cs create mode 100644 libs/common/RedisLengthEncodingUtils.cs diff --git a/libs/common/Crc64.cs b/libs/common/Crc64.cs new file mode 100644 index 0000000000..297fc90756 --- /dev/null +++ b/libs/common/Crc64.cs @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; + +namespace Garnet.common; + +/// +/// Port of redis crc64 from https://github.com/redis/redis/blob/7.2/src/crc64.c +/// +public static class Crc64 +{ + /// + /// Polynomial (same as redis) + /// + private const ulong POLY = 0xad93d23594c935a9UL; + + /// + /// Reverse all bits in a 64-bit value (bit reflection). + /// Only used for data_len == 64 in this code. + /// + private static ulong Reflect64(ulong data) + { + // swap odd/even bits + data = ((data >> 1) & 0x5555555555555555UL) | ((data & 0x5555555555555555UL) << 1); + // swap consecutive pairs + data = ((data >> 2) & 0x3333333333333333UL) | ((data & 0x3333333333333333UL) << 2); + // swap nibbles + data = ((data >> 4) & 0x0F0F0F0F0F0F0F0FUL) | ((data & 0x0F0F0F0F0F0F0F0FUL) << 4); + // swap bytes, then 2-byte pairs, then 4-byte pairs + data = System.Buffers.Binary.BinaryPrimitives.ReverseEndianness(data); + return data; + } + + /// + /// A direct bit-by-bit CRC64 calculation (like _crc64 in C). + /// + private static ulong Crc64Bitwise(ReadOnlySpan data) + { + ulong crc = 0; + + foreach (var c in data) + { + for (byte i = 1; i != 0; i <<= 1) + { + // interpret the top bit of 'crc' and current bit of 'c' + var bitSet = (crc & 0x8000000000000000UL) != 0; + var cbit = (c & i) != 0; + + // if cbit flips the sense, invert bitSet + if (cbit) + bitSet = !bitSet; + + // shift + crc <<= 1; + + // apply polynomial if needed + if (bitSet) + crc ^= POLY; + } + + // ensure it stays in 64 bits + crc &= 0xffffffffffffffffUL; + } + + // reflect and XOR, per standard + crc &= 0xffffffffffffffffUL; + crc = Reflect64(crc) ^ 0x0000000000000000UL; + return crc; + } + + /// + /// Computes crc64 + /// + /// + /// + public static byte[] Hash(ReadOnlySpan data) + { + var bitwiseCrc = Crc64Bitwise(data); + return BitConverter.GetBytes(bitwiseCrc); + } +} \ No newline at end of file diff --git a/libs/common/RedisLengthEncodingUtils.cs b/libs/common/RedisLengthEncodingUtils.cs new file mode 100644 index 0000000000..f7307624f5 --- /dev/null +++ b/libs/common/RedisLengthEncodingUtils.cs @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Linq; + +namespace Garnet.common; + +/// +/// Utils for working with redis length encoding +/// +public static class RedisLengthEncodingUtils +{ + /// + /// Decodes the redis length encoded length and returns payload start + /// + /// + /// + /// + public static (long length, byte payloadStart) DecodeLength(ref ReadOnlySpan buff) + { + // remove the value type byte + var encoded = buff.Slice(1); + + if (encoded.Length == 0) + throw new ArgumentException("Encoded length cannot be empty.", nameof(encoded)); + + var firstByte = encoded[0]; + return (firstByte >> 6) switch + { + // 6-bit encoding + 0 => (firstByte & 0x3F, 1), + // 14-bit encoding + 1 when encoded.Length < 2 => throw new ArgumentException("Not enough bytes for 14-bit encoding."), + 1 => (((firstByte & 0x3F) << 8) | encoded[1], 2), + // 32-bit encoding + 2 when encoded.Length < 5 => throw new ArgumentException("Not enough bytes for 32-bit encoding."), + 2 => ((long)((encoded[1] << 24) | (encoded[2] << 16) | (encoded[3] << 8) | encoded[4]), 5), + _ => throw new ArgumentException("Invalid encoding type.", nameof(encoded)) + }; + } + + /// + /// Encoded payload length to redis encoded payload length + /// + /// + /// + /// + public static byte[] EncodeLength(long length) + { + switch (length) + { + // 6-bit encoding (length ≤ 63) + case < 1 << 6: + return [(byte)(length & 0x3F)]; // 00xxxxxx + // 14-bit encoding (64 ≤ length ≤ 16,383) + case < 1 << 14: + { + var firstByte = (byte)(((length >> 8) & 0x3F) | (1 << 6)); // 01xxxxxx + var secondByte = (byte)(length & 0xFF); + return [firstByte, secondByte]; + } + // 32-bit encoding (length ≤ 4,294,967,295) + case <= 0xFFFFFFFF: + { + var firstByte = (byte)(2 << 6); // 10xxxxxx + var lengthBytes = BitConverter.GetBytes((uint)length); // Ensure unsigned + if (BitConverter.IsLittleEndian) + { + Array.Reverse(lengthBytes); // Convert to big-endian + } + return new[] { firstByte }.Concat(lengthBytes).ToArray(); + } + default: + throw new ArgumentOutOfRangeException("Length exceeds maximum allowed for Redis encoding (4,294,967,295)."); + } + } +} \ No newline at end of file diff --git a/libs/host/Configuration/Options.cs b/libs/host/Configuration/Options.cs index 96fdd62433..4199cd8fd5 100644 --- a/libs/host/Configuration/Options.cs +++ b/libs/host/Configuration/Options.cs @@ -499,6 +499,10 @@ internal sealed class Options [OptionValidation] [Option("fail-on-recovery-error", Default = false, Required = false, HelpText = "Server bootup should fail if errors happen during bootup of AOF and checkpointing")] public bool? FailOnRecoveryError { get; set; } + + [OptionValidation] + [Option("skip-checksum-validation", Default = false, Required = false, HelpText = "Skip checksum validation")] + public bool? SkipChecksumValidation { get; set; } /// /// This property contains all arguments that were not parsed by the command line argument parser @@ -708,7 +712,8 @@ public GarnetServerOptions GetServerOptions(ILogger logger = null) IndexResizeFrequencySecs = IndexResizeFrequencySecs, IndexResizeThreshold = IndexResizeThreshold, LoadModuleCS = LoadModuleCS, - FailOnRecoveryError = FailOnRecoveryError.GetValueOrDefault() + FailOnRecoveryError = FailOnRecoveryError.GetValueOrDefault(), + SkipChecksumValidation = SkipChecksumValidation.GetValueOrDefault(), }; } diff --git a/libs/resources/RespCommandsDocs.json b/libs/resources/RespCommandsDocs.json index b9701595ad..ac070f6eff 100644 --- a/libs/resources/RespCommandsDocs.json +++ b/libs/resources/RespCommandsDocs.json @@ -2586,6 +2586,50 @@ } ] }, + { + "Command": "DUMP", + "Name": "DUMP", + "Summary": "Returns a serialized representation of the value stored at a key.", + "Group": "Generic", + "Complexity": "O(1) to access the key and additional O(N*M) to serialize it, where N is the number of Redis objects composing the value and M their average size. For small string values the time complexity is thus O(1)+O(1*M) where M is small, so simply O(1).", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + } + ] + }, + { + "Command": "RESTORE", + "Name": "RESTORE", + "Summary": "Creates a key from the serialized representation of a value.", + "Group": "Generic", + "Complexity": "O(1) to create the new key and additional O(N*M) to reconstruct the serialized value, where N is the number of Redis objects composing the value and M their average size. For small string values the time complexity is thus O(1)+O(1*M) where M is small, so simply O(1). However for sorted set values the complexity is O(N*M*log(N)) because inserting values into sorted sets is O(log(N)).", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "TTL", + "DisplayText": "ttl", + "Type": "Integer" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "SERIALIZEDVALUE", + "DisplayText": "serialized-value", + "Type": "String" + } + ] + }, { "Command": "GET", "Name": "GET", diff --git a/libs/resources/RespCommandsInfo.json b/libs/resources/RespCommandsInfo.json index b131275ee7..2831fc3f35 100644 --- a/libs/resources/RespCommandsInfo.json +++ b/libs/resources/RespCommandsInfo.json @@ -1395,6 +1395,56 @@ } ] }, + { + "Command": "RESTORE", + "Name": "RESTORE", + "Arity": -4, + "Flags": "DenyOom, Write", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "KeySpace, Dangerous", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 0, + "Limit": 0 + }, + "Flags": "OW, Update" + } + ] + }, + { + "Command": "DUMP", + "Name": "DUMP", + "Arity": 2, + "Flags": "ReadOnly", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "KeySpace", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RO, Access" + } + ] + }, { "Command": "GET", "Name": "GET", diff --git a/libs/server/API/GarnetApi.cs b/libs/server/API/GarnetApi.cs index 34c91d00a5..4870143603 100644 --- a/libs/server/API/GarnetApi.cs +++ b/libs/server/API/GarnetApi.cs @@ -98,9 +98,9 @@ public GarnetStatus TTL(ref SpanByte key, StoreType storeType, ref SpanByteAndMe /// public GarnetStatus PTTL(ref SpanByte key, StoreType storeType, ref SpanByteAndMemory output) => storageSession.TTL(ref key, storeType, ref output, ref context, ref objectContext, milliseconds: true); - + #endregion - + #region EXPIRETIME /// diff --git a/libs/server/API/GarnetWatchApi.cs b/libs/server/API/GarnetWatchApi.cs index 2cec273074..523b4cf536 100644 --- a/libs/server/API/GarnetWatchApi.cs +++ b/libs/server/API/GarnetWatchApi.cs @@ -74,7 +74,7 @@ public GarnetStatus PTTL(ref SpanByte key, StoreType storeType, ref SpanByteAndM garnetApi.WATCH(new ArgSlice(ref key), storeType); return garnetApi.PTTL(ref key, storeType, ref output); } - + #endregion #region EXPIRETIME diff --git a/libs/server/Resp/BasicCommands.cs b/libs/server/Resp/BasicCommands.cs index 860dd691b2..3246ac17af 100644 --- a/libs/server/Resp/BasicCommands.cs +++ b/libs/server/Resp/BasicCommands.cs @@ -3,7 +3,6 @@ using System; using System.Diagnostics; -using System.Runtime.CompilerServices; using System.Text; using System.Threading.Tasks; using Garnet.common; diff --git a/libs/server/Resp/CmdStrings.cs b/libs/server/Resp/CmdStrings.cs index 22141be4d4..2f851e5f55 100644 --- a/libs/server/Resp/CmdStrings.cs +++ b/libs/server/Resp/CmdStrings.cs @@ -215,6 +215,7 @@ static partial class CmdStrings public static ReadOnlySpan RESP_ERR_INCR_SUPPORTS_ONLY_SINGLE_PAIR => "ERR INCR option supports a single increment-element pair"u8; public static ReadOnlySpan RESP_ERR_INVALID_BITFIELD_TYPE => "ERR Invalid bitfield type. Use something like i16 u8. Note that u64 is not supported but i64 is"u8; public static ReadOnlySpan RESP_ERR_SCRIPT_FLUSH_OPTIONS => "ERR SCRIPT FLUSH only support SYNC|ASYNC option"u8; + public static ReadOnlySpan RESP_ERR_KEY_ALREADY_EXISTS => "ERR Key already exists"u8; /// /// Response string templates diff --git a/libs/server/Resp/KeyAdminCommands.cs b/libs/server/Resp/KeyAdminCommands.cs index 8cf1434517..80f42afbd8 100644 --- a/libs/server/Resp/KeyAdminCommands.cs +++ b/libs/server/Resp/KeyAdminCommands.cs @@ -10,7 +10,183 @@ namespace Garnet.server { internal sealed unsafe partial class RespServerSession : ServerSessionBase { + /// + /// RDB format version + /// + private readonly byte RDB_VERSION = 11; + + /// + /// RESTORE + /// + /// + /// + /// + bool NetworkRESTORE(ref TGarnetApi storageApi) + where TGarnetApi : IGarnetApi + { + if (parseState.Count != 3) + { + return AbortWithWrongNumberOfArguments(nameof(RespCommand.RESTORE)); + } + + var key = parseState.GetArgSliceByRef(0); + + if (!parseState.TryGetDouble(parseState.Count - 2, out var ttl)) + { + while (!RespWriteUtils.WriteError(CmdStrings.RESP_ERR_TIMEOUT_NOT_VALID_FLOAT, ref dcurr, dend)) + SendAndReset(); + return true; + } + + var value = parseState.GetArgSliceByRef(2); + + // return error if key already exists + var keyExists = storageApi.EXISTS(key); + switch (keyExists) + { + case GarnetStatus.OK: + while (!RespWriteUtils.WriteDirect(CmdStrings.RESP_ERR_KEY_ALREADY_EXISTS, ref dcurr, dend)) + SendAndReset(); + return true; + } + + var valueSpan = value.ReadOnlySpan; + + // Restore is only implemented for string type + if (valueSpan[0] != 0x00) + { + while(!RespWriteUtils.WriteError("ERR RESTORE currently only supports string types", ref dcurr, dend)) + SendAndReset(); + return true; + } + + // get footer (2 bytes of rdb version + 8 bytes of crc) + var footer = valueSpan[^10..]; + + var rdbVersion = (footer[1] << 8) | footer[0]; + + if (rdbVersion > RDB_VERSION) + { + while(!RespWriteUtils.WriteError("ERR DUMP payload version or checksum are wrong", ref dcurr, dend)) + SendAndReset(); + return true; + } + if (storeWrapper.serverOptions.SkipChecksumValidation) + { + // crc is calculated over the encoded payload length, payload and the rdb version bytes + // skip's the value type byte and crc64 bytes + var calculatedCrc = new ReadOnlySpan(Crc64.Hash(valueSpan.Slice(0, valueSpan.Length - 8))); + + // skip's rdb version bytes + var payloadCrc = footer[2..]; + + if (calculatedCrc.SequenceCompareTo(payloadCrc) != 0) + { + while(!RespWriteUtils.WriteError("ERR DUMP payload version or checksum are wrong", ref dcurr, dend)) + SendAndReset(); + return true; + } + } + + // decode the length of payload + var (length, payloadStart) = RedisLengthEncodingUtils.DecodeLength(ref valueSpan); + + // Start from payload start and skip the value type byte + var val = value.ReadOnlySpan.Slice(payloadStart + 1, (int)length); + + var valArgSlice = scratchBufferManager.CreateArgSlice(val); + + if (ttl > 0) + { + storageApi.SETEX(key, valArgSlice, TimeSpan.FromMilliseconds(ttl)); + } + else + { + storageApi.SET(key, valArgSlice); + } + + while (!RespWriteUtils.WriteDirect(CmdStrings.RESP_OK, ref dcurr, dend)) + SendAndReset(); + + return true; + } + + /// + /// DUMP + /// + bool NetworkDUMP(ref TGarnetApi storageApi) + where TGarnetApi : IGarnetApi + { + if (parseState.Count != 1) + { + return AbortWithWrongNumberOfArguments(nameof(RespCommand.DUMP)); + } + + var key = parseState.GetArgSliceByRef(0); + + var status = storageApi.GET(key, out var value); + + switch (status) + { + case GarnetStatus.NOTFOUND: + while (!RespWriteUtils.WriteDirect(CmdStrings.RESP_ERRNOTFOUND, ref dcurr, dend)) + SendAndReset(); + return true; + } + + var encodedLength = RedisLengthEncodingUtils.EncodeLength(value.ReadOnlySpan.Length); + + // Len of the dump (payload type + redis encoded payload len + payload len + rdb version + crc64) + var len = 1 + encodedLength.Length + value.ReadOnlySpan.Length + 2 + 8; + var lengthInASCIIBytes = new Span(new byte[NumUtils.NumDigitsInLong(len)]); + var lengthInASCIIBytesLen = NumUtils.LongToSpanByte(len, lengthInASCIIBytes); + + // Total len (% + length of ascii bytes + CR LF + payload type + redis encoded payload len + payload len + rdb version + crc64 + CR LF) + var totalLength = 1 + lengthInASCIIBytesLen + 2 + 1 + encodedLength.Length + value.ReadOnlySpan.Length + 2 + 8 + 2; + Span buffer = stackalloc byte[totalLength]; + var offset = 0; + + // Write RESP bulk string prefix and length + buffer[offset++] = 0x24; // '$' + lengthInASCIIBytes.CopyTo(buffer[offset..]); + offset += lengthInASCIIBytes.Length; + buffer[offset++] = 0x0D; // CR + buffer[offset++] = 0x0A; // LF + + // value type byte + buffer[offset++] = 0x00; + + // length of the span + foreach (var b in encodedLength) + buffer[offset++] = b; + + // copy value to buffer + value.ReadOnlySpan.CopyTo(buffer[offset..]); + offset += value.ReadOnlySpan.Length; + + // Write RDB version + buffer[offset++] = (byte)(RDB_VERSION & 0xff); + buffer[offset++] = (byte)((RDB_VERSION >> 8) & 0xff); + + // Compute and write CRC64 checksum + var payloadToHash = buffer.Slice(1 + lengthInASCIIBytes.Length + 2 + 1, + encodedLength.Length + value.ReadOnlySpan.Length + 2); + + var crcBytes = Crc64.Hash(payloadToHash); + crcBytes.CopyTo(buffer[offset..]); + offset += crcBytes.Length; + + // Write final CRLF + buffer[offset++] = 0x0D; // CR + buffer[offset++] = 0x0A; // LF + + while (!RespWriteUtils.WriteDirect(buffer, ref dcurr, dend)) + SendAndReset(); + + return true; + } + /// /// TryRENAME /// diff --git a/libs/server/Resp/Parser/RespCommand.cs b/libs/server/Resp/Parser/RespCommand.cs index 9e706a5859..aad225f491 100644 --- a/libs/server/Resp/Parser/RespCommand.cs +++ b/libs/server/Resp/Parser/RespCommand.cs @@ -90,6 +90,7 @@ public enum RespCommand : ushort ZSCAN, ZSCORE, // Note: Last read command should immediately precede FirstWriteCommand ZUNION, + DUMP, // Write commands APPEND, // Note: Update FirstWriteCommand if adding new write commands before this @@ -174,6 +175,7 @@ public enum RespCommand : ushort ZREMRANGEBYRANK, ZREMRANGEBYSCORE, ZUNIONSTORE, + RESTORE, // BITOP is the true command, AND|OR|XOR|NOT are pseudo-subcommands BITOP, @@ -664,6 +666,7 @@ private RespCommand FastParseCommand(out int count) (1 << 4) | 3 when lastWord == MemoryMarshal.Read("3\r\nGET\r\n"u8) => RespCommand.GET, (1 << 4) | 3 when lastWord == MemoryMarshal.Read("3\r\nDEL\r\n"u8) => RespCommand.DEL, (1 << 4) | 3 when lastWord == MemoryMarshal.Read("3\r\nTTL\r\n"u8) => RespCommand.TTL, + (1 << 4) | 4 when lastWord == MemoryMarshal.Read("\r\nDUMP\r\n"u8) => RespCommand.DUMP, (1 << 4) | 4 when lastWord == MemoryMarshal.Read("\r\nINCR\r\n"u8) => RespCommand.INCR, (1 << 4) | 4 when lastWord == MemoryMarshal.Read("\r\nPTTL\r\n"u8) => RespCommand.PTTL, (1 << 4) | 4 when lastWord == MemoryMarshal.Read("\r\nDECR\r\n"u8) => RespCommand.DECR, @@ -687,6 +690,7 @@ private RespCommand FastParseCommand(out int count) (3 << 4) | 6 when lastWord == MemoryMarshal.Read("PSETEX\r\n"u8) => RespCommand.PSETEX, (3 << 4) | 6 when lastWord == MemoryMarshal.Read("SETBIT\r\n"u8) => RespCommand.SETBIT, (3 << 4) | 6 when lastWord == MemoryMarshal.Read("SUBSTR\r\n"u8) => RespCommand.SUBSTR, + (3 << 4) | 7 when lastWord == MemoryMarshal.Read("ESTORE\r\n"u8) && ptr[8] == 'R' => RespCommand.RESTORE, (3 << 4) | 8 when lastWord == MemoryMarshal.Read("TRANGE\r\n"u8) && *(ushort*)(ptr + 8) == MemoryMarshal.Read("SE"u8) => RespCommand.SETRANGE, (3 << 4) | 8 when lastWord == MemoryMarshal.Read("TRANGE\r\n"u8) && *(ushort*)(ptr + 8) == MemoryMarshal.Read("GE"u8) => RespCommand.GETRANGE, diff --git a/libs/server/Resp/RespServerSession.cs b/libs/server/Resp/RespServerSession.cs index f72be029bb..0351ae57c6 100644 --- a/libs/server/Resp/RespServerSession.cs +++ b/libs/server/Resp/RespServerSession.cs @@ -515,6 +515,8 @@ private bool ProcessBasicCommands(RespCommand cmd, ref TGarnetApi st */ _ = cmd switch { + RespCommand.RESTORE => NetworkRESTORE(ref storageApi), + RespCommand.DUMP => NetworkDUMP(ref storageApi), RespCommand.GET => NetworkGET(ref storageApi), RespCommand.GETEX => NetworkGETEX(ref storageApi), RespCommand.SET => NetworkSET(ref storageApi), diff --git a/libs/server/Servers/ServerOptions.cs b/libs/server/Servers/ServerOptions.cs index 342b563020..abf02ded44 100644 --- a/libs/server/Servers/ServerOptions.cs +++ b/libs/server/Servers/ServerOptions.cs @@ -97,6 +97,11 @@ public class ServerOptions /// Server bootup should fail if errors happen during bootup of AOF and checkpointing. /// public bool FailOnRecoveryError = false; + + /// + /// Skip checksum validation + /// + public bool SkipChecksumValidation = false; /// /// Logger diff --git a/test/Garnet.test/RespTests.cs b/test/Garnet.test/RespTests.cs index 86c0adfa96..fa5d9af557 100644 --- a/test/Garnet.test/RespTests.cs +++ b/test/Garnet.test/RespTests.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.Globalization; +using System.IO.Hashing; using System.Linq; using System.Text; using System.Threading; @@ -14,6 +15,7 @@ using NUnit.Framework; using NUnit.Framework.Legacy; using StackExchange.Redis; +using Crc64 = Garnet.common.Crc64; namespace Garnet.test { @@ -135,7 +137,291 @@ public void IsClusterSubCommand() ClassicAssert.AreEqual(expectedRes, actualRes, $"Mismatch for {cmd}"); } } + + /// + /// Tests RESTORE value that is not string + /// + [Test] + public void TryRestoreKeyNonStringType() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var payload = new byte[] + { + 0x14, 0xf, 0xf, 0x0, 0x0, 0x0, 0x1, 0x0, 0xc2, 0x86, 0x62, 0x69, 0x6b, 0x65, 0x3a, 0x31, 0x7, 0xc3, 0xbf, 0xb, 0x0, 0xc3, 0xaa, 0x33, 0x68, 0x7b, 0x2a, 0xc3, 0xa6, 0xc3, 0xbf, 0xc3, 0xb9 + }; + + Assert.Throws(() => db.KeyRestore("mykey", payload)); + } + + /// + /// Tests RESTORE command on existing key + /// + [Test] + public void TryRestoreExistingKey() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + db.StringSet("mykey", "val"); + + var dump = db.KeyDump("mykey")!; + + Assert.Throws(() => db.KeyRestore("mykey", dump)); + } + + /// + /// Tests RESTORE command that restores payload with 32 bit encoded length + /// + [Test] + public void SingleRestore32Bit() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var valueBuilder = new StringBuilder(); + + for (var i = 0; i < 16_383; i++) + valueBuilder.Append('a'); + + var val = valueBuilder.ToString(); + + db.StringSet("mykey", val); + + var dump = db.KeyDump("mykey")!; + + db.KeyDelete("mykey"); + + db.KeyRestore("mykey", dump); + + var value = db.StringGet("mykey"); + + ClassicAssert.AreEqual(val, value.ToString()); + } + + /// + /// Tests RESTORE command that restores payload with 14 bit encoded length + /// + [Test] + public void SingleRestore14Bit() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var valueBuilder = new StringBuilder(); + + for (var i = 0; i < 16_383 - 1; i++) + valueBuilder.Append('a'); + + var val = valueBuilder.ToString(); + + db.StringSet("mykey", val); + + var dump = db.KeyDump("mykey")!; + + db.KeyDelete("mykey"); + + db.KeyRestore("mykey", dump); + + var value = db.StringGet("mykey"); + + ClassicAssert.AreEqual(val, value.ToString()); + } + + /// + /// Tests RESTORE command that restores payload with 6 bit encoded length + /// + [Test] + public void SingleRestore6Bit() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + db.StringSet("mykey", "val"); + + var dump = db.KeyDump("mykey")!; + + db.KeyDelete("mykey"); + + db.KeyRestore("mykey", dump, TimeSpan.FromHours(3)); + + var value = db.StringGet("mykey"); + + ClassicAssert.AreEqual("val", value.ToString()); + } + + /// + /// Tests DUMP command that returns payload with 6 bit encoded length + /// + [Test] + public void SingleDump6Bit() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + db.StringSet("mykey", "val"); + + var dump = db.KeyDump("mykey"); + + var expectedValue = new byte[] + { + 0x00, // value type + 0x03, // length of payload + 0x76, 0x61, 0x6C, // 'v', 'a', 'l' + 0x0B, 0x00, // RDB version + }; + + var crc = new byte[] + { + 0xDB, + 0x82, + 0x3C, + 0x30, + 0x38, + 0x78, + 0x5A, + 0x99 + }; + + expectedValue = [..expectedValue, ..crc]; + + ClassicAssert.AreEqual(expectedValue, dump); + } + + /// + /// Tests DUMP command that returns payload with 14 bit encoded length + /// + [Test] + public void SingleDump14Bit() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var valueBuilder = new StringBuilder(); + + for (var i = 0; i < 16_383 - 1; i++) + valueBuilder.Append('a'); + + var value = valueBuilder.ToString(); + + db.StringSet("mykey", value); + + var dump = db.KeyDump("mykey"); + + var expectedValue = new byte[] + { + 0x00, // value type + 0x7F, 0xFE, // length of payload + }; + + expectedValue = [..expectedValue, ..Encoding.UTF8.GetBytes(value)]; + + var rdbVersion = new byte[] + { + 0x0B, 0x00, // RDB version + }; + + expectedValue = [..expectedValue, ..rdbVersion]; + + var crc = new byte[] + { + 0x7C, + 0x09, + 0x2D, + 0x16, + 0x73, + 0xAE, + 0x7C, + 0xCF + }; + + expectedValue = [..expectedValue, ..crc]; + + ClassicAssert.AreEqual(expectedValue, dump); + } + + /// + /// Tests DUMP command that returns payload with 32 bit encoded length + /// + [Test] + public void SingleDump32Bit() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var valueBuilder = new StringBuilder(); + + for (var i = 0; i < 16_383 + 1; i++) + valueBuilder.Append('a'); + + var value = valueBuilder.ToString(); + + db.StringSet("mykey", value); + + var dump = db.KeyDump("mykey"); + + var expectedValue = new byte[] + { + 0x00, // value type + 0x80, 0x00, 0x00, 0x40, 0x00, // length of payload + }; + + expectedValue = [..expectedValue, ..Encoding.UTF8.GetBytes(value)]; + + var rdbVersion = new byte[] + { + 0x0B, 0x00, // RDB version + }; + + expectedValue = [..expectedValue, ..rdbVersion]; + + var crc = new byte[] + { + 0x7F, + 0x73, + 0x7E, + 0xA9, + 0x87, + 0xD9, + 0x90, + 0x14 + }; + + expectedValue = [..expectedValue, ..crc]; + + ClassicAssert.AreEqual(expectedValue, dump); + } + + /// + /// Tests DUMP on non string type which is currently not supported + /// + [Test] + public void TryDumpKeyNonString() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + db.SetAdd("mykey", "val1"); + db.SetAdd("mykey", "val2"); + + var value = db.KeyDump("mykey"); + + ClassicAssert.AreEqual(null, value); + } + + /// + /// Try DUMP key that does not exist + /// + [Test] + public void TryDumpKeyThatDoesNotExist() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var value = db.KeyDump("mykey"); + + ClassicAssert.AreEqual(null, value); + } + [Test] public void SingleSetGet() { diff --git a/website/docs/commands/api-compatibility.md b/website/docs/commands/api-compatibility.md index 3e2e1f3d36..1b5bd982bc 100644 --- a/website/docs/commands/api-compatibility.md +++ b/website/docs/commands/api-compatibility.md @@ -125,13 +125,13 @@ Note that this list is subject to change as we continue to expand our API comman | **FUNCTIONS** | FCALL | ➖ | | | | FCALL_RO | ➖ | | | | DELETE | ➖ | -| | DUMP | ➖ | +| | DUMP | ➖ | | | | FLUSH | ➖ | | | HELP | ➖ | | | KILL | ➖ | | | LIST | ➖ | | | LOAD | ➖ | -| | RESTORE | ➖ | +| | RESTORE | ➖ | | | | STATS | ➖ | | **GENERIC** | [PERSIST](generic-commands.md#persist) | ➕ | | | | [PEXPIRE](generic-commands.md#pexpire) | ➕ | | @@ -141,7 +141,7 @@ Note that this list is subject to change as we continue to expand our API comman | | RANDOMKEY | ➖ | | | | [RENAME](generic-commands.md#rename) | ➕ | | | | [RENAMENX](generic-commands.md#renamenx) | ➕ | | -| | RESTORE | ➖ | | +| | [RESTORE](generic-commands.md#restore) | ➕ | | | [SCAN](generic-commands.md#scan) | ➕ | | | | SORT | ➖ | | | | SORT_RO | ➖ | | @@ -193,7 +193,7 @@ Note that this list is subject to change as we continue to expand our API comman | | PFSELFTEST | ➖ | Internal command | | **KEYS** | COPY | ➖ | | | | [DEL](generic-commands.md#del) | ➕ | | -| | DUMP | ➖ | | +| | [DUMP](generic-commands.md#dump) | ➕ | | | [EXISTS](generic-commands.md#exists) | ➕ | | | | [EXPIRE](generic-commands.md#expire) | ➕ | | | | [EXPIREAT](generic-commands.md#expireat) | ➕ | | diff --git a/website/docs/commands/generic-commands.md b/website/docs/commands/generic-commands.md index 59ebf9b933..d3d9851be1 100644 --- a/website/docs/commands/generic-commands.md +++ b/website/docs/commands/generic-commands.md @@ -475,5 +475,51 @@ Integer reply: the number of keys that were unlinked. --- +### DUMP + +> [!IMPORTANT] +> DUMP currently only supports string types without lzf compression + +#### Syntax + +```bash +DUMP mykey +``` + +Serialize the value stored at key in a Redis-specific format and return it to the user. The returned value can be synthesized back into a Redis key using the [RESTORE](#restore) command. + +#### Resp Reply + +String reply: The serialization format is opaque and non-standard, however it has a few semantic characteristics: + +- It contains a 64-bit checksum that is used to make sure errors will be detected. The [RESTORE](#restore) command makes sure to check the checksum before synthesizing a key using the serialized value. +- Values are encoded in the same format used by RDB. +- An RDB version is encoded inside the serialized value, so that different Redis versions with incompatible RDB formats will refuse to process the serialized value. + +--- + +### RESTORE + +> [!IMPORTANT] +> RESTORE currently only supports string types without lzf compression + +#### Syntax + +```bash +restore mykey 0 "\x00\x0evallllllllllll\x0b\x00|\xeb\xe2|\xd2.\xfa7" +``` + +Create a key associated with a value that is obtained by deserializing the provided serialized value (obtained via [DUMP](#dump)). + +If ttl is 0 the key is created without any expire, otherwise the specified expire time (in milliseconds) is set. + +#### Resp Reply + +Simple string reply: OK. + +--- + + +