From 4fefaf310093958a8a7a1658d238556f373ac549 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 16 Apr 2020 17:23:14 -0400 Subject: [PATCH 01/82] Create initial KafkaEndpoint --- Jasper.ConfluentKafka/KafkaEndpoint.cs | 89 ++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 Jasper.ConfluentKafka/KafkaEndpoint.cs diff --git a/Jasper.ConfluentKafka/KafkaEndpoint.cs b/Jasper.ConfluentKafka/KafkaEndpoint.cs new file mode 100644 index 000000000..bc20da9c0 --- /dev/null +++ b/Jasper.ConfluentKafka/KafkaEndpoint.cs @@ -0,0 +1,89 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Baseline; +using Confluent.Kafka; +using Jasper.Configuration; +using Jasper.ConfluentKafka.Internal; +using Jasper.Runtime; +using Jasper.Transports; +using Jasper.Transports.Sending; +using Jasper.Util; + +namespace Jasper.ConfluentKafka +{ + public class KafkaEndpoint : Endpoint + { + private const string TopicToken = "stream"; + public string TopicName { get; private set; } + public ProducerConfig ProducerConfig { get; private set; } + internal KafkaTransport Parent { get; set; } + public override Uri Uri => BuildUri(); + private Uri BuildUri(bool forReply = false) + { + var list = new List(); + + if (TopicName.IsNotEmpty()) + { + list.Add(TopicName); + list.Add(TopicName.ToLowerInvariant()); + } + + if (forReply && IsDurable) + { + list.Add(TransportConstants.Durable); + } + + var uri = $"{KafkaTransport.ProtocolName}://{list.Join("/")}".ToUri(); + + return uri; + } + + public override Uri ReplyUri() => BuildUri(); + + public override void Parse(Uri uri) + { + if (uri.Scheme != KafkaTransport.ProtocolName) + { + throw new ArgumentOutOfRangeException($"This is not a {nameof(KafkaTransport)} Uri"); + } + + var raw = uri.Segments.Where(x => x != "/").Select(x => x.Trim('/')); + var segments = new Queue(); + segments.Enqueue(uri.Host); + foreach (var segment in raw) + { + segments.Enqueue(segment); + } + + + while (segments.Any()) + { + if (segments.Peek().EqualsIgnoreCase(TopicToken)) + { + segments.Dequeue(); // token + TopicName = segments.Dequeue(); // value + } + else if (segments.Peek().EqualsIgnoreCase(TransportConstants.Durable)) + { + segments.Dequeue(); // token + IsDurable = true; + } + else + { + throw new InvalidOperationException($"The Uri '{uri}' is invalid for an {nameof(KafkaTransport)} endpoint"); + } + } + } + + protected internal override void StartListening(IMessagingRoot root, ITransportRuntime runtime) + { + throw new NotImplementedException(); + } + + protected override ISender CreateSender(IMessagingRoot root) + { + return new ConfluentKafkaSender(this, Parent, root.TransportLogger, root.Cancellation); + } + } +} From 0bb4b559b26bee2c89433dfcdeaf0cf878e4b0e6 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 16 Apr 2020 17:23:29 -0400 Subject: [PATCH 02/82] Create initial KafkaTransport & Protocol --- Jasper.ConfluentKafka/KafkaTransport.cs | 25 ++++++++++++++ .../KafkaTransportProtocol.cs | 33 +++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 Jasper.ConfluentKafka/KafkaTransport.cs create mode 100644 Jasper.ConfluentKafka/KafkaTransportProtocol.cs diff --git a/Jasper.ConfluentKafka/KafkaTransport.cs b/Jasper.ConfluentKafka/KafkaTransport.cs new file mode 100644 index 000000000..f6f335d6f --- /dev/null +++ b/Jasper.ConfluentKafka/KafkaTransport.cs @@ -0,0 +1,25 @@ +using System; +using System.Collections.Generic; +using Jasper.Transports; + +namespace Jasper.ConfluentKafka +{ + public class KafkaTransport : TransportBase> + { + public static readonly string ProtocolName = "kafka"; + + public KafkaTransport(string protocol) : base(protocol) + { + } + + protected override IEnumerable> endpoints() + { + throw new NotImplementedException(); + } + + protected override KafkaEndpoint findEndpointByUri(Uri uri) + { + throw new NotImplementedException(); + } + } +} diff --git a/Jasper.ConfluentKafka/KafkaTransportProtocol.cs b/Jasper.ConfluentKafka/KafkaTransportProtocol.cs new file mode 100644 index 000000000..517e8e530 --- /dev/null +++ b/Jasper.ConfluentKafka/KafkaTransportProtocol.cs @@ -0,0 +1,33 @@ +using System.Text; +using Confluent.Kafka; +using Jasper.Transports; + +namespace Jasper.ConfluentKafka +{ + public class KafkaTransportProtocol : ITransportProtocol> + { + public Message WriteFromEnvelope(Envelope envelope) + { + var msg = new Message(); + + msg.Value = envelope.Data; + + + return msg; + } + + public Envelope ReadEnvelope(Message message) + { + var env = new Envelope(); + + foreach (var header in message.Headers) + { + env.Headers.Add(header.Key, Encoding.UTF8.GetString(header.GetValueBytes())); + } + + env.Data = message.Value; + + return env; + } + } +} From f36697c5a966c848bc88ac713c89ffdc23172737 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 16 Apr 2020 17:23:52 -0400 Subject: [PATCH 03/82] Create initial ConfluentKafkaSender --- .../Exceptions/UnsupportedFeatureException.cs | 14 ++ .../Internal/ConfluentKafkaSender.cs | 129 ++++++++++++++++++ .../Jasper.ConfluentKafka.csproj | 16 +++ 3 files changed, 159 insertions(+) create mode 100644 Jasper.ConfluentKafka/Exceptions/UnsupportedFeatureException.cs create mode 100644 Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs create mode 100644 Jasper.ConfluentKafka/Jasper.ConfluentKafka.csproj diff --git a/Jasper.ConfluentKafka/Exceptions/UnsupportedFeatureException.cs b/Jasper.ConfluentKafka/Exceptions/UnsupportedFeatureException.cs new file mode 100644 index 000000000..6bb8a46be --- /dev/null +++ b/Jasper.ConfluentKafka/Exceptions/UnsupportedFeatureException.cs @@ -0,0 +1,14 @@ +using System; + +namespace Jasper.ConfluentKafka.Exceptions +{ + public class UnsupportedFeatureException : ApplicationException + { + public string Feature { get; } + public UnsupportedFeatureException(string feature) + : base ($"Confluent Kafka Transport does not support {feature}") + { + Feature = feature; + } + } +} diff --git a/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs b/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs new file mode 100644 index 000000000..945be72b2 --- /dev/null +++ b/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs @@ -0,0 +1,129 @@ +using System; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using System.Threading.Tasks.Dataflow; +using Confluent.Kafka; +using Jasper.ConfluentKafka.Exceptions; +using Jasper.Logging; +using Jasper.Transports; +using Jasper.Transports.Sending; + +namespace Jasper.ConfluentKafka.Internal +{ + public class ConfluentKafkaSender : ISender + { + private readonly ITransportProtocol> _protocol; + private readonly KafkaEndpoint _endpoint; + private readonly KafkaTransport _transport; + private readonly ITransportLogger _logger; + private readonly CancellationToken _cancellation; + private ActionBlock _sending; + private ISenderCallback _callback; + private IProducer _publisher; + + public ConfluentKafkaSender(KafkaEndpoint endpoint, KafkaTransport transport, ITransportLogger logger, CancellationToken cancellation) + { + _endpoint = endpoint; + _transport = transport; + _logger = logger; + _cancellation = cancellation; + Destination = endpoint.Uri; + } + + public void Dispose() + { + _publisher?.Dispose(); + } + + public Uri Destination { get; } + public int QueuedCount => _sending.InputCount; + public bool Latched { get; private set; } + + + public void Start(ISenderCallback callback) + { + _callback = callback; + + _publisher = new ProducerBuilder(_endpoint.ProducerConfig).Build(); + + _sending = new ActionBlock(sendBySession, new ExecutionDataflowBlockOptions + { + CancellationToken = _cancellation + }); + } + + + public Task Enqueue(Envelope envelope) + { + _sending.Post(envelope); + + return Task.CompletedTask; + } + + public async Task LatchAndDrain() + { + Latched = true; + + _publisher.Flush(_cancellation); + + _sending.Complete(); + + _logger.CircuitBroken(Destination); + } + + public void Unlatch() + { + _logger.CircuitResumed(Destination); + + Start(_callback); + Latched = false; + } + + public async Task Ping(CancellationToken cancellationToken) + { + Envelope envelope = Envelope.ForPing(Destination); + Message message = _protocol.WriteFromEnvelope(envelope); + + message.Headers.Add("MessageGroupId", Encoding.UTF8.GetBytes(Guid.NewGuid().ToString())); + message.Headers.Add("Jasper_SessionId", Encoding.UTF8.GetBytes(Guid.NewGuid().ToString())); + + await _publisher.ProduceAsync("jasper-ping", message, cancellationToken); + + return true; + } + + public bool SupportsNativeScheduledSend { get; } = false; + + private async Task sendBySession(Envelope envelope) + { + try + { + Message message = _protocol.WriteFromEnvelope(envelope); + message.Headers.Add("Jasper_SessionId", Encoding.UTF8.GetBytes(Guid.NewGuid().ToString())); + + if (envelope.IsDelayed(DateTime.UtcNow)) + { + throw new UnsupportedFeatureException("Delayed Message Delivery"); + } + else + { + await _publisher.ProduceAsync(_endpoint.TopicName, new Message(), _cancellation); + } + + await _callback.Successful(envelope); + } + catch (Exception e) + { + try + { + await _callback.ProcessingFailure(envelope, e); + } + catch (Exception exception) + { + _logger.LogException(exception); + } + } + } + } +} diff --git a/Jasper.ConfluentKafka/Jasper.ConfluentKafka.csproj b/Jasper.ConfluentKafka/Jasper.ConfluentKafka.csproj new file mode 100644 index 000000000..fa573677a --- /dev/null +++ b/Jasper.ConfluentKafka/Jasper.ConfluentKafka.csproj @@ -0,0 +1,16 @@ + + + + netstandard2.1 + portable + + + + + + + + + + + From 4c3214941456eac9b5b24d10dd3389a32f059051 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 16 Apr 2020 17:24:06 -0400 Subject: [PATCH 04/82] Expose internals to kafka proj --- src/Jasper/AssemblyAttributes.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Jasper/AssemblyAttributes.cs b/src/Jasper/AssemblyAttributes.cs index 54354baf4..5d879eeac 100644 --- a/src/Jasper/AssemblyAttributes.cs +++ b/src/Jasper/AssemblyAttributes.cs @@ -1,4 +1,4 @@ -using System.Runtime.CompilerServices; +using System.Runtime.CompilerServices; using Jasper.Attributes; using Jasper.Configuration; using Lamar; @@ -12,6 +12,7 @@ [assembly: InternalsVisibleTo("Jasper.RabbitMq")] [assembly: InternalsVisibleTo("Jasper.RabbitMq.Tests")] [assembly: InternalsVisibleTo("Jasper.AzureServiceBus")] +[assembly: InternalsVisibleTo("Jasper.ConfluentKafka")] [assembly: InternalsVisibleTo("Jasper.AzureServiceBus.Tests")] [assembly: InternalsVisibleTo("Jasper.Persistence.Testing")] [assembly: InternalsVisibleTo("Jasper.Persistence.Database")] From 817c8af9d2ac35d9a4da8ab2cf5e0aae52a18f6c Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 16 Apr 2020 17:24:19 -0400 Subject: [PATCH 05/82] Add a core ITransportProtocol --- src/Jasper/Transports/ITransportProtocol.cs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 src/Jasper/Transports/ITransportProtocol.cs diff --git a/src/Jasper/Transports/ITransportProtocol.cs b/src/Jasper/Transports/ITransportProtocol.cs new file mode 100644 index 000000000..21d780f63 --- /dev/null +++ b/src/Jasper/Transports/ITransportProtocol.cs @@ -0,0 +1,19 @@ +namespace Jasper.Transports +{ + public interface ITransportProtocol + { + /// + /// Creates a transport message object from a Jasper Envelope + /// + /// + /// + TTransportMsg WriteFromEnvelope(Envelope envelope); + + /// + /// Creates an Envelope from the incoming transport message + /// + /// + /// + Envelope ReadEnvelope(TTransportMsg message); + } +} From e9e640408806c04e1e025eeb40bcae2d2a948983 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 16 Apr 2020 17:24:25 -0400 Subject: [PATCH 06/82] Add project to sln --- Jasper.sln | 63 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 24 deletions(-) diff --git a/Jasper.sln b/Jasper.sln index 6cf006aa3..2f42a49b9 100644 --- a/Jasper.sln +++ b/Jasper.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 15 -VisualStudioVersion = 15.0.26730.16 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.29806.167 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Jasper", "src\Jasper\Jasper.csproj", "{E617B35A-B0D2-43BB-A723-D29B992EE744}" EndProject @@ -15,45 +15,47 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Samples", "Samples", "{A806 EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Publisher", "src\Publisher\Publisher.csproj", "{2AE00C65-E72F-4F95-9554-55E9B7D1A519}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Jasper.JsonCommands", "src\Jasper.JsonCommands\Jasper.JsonCommands.csproj", "{627A87BA-75B9-4C2E-861E-02880CA04295}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Jasper.JsonCommands", "src\Jasper.JsonCommands\Jasper.JsonCommands.csproj", "{627A87BA-75B9-4C2E-861E-02880CA04295}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Persistence", "Persistence", "{166943FC-7D19-4C4A-9E74-02A2CB49CD6B}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Jasper.Persistence.Marten", "src\Jasper.Persistence.Marten\Jasper.Persistence.Marten.csproj", "{79B5A55F-4AC6-46BA-829B-00A3ACE540D8}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Jasper.Persistence.Marten", "src\Jasper.Persistence.Marten\Jasper.Persistence.Marten.csproj", "{79B5A55F-4AC6-46BA-829B-00A3ACE540D8}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Jasper.Persistence.SqlServer", "src\Jasper.Persistence.SqlServer\Jasper.Persistence.SqlServer.csproj", "{A762EA81-6ECD-4DDD-B073-AC00BA4521CD}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Jasper.Persistence.SqlServer", "src\Jasper.Persistence.SqlServer\Jasper.Persistence.SqlServer.csproj", "{A762EA81-6ECD-4DDD-B073-AC00BA4521CD}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Extensions", "Extensions", "{CA5A0AA5-2CAD-4F42-AF73-980469934F27}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestingSupport", "src\TestingSupport\TestingSupport.csproj", "{9D1DBC42-96F7-4516-8C47-9DD35C6D9AAD}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestingSupport", "src\TestingSupport\TestingSupport.csproj", "{9D1DBC42-96F7-4516-8C47-9DD35C6D9AAD}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Samples", "src\Samples\Samples.csproj", "{32E92CD7-190E-4E9E-8230-1D7E5E19A539}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Samples", "src\Samples\Samples.csproj", "{32E92CD7-190E-4E9E-8230-1D7E5E19A539}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Testing", "Testing", "{2257A448-52A2-466A-ABC5-BD63018F004A}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Jasper.Persistence.Postgresql", "src\Jasper.Persistence.Postgresql\Jasper.Persistence.Postgresql.csproj", "{4860ADD6-EC91-4DA8-84AC-9882B6210D4D}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Jasper.Persistence.Postgresql", "src\Jasper.Persistence.Postgresql\Jasper.Persistence.Postgresql.csproj", "{4860ADD6-EC91-4DA8-84AC-9882B6210D4D}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StorytellerRunner", "src\StorytellerRunner\StorytellerRunner.csproj", "{0197CBCD-7A5F-4A52-9294-32336B312F9C}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "StorytellerRunner", "src\StorytellerRunner\StorytellerRunner.csproj", "{0197CBCD-7A5F-4A52-9294-32336B312F9C}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Jasper.Persistence.Database", "src\Jasper.Persistence.Database\Jasper.Persistence.Database.csproj", "{442F7B7D-C529-4C17-A6F6-C578CA5C06F5}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Jasper.Persistence.Database", "src\Jasper.Persistence.Database\Jasper.Persistence.Database.csproj", "{442F7B7D-C529-4C17-A6F6-C578CA5C06F5}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Jasper.Persistence.Testing", "src\Jasper.Persistence.Testing\Jasper.Persistence.Testing.csproj", "{899902B6-63DB-4FED-ABC7-9AE35CCE1DB6}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Jasper.Persistence.Testing", "src\Jasper.Persistence.Testing\Jasper.Persistence.Testing.csproj", "{899902B6-63DB-4FED-ABC7-9AE35CCE1DB6}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Jasper.Testing", "src\Jasper.Testing\Jasper.Testing.csproj", "{1C7783B1-CC8E-4225-9B9D-30C05A99B912}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Jasper.Testing", "src\Jasper.Testing\Jasper.Testing.csproj", "{1C7783B1-CC8E-4225-9B9D-30C05A99B912}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Jasper.Persistence.EntityFrameworkCore", "src\Jasper.Persistence.EntityFrameworkCore\Jasper.Persistence.EntityFrameworkCore.csproj", "{D830F62B-1031-47D5-AF3B-CC48A178FE43}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Jasper.Persistence.EntityFrameworkCore", "src\Jasper.Persistence.EntityFrameworkCore\Jasper.Persistence.EntityFrameworkCore.csproj", "{D830F62B-1031-47D5-AF3B-CC48A178FE43}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Jasper.RabbitMQ", "src\Jasper.RabbitMQ\Jasper.RabbitMQ.csproj", "{1B86F467-4DC6-4D30-9201-FD1BD44C3271}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Jasper.RabbitMQ", "src\Jasper.RabbitMQ\Jasper.RabbitMQ.csproj", "{1B86F467-4DC6-4D30-9201-FD1BD44C3271}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Jasper.RabbitMQ.Tests", "src\Jasper.RabbitMQ.Tests\Jasper.RabbitMQ.Tests.csproj", "{57273C2A-3F16-49B7-AB6C-80C6F44A60FE}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Jasper.RabbitMQ.Tests", "src\Jasper.RabbitMQ.Tests\Jasper.RabbitMQ.Tests.csproj", "{57273C2A-3F16-49B7-AB6C-80C6F44A60FE}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Jasper.TestSupport.Storyteller", "src\Jasper.TestSupport.Storyteller\Jasper.TestSupport.Storyteller.csproj", "{40670392-73E5-499C-8324-EE40BA6B5A10}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Jasper.TestSupport.Storyteller", "src\Jasper.TestSupport.Storyteller\Jasper.TestSupport.Storyteller.csproj", "{40670392-73E5-499C-8324-EE40BA6B5A10}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Jasper.TestSupport.Tests", "src\Jasper.TestSupport.Tests\Jasper.TestSupport.Tests.csproj", "{D09FBD2B-87AD-47CC-9191-5B4E06A48FBC}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Jasper.TestSupport.Tests", "src\Jasper.TestSupport.Tests\Jasper.TestSupport.Tests.csproj", "{D09FBD2B-87AD-47CC-9191-5B4E06A48FBC}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Jasper.AzureServiceBus", "src\Jasper.AzureServiceBus\Jasper.AzureServiceBus.csproj", "{CA4812BF-8580-4891-95FE-518930FCF859}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Jasper.AzureServiceBus", "src\Jasper.AzureServiceBus\Jasper.AzureServiceBus.csproj", "{CA4812BF-8580-4891-95FE-518930FCF859}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Jasper.AzureServiceBus.Tests", "src\Jasper.AzureServiceBus.Tests\Jasper.AzureServiceBus.Tests.csproj", "{6EC1EA66-63C7-4DF6-8DCB-40DFBEA4E07A}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Jasper.AzureServiceBus.Tests", "src\Jasper.AzureServiceBus.Tests\Jasper.AzureServiceBus.Tests.csproj", "{6EC1EA66-63C7-4DF6-8DCB-40DFBEA4E07A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Jasper.ConfluentKafka", "Jasper.ConfluentKafka\Jasper.ConfluentKafka.csproj", "{DD65A67C-2B52-416E-8017-ED1E4D678ED6}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -329,20 +331,32 @@ Global {6EC1EA66-63C7-4DF6-8DCB-40DFBEA4E07A}.Release|x64.Build.0 = Release|Any CPU {6EC1EA66-63C7-4DF6-8DCB-40DFBEA4E07A}.Release|x86.ActiveCfg = Release|Any CPU {6EC1EA66-63C7-4DF6-8DCB-40DFBEA4E07A}.Release|x86.Build.0 = Release|Any CPU + {DD65A67C-2B52-416E-8017-ED1E4D678ED6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {DD65A67C-2B52-416E-8017-ED1E4D678ED6}.Debug|Any CPU.Build.0 = Debug|Any CPU + {DD65A67C-2B52-416E-8017-ED1E4D678ED6}.Debug|x64.ActiveCfg = Debug|Any CPU + {DD65A67C-2B52-416E-8017-ED1E4D678ED6}.Debug|x64.Build.0 = Debug|Any CPU + {DD65A67C-2B52-416E-8017-ED1E4D678ED6}.Debug|x86.ActiveCfg = Debug|Any CPU + {DD65A67C-2B52-416E-8017-ED1E4D678ED6}.Debug|x86.Build.0 = Debug|Any CPU + {DD65A67C-2B52-416E-8017-ED1E4D678ED6}.Release|Any CPU.ActiveCfg = Release|Any CPU + {DD65A67C-2B52-416E-8017-ED1E4D678ED6}.Release|Any CPU.Build.0 = Release|Any CPU + {DD65A67C-2B52-416E-8017-ED1E4D678ED6}.Release|x64.ActiveCfg = Release|Any CPU + {DD65A67C-2B52-416E-8017-ED1E4D678ED6}.Release|x64.Build.0 = Release|Any CPU + {DD65A67C-2B52-416E-8017-ED1E4D678ED6}.Release|x86.ActiveCfg = Release|Any CPU + {DD65A67C-2B52-416E-8017-ED1E4D678ED6}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection GlobalSection(NestedProjects) = preSolution - {2AE00C65-E72F-4F95-9554-55E9B7D1A519} = {A806095A-9D66-4D55-8662-1FC67E90F6FB} - {3FA62C1A-7046-4CFB-8124-284EC7BD3660} = {A806095A-9D66-4D55-8662-1FC67E90F6FB} {4F18A2E4-5056-48C8-89BA-4837F6F983E4} = {A806095A-9D66-4D55-8662-1FC67E90F6FB} + {3FA62C1A-7046-4CFB-8124-284EC7BD3660} = {A806095A-9D66-4D55-8662-1FC67E90F6FB} + {51279890-5EDE-42A3-8D8D-EE0230CE0944} = {2257A448-52A2-466A-ABC5-BD63018F004A} + {2AE00C65-E72F-4F95-9554-55E9B7D1A519} = {A806095A-9D66-4D55-8662-1FC67E90F6FB} + {627A87BA-75B9-4C2E-861E-02880CA04295} = {CA5A0AA5-2CAD-4F42-AF73-980469934F27} {79B5A55F-4AC6-46BA-829B-00A3ACE540D8} = {166943FC-7D19-4C4A-9E74-02A2CB49CD6B} {A762EA81-6ECD-4DDD-B073-AC00BA4521CD} = {166943FC-7D19-4C4A-9E74-02A2CB49CD6B} - {627A87BA-75B9-4C2E-861E-02880CA04295} = {CA5A0AA5-2CAD-4F42-AF73-980469934F27} - {32E92CD7-190E-4E9E-8230-1D7E5E19A539} = {A806095A-9D66-4D55-8662-1FC67E90F6FB} - {51279890-5EDE-42A3-8D8D-EE0230CE0944} = {2257A448-52A2-466A-ABC5-BD63018F004A} {9D1DBC42-96F7-4516-8C47-9DD35C6D9AAD} = {2257A448-52A2-466A-ABC5-BD63018F004A} + {32E92CD7-190E-4E9E-8230-1D7E5E19A539} = {A806095A-9D66-4D55-8662-1FC67E90F6FB} {4860ADD6-EC91-4DA8-84AC-9882B6210D4D} = {166943FC-7D19-4C4A-9E74-02A2CB49CD6B} {0197CBCD-7A5F-4A52-9294-32336B312F9C} = {2257A448-52A2-466A-ABC5-BD63018F004A} {442F7B7D-C529-4C17-A6F6-C578CA5C06F5} = {166943FC-7D19-4C4A-9E74-02A2CB49CD6B} @@ -355,6 +369,7 @@ Global {D09FBD2B-87AD-47CC-9191-5B4E06A48FBC} = {CA5A0AA5-2CAD-4F42-AF73-980469934F27} {CA4812BF-8580-4891-95FE-518930FCF859} = {CA5A0AA5-2CAD-4F42-AF73-980469934F27} {6EC1EA66-63C7-4DF6-8DCB-40DFBEA4E07A} = {CA5A0AA5-2CAD-4F42-AF73-980469934F27} + {DD65A67C-2B52-416E-8017-ED1E4D678ED6} = {CA5A0AA5-2CAD-4F42-AF73-980469934F27} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {D92D723F-44EC-4C1E-AAC3-C162FCEAAA08} From f20b2f78987cb580feae6abdd99499931af2050e Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 17 Apr 2020 10:21:04 -0400 Subject: [PATCH 07/82] Add generics for Kafka producer --- Jasper.ConfluentKafka.Tests/Class1.cs | 8 ++++ .../Jasper.ConfluentKafka.Tests.csproj | 28 ++++++++++++++ .../Internal/KafkaTopicRouter.cs | 36 ++++++++++++++++++ Jasper.ConfluentKafka/KafkaEndpoint.cs | 38 +++++++++++++------ .../KafkaSubscriberConfiguration.cs | 12 ++++++ Jasper.ConfluentKafka/KafkaTransport.cs | 7 +++- Jasper.sln | 2 +- .../Jasper.AzureServiceBus.Tests.csproj | 5 +-- .../Internal/RabbitMqSender.cs | 4 +- 9 files changed, 119 insertions(+), 21 deletions(-) create mode 100644 Jasper.ConfluentKafka.Tests/Class1.cs create mode 100644 Jasper.ConfluentKafka.Tests/Jasper.ConfluentKafka.Tests.csproj create mode 100644 Jasper.ConfluentKafka/Internal/KafkaTopicRouter.cs create mode 100644 Jasper.ConfluentKafka/KafkaSubscriberConfiguration.cs diff --git a/Jasper.ConfluentKafka.Tests/Class1.cs b/Jasper.ConfluentKafka.Tests/Class1.cs new file mode 100644 index 000000000..7f058e200 --- /dev/null +++ b/Jasper.ConfluentKafka.Tests/Class1.cs @@ -0,0 +1,8 @@ +using System; + +namespace Jasper.ConfluentKafka.Tests +{ + public class Class1 + { + } +} diff --git a/Jasper.ConfluentKafka.Tests/Jasper.ConfluentKafka.Tests.csproj b/Jasper.ConfluentKafka.Tests/Jasper.ConfluentKafka.Tests.csproj new file mode 100644 index 000000000..c94cdc153 --- /dev/null +++ b/Jasper.ConfluentKafka.Tests/Jasper.ConfluentKafka.Tests.csproj @@ -0,0 +1,28 @@ + + + + netcoreapp3.0 + + false + + + + + + + + + + + + + + + + + + Servers.cs + + + + diff --git a/Jasper.ConfluentKafka/Internal/KafkaTopicRouter.cs b/Jasper.ConfluentKafka/Internal/KafkaTopicRouter.cs new file mode 100644 index 000000000..3f461d81a --- /dev/null +++ b/Jasper.ConfluentKafka/Internal/KafkaTopicRouter.cs @@ -0,0 +1,36 @@ +using System; +using Baseline; +using Jasper.Configuration; +using Jasper.Runtime.Routing; + +namespace Jasper.ConfluentKafka.Internal +{ + public class KafkaTopicRouter : TopicRouter + { + public KafkaTopicRouter() + { + } + + public override Uri BuildUriForTopic(string topicName) + { + var endpoint = new KafkaEndpoint + { + IsDurable = true, + TopicName = topicName + }; + + return endpoint.Uri; + } + + public override KafkaSubscriberConfiguration FindConfigurationForTopic(string topicName, + IEndpoints endpoints) + { + var uri = BuildUriForTopic(topicName); + var endpoint = endpoints.As().GetOrCreateEndpoint(uri); + + return new KafkaSubscriberConfiguration((KafkaEndpoint) endpoint); + } + + } + +} diff --git a/Jasper.ConfluentKafka/KafkaEndpoint.cs b/Jasper.ConfluentKafka/KafkaEndpoint.cs index bc20da9c0..c2dcecd19 100644 --- a/Jasper.ConfluentKafka/KafkaEndpoint.cs +++ b/Jasper.ConfluentKafka/KafkaEndpoint.cs @@ -12,12 +12,11 @@ namespace Jasper.ConfluentKafka { - public class KafkaEndpoint : Endpoint + public class KafkaEndpoint : Endpoint { - private const string TopicToken = "stream"; - public string TopicName { get; private set; } + private const string TopicToken = "topic"; + public string TopicName { get; set; } public ProducerConfig ProducerConfig { get; private set; } - internal KafkaTransport Parent { get; set; } public override Uri Uri => BuildUri(); private Uri BuildUri(bool forReply = false) { @@ -34,18 +33,16 @@ private Uri BuildUri(bool forReply = false) list.Add(TransportConstants.Durable); } - var uri = $"{KafkaTransport.ProtocolName}://{list.Join("/")}".ToUri(); + var uri = $"{Protocols.Kafka}://{list.Join("/")}".ToUri(); return uri; } - public override Uri ReplyUri() => BuildUri(); - public override void Parse(Uri uri) { - if (uri.Scheme != KafkaTransport.ProtocolName) + if (uri.Scheme != Protocols.Kafka) { - throw new ArgumentOutOfRangeException($"This is not a {nameof(KafkaTransport)} Uri"); + throw new ArgumentOutOfRangeException($"This is not a Kafka Transport Uri"); } var raw = uri.Segments.Where(x => x != "/").Select(x => x.Trim('/')); @@ -56,7 +53,6 @@ public override void Parse(Uri uri) segments.Enqueue(segment); } - while (segments.Any()) { if (segments.Peek().EqualsIgnoreCase(TopicToken)) @@ -71,7 +67,7 @@ public override void Parse(Uri uri) } else { - throw new InvalidOperationException($"The Uri '{uri}' is invalid for an {nameof(KafkaTransport)} endpoint"); + throw new InvalidOperationException($"The Uri '{uri}' is invalid for a Kafka Transport endpoint"); } } } @@ -81,6 +77,26 @@ protected internal override void StartListening(IMessagingRoot root, ITransportR throw new NotImplementedException(); } + protected override ISender CreateSender(IMessagingRoot root) + { + throw new NotImplementedException(); + } + + public override Uri ReplyUri() + { + throw new NotImplementedException(); + } + } + + public class KafkaEndpoint : KafkaEndpoint + { + internal KafkaTransport Parent { get; set; } + + protected internal override void StartListening(IMessagingRoot root, ITransportRuntime runtime) + { + throw new NotImplementedException(); + } + protected override ISender CreateSender(IMessagingRoot root) { return new ConfluentKafkaSender(this, Parent, root.TransportLogger, root.Cancellation); diff --git a/Jasper.ConfluentKafka/KafkaSubscriberConfiguration.cs b/Jasper.ConfluentKafka/KafkaSubscriberConfiguration.cs new file mode 100644 index 000000000..57b9ece0e --- /dev/null +++ b/Jasper.ConfluentKafka/KafkaSubscriberConfiguration.cs @@ -0,0 +1,12 @@ +using Jasper.Configuration; + +namespace Jasper.ConfluentKafka +{ + public class KafkaSubscriberConfiguration : SubscriberConfiguration + { + public KafkaSubscriberConfiguration(KafkaEndpoint endpoint) : base(endpoint) + { + } + + } +} diff --git a/Jasper.ConfluentKafka/KafkaTransport.cs b/Jasper.ConfluentKafka/KafkaTransport.cs index f6f335d6f..87fcbdb89 100644 --- a/Jasper.ConfluentKafka/KafkaTransport.cs +++ b/Jasper.ConfluentKafka/KafkaTransport.cs @@ -4,10 +4,13 @@ namespace Jasper.ConfluentKafka { - public class KafkaTransport : TransportBase> + public static class Protocols { - public static readonly string ProtocolName = "kafka"; + public const string Kafka = "kafka"; + } + public class KafkaTransport : TransportBase> + { public KafkaTransport(string protocol) : base(protocol) { } diff --git a/Jasper.sln b/Jasper.sln index 2f42a49b9..647ea0b0c 100644 --- a/Jasper.sln +++ b/Jasper.sln @@ -55,7 +55,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Jasper.AzureServiceBus", "s EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Jasper.AzureServiceBus.Tests", "src\Jasper.AzureServiceBus.Tests\Jasper.AzureServiceBus.Tests.csproj", "{6EC1EA66-63C7-4DF6-8DCB-40DFBEA4E07A}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Jasper.ConfluentKafka", "Jasper.ConfluentKafka\Jasper.ConfluentKafka.csproj", "{DD65A67C-2B52-416E-8017-ED1E4D678ED6}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Jasper.ConfluentKafka", "Jasper.ConfluentKafka\Jasper.ConfluentKafka.csproj", "{DD65A67C-2B52-416E-8017-ED1E4D678ED6}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution diff --git a/src/Jasper.AzureServiceBus.Tests/Jasper.AzureServiceBus.Tests.csproj b/src/Jasper.AzureServiceBus.Tests/Jasper.AzureServiceBus.Tests.csproj index 91ca1a574..5f10e44bc 100644 --- a/src/Jasper.AzureServiceBus.Tests/Jasper.AzureServiceBus.Tests.csproj +++ b/src/Jasper.AzureServiceBus.Tests/Jasper.AzureServiceBus.Tests.csproj @@ -1,4 +1,4 @@ - + netcoreapp3.0 @@ -13,9 +13,6 @@ - - - diff --git a/src/Jasper.RabbitMQ/Internal/RabbitMqSender.cs b/src/Jasper.RabbitMQ/Internal/RabbitMqSender.cs index 8b8ba8dbb..a986acf2e 100644 --- a/src/Jasper.RabbitMQ/Internal/RabbitMqSender.cs +++ b/src/Jasper.RabbitMQ/Internal/RabbitMqSender.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Security.Authentication.ExtendedProtection; using System.Threading; using System.Threading.Tasks; @@ -35,8 +35,6 @@ public RabbitMqSender(ITransportLogger logger, RabbitMqEndpoint endpoint, Rabbit _key = endpoint.RoutingKey ?? endpoint.QueueName ?? ""; } - - public void Start(ISenderCallback callback) { Connect(); From 3f4cd349f807facc5dc8097e642abaf5c76ac866 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 17 Apr 2020 13:19:28 -0400 Subject: [PATCH 08/82] Move projects to src dir --- Jasper.sln | 43 ++++--- .../Jasper.ConfluentKafka.Tests}/Class1.cs | 0 .../Jasper.ConfluentKafka.Tests.csproj | 0 .../KafkaEndpointTester.cs | 107 ++++++++++++++++++ .../Exceptions/UnsupportedFeatureException.cs | 0 .../Internal/ConfluentKafkaSender.cs | 0 .../Internal/KafkaTopicRouter.cs | 0 .../Jasper.ConfluentKafka.csproj | 0 .../Jasper.ConfluentKafka}/KafkaEndpoint.cs | 0 .../KafkaSubscriberConfiguration.cs | 0 .../Jasper.ConfluentKafka}/KafkaTransport.cs | 0 .../KafkaTransportProtocol.cs | 0 12 files changed, 136 insertions(+), 14 deletions(-) rename {Jasper.ConfluentKafka.Tests => src/Jasper.ConfluentKafka.Tests}/Class1.cs (100%) rename {Jasper.ConfluentKafka.Tests => src/Jasper.ConfluentKafka.Tests}/Jasper.ConfluentKafka.Tests.csproj (100%) create mode 100644 src/Jasper.ConfluentKafka.Tests/KafkaEndpointTester.cs rename {Jasper.ConfluentKafka => src/Jasper.ConfluentKafka}/Exceptions/UnsupportedFeatureException.cs (100%) rename {Jasper.ConfluentKafka => src/Jasper.ConfluentKafka}/Internal/ConfluentKafkaSender.cs (100%) rename {Jasper.ConfluentKafka => src/Jasper.ConfluentKafka}/Internal/KafkaTopicRouter.cs (100%) rename {Jasper.ConfluentKafka => src/Jasper.ConfluentKafka}/Jasper.ConfluentKafka.csproj (100%) rename {Jasper.ConfluentKafka => src/Jasper.ConfluentKafka}/KafkaEndpoint.cs (100%) rename {Jasper.ConfluentKafka => src/Jasper.ConfluentKafka}/KafkaSubscriberConfiguration.cs (100%) rename {Jasper.ConfluentKafka => src/Jasper.ConfluentKafka}/KafkaTransport.cs (100%) rename {Jasper.ConfluentKafka => src/Jasper.ConfluentKafka}/KafkaTransportProtocol.cs (100%) diff --git a/Jasper.sln b/Jasper.sln index 647ea0b0c..7added68e 100644 --- a/Jasper.sln +++ b/Jasper.sln @@ -55,7 +55,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Jasper.AzureServiceBus", "s EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Jasper.AzureServiceBus.Tests", "src\Jasper.AzureServiceBus.Tests\Jasper.AzureServiceBus.Tests.csproj", "{6EC1EA66-63C7-4DF6-8DCB-40DFBEA4E07A}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Jasper.ConfluentKafka", "Jasper.ConfluentKafka\Jasper.ConfluentKafka.csproj", "{DD65A67C-2B52-416E-8017-ED1E4D678ED6}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Jasper.ConfluentKafka", "src\Jasper.ConfluentKafka\Jasper.ConfluentKafka.csproj", "{ABBCB70C-9087-4A0C-A3DB-C9BB0DFAAC5A}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Jasper.ConfluentKafka.Tests", "src\Jasper.ConfluentKafka.Tests\Jasper.ConfluentKafka.Tests.csproj", "{B8F1BCB3-4A8A-4368-85BA-E69EB879BC5A}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -331,18 +333,30 @@ Global {6EC1EA66-63C7-4DF6-8DCB-40DFBEA4E07A}.Release|x64.Build.0 = Release|Any CPU {6EC1EA66-63C7-4DF6-8DCB-40DFBEA4E07A}.Release|x86.ActiveCfg = Release|Any CPU {6EC1EA66-63C7-4DF6-8DCB-40DFBEA4E07A}.Release|x86.Build.0 = Release|Any CPU - {DD65A67C-2B52-416E-8017-ED1E4D678ED6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {DD65A67C-2B52-416E-8017-ED1E4D678ED6}.Debug|Any CPU.Build.0 = Debug|Any CPU - {DD65A67C-2B52-416E-8017-ED1E4D678ED6}.Debug|x64.ActiveCfg = Debug|Any CPU - {DD65A67C-2B52-416E-8017-ED1E4D678ED6}.Debug|x64.Build.0 = Debug|Any CPU - {DD65A67C-2B52-416E-8017-ED1E4D678ED6}.Debug|x86.ActiveCfg = Debug|Any CPU - {DD65A67C-2B52-416E-8017-ED1E4D678ED6}.Debug|x86.Build.0 = Debug|Any CPU - {DD65A67C-2B52-416E-8017-ED1E4D678ED6}.Release|Any CPU.ActiveCfg = Release|Any CPU - {DD65A67C-2B52-416E-8017-ED1E4D678ED6}.Release|Any CPU.Build.0 = Release|Any CPU - {DD65A67C-2B52-416E-8017-ED1E4D678ED6}.Release|x64.ActiveCfg = Release|Any CPU - {DD65A67C-2B52-416E-8017-ED1E4D678ED6}.Release|x64.Build.0 = Release|Any CPU - {DD65A67C-2B52-416E-8017-ED1E4D678ED6}.Release|x86.ActiveCfg = Release|Any CPU - {DD65A67C-2B52-416E-8017-ED1E4D678ED6}.Release|x86.Build.0 = Release|Any CPU + {ABBCB70C-9087-4A0C-A3DB-C9BB0DFAAC5A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {ABBCB70C-9087-4A0C-A3DB-C9BB0DFAAC5A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {ABBCB70C-9087-4A0C-A3DB-C9BB0DFAAC5A}.Debug|x64.ActiveCfg = Debug|Any CPU + {ABBCB70C-9087-4A0C-A3DB-C9BB0DFAAC5A}.Debug|x64.Build.0 = Debug|Any CPU + {ABBCB70C-9087-4A0C-A3DB-C9BB0DFAAC5A}.Debug|x86.ActiveCfg = Debug|Any CPU + {ABBCB70C-9087-4A0C-A3DB-C9BB0DFAAC5A}.Debug|x86.Build.0 = Debug|Any CPU + {ABBCB70C-9087-4A0C-A3DB-C9BB0DFAAC5A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {ABBCB70C-9087-4A0C-A3DB-C9BB0DFAAC5A}.Release|Any CPU.Build.0 = Release|Any CPU + {ABBCB70C-9087-4A0C-A3DB-C9BB0DFAAC5A}.Release|x64.ActiveCfg = Release|Any CPU + {ABBCB70C-9087-4A0C-A3DB-C9BB0DFAAC5A}.Release|x64.Build.0 = Release|Any CPU + {ABBCB70C-9087-4A0C-A3DB-C9BB0DFAAC5A}.Release|x86.ActiveCfg = Release|Any CPU + {ABBCB70C-9087-4A0C-A3DB-C9BB0DFAAC5A}.Release|x86.Build.0 = Release|Any CPU + {B8F1BCB3-4A8A-4368-85BA-E69EB879BC5A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B8F1BCB3-4A8A-4368-85BA-E69EB879BC5A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B8F1BCB3-4A8A-4368-85BA-E69EB879BC5A}.Debug|x64.ActiveCfg = Debug|Any CPU + {B8F1BCB3-4A8A-4368-85BA-E69EB879BC5A}.Debug|x64.Build.0 = Debug|Any CPU + {B8F1BCB3-4A8A-4368-85BA-E69EB879BC5A}.Debug|x86.ActiveCfg = Debug|Any CPU + {B8F1BCB3-4A8A-4368-85BA-E69EB879BC5A}.Debug|x86.Build.0 = Debug|Any CPU + {B8F1BCB3-4A8A-4368-85BA-E69EB879BC5A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B8F1BCB3-4A8A-4368-85BA-E69EB879BC5A}.Release|Any CPU.Build.0 = Release|Any CPU + {B8F1BCB3-4A8A-4368-85BA-E69EB879BC5A}.Release|x64.ActiveCfg = Release|Any CPU + {B8F1BCB3-4A8A-4368-85BA-E69EB879BC5A}.Release|x64.Build.0 = Release|Any CPU + {B8F1BCB3-4A8A-4368-85BA-E69EB879BC5A}.Release|x86.ActiveCfg = Release|Any CPU + {B8F1BCB3-4A8A-4368-85BA-E69EB879BC5A}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -369,7 +383,8 @@ Global {D09FBD2B-87AD-47CC-9191-5B4E06A48FBC} = {CA5A0AA5-2CAD-4F42-AF73-980469934F27} {CA4812BF-8580-4891-95FE-518930FCF859} = {CA5A0AA5-2CAD-4F42-AF73-980469934F27} {6EC1EA66-63C7-4DF6-8DCB-40DFBEA4E07A} = {CA5A0AA5-2CAD-4F42-AF73-980469934F27} - {DD65A67C-2B52-416E-8017-ED1E4D678ED6} = {CA5A0AA5-2CAD-4F42-AF73-980469934F27} + {ABBCB70C-9087-4A0C-A3DB-C9BB0DFAAC5A} = {CA5A0AA5-2CAD-4F42-AF73-980469934F27} + {B8F1BCB3-4A8A-4368-85BA-E69EB879BC5A} = {CA5A0AA5-2CAD-4F42-AF73-980469934F27} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {D92D723F-44EC-4C1E-AAC3-C162FCEAAA08} diff --git a/Jasper.ConfluentKafka.Tests/Class1.cs b/src/Jasper.ConfluentKafka.Tests/Class1.cs similarity index 100% rename from Jasper.ConfluentKafka.Tests/Class1.cs rename to src/Jasper.ConfluentKafka.Tests/Class1.cs diff --git a/Jasper.ConfluentKafka.Tests/Jasper.ConfluentKafka.Tests.csproj b/src/Jasper.ConfluentKafka.Tests/Jasper.ConfluentKafka.Tests.csproj similarity index 100% rename from Jasper.ConfluentKafka.Tests/Jasper.ConfluentKafka.Tests.csproj rename to src/Jasper.ConfluentKafka.Tests/Jasper.ConfluentKafka.Tests.csproj diff --git a/src/Jasper.ConfluentKafka.Tests/KafkaEndpointTester.cs b/src/Jasper.ConfluentKafka.Tests/KafkaEndpointTester.cs new file mode 100644 index 000000000..7b1f14a34 --- /dev/null +++ b/src/Jasper.ConfluentKafka.Tests/KafkaEndpointTester.cs @@ -0,0 +1,107 @@ +using System; +using Jasper.ConfluentKafka; +using Shouldly; +using Xunit; + +namespace Jasper.Kafka.Tests +{ + public class KafkaEndpointTester + { + [Fact] + public void default_protocol_is_the_default_protocol() + { + var endpoint = new KafkaEndpoint(); + endpoint.Protocol.ShouldBeOfType(); + } + + [Fact] + public void parse_non_durable_uri() + { + var endpoint = new KafkaEndpoint(); + endpoint.Parse(new Uri("ckafka://subscription/sub1/topic/key1")); + + endpoint.IsDurable.ShouldBeFalse(); + endpoint.SubscriptionName.ShouldBe("sub1"); + endpoint.TopicName.ShouldBe("key1"); + } + + [Fact] + public void parse_durable_uri() + { + var endpoint = new KafkaEndpoint(); + endpoint.Parse(new Uri("ckafka://subscription/sub1/topic/key1/durable")); + + endpoint.IsDurable.ShouldBeTrue(); + endpoint.SubscriptionName.ShouldBe("sub1"); + endpoint.TopicName.ShouldBe("key1"); + } + + [Fact] + public void parse_durable_uri_with_only_queue() + { + var endpoint = new KafkaEndpoint(); + endpoint.Parse(new Uri("ckafka://queue/q1/durable")); + + endpoint.IsDurable.ShouldBeTrue(); + endpoint.QueueName.ShouldBe("q1"); + } + + [Fact] + public void build_uri_for_subscription_and_topic() + { + new KafkaEndpoint + { + SubscriptionName = "ex1", + TopicName = "key1" + } + .Uri.ShouldBe(new Uri("ckafka://subscription/ex1/topic/key1")); + } + + [Fact] + public void build_uri_for_queue_only() + { + new KafkaEndpoint + { + QueueName = "foo" + } + .Uri.ShouldBe(new Uri("ckafka://queue/foo")); + } + + + [Fact] + public void build_uri_for_queue_only_and_durable() + { + new KafkaEndpoint + { + QueueName = "foo", + IsDurable = true + } + .ReplyUri().ShouldBe(new Uri("ckafka://queue/foo/durable")); + } + + [Fact] + public void generate_reply_uri_for_non_durable() + { + new KafkaEndpoint + { + SubscriptionName = "ex1", + TopicName = "key1" + } + .ReplyUri().ShouldBe(new Uri("ckafka://topic/key1")); + } + + [Fact] + public void generate_reply_uri_for_durable() + { + new KafkaEndpoint + { + SubscriptionName = "ex1", + TopicName = "key1", + IsDurable = true + }.ReplyUri().ShouldBe(new Uri("ckafka://topic/key1/durable")); + + + } + + } +} diff --git a/Jasper.ConfluentKafka/Exceptions/UnsupportedFeatureException.cs b/src/Jasper.ConfluentKafka/Exceptions/UnsupportedFeatureException.cs similarity index 100% rename from Jasper.ConfluentKafka/Exceptions/UnsupportedFeatureException.cs rename to src/Jasper.ConfluentKafka/Exceptions/UnsupportedFeatureException.cs diff --git a/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs similarity index 100% rename from Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs rename to src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs diff --git a/Jasper.ConfluentKafka/Internal/KafkaTopicRouter.cs b/src/Jasper.ConfluentKafka/Internal/KafkaTopicRouter.cs similarity index 100% rename from Jasper.ConfluentKafka/Internal/KafkaTopicRouter.cs rename to src/Jasper.ConfluentKafka/Internal/KafkaTopicRouter.cs diff --git a/Jasper.ConfluentKafka/Jasper.ConfluentKafka.csproj b/src/Jasper.ConfluentKafka/Jasper.ConfluentKafka.csproj similarity index 100% rename from Jasper.ConfluentKafka/Jasper.ConfluentKafka.csproj rename to src/Jasper.ConfluentKafka/Jasper.ConfluentKafka.csproj diff --git a/Jasper.ConfluentKafka/KafkaEndpoint.cs b/src/Jasper.ConfluentKafka/KafkaEndpoint.cs similarity index 100% rename from Jasper.ConfluentKafka/KafkaEndpoint.cs rename to src/Jasper.ConfluentKafka/KafkaEndpoint.cs diff --git a/Jasper.ConfluentKafka/KafkaSubscriberConfiguration.cs b/src/Jasper.ConfluentKafka/KafkaSubscriberConfiguration.cs similarity index 100% rename from Jasper.ConfluentKafka/KafkaSubscriberConfiguration.cs rename to src/Jasper.ConfluentKafka/KafkaSubscriberConfiguration.cs diff --git a/Jasper.ConfluentKafka/KafkaTransport.cs b/src/Jasper.ConfluentKafka/KafkaTransport.cs similarity index 100% rename from Jasper.ConfluentKafka/KafkaTransport.cs rename to src/Jasper.ConfluentKafka/KafkaTransport.cs diff --git a/Jasper.ConfluentKafka/KafkaTransportProtocol.cs b/src/Jasper.ConfluentKafka/KafkaTransportProtocol.cs similarity index 100% rename from Jasper.ConfluentKafka/KafkaTransportProtocol.cs rename to src/Jasper.ConfluentKafka/KafkaTransportProtocol.cs From 3ecdc71eef02ba8992c85ba0e1cec22ee8f50afb Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 17 Apr 2020 13:25:24 -0400 Subject: [PATCH 09/82] Fix broken refs --- .../Jasper.AzureServiceBus.Tests.csproj | 35 +-- src/Jasper.ConfluentKafka.Tests/Class1.cs | 8 - .../Jasper.ConfluentKafka.Tests.csproj | 2 +- .../KafkaEndpointTester.cs | 214 +++++++++--------- .../Jasper.ConfluentKafka.csproj | 2 +- 5 files changed, 128 insertions(+), 133 deletions(-) delete mode 100644 src/Jasper.ConfluentKafka.Tests/Class1.cs diff --git a/src/Jasper.AzureServiceBus.Tests/Jasper.AzureServiceBus.Tests.csproj b/src/Jasper.AzureServiceBus.Tests/Jasper.AzureServiceBus.Tests.csproj index 5f10e44bc..7eb6d8d9c 100644 --- a/src/Jasper.AzureServiceBus.Tests/Jasper.AzureServiceBus.Tests.csproj +++ b/src/Jasper.AzureServiceBus.Tests/Jasper.AzureServiceBus.Tests.csproj @@ -1,25 +1,28 @@ - - netcoreapp3.0 + + netcoreapp3.0 - false - + false + - - - - - + + + + + - + + + + - + - - - Servers.cs - - + + + Servers.cs + + diff --git a/src/Jasper.ConfluentKafka.Tests/Class1.cs b/src/Jasper.ConfluentKafka.Tests/Class1.cs deleted file mode 100644 index 7f058e200..000000000 --- a/src/Jasper.ConfluentKafka.Tests/Class1.cs +++ /dev/null @@ -1,8 +0,0 @@ -using System; - -namespace Jasper.ConfluentKafka.Tests -{ - public class Class1 - { - } -} diff --git a/src/Jasper.ConfluentKafka.Tests/Jasper.ConfluentKafka.Tests.csproj b/src/Jasper.ConfluentKafka.Tests/Jasper.ConfluentKafka.Tests.csproj index c94cdc153..964180561 100644 --- a/src/Jasper.ConfluentKafka.Tests/Jasper.ConfluentKafka.Tests.csproj +++ b/src/Jasper.ConfluentKafka.Tests/Jasper.ConfluentKafka.Tests.csproj @@ -13,7 +13,7 @@ - + diff --git a/src/Jasper.ConfluentKafka.Tests/KafkaEndpointTester.cs b/src/Jasper.ConfluentKafka.Tests/KafkaEndpointTester.cs index 7b1f14a34..7a6486cbb 100644 --- a/src/Jasper.ConfluentKafka.Tests/KafkaEndpointTester.cs +++ b/src/Jasper.ConfluentKafka.Tests/KafkaEndpointTester.cs @@ -1,107 +1,107 @@ -using System; -using Jasper.ConfluentKafka; -using Shouldly; -using Xunit; - -namespace Jasper.Kafka.Tests -{ - public class KafkaEndpointTester - { - [Fact] - public void default_protocol_is_the_default_protocol() - { - var endpoint = new KafkaEndpoint(); - endpoint.Protocol.ShouldBeOfType(); - } - - [Fact] - public void parse_non_durable_uri() - { - var endpoint = new KafkaEndpoint(); - endpoint.Parse(new Uri("ckafka://subscription/sub1/topic/key1")); - - endpoint.IsDurable.ShouldBeFalse(); - endpoint.SubscriptionName.ShouldBe("sub1"); - endpoint.TopicName.ShouldBe("key1"); - } - - [Fact] - public void parse_durable_uri() - { - var endpoint = new KafkaEndpoint(); - endpoint.Parse(new Uri("ckafka://subscription/sub1/topic/key1/durable")); - - endpoint.IsDurable.ShouldBeTrue(); - endpoint.SubscriptionName.ShouldBe("sub1"); - endpoint.TopicName.ShouldBe("key1"); - } - - [Fact] - public void parse_durable_uri_with_only_queue() - { - var endpoint = new KafkaEndpoint(); - endpoint.Parse(new Uri("ckafka://queue/q1/durable")); - - endpoint.IsDurable.ShouldBeTrue(); - endpoint.QueueName.ShouldBe("q1"); - } - - [Fact] - public void build_uri_for_subscription_and_topic() - { - new KafkaEndpoint - { - SubscriptionName = "ex1", - TopicName = "key1" - } - .Uri.ShouldBe(new Uri("ckafka://subscription/ex1/topic/key1")); - } - - [Fact] - public void build_uri_for_queue_only() - { - new KafkaEndpoint - { - QueueName = "foo" - } - .Uri.ShouldBe(new Uri("ckafka://queue/foo")); - } - - - [Fact] - public void build_uri_for_queue_only_and_durable() - { - new KafkaEndpoint - { - QueueName = "foo", - IsDurable = true - } - .ReplyUri().ShouldBe(new Uri("ckafka://queue/foo/durable")); - } - - [Fact] - public void generate_reply_uri_for_non_durable() - { - new KafkaEndpoint - { - SubscriptionName = "ex1", - TopicName = "key1" - } - .ReplyUri().ShouldBe(new Uri("ckafka://topic/key1")); - } - - [Fact] - public void generate_reply_uri_for_durable() - { - new KafkaEndpoint - { - SubscriptionName = "ex1", - TopicName = "key1", - IsDurable = true - }.ReplyUri().ShouldBe(new Uri("ckafka://topic/key1/durable")); - - - } - - } -} +//using System; +//using Jasper.ConfluentKafka; +//using Shouldly; +//using Xunit; + +//namespace Jasper.Kafka.Tests +//{ +// public class KafkaEndpointTester +// { +// [Fact] +// public void default_protocol_is_the_default_protocol() +// { +// var endpoint = new KafkaEndpoint(); +// endpoint.Protocol.ShouldBeOfType(); +// } + +// [Fact] +// public void parse_non_durable_uri() +// { +// var endpoint = new KafkaEndpoint(); +// endpoint.Parse(new Uri("ckafka://subscription/sub1/topic/key1")); + +// endpoint.IsDurable.ShouldBeFalse(); +// endpoint.SubscriptionName.ShouldBe("sub1"); +// endpoint.TopicName.ShouldBe("key1"); +// } + +// [Fact] +// public void parse_durable_uri() +// { +// var endpoint = new KafkaEndpoint(); +// endpoint.Parse(new Uri("ckafka://subscription/sub1/topic/key1/durable")); + +// endpoint.IsDurable.ShouldBeTrue(); +// endpoint.SubscriptionName.ShouldBe("sub1"); +// endpoint.TopicName.ShouldBe("key1"); +// } + +// [Fact] +// public void parse_durable_uri_with_only_queue() +// { +// var endpoint = new KafkaEndpoint(); +// endpoint.Parse(new Uri("ckafka://queue/q1/durable")); + +// endpoint.IsDurable.ShouldBeTrue(); +// endpoint.QueueName.ShouldBe("q1"); +// } + +// [Fact] +// public void build_uri_for_subscription_and_topic() +// { +// new KafkaEndpoint +// { +// SubscriptionName = "ex1", +// TopicName = "key1" +// } +// .Uri.ShouldBe(new Uri("ckafka://subscription/ex1/topic/key1")); +// } + +// [Fact] +// public void build_uri_for_queue_only() +// { +// new KafkaEndpoint +// { +// QueueName = "foo" +// } +// .Uri.ShouldBe(new Uri("ckafka://queue/foo")); +// } + + +// [Fact] +// public void build_uri_for_queue_only_and_durable() +// { +// new KafkaEndpoint +// { +// QueueName = "foo", +// IsDurable = true +// } +// .ReplyUri().ShouldBe(new Uri("ckafka://queue/foo/durable")); +// } + +// [Fact] +// public void generate_reply_uri_for_non_durable() +// { +// new KafkaEndpoint +// { +// SubscriptionName = "ex1", +// TopicName = "key1" +// } +// .ReplyUri().ShouldBe(new Uri("ckafka://topic/key1")); +// } + +// [Fact] +// public void generate_reply_uri_for_durable() +// { +// new KafkaEndpoint +// { +// SubscriptionName = "ex1", +// TopicName = "key1", +// IsDurable = true +// }.ReplyUri().ShouldBe(new Uri("ckafka://topic/key1/durable")); + + +// } + +// } +//} diff --git a/src/Jasper.ConfluentKafka/Jasper.ConfluentKafka.csproj b/src/Jasper.ConfluentKafka/Jasper.ConfluentKafka.csproj index fa573677a..8417268ba 100644 --- a/src/Jasper.ConfluentKafka/Jasper.ConfluentKafka.csproj +++ b/src/Jasper.ConfluentKafka/Jasper.ConfluentKafka.csproj @@ -10,7 +10,7 @@ - + From 4d7f01ebd46b9fa632735c2ae38c4ec1af5af8fe Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 17 Apr 2020 13:32:06 -0400 Subject: [PATCH 10/82] Make KafkaTransport non-generic --- .../Internal/ConfluentKafkaSender.cs | 4 ++-- src/Jasper.ConfluentKafka/KafkaEndpoint.cs | 2 +- src/Jasper.ConfluentKafka/KafkaTransport.cs | 22 ++++++++++++------- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs index 945be72b2..d7f50270f 100644 --- a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs +++ b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs @@ -15,14 +15,14 @@ public class ConfluentKafkaSender : ISender { private readonly ITransportProtocol> _protocol; private readonly KafkaEndpoint _endpoint; - private readonly KafkaTransport _transport; + private readonly KafkaTransport _transport; private readonly ITransportLogger _logger; private readonly CancellationToken _cancellation; private ActionBlock _sending; private ISenderCallback _callback; private IProducer _publisher; - public ConfluentKafkaSender(KafkaEndpoint endpoint, KafkaTransport transport, ITransportLogger logger, CancellationToken cancellation) + public ConfluentKafkaSender(KafkaEndpoint endpoint, KafkaTransport transport, ITransportLogger logger, CancellationToken cancellation) { _endpoint = endpoint; _transport = transport; diff --git a/src/Jasper.ConfluentKafka/KafkaEndpoint.cs b/src/Jasper.ConfluentKafka/KafkaEndpoint.cs index c2dcecd19..de9b4db4a 100644 --- a/src/Jasper.ConfluentKafka/KafkaEndpoint.cs +++ b/src/Jasper.ConfluentKafka/KafkaEndpoint.cs @@ -90,7 +90,7 @@ public override Uri ReplyUri() public class KafkaEndpoint : KafkaEndpoint { - internal KafkaTransport Parent { get; set; } + internal KafkaTransport Parent { get; set; } protected internal override void StartListening(IMessagingRoot root, ITransportRuntime runtime) { diff --git a/src/Jasper.ConfluentKafka/KafkaTransport.cs b/src/Jasper.ConfluentKafka/KafkaTransport.cs index 87fcbdb89..847ef84f7 100644 --- a/src/Jasper.ConfluentKafka/KafkaTransport.cs +++ b/src/Jasper.ConfluentKafka/KafkaTransport.cs @@ -1,5 +1,7 @@ using System; using System.Collections.Generic; +using Baseline; +using Jasper.ConfluentKafka.Internal; using Jasper.Transports; namespace Jasper.ConfluentKafka @@ -9,20 +11,24 @@ public static class Protocols public const string Kafka = "kafka"; } - public class KafkaTransport : TransportBase> + public class KafkaTransport : TransportBase { - public KafkaTransport(string protocol) : base(protocol) - { - } + private readonly LightweightCache _endpoints; + + public KafkaTopicRouter Topics { get; } = new KafkaTopicRouter(); - protected override IEnumerable> endpoints() + public KafkaTransport() : base(Protocols.Kafka) { - throw new NotImplementedException(); } - protected override KafkaEndpoint findEndpointByUri(Uri uri) + protected override IEnumerable endpoints() => _endpoints; + + protected override KafkaEndpoint findEndpointByUri(Uri uri) => _endpoints[uri]; + + public KafkaEndpoint EndpointForTopic(string topicName) { - throw new NotImplementedException(); + var uri = new KafkaEndpoint { TopicName = topicName }.Uri; + return _endpoints[uri]; } } } From b91603945fb5db078624068dd8bea069940aacdc Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 17 Apr 2020 13:49:20 -0400 Subject: [PATCH 11/82] Test KafkaEndpoint --- .../KafkaEndpointTester.cs | 168 +++++++----------- src/Jasper.ConfluentKafka/KafkaEndpoint.cs | 7 +- 2 files changed, 63 insertions(+), 112 deletions(-) diff --git a/src/Jasper.ConfluentKafka.Tests/KafkaEndpointTester.cs b/src/Jasper.ConfluentKafka.Tests/KafkaEndpointTester.cs index 7a6486cbb..bbff770e9 100644 --- a/src/Jasper.ConfluentKafka.Tests/KafkaEndpointTester.cs +++ b/src/Jasper.ConfluentKafka.Tests/KafkaEndpointTester.cs @@ -1,107 +1,61 @@ -//using System; -//using Jasper.ConfluentKafka; -//using Shouldly; -//using Xunit; - -//namespace Jasper.Kafka.Tests -//{ -// public class KafkaEndpointTester -// { -// [Fact] -// public void default_protocol_is_the_default_protocol() -// { -// var endpoint = new KafkaEndpoint(); -// endpoint.Protocol.ShouldBeOfType(); -// } - -// [Fact] -// public void parse_non_durable_uri() -// { -// var endpoint = new KafkaEndpoint(); -// endpoint.Parse(new Uri("ckafka://subscription/sub1/topic/key1")); - -// endpoint.IsDurable.ShouldBeFalse(); -// endpoint.SubscriptionName.ShouldBe("sub1"); -// endpoint.TopicName.ShouldBe("key1"); -// } - -// [Fact] -// public void parse_durable_uri() -// { -// var endpoint = new KafkaEndpoint(); -// endpoint.Parse(new Uri("ckafka://subscription/sub1/topic/key1/durable")); - -// endpoint.IsDurable.ShouldBeTrue(); -// endpoint.SubscriptionName.ShouldBe("sub1"); -// endpoint.TopicName.ShouldBe("key1"); -// } - -// [Fact] -// public void parse_durable_uri_with_only_queue() -// { -// var endpoint = new KafkaEndpoint(); -// endpoint.Parse(new Uri("ckafka://queue/q1/durable")); - -// endpoint.IsDurable.ShouldBeTrue(); -// endpoint.QueueName.ShouldBe("q1"); -// } - -// [Fact] -// public void build_uri_for_subscription_and_topic() -// { -// new KafkaEndpoint -// { -// SubscriptionName = "ex1", -// TopicName = "key1" -// } -// .Uri.ShouldBe(new Uri("ckafka://subscription/ex1/topic/key1")); -// } - -// [Fact] -// public void build_uri_for_queue_only() -// { -// new KafkaEndpoint -// { -// QueueName = "foo" -// } -// .Uri.ShouldBe(new Uri("ckafka://queue/foo")); -// } - - -// [Fact] -// public void build_uri_for_queue_only_and_durable() -// { -// new KafkaEndpoint -// { -// QueueName = "foo", -// IsDurable = true -// } -// .ReplyUri().ShouldBe(new Uri("ckafka://queue/foo/durable")); -// } - -// [Fact] -// public void generate_reply_uri_for_non_durable() -// { -// new KafkaEndpoint -// { -// SubscriptionName = "ex1", -// TopicName = "key1" -// } -// .ReplyUri().ShouldBe(new Uri("ckafka://topic/key1")); -// } - -// [Fact] -// public void generate_reply_uri_for_durable() -// { -// new KafkaEndpoint -// { -// SubscriptionName = "ex1", -// TopicName = "key1", -// IsDurable = true -// }.ReplyUri().ShouldBe(new Uri("ckafka://topic/key1/durable")); - - -// } - -// } -//} +using System; +using Jasper.ConfluentKafka; +using Shouldly; +using Xunit; + +namespace Jasper.Kafka.Tests +{ + public class KafkaEndpointTester + { + [Fact] + public void parse_non_durable_uri() + { + var endpoint = new KafkaEndpoint(); + endpoint.Parse(new Uri("kafka://topic/key1")); + + endpoint.IsDurable.ShouldBeFalse(); + endpoint.TopicName.ShouldBe("key1"); + } + + [Fact] + public void parse_durable_uri() + { + var endpoint = new KafkaEndpoint(); + endpoint.Parse(new Uri("kafka://topic/key1/durable")); + + endpoint.IsDurable.ShouldBeTrue(); + endpoint.TopicName.ShouldBe("key1"); + } + + [Fact] + public void build_uri_for_subscription_and_topic() + { + new KafkaEndpoint + { + TopicName = "key1" + } + .Uri.ShouldBe(new Uri("kafka://topic/key1")); + } + + [Fact] + public void generate_reply_uri_for_non_durable() + { + new KafkaEndpoint + { + TopicName = "key1" + } + .ReplyUri().ShouldBe(new Uri("kafka://topic/key1")); + } + + [Fact] + public void generate_reply_uri_for_durable() + { + new KafkaEndpoint + { + TopicName = "key1", + IsDurable = true + }.ReplyUri().ShouldBe(new Uri("kafka://topic/key1/durable")); + } + + } +} diff --git a/src/Jasper.ConfluentKafka/KafkaEndpoint.cs b/src/Jasper.ConfluentKafka/KafkaEndpoint.cs index de9b4db4a..7c027ce68 100644 --- a/src/Jasper.ConfluentKafka/KafkaEndpoint.cs +++ b/src/Jasper.ConfluentKafka/KafkaEndpoint.cs @@ -24,7 +24,7 @@ private Uri BuildUri(bool forReply = false) if (TopicName.IsNotEmpty()) { - list.Add(TopicName); + list.Add(TopicToken); list.Add(TopicName.ToLowerInvariant()); } @@ -82,10 +82,7 @@ protected override ISender CreateSender(IMessagingRoot root) throw new NotImplementedException(); } - public override Uri ReplyUri() - { - throw new NotImplementedException(); - } + public override Uri ReplyUri() => BuildUri(true); } public class KafkaEndpoint : KafkaEndpoint From b6b5a59939d9e657a225e044bedef5bf174843a5 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 17 Apr 2020 15:22:56 -0400 Subject: [PATCH 12/82] Add a basic kafka serilaizer --- .../Serialization/DefaultJsonSerializer.cs | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 src/Jasper.ConfluentKafka/Serialization/DefaultJsonSerializer.cs diff --git a/src/Jasper.ConfluentKafka/Serialization/DefaultJsonSerializer.cs b/src/Jasper.ConfluentKafka/Serialization/DefaultJsonSerializer.cs new file mode 100644 index 000000000..1078da885 --- /dev/null +++ b/src/Jasper.ConfluentKafka/Serialization/DefaultJsonSerializer.cs @@ -0,0 +1,36 @@ +using System; +using System.Text; +using System.Threading.Tasks; +using Confluent.Kafka; +using Confluent.Kafka.SyncOverAsync; +using Newtonsoft.Json; + +namespace Jasper.ConfluentKafka.Serialization +{ + internal class DefaultJsonSerializer : IAsyncSerializer + { + public Task SerializeAsync(T data, SerializationContext context) + { + var json = JsonConvert.SerializeObject(data); + return Task.FromResult(Encoding.UTF8.GetBytes(json)); + } + + public ISerializer AsSyncOverAsync() + { + return new SyncOverAsyncSerializer(this); + } + } + + internal class DefaultJsonDeserializer : IAsyncDeserializer + { + public IDeserializer AsSyncOverAsync() + { + return new SyncOverAsyncDeserializer(this); + } + + public Task DeserializeAsync(ReadOnlyMemory data, bool isNull, SerializationContext context) + { + return Task.FromResult(JsonConvert.DeserializeObject(Encoding.UTF8.GetString(data.ToArray()))); + } + } +} From 12dd5f7fbe7468a34ffee03d26eeb3a37201cff7 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 17 Apr 2020 15:23:24 -0400 Subject: [PATCH 13/82] Remvoe transport from sender (for now) --- .../Internal/ConfluentKafkaSender.cs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs index d7f50270f..db282fef0 100644 --- a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs +++ b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs @@ -4,7 +4,9 @@ using System.Threading.Tasks; using System.Threading.Tasks.Dataflow; using Confluent.Kafka; +using Confluent.Kafka.SyncOverAsync; using Jasper.ConfluentKafka.Exceptions; +using Jasper.ConfluentKafka.Serialization; using Jasper.Logging; using Jasper.Transports; using Jasper.Transports.Sending; @@ -15,20 +17,19 @@ public class ConfluentKafkaSender : ISender { private readonly ITransportProtocol> _protocol; private readonly KafkaEndpoint _endpoint; - private readonly KafkaTransport _transport; private readonly ITransportLogger _logger; private readonly CancellationToken _cancellation; private ActionBlock _sending; private ISenderCallback _callback; private IProducer _publisher; - public ConfluentKafkaSender(KafkaEndpoint endpoint, KafkaTransport transport, ITransportLogger logger, CancellationToken cancellation) + public ConfluentKafkaSender(KafkaEndpoint endpoint, ITransportLogger logger, CancellationToken cancellation) { _endpoint = endpoint; - _transport = transport; _logger = logger; _cancellation = cancellation; Destination = endpoint.Uri; + _protocol = new KafkaTransportProtocol(); } public void Dispose() @@ -45,7 +46,10 @@ public void Start(ISenderCallback callback) { _callback = callback; - _publisher = new ProducerBuilder(_endpoint.ProducerConfig).Build(); + _publisher = new ProducerBuilder(_endpoint.ProducerConfig) + .SetKeySerializer(new DefaultJsonSerializer().AsSyncOverAsync()) + .SetValueSerializer(new DefaultJsonSerializer().AsSyncOverAsync()) + .Build(); _sending = new ActionBlock(sendBySession, new ExecutionDataflowBlockOptions { From 4526acf33cc39342b10b8b69175ac7878f765c8e Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 17 Apr 2020 15:23:58 -0400 Subject: [PATCH 14/82] Use generic KafkaEndpoint when configuring --- .../Jasper.ConfluentKafka.csproj | 1 + src/Jasper.ConfluentKafka/KafkaEndpoint.cs | 5 +- src/Jasper.ConfluentKafka/KafkaTransport.cs | 21 +++- .../KafkaTransportConfigurationExtensions.cs | 108 ++++++++++++++++++ .../KafkaTransportProtocol.cs | 21 ++-- 5 files changed, 135 insertions(+), 21 deletions(-) create mode 100644 src/Jasper.ConfluentKafka/KafkaTransportConfigurationExtensions.cs diff --git a/src/Jasper.ConfluentKafka/Jasper.ConfluentKafka.csproj b/src/Jasper.ConfluentKafka/Jasper.ConfluentKafka.csproj index 8417268ba..791a43570 100644 --- a/src/Jasper.ConfluentKafka/Jasper.ConfluentKafka.csproj +++ b/src/Jasper.ConfluentKafka/Jasper.ConfluentKafka.csproj @@ -7,6 +7,7 @@ + diff --git a/src/Jasper.ConfluentKafka/KafkaEndpoint.cs b/src/Jasper.ConfluentKafka/KafkaEndpoint.cs index 7c027ce68..22b18ca9b 100644 --- a/src/Jasper.ConfluentKafka/KafkaEndpoint.cs +++ b/src/Jasper.ConfluentKafka/KafkaEndpoint.cs @@ -16,7 +16,7 @@ public class KafkaEndpoint : Endpoint { private const string TopicToken = "topic"; public string TopicName { get; set; } - public ProducerConfig ProducerConfig { get; private set; } + public ProducerConfig ProducerConfig { get; set; } public override Uri Uri => BuildUri(); private Uri BuildUri(bool forReply = false) { @@ -91,12 +91,11 @@ public class KafkaEndpoint : KafkaEndpoint protected internal override void StartListening(IMessagingRoot root, ITransportRuntime runtime) { - throw new NotImplementedException(); } protected override ISender CreateSender(IMessagingRoot root) { - return new ConfluentKafkaSender(this, Parent, root.TransportLogger, root.Cancellation); + return new ConfluentKafkaSender(this, root.TransportLogger, root.Cancellation); } } } diff --git a/src/Jasper.ConfluentKafka/KafkaTransport.cs b/src/Jasper.ConfluentKafka/KafkaTransport.cs index 847ef84f7..67103c1eb 100644 --- a/src/Jasper.ConfluentKafka/KafkaTransport.cs +++ b/src/Jasper.ConfluentKafka/KafkaTransport.cs @@ -1,6 +1,6 @@ using System; using System.Collections.Generic; -using Baseline; +using Confluent.Kafka; using Jasper.ConfluentKafka.Internal; using Jasper.Transports; @@ -13,22 +13,31 @@ public static class Protocols } public class KafkaTransport : TransportBase { - private readonly LightweightCache _endpoints; + private readonly Dictionary _endpoints; public KafkaTopicRouter Topics { get; } = new KafkaTopicRouter(); + public Config Config { get; set; } public KafkaTransport() : base(Protocols.Kafka) { + _endpoints = new Dictionary(); } - protected override IEnumerable endpoints() => _endpoints; + protected override IEnumerable endpoints() => _endpoints.Values; protected override KafkaEndpoint findEndpointByUri(Uri uri) => _endpoints[uri]; - public KafkaEndpoint EndpointForTopic(string topicName) + public KafkaEndpoint EndpointForTopic(string topicName, ProducerConfig conifg) { - var uri = new KafkaEndpoint { TopicName = topicName }.Uri; - return _endpoints[uri]; + var endpoint = new KafkaEndpoint + { + TopicName = topicName, + ProducerConfig = conifg + }; + + _endpoints.Add(endpoint.Uri, endpoint); + + return endpoint; } } } diff --git a/src/Jasper.ConfluentKafka/KafkaTransportConfigurationExtensions.cs b/src/Jasper.ConfluentKafka/KafkaTransportConfigurationExtensions.cs new file mode 100644 index 000000000..61d2b5408 --- /dev/null +++ b/src/Jasper.ConfluentKafka/KafkaTransportConfigurationExtensions.cs @@ -0,0 +1,108 @@ +using System; +using Baseline; +using Confluent.Kafka; +using Jasper.Configuration; + +namespace Jasper.ConfluentKafka +{ + public static class KafkaTransportConfigurationExtensions + {/// + /// Quick access to the kafka Transport within this application. + /// This is for advanced usage + /// + /// + /// + internal static KafkaTransport KafkaTransport(this IEndpoints endpoints) + { + var transports = endpoints.As(); + + var transport = transports.Get(); + + if (transport == null) + { + transport = new KafkaTransport(); + transports.Add(transport); + } + + transports.Subscribers.Fill(transport.Topics); + + return transport; + } + /// + /// Configure connection and authentication information about the Azure Service Bus usage + /// within this Jasper application + /// + /// + /// + public static void ConfigureKafka(this IEndpoints endpoints, Action configure) + { + var transport = endpoints.KafkaTransport(); + endpoints.As().Subscribers.Fill(transport.Topics); + configure(transport); + } + + /// + /// Configure connection and authentication information about the Azure Service Bus usage + /// within this Jasper application + /// + /// + /// + public static void ConfigureKafka(this IEndpoints endpoints) + { + endpoints.ConfigureKafka(_ => + { + + }); + } + + /// + /// Listen for incoming messages at the designated Rabbit MQ queue by name + /// + /// + /// The name of the Rabbit MQ queue + /// + //public static KafkaListenerConfiguration ListenToKafkaTopic(this IEndpoints endpoints, string topicName, string subscriptionName) + //{ + // var raw = new KafkaEndpoint { TopicName = topicName, SubscriptionName = subscriptionName }.Uri; + // var endpoint = endpoints.KafkaTransport().GetOrCreateEndpoint(raw); + // endpoint.IsListener = true; + // return new KafkaListenerConfiguration((KafkaEndpoint)endpoint); + //} + + /// + /// Publish matching messages to Rabbit MQ using the named routing key or queue name and + /// optionally an exchange + /// + /// + /// This is used as the topic name when publishing. Can be either a binding key or a queue name or a static topic name if the exchange is topic-based + /// Optional, you only need to supply this if you are using a non-default exchange + /// + public static KafkaSubscriberConfiguration ToKafkaTopic(this IPublishToExpression publishing, string topicName, ProducerConfig producerConfig) + { + var transports = publishing.As().Parent; + var transport = transports.Get(); + var endpoint = transport.EndpointForTopic(topicName, producerConfig); + + // This is necessary unfortunately to hook up the subscription rules + publishing.To(endpoint.Uri); + + return new KafkaSubscriberConfiguration(endpoint); + } + + /// + /// Publish matching messages to Azure Service Bus using the topic name derived from the message and + /// + /// + /// + public static TopicRouterConfiguration ToKafkaTopics(this IPublishToExpression publishing) + { + var transports = publishing.As().Parent; + + var router = transports.KafkaTransport().Topics; + + publishing.ViaRouter(router); + + return new TopicRouterConfiguration(router, transports); + } + } +} diff --git a/src/Jasper.ConfluentKafka/KafkaTransportProtocol.cs b/src/Jasper.ConfluentKafka/KafkaTransportProtocol.cs index 517e8e530..bea2ee387 100644 --- a/src/Jasper.ConfluentKafka/KafkaTransportProtocol.cs +++ b/src/Jasper.ConfluentKafka/KafkaTransportProtocol.cs @@ -4,19 +4,16 @@ namespace Jasper.ConfluentKafka { - public class KafkaTransportProtocol : ITransportProtocol> + public class KafkaTransportProtocol : ITransportProtocol> { - public Message WriteFromEnvelope(Envelope envelope) - { - var msg = new Message(); - - msg.Value = envelope.Data; - - - return msg; - } + public Message WriteFromEnvelope(Envelope envelope) => + new Message + { + Headers = new Headers(), + Value = (TVal) envelope.Message + }; - public Envelope ReadEnvelope(Message message) + public Envelope ReadEnvelope(Message message) { var env = new Envelope(); @@ -25,7 +22,7 @@ public Envelope ReadEnvelope(Message message) env.Headers.Add(header.Key, Encoding.UTF8.GetString(header.GetValueBytes())); } - env.Data = message.Value; + env.Message = message.Value; return env; } From 4e345a0bb6181f3e7f1c61ea18e098adf2cc98c6 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 17 Apr 2020 15:24:09 -0400 Subject: [PATCH 15/82] Add basic end-to-end test --- src/Jasper.ConfluentKafka.Tests/end_to_end.cs | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 src/Jasper.ConfluentKafka.Tests/end_to_end.cs diff --git a/src/Jasper.ConfluentKafka.Tests/end_to_end.cs b/src/Jasper.ConfluentKafka.Tests/end_to_end.cs new file mode 100644 index 000000000..268c707c0 --- /dev/null +++ b/src/Jasper.ConfluentKafka.Tests/end_to_end.cs @@ -0,0 +1,110 @@ +using System; +using System.Threading.Tasks; +using Baseline.Dates; +using Confluent.Kafka; +using Jasper.Tracking; +using Jasper.Util; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Shouldly; +using TestingSupport; +using TestingSupport.Compliance; +using Xunit; + +namespace Jasper.ConfluentKafka.Tests +{ + [Obsolete("try to replace with compliance tests")] + public class end_to_end + { + private static ProducerConfig ProducerConfig = new ProducerConfig + { + BootstrapServers = + "b-1.jj-test.y7lv7k.c5.kafka.us-east-1.amazonaws.com:9094,b-2.jj-test.y7lv7k.c5.kafka.us-east-1.amazonaws.com:9094", + SecurityProtocol = SecurityProtocol.Ssl + }; + + public class Sender : JasperOptions + { + + public Sender() + { + Endpoints.ConfigureKafka(); + + } + + public string QueueName { get; set; } + } + + public class Receiver : JasperOptions + { + public Receiver(string queueName) + { + Endpoints.ConfigureKafka(); + } + } + + + public class KafkaSendingComplianceTests : SendingCompliance + { + public KafkaSendingComplianceTests() : base($"kafka://topic/messages".ToUri()) + { + var sender = new Sender(); + + SenderIs(sender); + + var receiver = new Receiver(sender.QueueName); + + ReceiverIs(receiver); + } + } + + + + + + // SAMPLE: can_stop_and_start_ASB + [Fact] + public async Task can_stop_and_start() + { + using (var host = JasperHost.For()) + { + await host + // The TrackActivity() method starts a Fluent Interface + // that gives you fine-grained control over the + // message tracking + .TrackActivity() + .Timeout(30.Seconds()) + // Include the external transports in the determination + // of "completion" + .IncludeExternalTransports() + .SendMessageAndWait(new ColorChosen {Name = "Red"}); + + var colors = host.Get(); + + colors.Name.ShouldBe("Red"); + } + } + + // ENDSAMPLE + public class KafkaUsingApp : JasperOptions + { + public KafkaUsingApp() + { + Endpoints.ConfigureKafka(); + Endpoints.PublishAllMessages().ToKafkaTopic("messages", ProducerConfig); + + Handlers.IncludeType(); + + Services.AddSingleton(); + + Extensions.UseMessageTrackingTestingSupport(); + } + + public override void Configure(IHostEnvironment hosting, IConfiguration config) + { + //Endpoints.ConfigureAzureServiceBus(config.GetValue("AzureServiceBusConnectionString")); + } + } + } +} From 1fc06e2093e0a4082436fa1fea039bf75c452fa1 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 17 Apr 2020 16:33:39 -0400 Subject: [PATCH 16/82] Create KafkaListener --- src/Jasper.ConfluentKafka.Tests/end_to_end.cs | 12 ++- .../Internal/ConfluentKafkaListener.cs | 96 +++++++++++++++++++ src/Jasper.ConfluentKafka/KafkaEndpoint.cs | 1 + .../KafkaListenerConfiguration.cs | 12 +++ src/Jasper.ConfluentKafka/KafkaTransport.cs | 29 ++++-- .../KafkaTransportConfigurationExtensions.cs | 17 ++-- 6 files changed, 150 insertions(+), 17 deletions(-) create mode 100644 src/Jasper.ConfluentKafka/Internal/ConfluentKafkaListener.cs create mode 100644 src/Jasper.ConfluentKafka/KafkaListenerConfiguration.cs diff --git a/src/Jasper.ConfluentKafka.Tests/end_to_end.cs b/src/Jasper.ConfluentKafka.Tests/end_to_end.cs index 268c707c0..41ae7d21f 100644 --- a/src/Jasper.ConfluentKafka.Tests/end_to_end.cs +++ b/src/Jasper.ConfluentKafka.Tests/end_to_end.cs @@ -17,13 +17,20 @@ namespace Jasper.ConfluentKafka.Tests [Obsolete("try to replace with compliance tests")] public class end_to_end { + private static string KafkaServer = "b-1.jj-test.y7lv7k.c5.kafka.us-east-1.amazonaws.com:9094,b-2.jj-test.y7lv7k.c5.kafka.us-east-1.amazonaws.com:9094"; private static ProducerConfig ProducerConfig = new ProducerConfig { - BootstrapServers = - "b-1.jj-test.y7lv7k.c5.kafka.us-east-1.amazonaws.com:9094,b-2.jj-test.y7lv7k.c5.kafka.us-east-1.amazonaws.com:9094", + BootstrapServers = KafkaServer, SecurityProtocol = SecurityProtocol.Ssl }; + private static ConsumerConfig ConsumerConfig = new ConsumerConfig + { + BootstrapServers = KafkaServer, + SecurityProtocol = SecurityProtocol.Ssl, + GroupId = nameof(end_to_end), + }; + public class Sender : JasperOptions { @@ -92,6 +99,7 @@ public class KafkaUsingApp : JasperOptions public KafkaUsingApp() { Endpoints.ConfigureKafka(); + Endpoints.ListenToKafkaTopic("messages", ConsumerConfig); Endpoints.PublishAllMessages().ToKafkaTopic("messages", ProducerConfig); Handlers.IncludeType(); diff --git a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaListener.cs b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaListener.cs new file mode 100644 index 000000000..3b5e84f8b --- /dev/null +++ b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaListener.cs @@ -0,0 +1,96 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Confluent.Kafka; +using Jasper.ConfluentKafka; +using Jasper.ConfluentKafka.Serialization; +using Jasper.Logging; +using Jasper.Transports; + +namespace Jasper.Kafka.Internal +{ + public class ConfluentKafkaListener : IListener + { + private readonly CancellationToken _cancellation; + + private readonly KafkaEndpoint _endpoint; + private readonly ITransportLogger _logger; + private readonly KafkaTransportProtocol _protocol = new KafkaTransportProtocol(); + private IReceiverCallback _callback; + private IConsumer _consumer; + private Task _consumerTask; + + public ConfluentKafkaListener(KafkaEndpoint endpoint, ITransportLogger logger, CancellationToken cancellation) + { + _endpoint = endpoint; + _logger = logger; + _cancellation = cancellation; + Address = endpoint.Uri; + } + + + public void Dispose() + { + _consumerTask?.Dispose(); + _consumer?.Dispose(); + } + + public Uri Address { get; } + public ListeningStatus Status { get; set; } + + public void Start(IReceiverCallback callback) + { + _callback = callback; + + _consumer = new ConsumerBuilder(_endpoint.ConsumerConfig) + .SetKeyDeserializer(new DefaultJsonDeserializer().AsSyncOverAsync()) + .SetValueDeserializer(new DefaultJsonDeserializer().AsSyncOverAsync()) + .Build(); + + _consumer.Subscribe(_endpoint.TopicName); + + _consumerTask = ConsumeAsync(); + } + + private async Task ConsumeAsync() + { + while (!_cancellation.IsCancellationRequested) + { + ConsumeResult message; + try + { + message = _consumer.Consume(); + } + catch (Exception ex) + { + _logger.LogException(ex, message: $"Error consuming message from Kafka topic {_endpoint.TopicName}"); + return; + } + + Envelope envelope; + + try + { + envelope = _protocol.ReadEnvelope(message.Message); + } + catch (Exception ex) + { + _logger.LogException(ex, message: $"Error trying to map an incoming Kafka {_endpoint.TopicName} Topic message to an Envelope. See the Dead Letter Queue"); + return; + } + + try + { + await _callback.Received(Address, new[] { envelope }); + + _consumer.Commit(); + } + catch (Exception e) + { + _logger.LogException(e, envelope.Id, "Error trying to receive a message from " + Address); + } + } + } + + } +} diff --git a/src/Jasper.ConfluentKafka/KafkaEndpoint.cs b/src/Jasper.ConfluentKafka/KafkaEndpoint.cs index 22b18ca9b..c1b3b83c9 100644 --- a/src/Jasper.ConfluentKafka/KafkaEndpoint.cs +++ b/src/Jasper.ConfluentKafka/KafkaEndpoint.cs @@ -17,6 +17,7 @@ public class KafkaEndpoint : Endpoint private const string TopicToken = "topic"; public string TopicName { get; set; } public ProducerConfig ProducerConfig { get; set; } + public ConsumerConfig ConsumerConfig { get; set; } public override Uri Uri => BuildUri(); private Uri BuildUri(bool forReply = false) { diff --git a/src/Jasper.ConfluentKafka/KafkaListenerConfiguration.cs b/src/Jasper.ConfluentKafka/KafkaListenerConfiguration.cs new file mode 100644 index 000000000..0541f62ad --- /dev/null +++ b/src/Jasper.ConfluentKafka/KafkaListenerConfiguration.cs @@ -0,0 +1,12 @@ +using Jasper.Configuration; +using Jasper.ConfluentKafka; + +namespace Jasper.Kafka +{ + public class KafkaListenerConfiguration : ListenerConfiguration + { + public KafkaListenerConfiguration(KafkaEndpoint endpoint) : base(endpoint) + { + } + } +} diff --git a/src/Jasper.ConfluentKafka/KafkaTransport.cs b/src/Jasper.ConfluentKafka/KafkaTransport.cs index 67103c1eb..4324271de 100644 --- a/src/Jasper.ConfluentKafka/KafkaTransport.cs +++ b/src/Jasper.ConfluentKafka/KafkaTransport.cs @@ -27,17 +27,32 @@ public KafkaTransport() : base(Protocols.Kafka) protected override KafkaEndpoint findEndpointByUri(Uri uri) => _endpoints[uri]; - public KafkaEndpoint EndpointForTopic(string topicName, ProducerConfig conifg) + public KafkaEndpoint EndpointForTopic(string topicName, ProducerConfig producerConifg) => + AddOrUpdateEndpoint(topicName, c => c.ProducerConfig = producerConifg); + + public KafkaEndpoint EndpointForTopic(string topicName, ConsumerConfig consumerConifg) => + AddOrUpdateEndpoint(topicName, c => c.ConsumerConfig = consumerConifg); + + KafkaEndpoint AddOrUpdateEndpoint(string topicName, Action> configure) { var endpoint = new KafkaEndpoint - { - TopicName = topicName, - ProducerConfig = conifg - }; - - _endpoints.Add(endpoint.Uri, endpoint); + { + TopicName = topicName + }; + + if (_endpoints.ContainsKey(endpoint.Uri)) + { + endpoint = (KafkaEndpoint)_endpoints[endpoint.Uri]; + configure(endpoint); + } + else + { + configure(endpoint); + _endpoints.Add(endpoint.Uri, endpoint); + } return endpoint; } + } } diff --git a/src/Jasper.ConfluentKafka/KafkaTransportConfigurationExtensions.cs b/src/Jasper.ConfluentKafka/KafkaTransportConfigurationExtensions.cs index 61d2b5408..949400e7e 100644 --- a/src/Jasper.ConfluentKafka/KafkaTransportConfigurationExtensions.cs +++ b/src/Jasper.ConfluentKafka/KafkaTransportConfigurationExtensions.cs @@ -2,6 +2,7 @@ using Baseline; using Confluent.Kafka; using Jasper.Configuration; +using Jasper.Kafka; namespace Jasper.ConfluentKafka { @@ -61,14 +62,13 @@ public static void ConfigureKafka(this IEndpoints endpoints) /// /// The name of the Rabbit MQ queue /// - //public static KafkaListenerConfiguration ListenToKafkaTopic(this IEndpoints endpoints, string topicName, string subscriptionName) - //{ - // var raw = new KafkaEndpoint { TopicName = topicName, SubscriptionName = subscriptionName }.Uri; - // var endpoint = endpoints.KafkaTransport().GetOrCreateEndpoint(raw); - // endpoint.IsListener = true; - // return new KafkaListenerConfiguration((KafkaEndpoint)endpoint); - //} - + public static KafkaListenerConfiguration ListenToKafkaTopic(this IEndpoints endpoints, string topicName, ConsumerConfig consumerConfig) + { + var endpoint = endpoints.KafkaTransport().EndpointForTopic(topicName, consumerConfig); + endpoint.IsListener = true; + return new KafkaListenerConfiguration((KafkaEndpoint)endpoint); + } + /// /// Publish matching messages to Rabbit MQ using the named routing key or queue name and /// optionally an exchange @@ -104,5 +104,6 @@ public static TopicRouterConfiguration ToKafkaTopi return new TopicRouterConfiguration(router, transports); } + } } From e8fef65da9f4e9761e1fc6b306eed386832d8344 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 17 Apr 2020 17:19:50 -0400 Subject: [PATCH 17/82] Make sure Jasper MessageId goes across the wire --- .../KafkaTransportProtocol.cs | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/Jasper.ConfluentKafka/KafkaTransportProtocol.cs b/src/Jasper.ConfluentKafka/KafkaTransportProtocol.cs index bea2ee387..c6d8de52b 100644 --- a/src/Jasper.ConfluentKafka/KafkaTransportProtocol.cs +++ b/src/Jasper.ConfluentKafka/KafkaTransportProtocol.cs @@ -1,3 +1,6 @@ +using System; +using System.Collections.Generic; +using System.Linq; using System.Text; using Confluent.Kafka; using Jasper.Transports; @@ -6,22 +9,37 @@ namespace Jasper.ConfluentKafka { public class KafkaTransportProtocol : ITransportProtocol> { - public Message WriteFromEnvelope(Envelope envelope) => - new Message + private const string JasperMessageIdHeader = "Jasper_MessageId"; + public Message WriteFromEnvelope(Envelope envelope) + { + var message = new Message { Headers = new Headers(), Value = (TVal) envelope.Message }; + foreach (KeyValuePair h in envelope.Headers) + { + Header header = new Header(h.Key, Encoding.UTF8.GetBytes(h.Value)); + message.Headers.Add(header); + } + + message.Headers.Add(JasperMessageIdHeader, Encoding.UTF8.GetBytes(envelope.Id.ToString())); + + return message; + } + public Envelope ReadEnvelope(Message message) { var env = new Envelope(); - foreach (var header in message.Headers) + foreach (var header in message.Headers.Where(h => !h.Key.StartsWith("Jasper"))) { env.Headers.Add(header.Key, Encoding.UTF8.GetString(header.GetValueBytes())); } + var messageIdHeader = message.Headers.Single(h => h.Key.Equals(JasperMessageIdHeader)); + env.Id = Guid.Parse(Encoding.UTF8.GetString(messageIdHeader.GetValueBytes())); env.Message = message.Value; return env; From d27513cd91fe4d491595f2e874a455b093fe73d2 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 17 Apr 2020 17:20:13 -0400 Subject: [PATCH 18/82] Handle commit exception in listener --- .../Internal/ConfluentKafkaListener.cs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaListener.cs b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaListener.cs index 3b5e84f8b..844fc224a 100644 --- a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaListener.cs +++ b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaListener.cs @@ -50,6 +50,8 @@ public void Start(IReceiverCallback callback) _consumer.Subscribe(_endpoint.TopicName); _consumerTask = ConsumeAsync(); + + Thread.Sleep(1000); // let the consumer start consuming } private async Task ConsumeAsync() @@ -59,7 +61,7 @@ private async Task ConsumeAsync() ConsumeResult message; try { - message = _consumer.Consume(); + message = await Task.Run(() => _consumer.Consume(), _cancellation); } catch (Exception ex) { @@ -81,10 +83,18 @@ private async Task ConsumeAsync() try { - await _callback.Received(Address, new[] { envelope }); + await _callback.Received(Address, new[] {envelope}); _consumer.Commit(); } + catch (KafkaException ke) + { + if (ke.Error?.Code == ErrorCode.Local_NoOffset) + { + return; + } + _logger.LogException(ke, envelope.Id, "Error trying to receive a message from " + Address); + } catch (Exception e) { _logger.LogException(e, envelope.Id, "Error trying to receive a message from " + Address); From 82489d153928104c7b7a1c68de9daa9a5eb695cc Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 17 Apr 2020 17:20:28 -0400 Subject: [PATCH 19/82] Make sure to pass message to publisher --- src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs index db282fef0..b1d48ccd1 100644 --- a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs +++ b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs @@ -112,7 +112,7 @@ private async Task sendBySession(Envelope envelope) } else { - await _publisher.ProduceAsync(_endpoint.TopicName, new Message(), _cancellation); + await _publisher.ProduceAsync(_endpoint.TopicName, message, _cancellation); } await _callback.Successful(envelope); From 5d6346f60bbd5b34887a94f13dd2bfd4117c40fb Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 17 Apr 2020 17:20:41 -0400 Subject: [PATCH 20/82] Add listener on StartListening --- src/Jasper.ConfluentKafka/KafkaEndpoint.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Jasper.ConfluentKafka/KafkaEndpoint.cs b/src/Jasper.ConfluentKafka/KafkaEndpoint.cs index c1b3b83c9..f5e5afd44 100644 --- a/src/Jasper.ConfluentKafka/KafkaEndpoint.cs +++ b/src/Jasper.ConfluentKafka/KafkaEndpoint.cs @@ -5,6 +5,7 @@ using Confluent.Kafka; using Jasper.Configuration; using Jasper.ConfluentKafka.Internal; +using Jasper.Kafka.Internal; using Jasper.Runtime; using Jasper.Transports; using Jasper.Transports.Sending; @@ -92,6 +93,10 @@ public class KafkaEndpoint : KafkaEndpoint protected internal override void StartListening(IMessagingRoot root, ITransportRuntime runtime) { + if (!IsListener) return; + + var listener = new ConfluentKafkaListener(this, root.TransportLogger, root.Cancellation); + runtime.AddListener(listener, this); } protected override ISender CreateSender(IMessagingRoot root) From f92f1d5663257a9baed4e512affe192b7d8ed6c2 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 17 Apr 2020 17:34:40 -0400 Subject: [PATCH 21/82] Leverage ITransporProtocol<> in AzureServiceBus ext --- src/Jasper.AzureServiceBus.Tests/Samples.cs | 4 +-- .../AzureServiceBusEndpoint.cs | 3 ++- .../AzureServiceBusListenerConfiguration.cs | 6 +++-- .../AzureServiceBusSubscriberConfiguration.cs | 6 +++-- .../IAzureServiceBusProtocol.cs | 27 ------------------- .../Internal/AzureServiceBusListener.cs | 2 +- .../Internal/AzureServiceBusSender.cs | 3 ++- .../DefaultAzureServiceBusProtocol.cs | 3 ++- 8 files changed, 17 insertions(+), 37 deletions(-) delete mode 100644 src/Jasper.AzureServiceBus/IAzureServiceBusProtocol.cs diff --git a/src/Jasper.AzureServiceBus.Tests/Samples.cs b/src/Jasper.AzureServiceBus.Tests/Samples.cs index bdb01f59d..21a8338f9 100644 --- a/src/Jasper.AzureServiceBus.Tests/Samples.cs +++ b/src/Jasper.AzureServiceBus.Tests/Samples.cs @@ -3,10 +3,10 @@ using Baseline; using Jasper.Attributes; using Jasper.AzureServiceBus.Internal; +using Jasper.Transports; using Microsoft.Azure.ServiceBus; using Microsoft.Azure.ServiceBus.Primitives; using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; namespace Jasper.AzureServiceBus.Tests @@ -64,7 +64,7 @@ public AzureServiceBusTopicSendingApp( ) - public class MySpecialProtocol : IAzureServiceBusProtocol + public class MySpecialProtocol : ITransportProtocol { public Message WriteFromEnvelope(Envelope envelope) { diff --git a/src/Jasper.AzureServiceBus/AzureServiceBusEndpoint.cs b/src/Jasper.AzureServiceBus/AzureServiceBusEndpoint.cs index 81e88493c..45e988d13 100644 --- a/src/Jasper.AzureServiceBus/AzureServiceBusEndpoint.cs +++ b/src/Jasper.AzureServiceBus/AzureServiceBusEndpoint.cs @@ -8,6 +8,7 @@ using Jasper.Transports; using Jasper.Transports.Sending; using Jasper.Util; +using Microsoft.Azure.ServiceBus; namespace Jasper.AzureServiceBus { @@ -33,7 +34,7 @@ public AzureServiceBusEndpoint(Uri uri) : base(uri) public string QueueName { get; set; } public string TopicName { get; set; } - public IAzureServiceBusProtocol Protocol { get; set; } = new DefaultAzureServiceBusProtocol(); + public ITransportProtocol Protocol { get; set; } = new DefaultAzureServiceBusProtocol(); public override Uri Uri => buildUri(false); diff --git a/src/Jasper.AzureServiceBus/AzureServiceBusListenerConfiguration.cs b/src/Jasper.AzureServiceBus/AzureServiceBusListenerConfiguration.cs index 3291239fc..8e6e3d80f 100644 --- a/src/Jasper.AzureServiceBus/AzureServiceBusListenerConfiguration.cs +++ b/src/Jasper.AzureServiceBus/AzureServiceBusListenerConfiguration.cs @@ -1,4 +1,6 @@ using Jasper.Configuration; +using Jasper.Transports; +using Microsoft.Azure.ServiceBus; namespace Jasper.AzureServiceBus { @@ -14,7 +16,7 @@ public AzureServiceBusListenerConfiguration(AzureServiceBusEndpoint endpoint) : /// /// /// - public AzureServiceBusListenerConfiguration Protocol() where T : IAzureServiceBusProtocol, new() + public AzureServiceBusListenerConfiguration Protocol() where T : ITransportProtocol, new() { return Protocol(new T()); } @@ -25,7 +27,7 @@ public AzureServiceBusListenerConfiguration(AzureServiceBusEndpoint endpoint) : /// /// /// - public AzureServiceBusListenerConfiguration Protocol(IAzureServiceBusProtocol protocol) + public AzureServiceBusListenerConfiguration Protocol(ITransportProtocol protocol) { endpoint.Protocol = protocol; return this; diff --git a/src/Jasper.AzureServiceBus/AzureServiceBusSubscriberConfiguration.cs b/src/Jasper.AzureServiceBus/AzureServiceBusSubscriberConfiguration.cs index 9824fb38a..95111e368 100644 --- a/src/Jasper.AzureServiceBus/AzureServiceBusSubscriberConfiguration.cs +++ b/src/Jasper.AzureServiceBus/AzureServiceBusSubscriberConfiguration.cs @@ -1,4 +1,6 @@ using Jasper.Configuration; +using Jasper.Transports; +using Microsoft.Azure.ServiceBus; namespace Jasper.AzureServiceBus { @@ -14,7 +16,7 @@ public AzureServiceBusSubscriberConfiguration(AzureServiceBusEndpoint endpoint) /// /// /// - public AzureServiceBusSubscriberConfiguration Protocol() where T : IAzureServiceBusProtocol, new() + public AzureServiceBusSubscriberConfiguration Protocol() where T : ITransportProtocol, new() { return Protocol(new T()); } @@ -25,7 +27,7 @@ public AzureServiceBusSubscriberConfiguration(AzureServiceBusEndpoint endpoint) /// /// /// - public AzureServiceBusSubscriberConfiguration Protocol(IAzureServiceBusProtocol protocol) + public AzureServiceBusSubscriberConfiguration Protocol(ITransportProtocol protocol) { _endpoint.Protocol = protocol; return this; diff --git a/src/Jasper.AzureServiceBus/IAzureServiceBusProtocol.cs b/src/Jasper.AzureServiceBus/IAzureServiceBusProtocol.cs deleted file mode 100644 index 331a50cbc..000000000 --- a/src/Jasper.AzureServiceBus/IAzureServiceBusProtocol.cs +++ /dev/null @@ -1,27 +0,0 @@ -using Microsoft.Azure.ServiceBus; - -namespace Jasper.AzureServiceBus -{ - // SAMPLE: IAzureServiceBusProtocol - /// - /// Used to "map" incoming Azure Service Bus Message objects to Jasper Envelopes. Can be implemented to - /// connect Jasper to non-Jasper applications - /// - public interface IAzureServiceBusProtocol - { - /// - /// Creates an Azure Service Bus Message object for a Jasper Envelope - /// - /// - /// - Message WriteFromEnvelope(Envelope envelope); - - /// - /// Creates an Envelope for the incoming Azure Service Bus Message - /// - /// - /// - Envelope ReadEnvelope(Message message); - } - // ENDSAMPLE -} diff --git a/src/Jasper.AzureServiceBus/Internal/AzureServiceBusListener.cs b/src/Jasper.AzureServiceBus/Internal/AzureServiceBusListener.cs index b29fa51db..bd7459d86 100644 --- a/src/Jasper.AzureServiceBus/Internal/AzureServiceBusListener.cs +++ b/src/Jasper.AzureServiceBus/Internal/AzureServiceBusListener.cs @@ -16,7 +16,7 @@ public class AzureServiceBusListener : IListener private readonly IList _clientEntities = new List(); private readonly AzureServiceBusEndpoint _endpoint; private readonly ITransportLogger _logger; - private readonly IAzureServiceBusProtocol _protocol; + private readonly ITransportProtocol _protocol; private readonly AzureServiceBusTransport _transport; private IReceiverCallback _callback; diff --git a/src/Jasper.AzureServiceBus/Internal/AzureServiceBusSender.cs b/src/Jasper.AzureServiceBus/Internal/AzureServiceBusSender.cs index 9c19b0b76..21fc4ad09 100644 --- a/src/Jasper.AzureServiceBus/Internal/AzureServiceBusSender.cs +++ b/src/Jasper.AzureServiceBus/Internal/AzureServiceBusSender.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks.Dataflow; using Baseline; using Jasper.Logging; +using Jasper.Transports; using Jasper.Transports.Sending; using Microsoft.Azure.ServiceBus; using Microsoft.Azure.ServiceBus.Core; @@ -14,7 +15,7 @@ namespace Jasper.AzureServiceBus.Internal { public class AzureServiceBusSender : ISender { - private readonly IAzureServiceBusProtocol _protocol; + private readonly ITransportProtocol _protocol; private readonly AzureServiceBusEndpoint _endpoint; private readonly AzureServiceBusTransport _transport; private readonly ITransportLogger _logger; diff --git a/src/Jasper.AzureServiceBus/Internal/DefaultAzureServiceBusProtocol.cs b/src/Jasper.AzureServiceBus/Internal/DefaultAzureServiceBusProtocol.cs index 010721af5..d85dee696 100644 --- a/src/Jasper.AzureServiceBus/Internal/DefaultAzureServiceBusProtocol.cs +++ b/src/Jasper.AzureServiceBus/Internal/DefaultAzureServiceBusProtocol.cs @@ -1,11 +1,12 @@ using System; using Baseline; +using Jasper.Transports; using Microsoft.Azure.ServiceBus; namespace Jasper.AzureServiceBus.Internal { // SAMPLE: DefaultAzureServiceBusProtocol - public class DefaultAzureServiceBusProtocol : IAzureServiceBusProtocol + public class DefaultAzureServiceBusProtocol : ITransportProtocol { public virtual Message WriteFromEnvelope(Envelope envelope) { From 0ff053277730318eb6f7b6729bc2709579545ac1 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Sat, 18 Apr 2020 13:48:11 -0400 Subject: [PATCH 22/82] Enable setting serializers on publisher --- .../Internal/ConfluentKafkaSender.cs | 16 ++++-- .../Internal/KafkaTopicRouter.cs | 9 ++-- src/Jasper.ConfluentKafka/KafkaEndpoint.cs | 8 ++- src/Jasper.ConfluentKafka/KafkaTransport.cs | 26 +++++++-- .../KafkaTransportConfigurationExtensions.cs | 53 ++++++++++++++----- 5 files changed, 86 insertions(+), 26 deletions(-) diff --git a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs index b1d48ccd1..f28df70bf 100644 --- a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs +++ b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs @@ -23,13 +23,18 @@ public class ConfluentKafkaSender : ISender private ISenderCallback _callback; private IProducer _publisher; - public ConfluentKafkaSender(KafkaEndpoint endpoint, ITransportLogger logger, CancellationToken cancellation) + private readonly ISerializer _keySerializer = new DefaultJsonSerializer().AsSyncOverAsync(); + private readonly ISerializer _valueSerializer = new DefaultJsonSerializer().AsSyncOverAsync(); + + public ConfluentKafkaSender(KafkaEndpoint endpoint, ITransportLogger logger, ISerializer keySerializer, ISerializer valueSerializer, CancellationToken cancellation) { _endpoint = endpoint; _logger = logger; _cancellation = cancellation; Destination = endpoint.Uri; _protocol = new KafkaTransportProtocol(); + _keySerializer = keySerializer; + _valueSerializer = valueSerializer; } public void Dispose() @@ -46,10 +51,11 @@ public void Start(ISenderCallback callback) { _callback = callback; - _publisher = new ProducerBuilder(_endpoint.ProducerConfig) - .SetKeySerializer(new DefaultJsonSerializer().AsSyncOverAsync()) - .SetValueSerializer(new DefaultJsonSerializer().AsSyncOverAsync()) - .Build(); + var publisherBuilder = new ProducerBuilder(_endpoint.ProducerConfig) + .SetKeySerializer(_keySerializer) + .SetValueSerializer(_valueSerializer); + + _publisher = publisherBuilder.Build(); _sending = new ActionBlock(sendBySession, new ExecutionDataflowBlockOptions { diff --git a/src/Jasper.ConfluentKafka/Internal/KafkaTopicRouter.cs b/src/Jasper.ConfluentKafka/Internal/KafkaTopicRouter.cs index 3f461d81a..483a50a01 100644 --- a/src/Jasper.ConfluentKafka/Internal/KafkaTopicRouter.cs +++ b/src/Jasper.ConfluentKafka/Internal/KafkaTopicRouter.cs @@ -1,5 +1,7 @@ using System; +using System.Collections.Generic; using Baseline; +using Confluent.Kafka; using Jasper.Configuration; using Jasper.Runtime.Routing; @@ -7,9 +9,10 @@ namespace Jasper.ConfluentKafka.Internal { public class KafkaTopicRouter : TopicRouter { - public KafkaTopicRouter() - { - } + //Dictionary producerConfigs; + //public KafkaTopicRouter(Dictionary producerConfigs) + //{ + //} public override Uri BuildUriForTopic(string topicName) { diff --git a/src/Jasper.ConfluentKafka/KafkaEndpoint.cs b/src/Jasper.ConfluentKafka/KafkaEndpoint.cs index f5e5afd44..4ae3c8f3e 100644 --- a/src/Jasper.ConfluentKafka/KafkaEndpoint.cs +++ b/src/Jasper.ConfluentKafka/KafkaEndpoint.cs @@ -5,6 +5,7 @@ using Confluent.Kafka; using Jasper.Configuration; using Jasper.ConfluentKafka.Internal; +using Jasper.ConfluentKafka.Serialization; using Jasper.Kafka.Internal; using Jasper.Runtime; using Jasper.Transports; @@ -91,6 +92,11 @@ public class KafkaEndpoint : KafkaEndpoint { internal KafkaTransport Parent { get; set; } + public ISerializer KeySerializer = new DefaultJsonSerializer().AsSyncOverAsync(); + public ISerializer ValueSerializer= new DefaultJsonSerializer().AsSyncOverAsync(); + public IDeserializer KeyDeserializer = new DefaultJsonDeserializer().AsSyncOverAsync(); + public IDeserializer ValueDeserializer = new DefaultJsonDeserializer().AsSyncOverAsync(); + protected internal override void StartListening(IMessagingRoot root, ITransportRuntime runtime) { if (!IsListener) return; @@ -101,7 +107,7 @@ protected internal override void StartListening(IMessagingRoot root, ITransportR protected override ISender CreateSender(IMessagingRoot root) { - return new ConfluentKafkaSender(this, root.TransportLogger, root.Cancellation); + return new ConfluentKafkaSender(this, root.TransportLogger, KeySerializer, ValueSerializer, root.Cancellation); } } } diff --git a/src/Jasper.ConfluentKafka/KafkaTransport.cs b/src/Jasper.ConfluentKafka/KafkaTransport.cs index 4324271de..114d98fe4 100644 --- a/src/Jasper.ConfluentKafka/KafkaTransport.cs +++ b/src/Jasper.ConfluentKafka/KafkaTransport.cs @@ -1,8 +1,10 @@ using System; using System.Collections.Generic; using Confluent.Kafka; +using Confluent.SchemaRegistry; using Jasper.ConfluentKafka.Internal; using Jasper.Transports; +using Lamar.IoC.Instances; namespace Jasper.ConfluentKafka { @@ -11,13 +13,12 @@ public static class Protocols public const string Kafka = "kafka"; } + public class KafkaTransport : TransportBase { private readonly Dictionary _endpoints; public KafkaTopicRouter Topics { get; } = new KafkaTopicRouter(); - public Config Config { get; set; } - public KafkaTransport() : base(Protocols.Kafka) { _endpoints = new Dictionary(); @@ -28,10 +29,27 @@ public KafkaTransport() : base(Protocols.Kafka) protected override KafkaEndpoint findEndpointByUri(Uri uri) => _endpoints[uri]; public KafkaEndpoint EndpointForTopic(string topicName, ProducerConfig producerConifg) => - AddOrUpdateEndpoint(topicName, c => c.ProducerConfig = producerConifg); + AddOrUpdateEndpoint(topicName, endpoint => endpoint.ProducerConfig = producerConifg); public KafkaEndpoint EndpointForTopic(string topicName, ConsumerConfig consumerConifg) => - AddOrUpdateEndpoint(topicName, c => c.ConsumerConfig = consumerConifg); + AddOrUpdateEndpoint(topicName, endpoint => endpoint.ConsumerConfig = consumerConifg); + + public KafkaEndpoint EndpointForTopic(string topicName, ProducerConfig producerConifg, ISerializer keySerializer, ISerializer valueSerializer) => + AddOrUpdateEndpoint(topicName, endpoint => + { + endpoint.KeySerializer = keySerializer; + endpoint.ValueSerializer = valueSerializer; + endpoint.ProducerConfig = producerConifg; + }); + + public KafkaEndpoint EndpointForTopic(string topicName, ConsumerConfig consumerConifg, IDeserializer keyDeserializer, IDeserializer valueDeserializer) => + AddOrUpdateEndpoint(topicName, endpoint => + { + endpoint.KeyDeserializer = keyDeserializer; + endpoint.ValueDeserializer = valueDeserializer; + endpoint.ConsumerConfig = consumerConifg; + }); + KafkaEndpoint AddOrUpdateEndpoint(string topicName, Action> configure) { diff --git a/src/Jasper.ConfluentKafka/KafkaTransportConfigurationExtensions.cs b/src/Jasper.ConfluentKafka/KafkaTransportConfigurationExtensions.cs index 949400e7e..ecf4d608e 100644 --- a/src/Jasper.ConfluentKafka/KafkaTransportConfigurationExtensions.cs +++ b/src/Jasper.ConfluentKafka/KafkaTransportConfigurationExtensions.cs @@ -57,21 +57,42 @@ public static void ConfigureKafka(this IEndpoints endpoints) } /// - /// Listen for incoming messages at the designated Rabbit MQ queue by name + /// Listen for incoming messages at the designated Kafka Topic by name /// + /// + /// /// - /// The name of the Rabbit MQ queue + /// + /// /// public static KafkaListenerConfiguration ListenToKafkaTopic(this IEndpoints endpoints, string topicName, ConsumerConfig consumerConfig) { var endpoint = endpoints.KafkaTransport().EndpointForTopic(topicName, consumerConfig); endpoint.IsListener = true; - return new KafkaListenerConfiguration((KafkaEndpoint)endpoint); + return new KafkaListenerConfiguration(endpoint); } - + /// - /// Publish matching messages to Rabbit MQ using the named routing key or queue name and - /// optionally an exchange + /// Listen for incoming messages at the designated Kafka Topic by name + /// + /// + /// + /// + /// + /// + /// + /// + /// + public static KafkaListenerConfiguration ListenToKafkaTopic(this IEndpoints endpoints, string topicName, ConsumerConfig consumerConfig, + IDeserializer keyDeserializer, IDeserializer valueDeserializer) + { + var endpoint = endpoints.KafkaTransport().EndpointForTopic(topicName, consumerConfig, keyDeserializer, valueDeserializer); + endpoint.IsListener = true; + return new KafkaListenerConfiguration(endpoint); + } + + /// + /// Publish matching messages to Kafka Topic using provided Producer Configuration /// /// /// This is used as the topic name when publishing. Can be either a binding key or a queue name or a static topic name if the exchange is topic-based @@ -90,20 +111,26 @@ public static KafkaSubscriberConfiguration ToKafkaTopic(this IPublis } /// - /// Publish matching messages to Azure Service Bus using the topic name derived from the message and + /// Publish matching messages to Kafka Topic using provided Producer Configuration and Serializers /// + /// + /// /// + /// + /// + /// + /// /// - public static TopicRouterConfiguration ToKafkaTopics(this IPublishToExpression publishing) + public static KafkaSubscriberConfiguration ToKafkaTopic(this IPublishToExpression publishing, string topicName, ProducerConfig producerConfig, ISerializer keySerializer, ISerializer valueSerializer) { var transports = publishing.As().Parent; + var transport = transports.Get(); + var endpoint = transport.EndpointForTopic(topicName, producerConfig, keySerializer, valueSerializer); - var router = transports.KafkaTransport().Topics; - - publishing.ViaRouter(router); + // This is necessary unfortunately to hook up the subscription rules + publishing.To(endpoint.Uri); - return new TopicRouterConfiguration(router, transports); + return new KafkaSubscriberConfiguration(endpoint); } - } } From 1af638ade2fc69d4ec322762ef94e3c43601301f Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Sat, 18 Apr 2020 13:50:47 -0400 Subject: [PATCH 23/82] Enable setting serializers on consumer --- .../Internal/ConfluentKafkaListener.cs | 12 +++++++++--- src/Jasper.ConfluentKafka/KafkaEndpoint.cs | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaListener.cs b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaListener.cs index 844fc224a..f861ebb69 100644 --- a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaListener.cs +++ b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaListener.cs @@ -6,6 +6,7 @@ using Jasper.ConfluentKafka.Serialization; using Jasper.Logging; using Jasper.Transports; +using Lamar.IoC.Instances; namespace Jasper.Kafka.Internal { @@ -18,14 +19,19 @@ public class ConfluentKafkaListener : IListener private readonly KafkaTransportProtocol _protocol = new KafkaTransportProtocol(); private IReceiverCallback _callback; private IConsumer _consumer; + private readonly IDeserializer _keyDeserializer = new DefaultJsonDeserializer().AsSyncOverAsync(); + private readonly IDeserializer _valueDeserializer = new DefaultJsonDeserializer().AsSyncOverAsync(); + private Task _consumerTask; - public ConfluentKafkaListener(KafkaEndpoint endpoint, ITransportLogger logger, CancellationToken cancellation) + public ConfluentKafkaListener(KafkaEndpoint endpoint, ITransportLogger logger, IDeserializer keyDeserializer, IDeserializer valueDeserializer, CancellationToken cancellation) { _endpoint = endpoint; _logger = logger; _cancellation = cancellation; Address = endpoint.Uri; + _keyDeserializer = keyDeserializer; + _valueDeserializer = valueDeserializer; } @@ -43,8 +49,8 @@ public void Start(IReceiverCallback callback) _callback = callback; _consumer = new ConsumerBuilder(_endpoint.ConsumerConfig) - .SetKeyDeserializer(new DefaultJsonDeserializer().AsSyncOverAsync()) - .SetValueDeserializer(new DefaultJsonDeserializer().AsSyncOverAsync()) + .SetKeyDeserializer(_keyDeserializer) + .SetValueDeserializer(_valueDeserializer) .Build(); _consumer.Subscribe(_endpoint.TopicName); diff --git a/src/Jasper.ConfluentKafka/KafkaEndpoint.cs b/src/Jasper.ConfluentKafka/KafkaEndpoint.cs index 4ae3c8f3e..247aff942 100644 --- a/src/Jasper.ConfluentKafka/KafkaEndpoint.cs +++ b/src/Jasper.ConfluentKafka/KafkaEndpoint.cs @@ -101,7 +101,7 @@ protected internal override void StartListening(IMessagingRoot root, ITransportR { if (!IsListener) return; - var listener = new ConfluentKafkaListener(this, root.TransportLogger, root.Cancellation); + var listener = new ConfluentKafkaListener(this, root.TransportLogger, KeyDeserializer, ValueDeserializer, root.Cancellation); runtime.AddListener(listener, this); } From f4141fb87f8a6187b985d26bf135c3bac3ab582b Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 20 Apr 2020 14:21:54 -0400 Subject: [PATCH 24/82] Honor endpoint TPL ExecutionOptions --- src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs index f28df70bf..c3b6d1e3a 100644 --- a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs +++ b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs @@ -4,7 +4,6 @@ using System.Threading.Tasks; using System.Threading.Tasks.Dataflow; using Confluent.Kafka; -using Confluent.Kafka.SyncOverAsync; using Jasper.ConfluentKafka.Exceptions; using Jasper.ConfluentKafka.Serialization; using Jasper.Logging; @@ -57,10 +56,7 @@ public void Start(ISenderCallback callback) _publisher = publisherBuilder.Build(); - _sending = new ActionBlock(sendBySession, new ExecutionDataflowBlockOptions - { - CancellationToken = _cancellation - }); + _sending = new ActionBlock(sendBySession, _endpoint.ExecutionOptions); } From 09ac0d9d4add2c24270566e398cf49137ed366b3 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 20 Apr 2020 14:37:08 -0400 Subject: [PATCH 25/82] Remove redundant if statement --- src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs index c3b6d1e3a..1807ddac3 100644 --- a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs +++ b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs @@ -112,10 +112,8 @@ private async Task sendBySession(Envelope envelope) { throw new UnsupportedFeatureException("Delayed Message Delivery"); } - else - { - await _publisher.ProduceAsync(_endpoint.TopicName, message, _cancellation); - } + + await _publisher.ProduceAsync(_endpoint.TopicName, message, _cancellation); await _callback.Successful(envelope); } From 346b39a9e446ae986c329b39924a0e479cb2e8a6 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 20 Apr 2020 14:38:31 -0400 Subject: [PATCH 26/82] Remove unnecessary async and return Task.CompletedTask; --- src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs index 1807ddac3..408ea0185 100644 --- a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs +++ b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs @@ -45,7 +45,6 @@ public void Dispose() public int QueuedCount => _sending.InputCount; public bool Latched { get; private set; } - public void Start(ISenderCallback callback) { _callback = callback; @@ -67,7 +66,7 @@ public Task Enqueue(Envelope envelope) return Task.CompletedTask; } - public async Task LatchAndDrain() + public Task LatchAndDrain() { Latched = true; @@ -76,6 +75,8 @@ public async Task LatchAndDrain() _sending.Complete(); _logger.CircuitBroken(Destination); + + return Task.CompletedTask; } public void Unlatch() From f764c18b8d84daabe69f1751dc7608d1050cf565 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 21 Apr 2020 09:18:25 -0400 Subject: [PATCH 27/82] Remove redundant property assignment --- src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs index 408ea0185..e30e16825 100644 --- a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs +++ b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs @@ -5,7 +5,6 @@ using System.Threading.Tasks.Dataflow; using Confluent.Kafka; using Jasper.ConfluentKafka.Exceptions; -using Jasper.ConfluentKafka.Serialization; using Jasper.Logging; using Jasper.Transports; using Jasper.Transports.Sending; @@ -22,8 +21,8 @@ public class ConfluentKafkaSender : ISender private ISenderCallback _callback; private IProducer _publisher; - private readonly ISerializer _keySerializer = new DefaultJsonSerializer().AsSyncOverAsync(); - private readonly ISerializer _valueSerializer = new DefaultJsonSerializer().AsSyncOverAsync(); + private readonly ISerializer _keySerializer; + private readonly ISerializer _valueSerializer; public ConfluentKafkaSender(KafkaEndpoint endpoint, ITransportLogger logger, ISerializer keySerializer, ISerializer valueSerializer, CancellationToken cancellation) { From 330c1212beeb623ce657a93d4296e45fc0a853a6 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 21 Apr 2020 09:18:38 -0400 Subject: [PATCH 28/82] Remove unused usings --- src/Jasper.AzureServiceBus/Internal/AzureServiceBusSender.cs | 2 -- src/Jasper.ConfluentKafka/KafkaTransport.cs | 2 -- 2 files changed, 4 deletions(-) diff --git a/src/Jasper.AzureServiceBus/Internal/AzureServiceBusSender.cs b/src/Jasper.AzureServiceBus/Internal/AzureServiceBusSender.cs index 21fc4ad09..47d1114d2 100644 --- a/src/Jasper.AzureServiceBus/Internal/AzureServiceBusSender.cs +++ b/src/Jasper.AzureServiceBus/Internal/AzureServiceBusSender.cs @@ -1,6 +1,4 @@ using System; -using System.Collections.Concurrent; -using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using System.Threading.Tasks.Dataflow; diff --git a/src/Jasper.ConfluentKafka/KafkaTransport.cs b/src/Jasper.ConfluentKafka/KafkaTransport.cs index 114d98fe4..e601e65fa 100644 --- a/src/Jasper.ConfluentKafka/KafkaTransport.cs +++ b/src/Jasper.ConfluentKafka/KafkaTransport.cs @@ -1,10 +1,8 @@ using System; using System.Collections.Generic; using Confluent.Kafka; -using Confluent.SchemaRegistry; using Jasper.ConfluentKafka.Internal; using Jasper.Transports; -using Lamar.IoC.Instances; namespace Jasper.ConfluentKafka { From 2cedd7771d1b4ab71f2250d87bc9e5c312178196 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 27 Apr 2020 20:41:42 -0400 Subject: [PATCH 29/82] Abstract away IProducer so we can have a common non-generic interface to leverage --- src/Jasper.ConfluentKafka.Tests/end_to_end.cs | 4 +- .../Internal/ConfluentKafkaListener.cs | 55 +++---------- .../Internal/ConfluentKafkaSender.cs | 80 +++++++------------ .../Internal/KafkaConsumer.cs | 77 ++++++++++++++++++ .../Internal/KafkaPublisher.cs | 48 +++++++++++ src/Jasper.ConfluentKafka/KafkaEndpoint.cs | 30 ++----- src/Jasper.ConfluentKafka/KafkaTransport.cs | 31 ++----- .../KafkaTransportConfigurationExtensions.cs | 49 +----------- .../KafkaTransportProtocol.cs | 2 + 9 files changed, 187 insertions(+), 189 deletions(-) create mode 100644 src/Jasper.ConfluentKafka/Internal/KafkaConsumer.cs create mode 100644 src/Jasper.ConfluentKafka/Internal/KafkaPublisher.cs diff --git a/src/Jasper.ConfluentKafka.Tests/end_to_end.cs b/src/Jasper.ConfluentKafka.Tests/end_to_end.cs index 41ae7d21f..09b79bf07 100644 --- a/src/Jasper.ConfluentKafka.Tests/end_to_end.cs +++ b/src/Jasper.ConfluentKafka.Tests/end_to_end.cs @@ -99,8 +99,8 @@ public class KafkaUsingApp : JasperOptions public KafkaUsingApp() { Endpoints.ConfigureKafka(); - Endpoints.ListenToKafkaTopic("messages", ConsumerConfig); - Endpoints.PublishAllMessages().ToKafkaTopic("messages", ProducerConfig); + Endpoints.ListenToKafkaTopic("messages", ConsumerConfig); + Endpoints.Publish(pub => pub.Message().ToKafkaTopic("messages", ProducerConfig)); Handlers.IncludeType(); diff --git a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaListener.cs b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaListener.cs index f861ebb69..287e59e69 100644 --- a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaListener.cs +++ b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaListener.cs @@ -1,8 +1,10 @@ using System; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Confluent.Kafka; using Jasper.ConfluentKafka; +using Jasper.ConfluentKafka.Internal; using Jasper.ConfluentKafka.Serialization; using Jasper.Logging; using Jasper.Transports; @@ -10,35 +12,28 @@ namespace Jasper.Kafka.Internal { - public class ConfluentKafkaListener : IListener + public class ConfluentKafkaListener : IListener { private readonly CancellationToken _cancellation; - - private readonly KafkaEndpoint _endpoint; + private readonly KafkaConsumer _consumer; private readonly ITransportLogger _logger; - private readonly KafkaTransportProtocol _protocol = new KafkaTransportProtocol(); private IReceiverCallback _callback; - private IConsumer _consumer; - private readonly IDeserializer _keyDeserializer = new DefaultJsonDeserializer().AsSyncOverAsync(); - private readonly IDeserializer _valueDeserializer = new DefaultJsonDeserializer().AsSyncOverAsync(); + private Task _consumerTask; - public ConfluentKafkaListener(KafkaEndpoint endpoint, ITransportLogger logger, IDeserializer keyDeserializer, IDeserializer valueDeserializer, CancellationToken cancellation) + public ConfluentKafkaListener(KafkaEndpoint endpoint, ITransportLogger logger, CancellationToken cancellation) { - _endpoint = endpoint; _logger = logger; _cancellation = cancellation; Address = endpoint.Uri; - _keyDeserializer = keyDeserializer; - _valueDeserializer = valueDeserializer; + _consumer= new KafkaConsumer(endpoint); } public void Dispose() { _consumerTask?.Dispose(); - _consumer?.Dispose(); } public Uri Address { get; } @@ -48,13 +43,6 @@ public void Start(IReceiverCallback callback) { _callback = callback; - _consumer = new ConsumerBuilder(_endpoint.ConsumerConfig) - .SetKeyDeserializer(_keyDeserializer) - .SetValueDeserializer(_valueDeserializer) - .Build(); - - _consumer.Subscribe(_endpoint.TopicName); - _consumerTask = ConsumeAsync(); Thread.Sleep(1000); // let the consumer start consuming @@ -64,34 +52,15 @@ private async Task ConsumeAsync() { while (!_cancellation.IsCancellationRequested) { - ConsumeResult message; + Envelope envelope = null; try { - message = await Task.Run(() => _consumer.Consume(), _cancellation); - } - catch (Exception ex) - { - _logger.LogException(ex, message: $"Error consuming message from Kafka topic {_endpoint.TopicName}"); - return; - } + (Envelope Envelope, TopicPartitionOffset TopicPartitionOffset) receivedMessage = await _consumer.ConsumeEnvelopeAsync(_cancellation); + envelope = receivedMessage.Envelope; - Envelope envelope; - - try - { - envelope = _protocol.ReadEnvelope(message.Message); - } - catch (Exception ex) - { - _logger.LogException(ex, message: $"Error trying to map an incoming Kafka {_endpoint.TopicName} Topic message to an Envelope. See the Dead Letter Queue"); - return; - } - - try - { - await _callback.Received(Address, new[] {envelope}); + await _callback.Received(Address, new[] { envelope }); - _consumer.Commit(); + _consumer.Commit(receivedMessage.TopicPartitionOffset); } catch (KafkaException ke) { diff --git a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs index e30e16825..d6e13315f 100644 --- a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs +++ b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs @@ -1,63 +1,49 @@ using System; -using System.Text; +using System.Collections.Generic; +using System.Net.NetworkInformation; using System.Threading; using System.Threading.Tasks; using System.Threading.Tasks.Dataflow; -using Confluent.Kafka; -using Jasper.ConfluentKafka.Exceptions; using Jasper.Logging; -using Jasper.Transports; using Jasper.Transports.Sending; +using LamarCodeGeneration.Util; namespace Jasper.ConfluentKafka.Internal { - public class ConfluentKafkaSender : ISender + public class ConfluentKafkaSender : ISender { - private readonly ITransportProtocol> _protocol; - private readonly KafkaEndpoint _endpoint; + private Dictionary _publishers = new Dictionary(); + private readonly KafkaEndpoint _endpoint; private readonly ITransportLogger _logger; private readonly CancellationToken _cancellation; private ActionBlock _sending; private ISenderCallback _callback; - private IProducer _publisher; - private readonly ISerializer _keySerializer; - private readonly ISerializer _valueSerializer; - - public ConfluentKafkaSender(KafkaEndpoint endpoint, ITransportLogger logger, ISerializer keySerializer, ISerializer valueSerializer, CancellationToken cancellation) + public ConfluentKafkaSender(KafkaEndpoint endpoint, ITransportLogger logger, CancellationToken cancellation) { _endpoint = endpoint; _logger = logger; _cancellation = cancellation; - Destination = endpoint.Uri; - _protocol = new KafkaTransportProtocol(); - _keySerializer = keySerializer; - _valueSerializer = valueSerializer; } public void Dispose() { - _publisher?.Dispose(); + foreach (KafkaPublisher publisher in _publishers.Values) + { + publisher.Dispose(); + } } - public Uri Destination { get; } - public int QueuedCount => _sending.InputCount; + public Uri Destination => _endpoint.Uri; + public int QueuedCount { get; } public bool Latched { get; private set; } - public void Start(ISenderCallback callback) { _callback = callback; - var publisherBuilder = new ProducerBuilder(_endpoint.ProducerConfig) - .SetKeySerializer(_keySerializer) - .SetValueSerializer(_valueSerializer); - - _publisher = publisherBuilder.Build(); - _sending = new ActionBlock(sendBySession, _endpoint.ExecutionOptions); } - public Task Enqueue(Envelope envelope) { _sending.Post(envelope); @@ -69,8 +55,6 @@ public Task LatchAndDrain() { Latched = true; - _publisher.Flush(_cancellation); - _sending.Complete(); _logger.CircuitBroken(Destination); @@ -86,34 +70,18 @@ public void Unlatch() Latched = false; } - public async Task Ping(CancellationToken cancellationToken) - { - Envelope envelope = Envelope.ForPing(Destination); - Message message = _protocol.WriteFromEnvelope(envelope); - - message.Headers.Add("MessageGroupId", Encoding.UTF8.GetBytes(Guid.NewGuid().ToString())); - message.Headers.Add("Jasper_SessionId", Encoding.UTF8.GetBytes(Guid.NewGuid().ToString())); - - await _publisher.ProduceAsync("jasper-ping", message, cancellationToken); - - return true; - } - - public bool SupportsNativeScheduledSend { get; } = false; - private async Task sendBySession(Envelope envelope) { try { - Message message = _protocol.WriteFromEnvelope(envelope); - message.Headers.Add("Jasper_SessionId", Encoding.UTF8.GetBytes(Guid.NewGuid().ToString())); - - if (envelope.IsDelayed(DateTime.UtcNow)) + Type messageType = envelope.Message.GetType(); + if (!_publishers.ContainsKey(messageType)) { - throw new UnsupportedFeatureException("Delayed Message Delivery"); + KafkaPublisher publisher = typeof(KafkaPublisher<,>).CloseAndBuildAs(_endpoint.ProducerConfig, typeof(string), messageType); + _publishers.Add(messageType, publisher); } - await _publisher.ProduceAsync(_endpoint.TopicName, message, _cancellation); + await _publishers[messageType].SendAsync(_endpoint.TopicName, envelope, CancellationToken.None); await _callback.Successful(envelope); } @@ -129,5 +97,17 @@ private async Task sendBySession(Envelope envelope) } } } + + public async Task Ping(CancellationToken cancellationToken) + { + Envelope envelope = Envelope.ForPing(Destination); + KafkaPublisher publisher = new KafkaPublisher(_endpoint.ProducerConfig); + await publisher.SendAsync("jasper-ping", envelope, cancellationToken); + return true; + } + + public bool SupportsNativeScheduledSend { get; } + } + } diff --git a/src/Jasper.ConfluentKafka/Internal/KafkaConsumer.cs b/src/Jasper.ConfluentKafka/Internal/KafkaConsumer.cs new file mode 100644 index 000000000..8f271e829 --- /dev/null +++ b/src/Jasper.ConfluentKafka/Internal/KafkaConsumer.cs @@ -0,0 +1,77 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Confluent.Kafka; +using Jasper.ConfluentKafka.Serialization; + +namespace Jasper.ConfluentKafka.Internal +{ + public interface IKafkaConsumer + { + Task<(Envelope, TopicPartitionOffset)> ConsumeEnvelopeAsync(CancellationToken cancel); + void Commit(TopicPartitionOffset topicPartitionOffset); + } + + public abstract class KafkaConsumer : IKafkaConsumer, IDisposable + { + public abstract Task<(Envelope, TopicPartitionOffset)> ConsumeEnvelopeAsync(CancellationToken cancel); + public abstract void Commit(TopicPartitionOffset offset); + public abstract void Dispose(); + } + + public class KafkaConsumer : KafkaConsumer + { + private readonly KafkaEndpoint _endpoint; + private IConsumer _consumer; + private readonly IDeserializer _keyDeserializer = new DefaultJsonDeserializer().AsSyncOverAsync(); + private readonly IDeserializer _valueDeserializer = new DefaultJsonDeserializer().AsSyncOverAsync(); + + public KafkaConsumer(KafkaEndpoint endpoint) + { + _endpoint = endpoint; + _consumer = new ConsumerBuilder(endpoint.ConsumerConfig) + .SetKeyDeserializer(_keyDeserializer) + .SetValueDeserializer(_valueDeserializer) + .Build(); + + _consumer.Subscribe(endpoint.TopicName); + } + + public override async Task<(Envelope, TopicPartitionOffset)> ConsumeEnvelopeAsync(CancellationToken cancel) + { + ConsumeResult message; + try + { + message = await Task.Run(() => _consumer.Consume(), cancel); + } + catch (Exception ex) + { + throw new Exception("failed to consume messages"); + } + + Envelope envelope; + + try + { + envelope = new KafkaTransportProtocol().ReadEnvelope(message.Message); + } + catch (Exception ex) + { + throw new Exception($"Error trying to map an incoming Kafka {_endpoint.TopicName} Topic message to an Envelope. See the Dead Letter Queue"); + } + + return (envelope, message.TopicPartitionOffset); + } + + public override void Commit(TopicPartitionOffset offset) + { + _consumer.Commit(new []{offset }); + } + + public override void Dispose() + { + throw new NotImplementedException(); + } + } +} diff --git a/src/Jasper.ConfluentKafka/Internal/KafkaPublisher.cs b/src/Jasper.ConfluentKafka/Internal/KafkaPublisher.cs new file mode 100644 index 000000000..5d7c22253 --- /dev/null +++ b/src/Jasper.ConfluentKafka/Internal/KafkaPublisher.cs @@ -0,0 +1,48 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Confluent.Kafka; +using Jasper.ConfluentKafka.Serialization; + +namespace Jasper.ConfluentKafka.Internal +{ + public interface IKafkaPublisher + { + Task SendAsync(string topic, Envelope envelope, CancellationToken cancel); + } + public abstract class KafkaPublisher : IKafkaPublisher, IDisposable + { + public abstract Task SendAsync(string topic, Envelope envelope, CancellationToken cancel); + + public abstract void Dispose(); + } + + public class KafkaPublisher : KafkaPublisher + { + private readonly IProducer _producer; + private readonly KafkaTransportProtocol _protocol = new KafkaTransportProtocol(); + // TODO: Move this logic somewhere else if we can. Can it be in Jasper's pipeline and we always expect byte[] here? + private readonly ISerializer _keySerializer = new DefaultJsonSerializer().AsSyncOverAsync(); + private readonly ISerializer _valueSerializer = new DefaultJsonSerializer().AsSyncOverAsync(); + + + public KafkaPublisher(ProducerConfig producerConifg) + { + _producer = new ProducerBuilder(producerConifg) + .SetKeySerializer(_keySerializer) + .SetValueSerializer(_valueSerializer) + .Build(); + } + + public override Task SendAsync(string topic, Envelope envelope, CancellationToken cancel) + { + Message message = _protocol.WriteFromEnvelope(envelope); + return _producer.ProduceAsync(topic, message, cancel); + } + + public override void Dispose() + { + _producer.Dispose(); + } + } +} diff --git a/src/Jasper.ConfluentKafka/KafkaEndpoint.cs b/src/Jasper.ConfluentKafka/KafkaEndpoint.cs index 247aff942..62e2a3c35 100644 --- a/src/Jasper.ConfluentKafka/KafkaEndpoint.cs +++ b/src/Jasper.ConfluentKafka/KafkaEndpoint.cs @@ -1,11 +1,11 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Reflection.Metadata.Ecma335; using Baseline; using Confluent.Kafka; using Jasper.Configuration; using Jasper.ConfluentKafka.Internal; -using Jasper.ConfluentKafka.Serialization; using Jasper.Kafka.Internal; using Jasper.Runtime; using Jasper.Transports; @@ -75,39 +75,19 @@ public override void Parse(Uri uri) } } - protected internal override void StartListening(IMessagingRoot root, ITransportRuntime runtime) - { - throw new NotImplementedException(); - } - - protected override ISender CreateSender(IMessagingRoot root) - { - throw new NotImplementedException(); - } - - public override Uri ReplyUri() => BuildUri(true); - } - - public class KafkaEndpoint : KafkaEndpoint - { - internal KafkaTransport Parent { get; set; } - - public ISerializer KeySerializer = new DefaultJsonSerializer().AsSyncOverAsync(); - public ISerializer ValueSerializer= new DefaultJsonSerializer().AsSyncOverAsync(); - public IDeserializer KeyDeserializer = new DefaultJsonDeserializer().AsSyncOverAsync(); - public IDeserializer ValueDeserializer = new DefaultJsonDeserializer().AsSyncOverAsync(); - protected internal override void StartListening(IMessagingRoot root, ITransportRuntime runtime) { if (!IsListener) return; - var listener = new ConfluentKafkaListener(this, root.TransportLogger, KeyDeserializer, ValueDeserializer, root.Cancellation); + var listener = new ConfluentKafkaListener(this, root.TransportLogger, root.Cancellation); runtime.AddListener(listener, this); } protected override ISender CreateSender(IMessagingRoot root) { - return new ConfluentKafkaSender(this, root.TransportLogger, KeySerializer, ValueSerializer, root.Cancellation); + return new ConfluentKafkaSender(this, root.TransportLogger, root.Cancellation); } + + public override Uri ReplyUri() => BuildUri(true); } } diff --git a/src/Jasper.ConfluentKafka/KafkaTransport.cs b/src/Jasper.ConfluentKafka/KafkaTransport.cs index e601e65fa..e8dc29cda 100644 --- a/src/Jasper.ConfluentKafka/KafkaTransport.cs +++ b/src/Jasper.ConfluentKafka/KafkaTransport.cs @@ -26,39 +26,22 @@ public KafkaTransport() : base(Protocols.Kafka) protected override KafkaEndpoint findEndpointByUri(Uri uri) => _endpoints[uri]; - public KafkaEndpoint EndpointForTopic(string topicName, ProducerConfig producerConifg) => - AddOrUpdateEndpoint(topicName, endpoint => endpoint.ProducerConfig = producerConifg); + public KafkaEndpoint EndpointForTopic(string topicName, ProducerConfig producerConifg) => + AddOrUpdateEndpoint(topicName, endpoint => endpoint.ProducerConfig = producerConifg); - public KafkaEndpoint EndpointForTopic(string topicName, ConsumerConfig consumerConifg) => - AddOrUpdateEndpoint(topicName, endpoint => endpoint.ConsumerConfig = consumerConifg); + public KafkaEndpoint EndpointForTopic(string topicName, ConsumerConfig consumerConifg) => + AddOrUpdateEndpoint(topicName, endpoint => endpoint.ConsumerConfig = consumerConifg); - public KafkaEndpoint EndpointForTopic(string topicName, ProducerConfig producerConifg, ISerializer keySerializer, ISerializer valueSerializer) => - AddOrUpdateEndpoint(topicName, endpoint => - { - endpoint.KeySerializer = keySerializer; - endpoint.ValueSerializer = valueSerializer; - endpoint.ProducerConfig = producerConifg; - }); - - public KafkaEndpoint EndpointForTopic(string topicName, ConsumerConfig consumerConifg, IDeserializer keyDeserializer, IDeserializer valueDeserializer) => - AddOrUpdateEndpoint(topicName, endpoint => - { - endpoint.KeyDeserializer = keyDeserializer; - endpoint.ValueDeserializer = valueDeserializer; - endpoint.ConsumerConfig = consumerConifg; - }); - - - KafkaEndpoint AddOrUpdateEndpoint(string topicName, Action> configure) + KafkaEndpoint AddOrUpdateEndpoint(string topicName, Action configure) { - var endpoint = new KafkaEndpoint + var endpoint = new KafkaEndpoint { TopicName = topicName }; if (_endpoints.ContainsKey(endpoint.Uri)) { - endpoint = (KafkaEndpoint)_endpoints[endpoint.Uri]; + endpoint = _endpoints[endpoint.Uri]; configure(endpoint); } else diff --git a/src/Jasper.ConfluentKafka/KafkaTransportConfigurationExtensions.cs b/src/Jasper.ConfluentKafka/KafkaTransportConfigurationExtensions.cs index ecf4d608e..e1d6b4516 100644 --- a/src/Jasper.ConfluentKafka/KafkaTransportConfigurationExtensions.cs +++ b/src/Jasper.ConfluentKafka/KafkaTransportConfigurationExtensions.cs @@ -65,28 +65,9 @@ public static void ConfigureKafka(this IEndpoints endpoints) /// /// /// - public static KafkaListenerConfiguration ListenToKafkaTopic(this IEndpoints endpoints, string topicName, ConsumerConfig consumerConfig) + public static KafkaListenerConfiguration ListenToKafkaTopic(this IEndpoints endpoints, string topicName, ConsumerConfig consumerConfig) { - var endpoint = endpoints.KafkaTransport().EndpointForTopic(topicName, consumerConfig); - endpoint.IsListener = true; - return new KafkaListenerConfiguration(endpoint); - } - - /// - /// Listen for incoming messages at the designated Kafka Topic by name - /// - /// - /// - /// - /// - /// - /// - /// - /// - public static KafkaListenerConfiguration ListenToKafkaTopic(this IEndpoints endpoints, string topicName, ConsumerConfig consumerConfig, - IDeserializer keyDeserializer, IDeserializer valueDeserializer) - { - var endpoint = endpoints.KafkaTransport().EndpointForTopic(topicName, consumerConfig, keyDeserializer, valueDeserializer); + var endpoint = endpoints.KafkaTransport().EndpointForTopic(topicName, consumerConfig); endpoint.IsListener = true; return new KafkaListenerConfiguration(endpoint); } @@ -98,11 +79,11 @@ public static KafkaListenerConfiguration ListenToKafkaTopic(this IEn /// This is used as the topic name when publishing. Can be either a binding key or a queue name or a static topic name if the exchange is topic-based /// Optional, you only need to supply this if you are using a non-default exchange /// - public static KafkaSubscriberConfiguration ToKafkaTopic(this IPublishToExpression publishing, string topicName, ProducerConfig producerConfig) + public static KafkaSubscriberConfiguration ToKafkaTopic(this IPublishToExpression publishing, string topicName, ProducerConfig producerConfig) { var transports = publishing.As().Parent; var transport = transports.Get(); - var endpoint = transport.EndpointForTopic(topicName, producerConfig); + var endpoint = transport.EndpointForTopic(topicName, producerConfig); // This is necessary unfortunately to hook up the subscription rules publishing.To(endpoint.Uri); @@ -110,27 +91,5 @@ public static KafkaSubscriberConfiguration ToKafkaTopic(this IPublis return new KafkaSubscriberConfiguration(endpoint); } - /// - /// Publish matching messages to Kafka Topic using provided Producer Configuration and Serializers - /// - /// - /// - /// - /// - /// - /// - /// - /// - public static KafkaSubscriberConfiguration ToKafkaTopic(this IPublishToExpression publishing, string topicName, ProducerConfig producerConfig, ISerializer keySerializer, ISerializer valueSerializer) - { - var transports = publishing.As().Parent; - var transport = transports.Get(); - var endpoint = transport.EndpointForTopic(topicName, producerConfig, keySerializer, valueSerializer); - - // This is necessary unfortunately to hook up the subscription rules - publishing.To(endpoint.Uri); - - return new KafkaSubscriberConfiguration(endpoint); - } } } diff --git a/src/Jasper.ConfluentKafka/KafkaTransportProtocol.cs b/src/Jasper.ConfluentKafka/KafkaTransportProtocol.cs index c6d8de52b..727e8693a 100644 --- a/src/Jasper.ConfluentKafka/KafkaTransportProtocol.cs +++ b/src/Jasper.ConfluentKafka/KafkaTransportProtocol.cs @@ -7,6 +7,8 @@ namespace Jasper.ConfluentKafka { + + public class KafkaTransportProtocol : ITransportProtocol> { private const string JasperMessageIdHeader = "Jasper_MessageId"; From 9f1191703bae3071718ea194eff3e6933c27641b Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 27 Apr 2020 21:19:40 -0400 Subject: [PATCH 30/82] Remove producer/consumer abstractions --- .../Internal/KafkaConsumer.cs | 77 ------------------- .../Internal/KafkaPublisher.cs | 48 ------------ 2 files changed, 125 deletions(-) delete mode 100644 src/Jasper.ConfluentKafka/Internal/KafkaConsumer.cs delete mode 100644 src/Jasper.ConfluentKafka/Internal/KafkaPublisher.cs diff --git a/src/Jasper.ConfluentKafka/Internal/KafkaConsumer.cs b/src/Jasper.ConfluentKafka/Internal/KafkaConsumer.cs deleted file mode 100644 index 8f271e829..000000000 --- a/src/Jasper.ConfluentKafka/Internal/KafkaConsumer.cs +++ /dev/null @@ -1,77 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; -using Confluent.Kafka; -using Jasper.ConfluentKafka.Serialization; - -namespace Jasper.ConfluentKafka.Internal -{ - public interface IKafkaConsumer - { - Task<(Envelope, TopicPartitionOffset)> ConsumeEnvelopeAsync(CancellationToken cancel); - void Commit(TopicPartitionOffset topicPartitionOffset); - } - - public abstract class KafkaConsumer : IKafkaConsumer, IDisposable - { - public abstract Task<(Envelope, TopicPartitionOffset)> ConsumeEnvelopeAsync(CancellationToken cancel); - public abstract void Commit(TopicPartitionOffset offset); - public abstract void Dispose(); - } - - public class KafkaConsumer : KafkaConsumer - { - private readonly KafkaEndpoint _endpoint; - private IConsumer _consumer; - private readonly IDeserializer _keyDeserializer = new DefaultJsonDeserializer().AsSyncOverAsync(); - private readonly IDeserializer _valueDeserializer = new DefaultJsonDeserializer().AsSyncOverAsync(); - - public KafkaConsumer(KafkaEndpoint endpoint) - { - _endpoint = endpoint; - _consumer = new ConsumerBuilder(endpoint.ConsumerConfig) - .SetKeyDeserializer(_keyDeserializer) - .SetValueDeserializer(_valueDeserializer) - .Build(); - - _consumer.Subscribe(endpoint.TopicName); - } - - public override async Task<(Envelope, TopicPartitionOffset)> ConsumeEnvelopeAsync(CancellationToken cancel) - { - ConsumeResult message; - try - { - message = await Task.Run(() => _consumer.Consume(), cancel); - } - catch (Exception ex) - { - throw new Exception("failed to consume messages"); - } - - Envelope envelope; - - try - { - envelope = new KafkaTransportProtocol().ReadEnvelope(message.Message); - } - catch (Exception ex) - { - throw new Exception($"Error trying to map an incoming Kafka {_endpoint.TopicName} Topic message to an Envelope. See the Dead Letter Queue"); - } - - return (envelope, message.TopicPartitionOffset); - } - - public override void Commit(TopicPartitionOffset offset) - { - _consumer.Commit(new []{offset }); - } - - public override void Dispose() - { - throw new NotImplementedException(); - } - } -} diff --git a/src/Jasper.ConfluentKafka/Internal/KafkaPublisher.cs b/src/Jasper.ConfluentKafka/Internal/KafkaPublisher.cs deleted file mode 100644 index 5d7c22253..000000000 --- a/src/Jasper.ConfluentKafka/Internal/KafkaPublisher.cs +++ /dev/null @@ -1,48 +0,0 @@ -using System; -using System.Threading; -using System.Threading.Tasks; -using Confluent.Kafka; -using Jasper.ConfluentKafka.Serialization; - -namespace Jasper.ConfluentKafka.Internal -{ - public interface IKafkaPublisher - { - Task SendAsync(string topic, Envelope envelope, CancellationToken cancel); - } - public abstract class KafkaPublisher : IKafkaPublisher, IDisposable - { - public abstract Task SendAsync(string topic, Envelope envelope, CancellationToken cancel); - - public abstract void Dispose(); - } - - public class KafkaPublisher : KafkaPublisher - { - private readonly IProducer _producer; - private readonly KafkaTransportProtocol _protocol = new KafkaTransportProtocol(); - // TODO: Move this logic somewhere else if we can. Can it be in Jasper's pipeline and we always expect byte[] here? - private readonly ISerializer _keySerializer = new DefaultJsonSerializer().AsSyncOverAsync(); - private readonly ISerializer _valueSerializer = new DefaultJsonSerializer().AsSyncOverAsync(); - - - public KafkaPublisher(ProducerConfig producerConifg) - { - _producer = new ProducerBuilder(producerConifg) - .SetKeySerializer(_keySerializer) - .SetValueSerializer(_valueSerializer) - .Build(); - } - - public override Task SendAsync(string topic, Envelope envelope, CancellationToken cancel) - { - Message message = _protocol.WriteFromEnvelope(envelope); - return _producer.ProduceAsync(topic, message, cancel); - } - - public override void Dispose() - { - _producer.Dispose(); - } - } -} From 5457e56db705a2773a64258d4f5534985f8ec538 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 27 Apr 2020 21:19:58 -0400 Subject: [PATCH 31/82] Rely entirely on byte arrays for kafka --- .../Internal/ConfluentKafkaListener.cs | 48 +++++++++----- .../Internal/ConfluentKafkaSender.cs | 63 ++++++++++--------- src/Jasper.ConfluentKafka/KafkaEndpoint.cs | 1 - .../KafkaTransportProtocol.cs | 14 ++--- 4 files changed, 73 insertions(+), 53 deletions(-) diff --git a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaListener.cs b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaListener.cs index 287e59e69..2f4a353f6 100644 --- a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaListener.cs +++ b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaListener.cs @@ -1,48 +1,46 @@ using System; -using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Confluent.Kafka; using Jasper.ConfluentKafka; -using Jasper.ConfluentKafka.Internal; -using Jasper.ConfluentKafka.Serialization; using Jasper.Logging; using Jasper.Transports; -using Lamar.IoC.Instances; namespace Jasper.Kafka.Internal { public class ConfluentKafkaListener : IListener { private readonly CancellationToken _cancellation; - private readonly KafkaConsumer _consumer; + private readonly IConsumer _consumer; + private readonly KafkaEndpoint _endpoint; private readonly ITransportLogger _logger; private IReceiverCallback _callback; - - + private readonly ITransportProtocol> _protocol; private Task _consumerTask; public ConfluentKafkaListener(KafkaEndpoint endpoint, ITransportLogger logger, CancellationToken cancellation) { + _endpoint = endpoint; _logger = logger; _cancellation = cancellation; - Address = endpoint.Uri; - _consumer= new KafkaConsumer(endpoint); + _protocol = new KafkaTransportProtocol(); + _consumer = new ConsumerBuilder(endpoint.ConsumerConfig).Build(); } - public void Dispose() { _consumerTask?.Dispose(); } - public Uri Address { get; } + public Uri Address => _endpoint.Uri; public ListeningStatus Status { get; set; } public void Start(IReceiverCallback callback) { _callback = callback; + _consumer.Subscribe(new []{ _endpoint.TopicName }); + _consumerTask = ConsumeAsync(); Thread.Sleep(1000); // let the consumer start consuming @@ -52,15 +50,34 @@ private async Task ConsumeAsync() { while (!_cancellation.IsCancellationRequested) { - Envelope envelope = null; + ConsumeResult message; + try + { + message = await Task.Run(() => _consumer.Consume(), _cancellation); + } + catch (Exception ex) + { + _logger.LogException(ex, message: $"Error consuming message from Kafka topic {_endpoint.TopicName}"); + return; + } + + Envelope envelope; + try { - (Envelope Envelope, TopicPartitionOffset TopicPartitionOffset) receivedMessage = await _consumer.ConsumeEnvelopeAsync(_cancellation); - envelope = receivedMessage.Envelope; + envelope = _protocol.ReadEnvelope(message.Message); + } + catch (Exception ex) + { + _logger.LogException(ex, message: $"Error trying to map an incoming Kafka {_endpoint.TopicName} Topic message to an Envelope. See the Dead Letter Queue"); + return; + } + try + { await _callback.Received(Address, new[] { envelope }); - _consumer.Commit(receivedMessage.TopicPartitionOffset); + _consumer.Commit(); } catch (KafkaException ke) { @@ -76,6 +93,5 @@ private async Task ConsumeAsync() } } } - } } diff --git a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs index d6e13315f..87fef2c4c 100644 --- a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs +++ b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs @@ -1,18 +1,20 @@ using System; -using System.Collections.Generic; -using System.Net.NetworkInformation; +using System.Text; using System.Threading; using System.Threading.Tasks; using System.Threading.Tasks.Dataflow; +using Confluent.Kafka; +using Jasper.ConfluentKafka.Exceptions; using Jasper.Logging; +using Jasper.Transports; using Jasper.Transports.Sending; -using LamarCodeGeneration.Util; namespace Jasper.ConfluentKafka.Internal { public class ConfluentKafkaSender : ISender { - private Dictionary _publishers = new Dictionary(); + private ITransportProtocol> _protocol; + private IProducer _publisher; private readonly KafkaEndpoint _endpoint; private readonly ITransportLogger _logger; private readonly CancellationToken _cancellation; @@ -24,24 +26,23 @@ public ConfluentKafkaSender(KafkaEndpoint endpoint, ITransportLogger logger, Can _endpoint = endpoint; _logger = logger; _cancellation = cancellation; + _publisher = new ProducerBuilder(endpoint.ProducerConfig).Build(); + _sending = new ActionBlock(sendBySession, _endpoint.ExecutionOptions); + _protocol = new KafkaTransportProtocol(); } public void Dispose() { - foreach (KafkaPublisher publisher in _publishers.Values) - { - publisher.Dispose(); - } + _publisher?.Dispose(); } public Uri Destination => _endpoint.Uri; - public int QueuedCount { get; } + public int QueuedCount => _sending.InputCount; public bool Latched { get; private set; } + public void Start(ISenderCallback callback) { _callback = callback; - - _sending = new ActionBlock(sendBySession, _endpoint.ExecutionOptions); } public Task Enqueue(Envelope envelope) @@ -55,6 +56,8 @@ public Task LatchAndDrain() { Latched = true; + _publisher.Flush(_cancellation); + _sending.Complete(); _logger.CircuitBroken(Destination); @@ -70,18 +73,34 @@ public void Unlatch() Latched = false; } + public async Task Ping(CancellationToken cancellationToken) + { + Envelope envelope = Envelope.ForPing(Destination); + Message message = _protocol.WriteFromEnvelope(envelope); + + message.Headers.Add("MessageGroupId", Encoding.UTF8.GetBytes(Guid.NewGuid().ToString())); + message.Headers.Add("Jasper_SessionId", Encoding.UTF8.GetBytes(Guid.NewGuid().ToString())); + + await _publisher.ProduceAsync("jasper-ping", message, cancellationToken); + + return true; + } + + public bool SupportsNativeScheduledSend { get; } = false; + private async Task sendBySession(Envelope envelope) { try { - Type messageType = envelope.Message.GetType(); - if (!_publishers.ContainsKey(messageType)) + Message message = _protocol.WriteFromEnvelope(envelope); + message.Headers.Add("Jasper_SessionId", Encoding.UTF8.GetBytes(Guid.NewGuid().ToString())); + + if (envelope.IsDelayed(DateTime.UtcNow)) { - KafkaPublisher publisher = typeof(KafkaPublisher<,>).CloseAndBuildAs(_endpoint.ProducerConfig, typeof(string), messageType); - _publishers.Add(messageType, publisher); + throw new UnsupportedFeatureException("Delayed Message Delivery"); } - await _publishers[messageType].SendAsync(_endpoint.TopicName, envelope, CancellationToken.None); + await _publisher.ProduceAsync(_endpoint.TopicName, message, _cancellation); await _callback.Successful(envelope); } @@ -97,17 +116,5 @@ private async Task sendBySession(Envelope envelope) } } } - - public async Task Ping(CancellationToken cancellationToken) - { - Envelope envelope = Envelope.ForPing(Destination); - KafkaPublisher publisher = new KafkaPublisher(_endpoint.ProducerConfig); - await publisher.SendAsync("jasper-ping", envelope, cancellationToken); - return true; - } - - public bool SupportsNativeScheduledSend { get; } - } - } diff --git a/src/Jasper.ConfluentKafka/KafkaEndpoint.cs b/src/Jasper.ConfluentKafka/KafkaEndpoint.cs index 62e2a3c35..7d52914f7 100644 --- a/src/Jasper.ConfluentKafka/KafkaEndpoint.cs +++ b/src/Jasper.ConfluentKafka/KafkaEndpoint.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Reflection.Metadata.Ecma335; using Baseline; using Confluent.Kafka; using Jasper.Configuration; diff --git a/src/Jasper.ConfluentKafka/KafkaTransportProtocol.cs b/src/Jasper.ConfluentKafka/KafkaTransportProtocol.cs index 727e8693a..a5f9262f7 100644 --- a/src/Jasper.ConfluentKafka/KafkaTransportProtocol.cs +++ b/src/Jasper.ConfluentKafka/KafkaTransportProtocol.cs @@ -7,17 +7,15 @@ namespace Jasper.ConfluentKafka { - - - public class KafkaTransportProtocol : ITransportProtocol> + public class KafkaTransportProtocol : ITransportProtocol> { private const string JasperMessageIdHeader = "Jasper_MessageId"; - public Message WriteFromEnvelope(Envelope envelope) + public Message WriteFromEnvelope(Envelope envelope) { - var message = new Message + var message = new Message { Headers = new Headers(), - Value = (TVal) envelope.Message + Value = envelope.Data }; foreach (KeyValuePair h in envelope.Headers) @@ -31,7 +29,7 @@ public Message WriteFromEnvelope(Envelope envelope) return message; } - public Envelope ReadEnvelope(Message message) + public Envelope ReadEnvelope(Message message) { var env = new Envelope(); @@ -42,7 +40,7 @@ public Envelope ReadEnvelope(Message message) var messageIdHeader = message.Headers.Single(h => h.Key.Equals(JasperMessageIdHeader)); env.Id = Guid.Parse(Encoding.UTF8.GetString(messageIdHeader.GetValueBytes())); - env.Message = message.Value; + env.Data = message.Value; return env; } From 06f3769d01d7005504df0038710749edc7bc1222 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 28 Apr 2020 12:06:24 -0400 Subject: [PATCH 32/82] Remove debugging sleep --- src/Jasper.ConfluentKafka/Internal/ConfluentKafkaListener.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaListener.cs b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaListener.cs index 2f4a353f6..32513b6b6 100644 --- a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaListener.cs +++ b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaListener.cs @@ -42,8 +42,6 @@ public void Start(IReceiverCallback callback) _consumer.Subscribe(new []{ _endpoint.TopicName }); _consumerTask = ConsumeAsync(); - - Thread.Sleep(1000); // let the consumer start consuming } private async Task ConsumeAsync() From eb2c9c5bb1ea9dc7e18219430fb53c68d814521d Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 28 Apr 2020 12:06:53 -0400 Subject: [PATCH 33/82] Map outgoing/incoming headers from envelope --- .../KafkaTransportProtocol.cs | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/Jasper.ConfluentKafka/KafkaTransportProtocol.cs b/src/Jasper.ConfluentKafka/KafkaTransportProtocol.cs index a5f9262f7..e3d2df0c3 100644 --- a/src/Jasper.ConfluentKafka/KafkaTransportProtocol.cs +++ b/src/Jasper.ConfluentKafka/KafkaTransportProtocol.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics.Eventing.Reader; using System.Linq; using System.Text; using Confluent.Kafka; @@ -18,30 +19,31 @@ public Message WriteFromEnvelope(Envelope envelope) Value = envelope.Data }; - foreach (KeyValuePair h in envelope.Headers) + IDictionary envelopHeaders = new Dictionary(); + envelope.WriteToDictionary(envelopHeaders); + var headers = new Headers(); + foreach (Header header in envelopHeaders.Select(h => new Header(h.Key, Encoding.UTF8.GetBytes(h.Value.ToString())))) { - Header header = new Header(h.Key, Encoding.UTF8.GetBytes(h.Value)); - message.Headers.Add(header); + headers.Add(header); } - message.Headers.Add(JasperMessageIdHeader, Encoding.UTF8.GetBytes(envelope.Id.ToString())); + message.Headers = headers; return message; } public Envelope ReadEnvelope(Message message) { - var env = new Envelope(); - - foreach (var header in message.Headers.Where(h => !h.Key.StartsWith("Jasper"))) + var env = new Envelope() { - env.Headers.Add(header.Key, Encoding.UTF8.GetString(header.GetValueBytes())); - } - - var messageIdHeader = message.Headers.Single(h => h.Key.Equals(JasperMessageIdHeader)); - env.Id = Guid.Parse(Encoding.UTF8.GetString(messageIdHeader.GetValueBytes())); - env.Data = message.Value; + Data = message.Value + }; + Dictionary incomingHeaders = message.Headers.Select(h => new {h.Key, Value = h.GetValueBytes()}) + .ToDictionary(k => k.Key, v => (object)Encoding.UTF8.GetString(v.Value)); + + env.ReadPropertiesFromDictionary(incomingHeaders); + return env; } } From 80de3a5a719dccee9897b1df26665168f9ca9b67 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 28 Apr 2020 12:08:35 -0400 Subject: [PATCH 34/82] Remove commented code and unused usings --- src/Jasper.ConfluentKafka/Internal/KafkaTopicRouter.cs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/Jasper.ConfluentKafka/Internal/KafkaTopicRouter.cs b/src/Jasper.ConfluentKafka/Internal/KafkaTopicRouter.cs index 483a50a01..747b924ad 100644 --- a/src/Jasper.ConfluentKafka/Internal/KafkaTopicRouter.cs +++ b/src/Jasper.ConfluentKafka/Internal/KafkaTopicRouter.cs @@ -1,7 +1,5 @@ using System; -using System.Collections.Generic; using Baseline; -using Confluent.Kafka; using Jasper.Configuration; using Jasper.Runtime.Routing; @@ -9,11 +7,6 @@ namespace Jasper.ConfluentKafka.Internal { public class KafkaTopicRouter : TopicRouter { - //Dictionary producerConfigs; - //public KafkaTopicRouter(Dictionary producerConfigs) - //{ - //} - public override Uri BuildUriForTopic(string topicName) { var endpoint = new KafkaEndpoint @@ -33,7 +26,5 @@ public override KafkaSubscriberConfiguration FindConfigurationForTopic(string to return new KafkaSubscriberConfiguration((KafkaEndpoint) endpoint); } - } - } From 5aba77ab4766aa871807f1fba37c24bf7241b947 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 28 Apr 2020 12:09:33 -0400 Subject: [PATCH 35/82] Remove unused constant --- src/Jasper.ConfluentKafka/KafkaTransportProtocol.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Jasper.ConfluentKafka/KafkaTransportProtocol.cs b/src/Jasper.ConfluentKafka/KafkaTransportProtocol.cs index e3d2df0c3..60f13d8b4 100644 --- a/src/Jasper.ConfluentKafka/KafkaTransportProtocol.cs +++ b/src/Jasper.ConfluentKafka/KafkaTransportProtocol.cs @@ -1,6 +1,4 @@ -using System; using System.Collections.Generic; -using System.Diagnostics.Eventing.Reader; using System.Linq; using System.Text; using Confluent.Kafka; @@ -10,7 +8,6 @@ namespace Jasper.ConfluentKafka { public class KafkaTransportProtocol : ITransportProtocol> { - private const string JasperMessageIdHeader = "Jasper_MessageId"; public Message WriteFromEnvelope(Envelope envelope) { var message = new Message From 45cea592a5808d9f323201aeb91f4101dbc0ee52 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 28 Apr 2020 12:10:15 -0400 Subject: [PATCH 36/82] Remove unused class --- .../Serialization/DefaultJsonSerializer.cs | 36 ------------------- 1 file changed, 36 deletions(-) delete mode 100644 src/Jasper.ConfluentKafka/Serialization/DefaultJsonSerializer.cs diff --git a/src/Jasper.ConfluentKafka/Serialization/DefaultJsonSerializer.cs b/src/Jasper.ConfluentKafka/Serialization/DefaultJsonSerializer.cs deleted file mode 100644 index 1078da885..000000000 --- a/src/Jasper.ConfluentKafka/Serialization/DefaultJsonSerializer.cs +++ /dev/null @@ -1,36 +0,0 @@ -using System; -using System.Text; -using System.Threading.Tasks; -using Confluent.Kafka; -using Confluent.Kafka.SyncOverAsync; -using Newtonsoft.Json; - -namespace Jasper.ConfluentKafka.Serialization -{ - internal class DefaultJsonSerializer : IAsyncSerializer - { - public Task SerializeAsync(T data, SerializationContext context) - { - var json = JsonConvert.SerializeObject(data); - return Task.FromResult(Encoding.UTF8.GetBytes(json)); - } - - public ISerializer AsSyncOverAsync() - { - return new SyncOverAsyncSerializer(this); - } - } - - internal class DefaultJsonDeserializer : IAsyncDeserializer - { - public IDeserializer AsSyncOverAsync() - { - return new SyncOverAsyncDeserializer(this); - } - - public Task DeserializeAsync(ReadOnlyMemory data, bool isNull, SerializationContext context) - { - return Task.FromResult(JsonConvert.DeserializeObject(Encoding.UTF8.GetString(data.ToArray()))); - } - } -} From b3e6fbbbbc4f190624b518ba3151ac3d1de1a6f5 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 28 Apr 2020 16:41:33 -0400 Subject: [PATCH 37/82] Write test for ordering guarantees --- .../Jasper.ConfluentKafka.Tests.csproj | 2 + src/Jasper.ConfluentKafka.Tests/end_to_end.cs | 74 ++++++++++++++----- .../KafkaTransportProtocol.cs | 14 +++- 3 files changed, 72 insertions(+), 18 deletions(-) diff --git a/src/Jasper.ConfluentKafka.Tests/Jasper.ConfluentKafka.Tests.csproj b/src/Jasper.ConfluentKafka.Tests/Jasper.ConfluentKafka.Tests.csproj index 964180561..4bf5e2989 100644 --- a/src/Jasper.ConfluentKafka.Tests/Jasper.ConfluentKafka.Tests.csproj +++ b/src/Jasper.ConfluentKafka.Tests/Jasper.ConfluentKafka.Tests.csproj @@ -7,7 +7,9 @@ + + diff --git a/src/Jasper.ConfluentKafka.Tests/end_to_end.cs b/src/Jasper.ConfluentKafka.Tests/end_to_end.cs index 09b79bf07..cc6cd988a 100644 --- a/src/Jasper.ConfluentKafka.Tests/end_to_end.cs +++ b/src/Jasper.ConfluentKafka.Tests/end_to_end.cs @@ -1,5 +1,9 @@ using System; +using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; +using AutoFixture; +using AutoFixture.Xunit2; using Baseline.Dates; using Confluent.Kafka; using Jasper.Tracking; @@ -72,25 +76,61 @@ public KafkaSendingComplianceTests() : base($"kafka://topic/messages".ToUri()) // SAMPLE: can_stop_and_start_ASB [Fact] - public async Task can_stop_and_start() + public async Task can_send_and_receive_from_kafka() { - using (var host = JasperHost.For()) + using var host = JasperHost.For(); + await host + // The TrackActivity() method starts a Fluent Interface + // that gives you fine-grained control over the + // message tracking + .TrackActivity() + .Timeout(30.Seconds()) + // Include the external transports in the determination + // of "completion" + .IncludeExternalTransports() + .SendMessageAndWait(new ColorChosen { Name = "Red" }); + + var colors = host.Get(); + + colors.Name.ShouldBe("Red"); + } + + + [Fact] + public async Task send_multiple_messages_in_order() + { + var colorsChosens = Enumerable.Range(0, 100).Select(i => new ColorChosen {Name = i.ToString()}); + var sequence = Guid.NewGuid().ToString(); + using var host = JasperHost.For(host => { - await host - // The TrackActivity() method starts a Fluent Interface - // that gives you fine-grained control over the - // message tracking - .TrackActivity() - .Timeout(30.Seconds()) - // Include the external transports in the determination - // of "completion" - .IncludeExternalTransports() - .SendMessageAndWait(new ColorChosen {Name = "Red"}); - - var colors = host.Get(); - - colors.Name.ShouldBe("Red"); - } + host.Endpoints.ConfigureKafka(); + host.Endpoints.ListenToKafkaTopic("messages", ConsumerConfig).Sequential(); + host.Endpoints.Publish(pub => pub.Message().ToKafkaTopic("messages", ProducerConfig) + .CustomizeOutgoing(e => e.Headers.Add("MessageKey", sequence)) // use the same message key in Kafka + ); + host.Handlers.IncludeType(); + host.Services.AddSingleton(); + host.Extensions.UseMessageTrackingTestingSupport(); + }); + + ITrackedSession session = await host + .TrackActivity() + .Timeout(60.Seconds()) + .IncludeExternalTransports() + .ExecuteAndWait(async ctx => + { + foreach (ColorChosen colorsChosen in colorsChosens) + { + await ctx.Publish(colorsChosen); + } + }); + + IEnumerable colorsSent = session.AllRecordsInOrder() + .Where(e => e.EventType == EventType.Sent) + .Select(e => e.Envelope.Message).Cast().Select(c => c.Name); + IEnumerable colorsPublished = colorsChosens.Select(c => c.Name); + + colorsSent.ShouldBe(colorsPublished); } // ENDSAMPLE diff --git a/src/Jasper.ConfluentKafka/KafkaTransportProtocol.cs b/src/Jasper.ConfluentKafka/KafkaTransportProtocol.cs index 60f13d8b4..3db70a73e 100644 --- a/src/Jasper.ConfluentKafka/KafkaTransportProtocol.cs +++ b/src/Jasper.ConfluentKafka/KafkaTransportProtocol.cs @@ -26,6 +26,18 @@ public Message WriteFromEnvelope(Envelope envelope) message.Headers = headers; + if (envelopHeaders.TryGetValue("MessageKey", out var msgKey)) + { + if (msgKey is byte[]) + { + message.Key = (byte[])msgKey; + } + else + { + message.Key = Encoding.UTF8.GetBytes(msgKey.ToString()); + } + } + return message; } @@ -40,7 +52,7 @@ public Envelope ReadEnvelope(Message message) .ToDictionary(k => k.Key, v => (object)Encoding.UTF8.GetString(v.Value)); env.ReadPropertiesFromDictionary(incomingHeaders); - + return env; } } From ff0006fec98213855beacc528bbcf271999f14bb Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 30 Apr 2020 15:03:07 -0400 Subject: [PATCH 38/82] Strip ISender of queue/buffer related methods/props --- src/Jasper/Transports/Sending/ISender.cs | 26 +++--------------------- 1 file changed, 3 insertions(+), 23 deletions(-) diff --git a/src/Jasper/Transports/Sending/ISender.cs b/src/Jasper/Transports/Sending/ISender.cs index cfb8ae9a6..1459969d1 100644 --- a/src/Jasper/Transports/Sending/ISender.cs +++ b/src/Jasper/Transports/Sending/ISender.cs @@ -1,5 +1,4 @@ -using System; -using System.Threading; +using System; using System.Threading.Tasks; namespace Jasper.Transports.Sending @@ -7,26 +6,7 @@ namespace Jasper.Transports.Sending public interface ISender : IDisposable { Uri Destination { get; } - - int QueuedCount { get; } - - bool Latched { get; } - void Start(ISenderCallback callback); - - Task Enqueue(Envelope envelope); - - Task LatchAndDrain(); - void Unlatch(); - - /// - /// Simply try to reach the endpoint to verify it can receive - /// - /// - /// - Task Ping(CancellationToken cancellationToken); - - bool SupportsNativeScheduledSend { get; } - - + + Task Send(Envelope envelope); } } From b34670752d03fc62b031a7bff037f1d0b90cc62c Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 30 Apr 2020 15:03:23 -0400 Subject: [PATCH 39/82] Rename ISender.Enqueue to ISender.Send for clarity --- .../Internal/AzureServiceBusSender.cs | 2 +- src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs | 2 +- src/Jasper.RabbitMQ/Internal/RabbitMqSender.cs | 2 +- src/Jasper.Testing/Transports/Sending/NulloSenderTester.cs | 2 +- src/Jasper/Persistence/Durability/DurableSendingAgent.cs | 6 +++--- src/Jasper/Transports/Sending/BatchedSender.cs | 4 ++-- src/Jasper/Transports/Sending/LightweightSendingAgent.cs | 6 +++--- src/Jasper/Transports/Sending/NulloSender.cs | 2 +- src/Jasper/Transports/Sending/SendingAgent.cs | 6 +++--- src/StorytellerSpecs/Fixtures/RetryAgentFixture.cs | 4 ++-- src/StorytellerSpecs/Stub/StubEndpoint.cs | 4 ++-- 11 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/Jasper.AzureServiceBus/Internal/AzureServiceBusSender.cs b/src/Jasper.AzureServiceBus/Internal/AzureServiceBusSender.cs index 47d1114d2..c43f67150 100644 --- a/src/Jasper.AzureServiceBus/Internal/AzureServiceBusSender.cs +++ b/src/Jasper.AzureServiceBus/Internal/AzureServiceBusSender.cs @@ -76,7 +76,7 @@ public void Start(ISenderCallback callback) } - public Task Enqueue(Envelope envelope) + public Task Send(Envelope envelope) { _sending.Post(envelope); diff --git a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs index 87fef2c4c..e3c0fa5e8 100644 --- a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs +++ b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs @@ -45,7 +45,7 @@ public void Start(ISenderCallback callback) _callback = callback; } - public Task Enqueue(Envelope envelope) + public Task Send(Envelope envelope) { _sending.Post(envelope); diff --git a/src/Jasper.RabbitMQ/Internal/RabbitMqSender.cs b/src/Jasper.RabbitMQ/Internal/RabbitMqSender.cs index a986acf2e..aa1998f1c 100644 --- a/src/Jasper.RabbitMQ/Internal/RabbitMqSender.cs +++ b/src/Jasper.RabbitMQ/Internal/RabbitMqSender.cs @@ -47,7 +47,7 @@ public void Start(ISenderCallback callback) }); } - public Task Enqueue(Envelope envelope) + public Task Send(Envelope envelope) { _sending.Post(envelope); diff --git a/src/Jasper.Testing/Transports/Sending/NulloSenderTester.cs b/src/Jasper.Testing/Transports/Sending/NulloSenderTester.cs index 88da40996..9db302d56 100644 --- a/src/Jasper.Testing/Transports/Sending/NulloSenderTester.cs +++ b/src/Jasper.Testing/Transports/Sending/NulloSenderTester.cs @@ -19,7 +19,7 @@ public async Task enqueue_automatically_marks_envelope_as_successful() var env = ObjectMother.Envelope(); - await sender.Enqueue(env); + await sender.Send(env); callback.Received().Successful(env); } diff --git a/src/Jasper/Persistence/Durability/DurableSendingAgent.cs b/src/Jasper/Persistence/Durability/DurableSendingAgent.cs index 846102eb7..225a05076 100644 --- a/src/Jasper/Persistence/Durability/DurableSendingAgent.cs +++ b/src/Jasper/Persistence/Durability/DurableSendingAgent.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Threading; @@ -77,7 +77,7 @@ protected override async Task afterRestarting(ISender sender) foreach (var envelope in toRetry) { - await _sender.Enqueue(envelope); + await _sender.Send(envelope); } } @@ -97,7 +97,7 @@ protected override async Task storeAndForward(Envelope envelope) { await _persistence.StoreOutgoing(envelope, _settings.UniqueNodeId); - await _sender.Enqueue(envelope); + await _sender.Send(envelope); } } diff --git a/src/Jasper/Transports/Sending/BatchedSender.cs b/src/Jasper/Transports/Sending/BatchedSender.cs index 9e8e4927d..ae60cc52d 100644 --- a/src/Jasper/Transports/Sending/BatchedSender.cs +++ b/src/Jasper/Transports/Sending/BatchedSender.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Threading; using System.Threading.Tasks; using System.Threading.Tasks.Dataflow; @@ -133,7 +133,7 @@ public async Task Ping(CancellationToken cancellationToken) public bool SupportsNativeScheduledSend { get; } = true; - public Task Enqueue(Envelope message) + public Task Send(Envelope message) { if (_batching == null) throw new InvalidOperationException("This agent has not been started"); diff --git a/src/Jasper/Transports/Sending/LightweightSendingAgent.cs b/src/Jasper/Transports/Sending/LightweightSendingAgent.cs index 8ab9e2719..0d6c0b910 100644 --- a/src/Jasper/Transports/Sending/LightweightSendingAgent.cs +++ b/src/Jasper/Transports/Sending/LightweightSendingAgent.cs @@ -1,4 +1,4 @@ -using System.Collections.Generic; +using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using Baseline; @@ -39,7 +39,7 @@ protected override Task afterRestarting(ISender sender) foreach (var envelope in toRetry) { // It's perfectly okay to not wait on the task here - _sender.Enqueue(envelope); + _sender.Send(envelope); } return Task.CompletedTask; @@ -57,7 +57,7 @@ public override Task Successful(Envelope outgoing) protected override Task storeAndForward(Envelope envelope) { - return _sender.Enqueue(envelope); + return _sender.Send(envelope); } public override bool IsDurable { get; } = false; diff --git a/src/Jasper/Transports/Sending/NulloSender.cs b/src/Jasper/Transports/Sending/NulloSender.cs index 860cfbbf2..cc9536b4b 100644 --- a/src/Jasper/Transports/Sending/NulloSender.cs +++ b/src/Jasper/Transports/Sending/NulloSender.cs @@ -26,7 +26,7 @@ public void Start(ISenderCallback callback) _callback = callback; } - public Task Enqueue(Envelope envelope) + public Task Send(Envelope envelope) { _callback.Successful(envelope); return Task.CompletedTask; diff --git a/src/Jasper/Transports/Sending/SendingAgent.cs b/src/Jasper/Transports/Sending/SendingAgent.cs index b69dd9095..68c51452e 100644 --- a/src/Jasper/Transports/Sending/SendingAgent.cs +++ b/src/Jasper/Transports/Sending/SendingAgent.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -52,7 +52,7 @@ private void setDefaults(Envelope envelope) public async Task EnqueueOutgoing(Envelope envelope) { setDefaults(envelope); - await _sender.Enqueue(envelope); + await _sender.Send(envelope); _messageLogger.Sent(envelope); } @@ -94,7 +94,7 @@ public async Task MarkFailed(OutgoingMessageBatch batch) foreach (var envelope in batch.Messages) { #pragma warning disable 4014 - _sender.Enqueue(envelope); + _sender.Send(envelope); #pragma warning restore 4014 } } diff --git a/src/StorytellerSpecs/Fixtures/RetryAgentFixture.cs b/src/StorytellerSpecs/Fixtures/RetryAgentFixture.cs index c1d8ac453..c29a120cb 100644 --- a/src/StorytellerSpecs/Fixtures/RetryAgentFixture.cs +++ b/src/StorytellerSpecs/Fixtures/RetryAgentFixture.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Threading; @@ -42,7 +42,7 @@ void ISender.Start(ISenderCallback callback) { } - Task ISender.Enqueue(Envelope envelope) + Task ISender.Send(Envelope envelope) { _enqueued.Add(envelope); return Task.CompletedTask; diff --git a/src/StorytellerSpecs/Stub/StubEndpoint.cs b/src/StorytellerSpecs/Stub/StubEndpoint.cs index d79cb9cdc..53f6dbf00 100644 --- a/src/StorytellerSpecs/Stub/StubEndpoint.cs +++ b/src/StorytellerSpecs/Stub/StubEndpoint.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; @@ -41,7 +41,7 @@ public void Start(ISenderCallback callback) _callback = callback; } - public Task Enqueue(Envelope envelope) + public Task Send(Envelope envelope) { Sent.Add(envelope); return _pipeline?.Invoke(envelope, new StubChannelCallback(this, envelope)) ?? Task.CompletedTask; From b9225eb8b23186083c5241d3b03356840d884323 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 6 May 2020 15:18:46 -0400 Subject: [PATCH 40/82] Add IRequireSenderCallback and implementation --- .../Transports/Sending/BatchedSender.cs | 38 ++++++++----------- src/Jasper/Transports/Sending/ISender.cs | 9 ++++- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/src/Jasper/Transports/Sending/BatchedSender.cs b/src/Jasper/Transports/Sending/BatchedSender.cs index ae60cc52d..44ceed35c 100644 --- a/src/Jasper/Transports/Sending/BatchedSender.cs +++ b/src/Jasper/Transports/Sending/BatchedSender.cs @@ -5,11 +5,10 @@ using Jasper.Logging; using Jasper.Transports.Tcp; using Jasper.Transports.Util; -using LamarCodeGeneration.Frames; namespace Jasper.Transports.Sending { - public class BatchedSender : ISender + public class BatchedSender : ISender, ISenderRequiresCallback { private readonly CancellationToken _cancellation; private readonly ITransportLogger _logger; @@ -22,20 +21,12 @@ public class BatchedSender : ISender private ActionBlock _sender; private ActionBlock _serializing; - public BatchedSender(Uri destination, ISenderProtocol protocol, CancellationToken cancellation, - ITransportLogger logger) + public BatchedSender(Uri destination, ISenderProtocol protocol, CancellationToken cancellation, ITransportLogger logger) { Destination = destination; _protocol = protocol; _cancellation = cancellation; _logger = logger; - } - - public Uri Destination { get; } - - public void Start(ISenderCallback callback) - { - _callback = callback; _sender = new ActionBlock(SendBatch, new ExecutionDataflowBlockOptions { @@ -50,16 +41,16 @@ public void Start(ISenderCallback callback) }, _cancellation); _serializing = new ActionBlock(async e => + { + try { - try - { - await _batching.SendAsync(e); - } - catch (Exception ex) - { - _logger.LogException(ex, message: $"Error while trying to serialize envelope {e}"); - } - }, + await _batching.SendAsync(e); + } + catch (Exception ex) + { + _logger.LogException(ex, message: $"Error while trying to serialize envelope {e}"); + } + }, new ExecutionDataflowBlockOptions { CancellationToken = _cancellation, @@ -81,7 +72,7 @@ public void Start(ISenderCallback callback) return batch; }, new ExecutionDataflowBlockOptions - {BoundedCapacity = DataflowBlockOptions.Unbounded, MaxDegreeOfParallelism = 10, CancellationToken = _cancellation}); + { BoundedCapacity = DataflowBlockOptions.Unbounded, MaxDegreeOfParallelism = 10, CancellationToken = _cancellation }); _batchWriting.Completion.ContinueWith(x => { @@ -97,6 +88,8 @@ public void Start(ISenderCallback callback) }, _cancellation); } + public Uri Destination { get; } + public int QueuedCount => _queued + _batching.ItemCount + _serializing.InputCount; public bool Latched { get; private set; } @@ -119,7 +112,6 @@ public void Unlatch() { _logger.CircuitResumed(Destination); - Start(_callback); Latched = false; } @@ -150,6 +142,8 @@ public void Dispose() _batching?.Dispose(); } + public void RegisterCallback(ISenderCallback senderCallback) => _callback = senderCallback; + public async Task SendBatch(OutgoingMessageBatch batch) { if (_cancellation.IsCancellationRequested) return; diff --git a/src/Jasper/Transports/Sending/ISender.cs b/src/Jasper/Transports/Sending/ISender.cs index 1459969d1..d6d600330 100644 --- a/src/Jasper/Transports/Sending/ISender.cs +++ b/src/Jasper/Transports/Sending/ISender.cs @@ -1,12 +1,19 @@ using System; +using System.Threading; using System.Threading.Tasks; namespace Jasper.Transports.Sending { + public interface ISenderRequiresCallback : IDisposable + { + void RegisterCallback(ISenderCallback senderCallback); + } + public interface ISender : IDisposable { + bool SupportsNativeScheduledSend { get; } Uri Destination { get; } - + Task Ping(CancellationToken cancellationToken); Task Send(Envelope envelope); } } From 68a12cb3b9f683ecb9aeda4ecf43189f739fcbcc Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 6 May 2020 15:19:48 -0400 Subject: [PATCH 41/82] Remove queueing logic from senders --- .../Internal/AzureServiceBusSender.cs | 108 +++--------------- .../Internal/ConfluentKafkaSender.cs | 103 ++++------------- .../Internal/RabbitMqSender.cs | 98 +++------------- 3 files changed, 56 insertions(+), 253 deletions(-) diff --git a/src/Jasper.AzureServiceBus/Internal/AzureServiceBusSender.cs b/src/Jasper.AzureServiceBus/Internal/AzureServiceBusSender.cs index c43f67150..d122176cc 100644 --- a/src/Jasper.AzureServiceBus/Internal/AzureServiceBusSender.cs +++ b/src/Jasper.AzureServiceBus/Internal/AzureServiceBusSender.cs @@ -1,7 +1,6 @@ using System; using System.Threading; using System.Threading.Tasks; -using System.Threading.Tasks.Dataflow; using Baseline; using Jasper.Logging; using Jasper.Transports; @@ -16,37 +15,16 @@ public class AzureServiceBusSender : ISender private readonly ITransportProtocol _protocol; private readonly AzureServiceBusEndpoint _endpoint; private readonly AzureServiceBusTransport _transport; - private readonly ITransportLogger _logger; - private readonly CancellationToken _cancellation; private ISenderClient _sender; - private ActionBlock _sending; - private ISenderCallback _callback; - - public AzureServiceBusSender(AzureServiceBusEndpoint endpoint, AzureServiceBusTransport transport, ITransportLogger logger, - CancellationToken cancellation) + public bool SupportsNativeScheduledSend { get; } = true; + public Uri Destination => _endpoint.Uri; + + public AzureServiceBusSender(AzureServiceBusEndpoint endpoint, AzureServiceBusTransport transport) { _protocol = endpoint.Protocol; _endpoint = endpoint; _transport = transport; - _logger = logger; - _cancellation = cancellation; - Destination = endpoint.Uri; - } - - public void Dispose() - { - _sender?.CloseAsync().GetAwaiter().GetResult(); - } - - public Uri Destination { get; } - public int QueuedCount => _sending.InputCount; - public bool Latched { get; private set; } - - - public void Start(ISenderCallback callback) - { - _callback = callback; - + // The variance here should be in constructing the sending & buffer blocks if (_endpoint.TopicName.IsEmpty()) { @@ -54,11 +32,6 @@ public void Start(ISenderCallback callback) ? new MessageSender(_transport.ConnectionString, _endpoint.QueueName, _transport.TokenProvider, _transport.TransportType, _transport.RetryPolicy) : new MessageSender(_transport.ConnectionString, _endpoint.QueueName, _transport.RetryPolicy); - - _sending = new ActionBlock(sendBySession, new ExecutionDataflowBlockOptions - { - CancellationToken = _cancellation - }); } else { @@ -67,44 +40,31 @@ public void Start(ISenderCallback callback) _transport.TransportType, _transport.RetryPolicy) : new TopicClient(_transport.ConnectionString, _endpoint.TopicName, _transport.RetryPolicy); - - _sending = new ActionBlock(sendBySession, new ExecutionDataflowBlockOptions - { - CancellationToken = _cancellation - }); } } - - public Task Send(Envelope envelope) + public void Dispose() { - _sending.Post(envelope); - - return Task.CompletedTask; + _sender?.CloseAsync().GetAwaiter().GetResult(); } - public async Task LatchAndDrain() + public Task Send(Envelope envelope) { - Latched = true; - - await _sender.CloseAsync(); - - _sending.Complete(); + var message = _protocol.WriteFromEnvelope(envelope); + message.SessionId = Guid.NewGuid().ToString(); - _logger.CircuitBroken(Destination); - } - public void Unlatch() - { - _logger.CircuitResumed(Destination); + if (envelope.IsDelayed(DateTime.UtcNow)) + { + return _sender.ScheduleMessageAsync(message, envelope.ExecutionTime.Value); + } - Start(_callback); - Latched = false; + return _sender.SendAsync(message); } - + public async Task Ping(CancellationToken cancellationToken) { - var envelope = Envelope.ForPing(Destination); + var envelope = Envelope.ForPing(_endpoint.Uri); var message = _protocol.WriteFromEnvelope(envelope); message.SessionId = Guid.NewGuid().ToString(); @@ -112,39 +72,5 @@ public async Task Ping(CancellationToken cancellationToken) return true; } - - public bool SupportsNativeScheduledSend { get; } = true; - - private async Task sendBySession(Envelope envelope) - { - try - { - var message = _protocol.WriteFromEnvelope(envelope); - message.SessionId = Guid.NewGuid().ToString(); - - - if (envelope.IsDelayed(DateTime.UtcNow)) - { - await _sender.ScheduleMessageAsync(message, envelope.ExecutionTime.Value); - } - else - { - await _sender.SendAsync(message); - } - - await _callback.Successful(envelope); - } - catch (Exception e) - { - try - { - await _callback.ProcessingFailure(envelope, e); - } - catch (Exception exception) - { - _logger.LogException(exception); - } - } - } } } diff --git a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs index e3c0fa5e8..fe15da110 100644 --- a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs +++ b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs @@ -1,11 +1,8 @@ using System; -using System.Text; using System.Threading; using System.Threading.Tasks; -using System.Threading.Tasks.Dataflow; using Confluent.Kafka; using Jasper.ConfluentKafka.Exceptions; -using Jasper.Logging; using Jasper.Transports; using Jasper.Transports.Sending; @@ -13,21 +10,15 @@ namespace Jasper.ConfluentKafka.Internal { public class ConfluentKafkaSender : ISender { - private ITransportProtocol> _protocol; - private IProducer _publisher; + private readonly ITransportProtocol> _protocol; + private readonly IProducer _publisher; private readonly KafkaEndpoint _endpoint; - private readonly ITransportLogger _logger; - private readonly CancellationToken _cancellation; - private ActionBlock _sending; - private ISenderCallback _callback; - - public ConfluentKafkaSender(KafkaEndpoint endpoint, ITransportLogger logger, CancellationToken cancellation) + public bool SupportsNativeScheduledSend { get; } = false; + public Uri Destination => _endpoint.Uri; + public ConfluentKafkaSender(KafkaEndpoint endpoint) { _endpoint = endpoint; - _logger = logger; - _cancellation = cancellation; _publisher = new ProducerBuilder(endpoint.ProducerConfig).Build(); - _sending = new ActionBlock(sendBySession, _endpoint.ExecutionOptions); _protocol = new KafkaTransportProtocol(); } @@ -36,84 +27,40 @@ public void Dispose() _publisher?.Dispose(); } - public Uri Destination => _endpoint.Uri; - public int QueuedCount => _sending.InputCount; - public bool Latched { get; private set; } - - public void Start(ISenderCallback callback) - { - _callback = callback; - } - - public Task Send(Envelope envelope) - { - _sending.Post(envelope); - - return Task.CompletedTask; - } - - public Task LatchAndDrain() - { - Latched = true; - - _publisher.Flush(_cancellation); - - _sending.Complete(); - - _logger.CircuitBroken(Destination); - - return Task.CompletedTask; - } - - public void Unlatch() - { - _logger.CircuitResumed(Destination); - - Start(_callback); - Latched = false; - } - public async Task Ping(CancellationToken cancellationToken) { Envelope envelope = Envelope.ForPing(Destination); - Message message = _protocol.WriteFromEnvelope(envelope); - - message.Headers.Add("MessageGroupId", Encoding.UTF8.GetBytes(Guid.NewGuid().ToString())); - message.Headers.Add("Jasper_SessionId", Encoding.UTF8.GetBytes(Guid.NewGuid().ToString())); + try + { + Message message = _protocol.WriteFromEnvelope(envelope); - await _publisher.ProduceAsync("jasper-ping", message, cancellationToken); + await _publisher.ProduceAsync("jasper-ping", message, cancellationToken); + } + catch (Exception e) + { + return false; + } return true; } - public bool SupportsNativeScheduledSend { get; } = false; - - private async Task sendBySession(Envelope envelope) + public async Task Send(Envelope envelope) { - try + if (envelope.IsDelayed(DateTime.UtcNow)) { - Message message = _protocol.WriteFromEnvelope(envelope); - message.Headers.Add("Jasper_SessionId", Encoding.UTF8.GetBytes(Guid.NewGuid().ToString())); - - if (envelope.IsDelayed(DateTime.UtcNow)) - { - throw new UnsupportedFeatureException("Delayed Message Delivery"); - } - - await _publisher.ProduceAsync(_endpoint.TopicName, message, _cancellation); + throw new UnsupportedFeatureException("Delayed Message Delivery"); + } - await _callback.Successful(envelope); + Message message = _protocol.WriteFromEnvelope(envelope); + try + { + var result = await _publisher.ProduceAsync(_endpoint.TopicName, message); + Console.WriteLine(result.Status); } catch (Exception e) { - try - { - await _callback.ProcessingFailure(envelope, e); - } - catch (Exception exception) - { - _logger.LogException(exception); - } + Console.WriteLine(e); + throw; } } } diff --git a/src/Jasper.RabbitMQ/Internal/RabbitMqSender.cs b/src/Jasper.RabbitMQ/Internal/RabbitMqSender.cs index aa1998f1c..3e242841f 100644 --- a/src/Jasper.RabbitMQ/Internal/RabbitMqSender.cs +++ b/src/Jasper.RabbitMQ/Internal/RabbitMqSender.cs @@ -1,9 +1,6 @@ using System; -using System.Security.Authentication.ExtendedProtection; using System.Threading; using System.Threading.Tasks; -using System.Threading.Tasks.Dataflow; -using Jasper.Logging; using Jasper.Transports; using Jasper.Transports.Sending; using RabbitMQ.Client; @@ -12,75 +9,39 @@ namespace Jasper.RabbitMQ.Internal { public class RabbitMqSender : RabbitMqConnectionAgent, ISender { - private readonly CancellationToken _cancellation; - private readonly ITransportLogger _logger; private readonly IRabbitMqProtocol _protocol; - private ISenderCallback _callback; - private ActionBlock _sending; private readonly string _exchangeName; private readonly string _key; private readonly bool _isDurable; + public bool SupportsNativeScheduledSend { get; } = false; + public Uri Destination { get; } - public RabbitMqSender(ITransportLogger logger, RabbitMqEndpoint endpoint, RabbitMqTransport transport, - CancellationToken cancellation) : base(transport) + public RabbitMqSender(RabbitMqEndpoint endpoint, RabbitMqTransport transport) : base(transport) { _protocol = endpoint.Protocol; - _logger = logger; - _cancellation = cancellation; Destination = endpoint.Uri; - + _isDurable = endpoint.IsDurable; _exchangeName = endpoint.ExchangeName == TransportConstants.Default ? "" : endpoint.ExchangeName; _key = endpoint.RoutingKey ?? endpoint.QueueName ?? ""; } - public void Start(ISenderCallback callback) - { - Connect(); - - _callback = callback; - - _sending = new ActionBlock(send, new ExecutionDataflowBlockOptions - { - CancellationToken = _cancellation - }); - } - - public Task Send(Envelope envelope) - { - _sending.Post(envelope); - - return Task.CompletedTask; - } - - public Uri Destination { get; } - public int QueuedCount => _sending.InputCount; - - public bool Latched { get; private set; } - - public Task LatchAndDrain() +#pragma warning disable 1998 + public async Task Send(Envelope envelope) +#pragma warning restore 1998 { - Latched = true; - - Stop(); - - _sending.Complete(); - - - _logger.CircuitBroken(Destination); - - return Task.CompletedTask; - } + EnsureConnected(); + if (State == AgentState.Disconnected) + throw new InvalidOperationException($"The RabbitMQ agent for {Destination} is disconnected"); + var props = Channel.CreateBasicProperties(); + props.Persistent = _isDurable; - public void Unlatch() - { - _logger.CircuitResumed(Destination); + _protocol.WriteFromEnvelope(envelope, props); - Start(_callback); - Latched = false; + Channel.BasicPublish(_exchangeName, _key, props, envelope.Data); } public Task Ping(CancellationToken cancellationToken) @@ -102,36 +63,5 @@ public Task Ping(CancellationToken cancellationToken) } } } - - public bool SupportsNativeScheduledSend { get; } = false; - - private async Task send(Envelope envelope) - { - if (State == AgentState.Disconnected) - throw new InvalidOperationException($"The RabbitMQ agent for {Destination} is disconnected"); - - try - { - var props = Channel.CreateBasicProperties(); - props.Persistent = _isDurable; - - _protocol.WriteFromEnvelope(envelope, props); - - Channel.BasicPublish(_exchangeName, _key, props, envelope.Data); - - await _callback.Successful(envelope); - } - catch (Exception e) - { - try - { - await _callback.ProcessingFailure(envelope, e); - } - catch (Exception exception) - { - _logger.LogException(exception); - } - } - } } } From fc80140a3c3cfaaeb291d27a786912c5a58d52d8 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 6 May 2020 15:20:28 -0400 Subject: [PATCH 42/82] Refactor for changed sender ctor --- src/Jasper.AzureServiceBus/AzureServiceBusEndpoint.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Jasper.AzureServiceBus/AzureServiceBusEndpoint.cs b/src/Jasper.AzureServiceBus/AzureServiceBusEndpoint.cs index 45e988d13..a85024d39 100644 --- a/src/Jasper.AzureServiceBus/AzureServiceBusEndpoint.cs +++ b/src/Jasper.AzureServiceBus/AzureServiceBusEndpoint.cs @@ -140,7 +140,7 @@ protected internal override void StartListening(IMessagingRoot root, ITransportR protected override ISender CreateSender(IMessagingRoot root) { if (Parent.ConnectionString == null) throw new InvalidOperationException("There is no configured connection string for Azure Service Bus, or it is empty"); - return new AzureServiceBusSender(this, Parent, root.TransportLogger, root.Cancellation); + return new AzureServiceBusSender(this, Parent); } } } From b85ba13be884d452f822beb2f5519c0a6105e86f Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 6 May 2020 15:20:51 -0400 Subject: [PATCH 43/82] Refactor for sender ctor change --- src/Jasper.ConfluentKafka/KafkaEndpoint.cs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/Jasper.ConfluentKafka/KafkaEndpoint.cs b/src/Jasper.ConfluentKafka/KafkaEndpoint.cs index 7d52914f7..36e43cd4b 100644 --- a/src/Jasper.ConfluentKafka/KafkaEndpoint.cs +++ b/src/Jasper.ConfluentKafka/KafkaEndpoint.cs @@ -17,9 +17,20 @@ public class KafkaEndpoint : Endpoint { private const string TopicToken = "topic"; public string TopicName { get; set; } - public ProducerConfig ProducerConfig { get; set; } - public ConsumerConfig ConsumerConfig { get; set; } + public ProducerConfig ProducerConfig { get; set; } = new ProducerConfig(); + public ConsumerConfig ConsumerConfig { get; set; } = new ConsumerConfig(); public override Uri Uri => BuildUri(); + + public KafkaEndpoint() + { + + } + public KafkaEndpoint(Uri uri) : base(uri) + { + + } + + private Uri BuildUri(bool forReply = false) { var list = new List(); @@ -84,7 +95,7 @@ protected internal override void StartListening(IMessagingRoot root, ITransportR protected override ISender CreateSender(IMessagingRoot root) { - return new ConfluentKafkaSender(this, root.TransportLogger, root.Cancellation); + return new ConfluentKafkaSender(this); } public override Uri ReplyUri() => BuildUri(true); From 0287cff433d2b15cc6c78f3979c6a00b96e821b5 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 6 May 2020 15:21:16 -0400 Subject: [PATCH 44/82] Override findEndpointByUri --- src/Jasper.ConfluentKafka/KafkaTransport.cs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/Jasper.ConfluentKafka/KafkaTransport.cs b/src/Jasper.ConfluentKafka/KafkaTransport.cs index e8dc29cda..f1e4e2bcb 100644 --- a/src/Jasper.ConfluentKafka/KafkaTransport.cs +++ b/src/Jasper.ConfluentKafka/KafkaTransport.cs @@ -24,7 +24,15 @@ public KafkaTransport() : base(Protocols.Kafka) protected override IEnumerable endpoints() => _endpoints.Values; - protected override KafkaEndpoint findEndpointByUri(Uri uri) => _endpoints[uri]; + protected override KafkaEndpoint findEndpointByUri(Uri uri) + { + if (!_endpoints.ContainsKey(uri)) + { + _endpoints.Add(uri, new KafkaEndpoint(uri)); + } + + return _endpoints[uri]; + } public KafkaEndpoint EndpointForTopic(string topicName, ProducerConfig producerConifg) => AddOrUpdateEndpoint(topicName, endpoint => endpoint.ProducerConfig = producerConifg); From cb086aba1f91f355c0c7ff89e4eceecd2fc0c5e1 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 6 May 2020 15:21:36 -0400 Subject: [PATCH 45/82] Make sure not to return inside of consumer loop --- .../Internal/ConfluentKafkaListener.cs | 37 +++++++++++++------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaListener.cs b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaListener.cs index 32513b6b6..afa6b702d 100644 --- a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaListener.cs +++ b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaListener.cs @@ -29,6 +29,7 @@ public ConfluentKafkaListener(KafkaEndpoint endpoint, ITransportLogger logger, C public void Dispose() { + _consumer?.Dispose(); _consumerTask?.Dispose(); } @@ -53,10 +54,19 @@ private async Task ConsumeAsync() { message = await Task.Run(() => _consumer.Consume(), _cancellation); } + catch (Confluent.Kafka.ConsumeException cex) + { + if (cex.Error.Code == ErrorCode.PolicyViolation) + { + throw; + } + + continue; + } catch (Exception ex) { _logger.LogException(ex, message: $"Error consuming message from Kafka topic {_endpoint.TopicName}"); - return; + continue; } Envelope envelope; @@ -68,22 +78,25 @@ private async Task ConsumeAsync() catch (Exception ex) { _logger.LogException(ex, message: $"Error trying to map an incoming Kafka {_endpoint.TopicName} Topic message to an Envelope. See the Dead Letter Queue"); - return; + continue; } try { - await _callback.Received(Address, new[] { envelope }); - - _consumer.Commit(); - } - catch (KafkaException ke) - { - if (ke.Error?.Code == ErrorCode.Local_NoOffset) + await _callback.Received(Address, new[] {envelope}).ContinueWith(t => { - return; - } - _logger.LogException(ke, envelope.Id, "Error trying to receive a message from " + Address); + try + { + _consumer.Commit(); + } + catch (KafkaException ke) + { + if (ke.Error?.Code != ErrorCode.Local_NoOffset) + { + throw; + } + } + }); } catch (Exception e) { From a8ea728e87ef0d1bcf09886d8cade90da1a43a23 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 6 May 2020 15:22:05 -0400 Subject: [PATCH 46/82] Rename method for clarity --- src/Jasper.RabbitMQ/Internal/RabbitMqConnectionAgent.cs | 2 +- src/Jasper.RabbitMQ/Internal/RabbitMqListener.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Jasper.RabbitMQ/Internal/RabbitMqConnectionAgent.cs b/src/Jasper.RabbitMQ/Internal/RabbitMqConnectionAgent.cs index 3c785d8dc..71bbe2576 100644 --- a/src/Jasper.RabbitMQ/Internal/RabbitMqConnectionAgent.cs +++ b/src/Jasper.RabbitMQ/Internal/RabbitMqConnectionAgent.cs @@ -21,7 +21,7 @@ public void Dispose() teardownConnection(); } - internal void Connect() + internal void EnsureConnected() { lock (_locker) { diff --git a/src/Jasper.RabbitMQ/Internal/RabbitMqListener.cs b/src/Jasper.RabbitMQ/Internal/RabbitMqListener.cs index 8892487bb..39daba259 100644 --- a/src/Jasper.RabbitMQ/Internal/RabbitMqListener.cs +++ b/src/Jasper.RabbitMQ/Internal/RabbitMqListener.cs @@ -1,4 +1,4 @@ -using System; +using System; using Baseline; using Jasper.Logging; using Jasper.Transports; @@ -48,7 +48,7 @@ public void Start(IReceiverCallback callback) { if (callback == null) return; - Connect(); + EnsureConnected(); _callback = callback; _consumer = new MessageConsumer(callback, _logger, Channel, _mapper, Address) From abef8bc5b8dd03b1afa73349a63fd487435d92f3 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 6 May 2020 15:22:39 -0400 Subject: [PATCH 47/82] Adjust for refactored ctor --- src/Jasper.RabbitMQ/Internal/RabbitMqEndpoint.cs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/Jasper.RabbitMQ/Internal/RabbitMqEndpoint.cs b/src/Jasper.RabbitMQ/Internal/RabbitMqEndpoint.cs index 5531ca95c..1990da238 100644 --- a/src/Jasper.RabbitMQ/Internal/RabbitMqEndpoint.cs +++ b/src/Jasper.RabbitMQ/Internal/RabbitMqEndpoint.cs @@ -2,13 +2,11 @@ using System.Collections.Generic; using System.Linq; using Baseline; -using ImTools; using Jasper.Configuration; using Jasper.Runtime; using Jasper.Transports; using Jasper.Transports.Sending; using Jasper.Util; -using Microsoft.CodeAnalysis.CSharp.Syntax; namespace Jasper.RabbitMQ.Internal { @@ -127,10 +125,8 @@ protected internal override void StartListening(IMessagingRoot root, ITransportR protected override ISender CreateSender(IMessagingRoot root) { - return new RabbitMqSender(root.TransportLogger, this, Parent, root.Cancellation); + return new RabbitMqSender(this, this.Parent); } - - } From 4d48680b851cee7e96255f0a42b10b1b8ce8c80b Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 6 May 2020 15:23:15 -0400 Subject: [PATCH 48/82] Make Destination field a public property --- .../Transports/Tcp/Protocol/ProtocolContext.cs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Jasper.Testing/Transports/Tcp/Protocol/ProtocolContext.cs b/src/Jasper.Testing/Transports/Tcp/Protocol/ProtocolContext.cs index 46a3ae3ed..b1676a80e 100644 --- a/src/Jasper.Testing/Transports/Tcp/Protocol/ProtocolContext.cs +++ b/src/Jasper.Testing/Transports/Tcp/Protocol/ProtocolContext.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Linq; using System.Net; using System.Net.Sockets; @@ -21,7 +21,7 @@ public abstract class ProtocolContext : IDisposable private readonly IPAddress theAddress = IPAddress.Loopback; private readonly int thePort = ++NextPort; private readonly ListeningAgent _listener; - private readonly Uri destination; + public readonly Uri Destination; private readonly OutgoingMessageBatch theMessageBatch; @@ -30,7 +30,7 @@ public abstract class ProtocolContext : IDisposable public ProtocolContext() { - destination = $"durable://localhost:{thePort}/incoming".ToUri(); + Destination = $"durable://localhost:{thePort}/incoming".ToUri(); _listener = new ListeningAgent(theReceiver, theAddress, thePort, "durable", CancellationToken.None); @@ -44,7 +44,7 @@ public ProtocolContext() outgoingMessage() }; - theMessageBatch = new OutgoingMessageBatch(destination, messages); + theMessageBatch = new OutgoingMessageBatch(Destination, messages); } public void Dispose() @@ -57,8 +57,8 @@ private Envelope outgoingMessage() { return new Envelope { - Destination = destination, - Data = new byte[] {1, 2, 3, 4, 5, 6, 7}, + Destination = Destination, + Data = new byte[] { 1, 2, 3, 4, 5, 6, 7 }, SentAt = DateTime.Today.ToUniversalTime() }; } @@ -69,10 +69,10 @@ protected async Task afterSending() using (var client = new TcpClient()) { - if (Dns.GetHostName() == destination.Host) - await client.ConnectAsync(IPAddress.Loopback, destination.Port); + if (Dns.GetHostName() == Destination.Host) + await client.ConnectAsync(IPAddress.Loopback, Destination.Port); - await client.ConnectAsync(destination.Host, destination.Port); + await client.ConnectAsync(Destination.Host, Destination.Port); await WireProtocol.Send(client.GetStream(), theMessageBatch, null, theSender); } From e8e68b4b8f1c353ce9861e7181243cd969886f33 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 6 May 2020 15:23:57 -0400 Subject: [PATCH 49/82] Set sender callbacks --- src/Jasper.Testing/Runtime/ping_handling.cs | 2 +- src/Jasper.Testing/Transports/Sending/BatchedSenderTests.cs | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Jasper.Testing/Runtime/ping_handling.cs b/src/Jasper.Testing/Runtime/ping_handling.cs index c01f08cc4..f2ebf8b9c 100644 --- a/src/Jasper.Testing/Runtime/ping_handling.cs +++ b/src/Jasper.Testing/Runtime/ping_handling.cs @@ -20,7 +20,7 @@ public async Task ping_happy_path_with_tcp() var sender = new BatchedSender("tcp://localhost:2222".ToUri(), new SocketSenderProtocol(), CancellationToken.None, TransportLogger.Empty()); - sender.Start(new StubSenderCallback()); + sender.RegisterCallback(new StubSenderCallback()); await sender.Ping(CancellationToken.None); } diff --git a/src/Jasper.Testing/Transports/Sending/BatchedSenderTests.cs b/src/Jasper.Testing/Transports/Sending/BatchedSenderTests.cs index 9e0175487..7bda03c00 100644 --- a/src/Jasper.Testing/Transports/Sending/BatchedSenderTests.cs +++ b/src/Jasper.Testing/Transports/Sending/BatchedSenderTests.cs @@ -1,4 +1,4 @@ -using System.Threading; +using System.Threading; using System.Threading.Tasks; using Baseline; using Jasper.Logging; @@ -16,7 +16,8 @@ public BatchedSenderTests() { theSender = new BatchedSender(TransportConstants.RepliesUri, theProtocol, theCancellation.Token, TransportLogger.Empty()); - theSender.Start(theSenderCallback); + + theSender.RegisterCallback(theSenderCallback); theBatch = new OutgoingMessageBatch(theSender.Destination, new[] { From 95c180223ed159f0316d0453b7d4c8c8a8ef1668 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 6 May 2020 15:24:25 -0400 Subject: [PATCH 50/82] Allow passing port number to test setup --- .../Tcp/LightweightTcpTransportCompliance.cs | 44 ++++++++++++++++--- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/src/Jasper.Testing/Transports/Tcp/LightweightTcpTransportCompliance.cs b/src/Jasper.Testing/Transports/Tcp/LightweightTcpTransportCompliance.cs index 5bd3eac5c..ab9766089 100644 --- a/src/Jasper.Testing/Transports/Tcp/LightweightTcpTransportCompliance.cs +++ b/src/Jasper.Testing/Transports/Tcp/LightweightTcpTransportCompliance.cs @@ -1,3 +1,9 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.NetworkInformation; +using System.Net.Sockets; using Jasper.Util; using TestingSupport.Compliance; using Xunit; @@ -6,28 +12,54 @@ namespace Jasper.Testing.Transports.Tcp { public class Sender : JasperOptions { + public Sender(int portNumber) + { + Endpoints.ListenForMessagesFrom($"tcp://localhost:{portNumber}/incoming".ToUri()); + } + public Sender() + : this(2389) { - Endpoints.ListenForMessagesFrom($"tcp://localhost:2289/incoming".ToUri()); + } } public class Receiver : JasperOptions { - public Receiver() + public Receiver(int portNumber) + { + Endpoints.ListenForMessagesFrom($"tcp://localhost:{portNumber}/incoming".ToUri()); + } + + public Receiver() : this(2388) + { + + } + } + + + public class PortFinder + { + private static readonly IPEndPoint DefaultLoopbackEndpoint = new IPEndPoint(IPAddress.Loopback, port: 0); + + public static int GetAvailablePort() { - Endpoints.ListenForMessagesFrom($"tcp://localhost:2288/incoming".ToUri()); + using var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + socket.Bind(DefaultLoopbackEndpoint); + var port = ((IPEndPoint)socket.LocalEndPoint).Port; + return port; } } + [Collection("compliance")] public class LightweightTcpTransportCompliance : SendingCompliance { - public LightweightTcpTransportCompliance() : base($"tcp://localhost:2288/incoming".ToUri()) + public LightweightTcpTransportCompliance() : base($"tcp://localhost:{PortFinder.GetAvailablePort()}/incoming".ToUri()) { - SenderIs(); + SenderIs(new Sender(PortFinder.GetAvailablePort())); - ReceiverIs(); + ReceiverIs(new Receiver(theOutboundAddress.Port)); } } } From ffdc3e12ba72b002559f213409215ba3a45cab30 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 6 May 2020 15:24:44 -0400 Subject: [PATCH 51/82] Remove ISenderCallback stub --- src/Jasper.Testing/Transports/Sending/NulloSenderTester.cs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/Jasper.Testing/Transports/Sending/NulloSenderTester.cs b/src/Jasper.Testing/Transports/Sending/NulloSenderTester.cs index 9db302d56..5fc5f9a68 100644 --- a/src/Jasper.Testing/Transports/Sending/NulloSenderTester.cs +++ b/src/Jasper.Testing/Transports/Sending/NulloSenderTester.cs @@ -1,8 +1,8 @@ +using System.Threading; using System.Threading.Tasks; using Jasper.Testing.Messaging; using Jasper.Transports.Sending; using Jasper.Util; -using NSubstitute; using Xunit; namespace Jasper.Testing.Transports.Sending @@ -14,14 +14,9 @@ public async Task enqueue_automatically_marks_envelope_as_successful() { var sender = new NulloSender("tcp://localhost:3333".ToUri()); - var callback = Substitute.For(); - sender.Start(callback); - var env = ObjectMother.Envelope(); await sender.Send(env); - - callback.Received().Successful(env); } } } From 6e8d8db83801a2e8e650066d4010052d62a31549 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 6 May 2020 15:25:22 -0400 Subject: [PATCH 52/82] Pull queue logic up from sender to agent --- src/StorytellerSpecs/Fixtures/RetryAgentFixture.cs | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/StorytellerSpecs/Fixtures/RetryAgentFixture.cs b/src/StorytellerSpecs/Fixtures/RetryAgentFixture.cs index c29a120cb..d1fc1d265 100644 --- a/src/StorytellerSpecs/Fixtures/RetryAgentFixture.cs +++ b/src/StorytellerSpecs/Fixtures/RetryAgentFixture.cs @@ -38,10 +38,6 @@ void IDisposable.Dispose() { } - void ISender.Start(ISenderCallback callback) - { - } - Task ISender.Send(Envelope envelope) { _enqueued.Add(envelope); @@ -50,17 +46,15 @@ Task ISender.Send(Envelope envelope) Uri ISender.Destination { get; } - int ISender.QueuedCount { get; } - - bool ISender.Latched => _latched; + bool Latched => _latched; - Task ISender.LatchAndDrain() + Task LatchAndDrain() { _latched = true; return Task.CompletedTask; } - void ISender.Unlatch() + void Unlatch() { _unlatched = true; } From 7f9c3292fc58fb0e0e5f9c2665e2d17ec0c74820 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 6 May 2020 15:25:43 -0400 Subject: [PATCH 53/82] Remove queue logic from sender --- src/StorytellerSpecs/Stub/StubEndpoint.cs | 29 +++-------------------- 1 file changed, 3 insertions(+), 26 deletions(-) diff --git a/src/StorytellerSpecs/Stub/StubEndpoint.cs b/src/StorytellerSpecs/Stub/StubEndpoint.cs index 53f6dbf00..ec7e1630b 100644 --- a/src/StorytellerSpecs/Stub/StubEndpoint.cs +++ b/src/StorytellerSpecs/Stub/StubEndpoint.cs @@ -17,7 +17,6 @@ public class StubEndpoint : Endpoint, ISendingAgent, ISender, IDisposable public readonly IList Callbacks = new List(); public readonly IList Sent = new List(); - private ISenderCallback _callback; private IMessageLogger _logger; private IHandlerPipeline _pipeline; private Uri _replyUri; @@ -31,44 +30,22 @@ public StubEndpoint(Uri destination, StubTransport stubTransport) } public Endpoint Endpoint => this; + public bool Latched { get; set; } public override Uri Uri => $"stub://{Name}".ToUri(); - - public int QueuedCount { get; } - - public void Start(ISenderCallback callback) - { - _callback = callback; - } - + public Task Send(Envelope envelope) { Sent.Add(envelope); return _pipeline?.Invoke(envelope, new StubChannelCallback(this, envelope)) ?? Task.CompletedTask; } - public Task LatchAndDrain() - { - return Task.CompletedTask; - } - - public void Unlatch() - { - Latched = false; - } - - public Task Ping(CancellationToken cancellationToken) - { - return Task.FromResult(true); - } - + public Task Ping(CancellationToken cancellationToken) => Task.FromResult(true); public void Dispose() { } - public bool Latched { get; set; } - public Uri Destination { get; } Uri ISendingAgent.ReplyUri From 23ed6fc3b84cc31815a32a277f44a685f207b3de Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 6 May 2020 15:26:03 -0400 Subject: [PATCH 54/82] Allow overriding timeout value --- .../Compliance/SendingCompliance.cs | 36 ++++++++++++------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/src/TestingSupport/Compliance/SendingCompliance.cs b/src/TestingSupport/Compliance/SendingCompliance.cs index cc3f20ddf..2e1f48464 100644 --- a/src/TestingSupport/Compliance/SendingCompliance.cs +++ b/src/TestingSupport/Compliance/SendingCompliance.cs @@ -2,6 +2,7 @@ using System.Data; using System.IO; using System.Linq; +using System.Threading; using System.Threading.Tasks; using Baseline.Dates; using Jasper; @@ -14,6 +15,8 @@ using TestingSupport.ErrorHandling; using TestMessages; using Xunit; +using Xunit.Abstractions; +using Xunit.Sdk; namespace TestingSupport.Compliance { @@ -22,6 +25,8 @@ public abstract class SendingCompliance : IDisposable protected IHost theSender; protected IHost theReceiver; protected Uri theOutboundAddress; + private readonly ITestOutputHelper _testOutputHelper = new TestOutputHelper(); + private readonly TimeSpan _defaultTimeout = 5.Seconds(); protected readonly ErrorCausingMessage theMessage = new ErrorCausingMessage(); private ITrackedSession _session; @@ -31,10 +36,15 @@ protected SendingCompliance(Uri destination) theOutboundAddress = destination; } + protected SendingCompliance(Uri destination, TimeSpan defaultTimeout) + { + theOutboundAddress = destination; + _defaultTimeout = defaultTimeout; + } + public void SenderIs() where T : JasperOptions, new() { theSender = JasperHost.For(configureSender); - } public void TheOnlyAppIs() where T : JasperOptions, new() @@ -113,13 +123,13 @@ public void Dispose() public async Task can_apply_requeue_mechanics() { - var session = await theSender.TrackActivity() + var session = await theSender.TrackActivity(_defaultTimeout) .AlsoTrack(theReceiver) .DoNotAssertOnExceptionsDetected() .ExecuteAndWait(c => c.SendToDestination(theOutboundAddress, new Message2())); - + session.FindSingleTrackedMessageOfType(EventType.MessageSucceeded) .ShouldNotBeNull(); @@ -128,7 +138,7 @@ public async Task can_apply_requeue_mechanics() [Fact] public async Task can_send_from_one_node_to_another_by_destination() { - var session = await theSender.TrackActivity() + var session = await theSender.TrackActivity(_defaultTimeout) .AlsoTrack(theReceiver) .DoNotAssertOnExceptionsDetected() .ExecuteAndWait(c => c.SendToDestination(theOutboundAddress, new Message1())); @@ -143,7 +153,7 @@ public async Task can_send_from_one_node_to_another_by_publishing_rule() { var message1 = new Message1(); - var session = await theSender.TrackActivity() + var session = await theSender.TrackActivity(_defaultTimeout) .AlsoTrack(theReceiver) .DoNotAssertOnExceptionsDetected() .SendMessageAndWait(message1); @@ -156,7 +166,7 @@ public async Task can_send_from_one_node_to_another_by_publishing_rule() [Fact] public async Task tags_the_envelope_with_the_source() { - var session = await theSender.TrackActivity() + var session = await theSender.TrackActivity(_defaultTimeout) .AlsoTrack(theReceiver) .DoNotAssertOnExceptionsDetected() .ExecuteAndWait(c => c.SendToDestination(theOutboundAddress, new Message1())); @@ -175,7 +185,7 @@ public async Task tracking_correlation_id_on_everything() var id2 = Guid.Empty; var session2 = await theSender - .TrackActivity() + .TrackActivity(_defaultTimeout) .AlsoTrack(theReceiver) .ExecuteAndWait(async context => @@ -200,7 +210,7 @@ public async Task tracking_correlation_id_on_everything() public async Task schedule_send() { var session = await theSender - .TrackActivity() + .TrackActivity(_defaultTimeout) .AlsoTrack(theReceiver) .Timeout(15.Seconds()) .WaitForMessageToBeReceivedAt(theReceiver ?? theSender) @@ -219,7 +229,7 @@ public async Task schedule_send() protected async Task afterProcessingIsComplete() { _session = await theSender - .TrackActivity() + .TrackActivity(_defaultTimeout) .AlsoTrack(theReceiver) .DoNotAssertOnExceptionsDetected() .SendMessageAndWait(theMessage); @@ -232,7 +242,7 @@ protected async Task afterProcessingIsComplete() protected async Task shouldSucceedOnAttempt(int attempt) { var session = await theSender - .TrackActivity() + .TrackActivity(_defaultTimeout) .AlsoTrack(theReceiver) .Timeout(15.Seconds()) .DoNotAssertOnExceptionsDetected() @@ -266,7 +276,7 @@ protected async Task shouldSucceedOnAttempt(int attempt) protected async Task shouldMoveToErrorQueueOnAttempt(int attempt) { var session = await theSender - .TrackActivity() + .TrackActivity(_defaultTimeout) .AlsoTrack(theReceiver) .DoNotAssertOnExceptionsDetected() .Timeout(30.Seconds()) @@ -343,7 +353,7 @@ public async Task explicit_respond_to_sender() var ping = new PingMessage(); var session = await theSender - .TrackActivity() + .TrackActivity(_defaultTimeout) .AlsoTrack(theReceiver) .Timeout(30.Seconds()) .SendMessageAndWait(ping); @@ -358,7 +368,7 @@ public async Task requested_response() var ping = new ImplicitPing(); var session = await theSender - .TrackActivity() + .TrackActivity(_defaultTimeout) .AlsoTrack(theReceiver) .Timeout(30.Seconds()) .ExecuteAndWait(x => x.SendAndExpectResponseFor(ping)); From 1a408574b1a51f11d531d218c8e38902efadfcb3 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 6 May 2020 15:26:19 -0400 Subject: [PATCH 55/82] Tweak setup of test configuration --- src/Jasper.ConfluentKafka.Tests/end_to_end.cs | 11 +- .../Durability/DurableSendingAgent.cs | 1 - .../JasperHostMessageTrackingExtensions.cs | 9 +- src/Jasper/Tracking/TrackedSession.cs | 11 ++ src/Jasper/Transports/Sending/NulloSender.cs | 37 +------ src/Jasper/Transports/Sending/SendingAgent.cs | 103 +++++++++++++----- src/Jasper/Transports/TransportBase.cs | 6 +- src/Jasper/Transports/TransportRuntime.cs | 10 +- 8 files changed, 113 insertions(+), 75 deletions(-) diff --git a/src/Jasper.ConfluentKafka.Tests/end_to_end.cs b/src/Jasper.ConfluentKafka.Tests/end_to_end.cs index cc6cd988a..23da82885 100644 --- a/src/Jasper.ConfluentKafka.Tests/end_to_end.cs +++ b/src/Jasper.ConfluentKafka.Tests/end_to_end.cs @@ -41,30 +41,31 @@ public class Sender : JasperOptions public Sender() { Endpoints.ConfigureKafka(); - + Endpoints.PublishAllMessages().ToKafkaTopic(Topic, ProducerConfig); } - public string QueueName { get; set; } + public string Topic = "messages"; } public class Receiver : JasperOptions { - public Receiver(string queueName) + public Receiver(string topic) { Endpoints.ConfigureKafka(); + Endpoints.ListenToKafkaTopic(topic, ConsumerConfig).UseForReplies(); } } public class KafkaSendingComplianceTests : SendingCompliance { - public KafkaSendingComplianceTests() : base($"kafka://topic/messages".ToUri()) + public KafkaSendingComplianceTests() : base($"kafka://topic/messages".ToUri(), 15.Seconds()) { var sender = new Sender(); SenderIs(sender); - var receiver = new Receiver(sender.QueueName); + var receiver = new Receiver(sender.Topic); ReceiverIs(receiver); } diff --git a/src/Jasper/Persistence/Durability/DurableSendingAgent.cs b/src/Jasper/Persistence/Durability/DurableSendingAgent.cs index 225a05076..57f17eb81 100644 --- a/src/Jasper/Persistence/Durability/DurableSendingAgent.cs +++ b/src/Jasper/Persistence/Durability/DurableSendingAgent.cs @@ -80,7 +80,6 @@ protected override async Task afterRestarting(ISender sender) await _sender.Send(envelope); } } - public override Task Successful(OutgoingMessageBatch outgoing) { return _policy.ExecuteAsync(c => _persistence.DeleteOutgoing(outgoing.Messages.ToArray()), _settings.Cancellation); diff --git a/src/Jasper/Tracking/JasperHostMessageTrackingExtensions.cs b/src/Jasper/Tracking/JasperHostMessageTrackingExtensions.cs index 5f0d02c27..92b54ea44 100644 --- a/src/Jasper/Tracking/JasperHostMessageTrackingExtensions.cs +++ b/src/Jasper/Tracking/JasperHostMessageTrackingExtensions.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Threading.Tasks; using Baseline.Dates; using Jasper.Logging; @@ -33,6 +33,13 @@ public static TrackedSessionConfiguration TrackActivity(this IHost host) return new TrackedSessionConfiguration(session); } + public static TrackedSessionConfiguration TrackActivity(this IHost host, TimeSpan trackingTimeout) + { + var session = new TrackedSession(host); + session.Timeout = trackingTimeout; + return new TrackedSessionConfiguration(session); + } + /// /// Send a message through the service bus and wait until that message /// and all cascading messages have been successfully processed diff --git a/src/Jasper/Tracking/TrackedSession.cs b/src/Jasper/Tracking/TrackedSession.cs index 49045ffe1..64f370513 100644 --- a/src/Jasper/Tracking/TrackedSession.cs +++ b/src/Jasper/Tracking/TrackedSession.cs @@ -262,12 +262,23 @@ public bool IsCompleted() public void LogException(Exception exception, string serviceName) { + Debug.WriteLine($"Exception Occurred in {serviceName}: {exception}"); _exceptions.Add(exception); } public void AddCondition(ITrackedCondition condition) { + Debug.WriteLine($"Condition Added: {condition}"); _conditions.Add(condition); } + + public override string ToString() + { + var conditionas = $"Conditions:\n{ _conditions.Select(x => x.ToString()).Join("\n")}"; + var activity = $"Activity:\n{ AllRecordsInOrder().Select(x => x.ToString()).Join("\n")}"; + var exceptions = $"Exceptions:\n{ _exceptions.Select(x => x.ToString()).Join("\n")}"; + + return $"{conditionas}\n\n{activity}\\{exceptions}"; + } } } diff --git a/src/Jasper/Transports/Sending/NulloSender.cs b/src/Jasper/Transports/Sending/NulloSender.cs index cc9536b4b..73a4b8a5f 100644 --- a/src/Jasper/Transports/Sending/NulloSender.cs +++ b/src/Jasper/Transports/Sending/NulloSender.cs @@ -1,14 +1,11 @@ using System; using System.Threading; using System.Threading.Tasks; -using LamarCodeGeneration.Frames; namespace Jasper.Transports.Sending { - public class NulloSender : ISender + public class NulloSender : ISender { - private ISenderCallback _callback; - public NulloSender(Uri destination) { Destination = destination; @@ -18,35 +15,9 @@ public void Dispose() { } - public Uri Destination { get; } - public int QueuedCount => 0; - public bool Latched => false; - public void Start(ISenderCallback callback) - { - _callback = callback; - } - - public Task Send(Envelope envelope) - { - _callback.Successful(envelope); - return Task.CompletedTask; - } - - public Task LatchAndDrain() - { - return Task.CompletedTask; - } - - public void Unlatch() - { - - } - - public Task Ping(CancellationToken cancellationToken) - { - return Task.FromResult(true); - } - public bool SupportsNativeScheduledSend { get; } = false; + public Uri Destination { get; } + public Task Send(Envelope envelope) => Task.CompletedTask; + public Task Ping(CancellationToken cancellationToken) => Task.FromResult(true); } } diff --git a/src/Jasper/Transports/Sending/SendingAgent.cs b/src/Jasper/Transports/Sending/SendingAgent.cs index 68c51452e..838040c26 100644 --- a/src/Jasper/Transports/Sending/SendingAgent.cs +++ b/src/Jasper/Transports/Sending/SendingAgent.cs @@ -2,6 +2,7 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; +using System.Threading.Tasks.Dataflow; using Baseline; using Jasper.Configuration; using Jasper.Logging; @@ -18,14 +19,19 @@ public abstract class SendingAgent : ISendingAgent, ISenderCallback, ICircuit private int _failureCount; private CircuitWatcher _circuitWatcher; - public SendingAgent(ITransportLogger logger, IMessageLogger messageLogger, ISender sender, - AdvancedSettings settings, Endpoint endpoint) + public SendingAgent(ITransportLogger logger, IMessageLogger messageLogger, ISender sender, AdvancedSettings settings, Endpoint endpoint) { _logger = logger; _messageLogger = messageLogger; _sender = sender; _settings = settings; Endpoint = endpoint; + _sending = new ActionBlock(sendViaSender, Endpoint.ExecutionOptions); + + _sending.Completion.ContinueWith(t => + { + Console.WriteLine(t.Exception?.ToString()); + }); } public Endpoint Endpoint { get; } @@ -39,7 +45,7 @@ public void Dispose() _sender.Dispose(); } - public bool Latched => _sender.Latched; + public bool Latched { get; private set; } public abstract bool IsDurable { get; } private void setDefaults(Envelope envelope) @@ -52,8 +58,8 @@ private void setDefaults(Envelope envelope) public async Task EnqueueOutgoing(Envelope envelope) { setDefaults(envelope); - await _sender.Send(envelope); - _messageLogger.Sent(envelope); + _sending.Post(envelope); + _messageLogger.Sent(envelope); } public async Task StoreAndForward(Envelope envelope) @@ -67,13 +73,69 @@ public async Task StoreAndForward(Envelope envelope) protected abstract Task storeAndForward(Envelope envelope); - public bool SupportsNativeScheduledSend => _sender.SupportsNativeScheduledSend; + public Task TryToResume(CancellationToken cancellationToken) + { + return _sender.Ping(cancellationToken); + } + TimeSpan ICircuit.RetryInterval => Endpoint.PingIntervalForCircuitResume; + + Task ICircuit.Resume(CancellationToken cancellationToken) + { + _circuitWatcher = null; + + Unlatch(); + + return afterRestarting(_sender); + } + + protected abstract Task afterRestarting(ISender sender); + + public abstract Task Successful(Envelope outgoing); + + private ActionBlock _sending; + public Task LatchAndDrain() + { + Latched = true; + + _sending.Complete(); + + _logger.CircuitBroken(Destination); + + return Task.CompletedTask; + } + + public void Unlatch() + { + _logger.CircuitResumed(Destination); + + Latched = false; + } + private async Task sendViaSender(Envelope envelope) + { + try + { + await _sender.Send(envelope); + + await Successful(envelope); + } + catch (Exception e) + { + try + { + await ((ISenderCallback)this).ProcessingFailure(envelope, e); + } + catch (Exception exception) + { + _logger.LogException(exception); + } + } + } public async Task MarkFailed(OutgoingMessageBatch batch) { // If it's already latched, just enqueue again - if (_sender.Latched) + if (Latched) { await EnqueueForRetry(batch); return; @@ -83,11 +145,10 @@ public async Task MarkFailed(OutgoingMessageBatch batch) if (_failureCount >= Endpoint.FailuresBeforeCircuitBreaks) { - await _sender.LatchAndDrain(); + await LatchAndDrain(); await EnqueueForRetry(batch); _circuitWatcher = new CircuitWatcher(this, _settings.Cancellation); - //_circuitWatcher = new CircuitWatcher(_sender, _endpoint.PingIntervalForCircuitResume, restartSending); } else { @@ -100,34 +161,19 @@ public async Task MarkFailed(OutgoingMessageBatch batch) } } - public Task TryToResume(CancellationToken cancellationToken) - { - return _sender.Ping(cancellationToken); - } - - TimeSpan ICircuit.RetryInterval => Endpoint.PingIntervalForCircuitResume; public abstract Task EnqueueForRetry(OutgoingMessageBatch batch); - Task ICircuit.Resume(CancellationToken cancellationToken) - { - _circuitWatcher = null; - - _sender.Unlatch(); - - return afterRestarting(_sender); - } - - protected abstract Task afterRestarting(ISender sender); public Task MarkSuccess() { _failureCount = 0; - _sender.Unlatch(); + Unlatch(); _circuitWatcher = null; return Task.CompletedTask; } + Task ISenderCallback.TimedOut(OutgoingMessageBatch outgoing) { @@ -160,7 +206,7 @@ Task ISenderCallback.ProcessingFailure(OutgoingMessageBatch outgoing) Task ISenderCallback.ProcessingFailure(Envelope outgoing, Exception exception) { - var batch = new OutgoingMessageBatch(outgoing.Destination, new[] {outgoing}); + var batch = new OutgoingMessageBatch(outgoing.Destination, new[] { outgoing }); _logger.OutgoingBatchFailed(batch, exception); return MarkFailed(batch); } @@ -180,8 +226,7 @@ Task ISenderCallback.SenderIsLatched(OutgoingMessageBatch outgoing) public abstract Task Successful(OutgoingMessageBatch outgoing); - public abstract Task Successful(Envelope outgoing); - + public bool SupportsNativeScheduledSend => _sender.SupportsNativeScheduledSend; } } diff --git a/src/Jasper/Transports/TransportBase.cs b/src/Jasper/Transports/TransportBase.cs index b9b322baa..82bde3d2b 100644 --- a/src/Jasper/Transports/TransportBase.cs +++ b/src/Jasper/Transports/TransportBase.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using Jasper.Configuration; @@ -82,7 +82,9 @@ public Endpoint GetOrCreateEndpoint(Uri uri) var endpoint = findEndpointByUri(canonicizeUri(uri)); - // It's coded this way so you don't override + if(endpoint == null) + + // It's coded this way so you don't override // durability if it's already set if (shouldBeDurable) endpoint.IsDurable = true; diff --git a/src/Jasper/Transports/TransportRuntime.cs b/src/Jasper/Transports/TransportRuntime.cs index 449720e23..94da66aa5 100644 --- a/src/Jasper/Transports/TransportRuntime.cs +++ b/src/Jasper/Transports/TransportRuntime.cs @@ -67,10 +67,14 @@ public ISendingAgent AddSubscriber(Uri replyUri, ISender sender, Endpoint endpoi : new LightweightSendingAgent(_root.TransportLogger, _root.MessageLogger, sender, _root.Settings, endpoint); agent.ReplyUri = replyUri; - sender.Start((ISenderCallback) agent); endpoint.Agent = agent; + if (sender is ISenderRequiresCallback senderRequiringCallback && agent is ISenderCallback callbackAgent) + { + senderRequiringCallback.RegisterCallback(callbackAgent); + } + AddSendingAgent(agent); AddSubscriber(endpoint); @@ -156,11 +160,9 @@ public void AddListener(IListener listener, Endpoint settings) ? (IWorkerQueue) new DurableWorkerQueue(settings, _root.Pipeline, _root.Settings, _root.Persistence, _root.TransportLogger) : new LightweightWorkerQueue(settings, _root.TransportLogger, _root.Pipeline, _root.Settings); - - + _listeners.Add(worker); - worker.StartListening(listener); } From 2732184084555e9eb695db76ecbe2ffac26cf6ed Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 6 May 2020 17:18:26 -0400 Subject: [PATCH 56/82] Configure ping/pong compliance tests --- src/Jasper.ConfluentKafka.Tests/end_to_end.cs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/Jasper.ConfluentKafka.Tests/end_to_end.cs b/src/Jasper.ConfluentKafka.Tests/end_to_end.cs index 23da82885..5f73e036f 100644 --- a/src/Jasper.ConfluentKafka.Tests/end_to_end.cs +++ b/src/Jasper.ConfluentKafka.Tests/end_to_end.cs @@ -37,14 +37,14 @@ public class end_to_end public class Sender : JasperOptions { - + public const string Topic = "jasper-compliance"; + public static string ReplyTopic = $"{Topic}-reply"; public Sender() { Endpoints.ConfigureKafka(); Endpoints.PublishAllMessages().ToKafkaTopic(Topic, ProducerConfig); + Endpoints.ListenToKafkaTopic(ReplyTopic, ConsumerConfig).UseForReplies(); } - - public string Topic = "messages"; } public class Receiver : JasperOptions @@ -52,20 +52,21 @@ public class Receiver : JasperOptions public Receiver(string topic) { Endpoints.ConfigureKafka(); - Endpoints.ListenToKafkaTopic(topic, ConsumerConfig).UseForReplies(); + Endpoints.PublishAllMessages().ToKafkaTopic(Sender.ReplyTopic, ProducerConfig); + Endpoints.ListenToKafkaTopic(topic, ConsumerConfig); } } public class KafkaSendingComplianceTests : SendingCompliance { - public KafkaSendingComplianceTests() : base($"kafka://topic/messages".ToUri(), 15.Seconds()) + public KafkaSendingComplianceTests() : base($"kafka://topic/{Sender.Topic}".ToUri(), 15.Seconds()) { var sender = new Sender(); SenderIs(sender); - var receiver = new Receiver(sender.Topic); + var receiver = new Receiver(Sender.Topic); ReceiverIs(receiver); } From e9c9517236bea9d65d5757aa605528126eea115e Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 6 May 2020 17:18:53 -0400 Subject: [PATCH 57/82] Commit received message --- .../Internal/ConfluentKafkaListener.cs | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaListener.cs b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaListener.cs index afa6b702d..7df66488a 100644 --- a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaListener.cs +++ b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaListener.cs @@ -83,20 +83,9 @@ private async Task ConsumeAsync() try { - await _callback.Received(Address, new[] {envelope}).ContinueWith(t => - { - try - { - _consumer.Commit(); - } - catch (KafkaException ke) - { - if (ke.Error?.Code != ErrorCode.Local_NoOffset) - { - throw; - } - } - }); + await _callback.Received(Address, new[] {envelope}); + + _consumer.Commit(message); } catch (Exception e) { From bb8c7391ef9d26d8ca0c72ef8bd5bff0b9df01dd Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 6 May 2020 17:21:42 -0400 Subject: [PATCH 58/82] Raise error on Fatal producer errors --- .../Exceptions/KafkaSenderException.cs | 15 ++++++++++ .../Internal/ConfluentKafkaSender.cs | 30 +++++++++++-------- 2 files changed, 32 insertions(+), 13 deletions(-) create mode 100644 src/Jasper.ConfluentKafka/Exceptions/KafkaSenderException.cs diff --git a/src/Jasper.ConfluentKafka/Exceptions/KafkaSenderException.cs b/src/Jasper.ConfluentKafka/Exceptions/KafkaSenderException.cs new file mode 100644 index 000000000..bd538219e --- /dev/null +++ b/src/Jasper.ConfluentKafka/Exceptions/KafkaSenderException.cs @@ -0,0 +1,15 @@ +using System; +using Confluent.Kafka; + +namespace Jasper.ConfluentKafka.Exceptions +{ + public class KafkaSenderException : ApplicationException + { + public Error Error { get; } + + public KafkaSenderException(Error error) : base(error.Reason) + { + Error = error; + } + } +} diff --git a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs index fe15da110..a63a1e346 100644 --- a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs +++ b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs @@ -3,6 +3,7 @@ using System.Threading.Tasks; using Confluent.Kafka; using Jasper.ConfluentKafka.Exceptions; +using Jasper.Logging; using Jasper.Transports; using Jasper.Transports.Sending; @@ -17,8 +18,19 @@ public class ConfluentKafkaSender : ISender public Uri Destination => _endpoint.Uri; public ConfluentKafkaSender(KafkaEndpoint endpoint) { + if(endpoint?.ProducerConfig == null) + throw new ArgumentNullException(nameof(KafkaEndpoint.ProducerConfig)); + _endpoint = endpoint; - _publisher = new ProducerBuilder(endpoint.ProducerConfig).Build(); + _publisher = new ProducerBuilder(endpoint.ProducerConfig) + .SetErrorHandler((producer, error) => + { + if (error.IsFatal) + { + throw new KafkaSenderException(error); + } + }) + .Build(); _protocol = new KafkaTransportProtocol(); } @@ -36,7 +48,7 @@ public async Task Ping(CancellationToken cancellationToken) await _publisher.ProduceAsync("jasper-ping", message, cancellationToken); } - catch (Exception e) + catch { return false; } @@ -44,7 +56,7 @@ public async Task Ping(CancellationToken cancellationToken) return true; } - public async Task Send(Envelope envelope) + public Task Send(Envelope envelope) { if (envelope.IsDelayed(DateTime.UtcNow)) { @@ -52,16 +64,8 @@ public async Task Send(Envelope envelope) } Message message = _protocol.WriteFromEnvelope(envelope); - try - { - var result = await _publisher.ProduceAsync(_endpoint.TopicName, message); - Console.WriteLine(result.Status); - } - catch (Exception e) - { - Console.WriteLine(e); - throw; - } + + return _publisher.ProduceAsync(_endpoint.TopicName, message); } } } From 16bfb1f5267e376c9d1a7ea9fba5512628df285c Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 6 May 2020 17:22:02 -0400 Subject: [PATCH 59/82] Don't default producer/consumer configs to make it easier to fail fast --- src/Jasper.ConfluentKafka/KafkaEndpoint.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Jasper.ConfluentKafka/KafkaEndpoint.cs b/src/Jasper.ConfluentKafka/KafkaEndpoint.cs index 36e43cd4b..19f1ce3a5 100644 --- a/src/Jasper.ConfluentKafka/KafkaEndpoint.cs +++ b/src/Jasper.ConfluentKafka/KafkaEndpoint.cs @@ -17,8 +17,8 @@ public class KafkaEndpoint : Endpoint { private const string TopicToken = "topic"; public string TopicName { get; set; } - public ProducerConfig ProducerConfig { get; set; } = new ProducerConfig(); - public ConsumerConfig ConsumerConfig { get; set; } = new ConsumerConfig(); + public ProducerConfig ProducerConfig { get; set; } + public ConsumerConfig ConsumerConfig { get; set; } public override Uri Uri => BuildUri(); public KafkaEndpoint() From 676efda78d6abe91f591b29f8c026f3b709d7f9f Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 7 May 2020 08:24:11 -0400 Subject: [PATCH 60/82] Add message key to jasper headers if set on inbound kafka message --- .../KafkaTransportProtocol.cs | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/Jasper.ConfluentKafka/KafkaTransportProtocol.cs b/src/Jasper.ConfluentKafka/KafkaTransportProtocol.cs index 3db70a73e..b7f4b7281 100644 --- a/src/Jasper.ConfluentKafka/KafkaTransportProtocol.cs +++ b/src/Jasper.ConfluentKafka/KafkaTransportProtocol.cs @@ -8,6 +8,8 @@ namespace Jasper.ConfluentKafka { public class KafkaTransportProtocol : ITransportProtocol> { + public const string KafkaMessageKeyHeader = "Confluent.Kafka.Message.Key"; + public Message WriteFromEnvelope(Envelope envelope) { var message = new Message @@ -18,6 +20,7 @@ public Message WriteFromEnvelope(Envelope envelope) IDictionary envelopHeaders = new Dictionary(); envelope.WriteToDictionary(envelopHeaders); + var headers = new Headers(); foreach (Header header in envelopHeaders.Select(h => new Header(h.Key, Encoding.UTF8.GetBytes(h.Value.ToString())))) { @@ -26,16 +29,15 @@ public Message WriteFromEnvelope(Envelope envelope) message.Headers = headers; - if (envelopHeaders.TryGetValue("MessageKey", out var msgKey)) + if (!envelopHeaders.TryGetValue(KafkaMessageKeyHeader, out object msgKey)) return message; + + if (msgKey is byte[] key) + { + message.Key = key; + } + else { - if (msgKey is byte[]) - { - message.Key = (byte[])msgKey; - } - else - { - message.Key = Encoding.UTF8.GetBytes(msgKey.ToString()); - } + message.Key = Encoding.UTF8.GetBytes(msgKey.ToString()); } return message; @@ -50,6 +52,11 @@ public Envelope ReadEnvelope(Message message) Dictionary incomingHeaders = message.Headers.Select(h => new {h.Key, Value = h.GetValueBytes()}) .ToDictionary(k => k.Key, v => (object)Encoding.UTF8.GetString(v.Value)); + + if(message.Key != null && !incomingHeaders.ContainsKey(KafkaMessageKeyHeader)) + { + env.Headers.Add(KafkaMessageKeyHeader, Encoding.UTF8.GetString(message.Key)); + } env.ReadPropertiesFromDictionary(incomingHeaders); From 9c609c32ea765282954a10fff08164b9fbae78e4 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 7 May 2020 15:17:00 -0400 Subject: [PATCH 61/82] Enable not checking timeout in tracked session --- src/Jasper/Tracking/TrackedSession.cs | 6 ++---- src/Jasper/Tracking/TrackedSessionConfiguration.cs | 13 +++++++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/Jasper/Tracking/TrackedSession.cs b/src/Jasper/Tracking/TrackedSession.cs index 64f370513..58023fda7 100644 --- a/src/Jasper/Tracking/TrackedSession.cs +++ b/src/Jasper/Tracking/TrackedSession.cs @@ -45,6 +45,7 @@ public void WatchOther(IHost host) public TimeSpan Timeout { get; set; } = 5.Seconds(); public bool AssertNoExceptions { get; set; } = true; + public bool AssertNoTimeout { get; set; } = true; public Func Execution { get; set; } = c => Task.CompletedTask; @@ -166,9 +167,6 @@ public async Task ExecuteAndTrack() _stopwatch.Start(); - - - try { using (var scope = _primaryHost.Services.As().GetNestedContainer()) @@ -191,7 +189,7 @@ public async Task ExecuteAndTrack() if (AssertNoExceptions) AssertNoExceptionsWereThrown(); - AssertNotTimedOut(); + if (AssertNoExceptions) AssertNotTimedOut(); } public Task Track() diff --git a/src/Jasper/Tracking/TrackedSessionConfiguration.cs b/src/Jasper/Tracking/TrackedSessionConfiguration.cs index 941875db9..66671341b 100644 --- a/src/Jasper/Tracking/TrackedSessionConfiguration.cs +++ b/src/Jasper/Tracking/TrackedSessionConfiguration.cs @@ -68,6 +68,12 @@ public TrackedSessionConfiguration DoNotAssertOnExceptionsDetected() return this; } + public TrackedSessionConfiguration DoNotAssertTimeout() + { + _session.AssertNoTimeout = false; + return this; + } + public TrackedSessionConfiguration WaitForMessageToBeReceivedAt(IHost host) { var condition = new WaitForMessage @@ -93,6 +99,13 @@ public async Task ExecuteAndWait(Func ac return _session; } + public async Task ExecuteWithoutWaiting(Func action) + { + _session.Execution = action; + await _session.ExecuteAndTrack(); + return _session; + } + /// /// Invoke a message inline from the current Jasper application /// and wait for all cascading activity to complete From 61994e95de365f1fbe0265abdce9bf3d11afabec Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 7 May 2020 15:19:00 -0400 Subject: [PATCH 62/82] Rename ISendingAgent.StoreAndForward -> Forward for clarity Not all agents were storing the message first --- src/Jasper/Envelope.Internals.cs | 2 +- src/Jasper/Transports/Local/DurableLocalSendingAgent.cs | 2 +- src/Jasper/Transports/Local/LightweightLocalSendingAgent.cs | 4 ++-- src/Jasper/Transports/Sending/ISendingAgent.cs | 6 ++---- src/Jasper/Transports/Sending/SendingAgent.cs | 2 +- src/StorytellerSpecs/Stub/StubEndpoint.cs | 2 +- 6 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/Jasper/Envelope.Internals.cs b/src/Jasper/Envelope.Internals.cs index fa20aa4f0..fc927a071 100644 --- a/src/Jasper/Envelope.Internals.cs +++ b/src/Jasper/Envelope.Internals.cs @@ -136,7 +136,7 @@ internal Task Send() _enqueued = true; - return Sender.StoreAndForward(this); + return Sender.Forward(this); } internal Task QuickSend() diff --git a/src/Jasper/Transports/Local/DurableLocalSendingAgent.cs b/src/Jasper/Transports/Local/DurableLocalSendingAgent.cs index 5ba564df0..888fa6807 100644 --- a/src/Jasper/Transports/Local/DurableLocalSendingAgent.cs +++ b/src/Jasper/Transports/Local/DurableLocalSendingAgent.cs @@ -52,7 +52,7 @@ public Task EnqueueOutgoing(Envelope envelope) return Enqueue(envelope); } - public async Task StoreAndForward(Envelope envelope) + public async Task Forward(Envelope envelope) { _messageLogger.Sent(envelope); writeMessageData(envelope); diff --git a/src/Jasper/Transports/Local/LightweightLocalSendingAgent.cs b/src/Jasper/Transports/Local/LightweightLocalSendingAgent.cs index 81c076680..557e871ce 100644 --- a/src/Jasper/Transports/Local/LightweightLocalSendingAgent.cs +++ b/src/Jasper/Transports/Local/LightweightLocalSendingAgent.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Threading.Tasks; using Jasper.Configuration; using Jasper.Logging; @@ -41,7 +41,7 @@ public Task EnqueueOutgoing(Envelope envelope) : Enqueue(envelope); } - public Task StoreAndForward(Envelope envelope) + public Task Forward(Envelope envelope) { return EnqueueOutgoing(envelope); } diff --git a/src/Jasper/Transports/Sending/ISendingAgent.cs b/src/Jasper/Transports/Sending/ISendingAgent.cs index d9e1f615a..997cc73b9 100644 --- a/src/Jasper/Transports/Sending/ISendingAgent.cs +++ b/src/Jasper/Transports/Sending/ISendingAgent.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Threading.Tasks; using Jasper.Configuration; @@ -15,15 +15,13 @@ public interface ISendingAgent : IDisposable bool SupportsNativeScheduledSend { get; } - - // This would be called in the future by the outbox, assuming // that the envelope is already persisted and just needs to be sent out Task EnqueueOutgoing(Envelope envelope); // This would be called by the EnvelopeSender if invoked // indirectly - Task StoreAndForward(Envelope envelope); + Task Forward(Envelope envelope); Endpoint Endpoint { get; } diff --git a/src/Jasper/Transports/Sending/SendingAgent.cs b/src/Jasper/Transports/Sending/SendingAgent.cs index 838040c26..3407f0075 100644 --- a/src/Jasper/Transports/Sending/SendingAgent.cs +++ b/src/Jasper/Transports/Sending/SendingAgent.cs @@ -62,7 +62,7 @@ public async Task EnqueueOutgoing(Envelope envelope) _messageLogger.Sent(envelope); } - public async Task StoreAndForward(Envelope envelope) + public async Task Forward(Envelope envelope) { setDefaults(envelope); diff --git a/src/StorytellerSpecs/Stub/StubEndpoint.cs b/src/StorytellerSpecs/Stub/StubEndpoint.cs index ec7e1630b..24c29b36f 100644 --- a/src/StorytellerSpecs/Stub/StubEndpoint.cs +++ b/src/StorytellerSpecs/Stub/StubEndpoint.cs @@ -77,7 +77,7 @@ public Task EnqueueOutgoing(Envelope envelope) return Task.CompletedTask; } - public Task StoreAndForward(Envelope envelope) + public Task Forward(Envelope envelope) { return EnqueueOutgoing(envelope); } From 7ba06ef8833dd00f819050679f8e5912fdfdacda Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 7 May 2020 15:19:32 -0400 Subject: [PATCH 63/82] Add test to make sure publish failures reported to publisher when not durable --- src/Jasper.ConfluentKafka.Tests/end_to_end.cs | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/src/Jasper.ConfluentKafka.Tests/end_to_end.cs b/src/Jasper.ConfluentKafka.Tests/end_to_end.cs index 5f73e036f..52f9d9e48 100644 --- a/src/Jasper.ConfluentKafka.Tests/end_to_end.cs +++ b/src/Jasper.ConfluentKafka.Tests/end_to_end.cs @@ -14,6 +14,8 @@ using Shouldly; using TestingSupport; using TestingSupport.Compliance; +using TestingSupport.ErrorHandling; +using TestMessages; using Xunit; namespace Jasper.ConfluentKafka.Tests @@ -27,6 +29,11 @@ public class end_to_end BootstrapServers = KafkaServer, SecurityProtocol = SecurityProtocol.Ssl }; + private static ProducerConfig FailureProducerConfig = new ProducerConfig + { + BootstrapServers = "badaddress", + MessageTimeoutMs = 1000 + }; private static ConsumerConfig ConsumerConfig = new ConsumerConfig { @@ -47,6 +54,16 @@ public Sender() } } + public class FailureSender : JasperOptions + { + public const string Topic = "jasper-compliance"; + public FailureSender() + { + Endpoints.ConfigureKafka(); + Endpoints.PublishAllMessages().ToKafkaTopic(Topic, FailureProducerConfig); + } + } + public class Receiver : JasperOptions { public Receiver(string topic) @@ -70,6 +87,22 @@ public KafkaSendingComplianceTests() : base($"kafka://topic/{Sender.Topic}".ToUr ReceiverIs(receiver); } + + [Fact] + public async Task publish_failures_reported_to_caller() + { + theSender = null; + SenderIs(); + + _ = await theSender.TrackActivity(60.Seconds()) + .DoNotAssertOnExceptionsDetected() + .DoNotAssertTimeout() + .ExecuteAndWait(c => + { + Should.Throw(c.Publish(new Message1())); + return Task.CompletedTask; + }); + } } @@ -97,7 +130,6 @@ await host colors.Name.ShouldBe("Red"); } - [Fact] public async Task send_multiple_messages_in_order() { From 4d6eb3e8e758962fa154142bd757f6d3c9275dc9 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 7 May 2020 17:04:49 -0400 Subject: [PATCH 64/82] Check for null Uris to avoid null ref in MS logger abstraction --- src/Jasper/Logging/MessageLogger.cs | 34 ++++++++++++++--------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/Jasper/Logging/MessageLogger.cs b/src/Jasper/Logging/MessageLogger.cs index 2d5bab2a2..ba21f75a5 100644 --- a/src/Jasper/Logging/MessageLogger.cs +++ b/src/Jasper/Logging/MessageLogger.cs @@ -1,4 +1,4 @@ -using System; +using System; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; @@ -20,14 +20,14 @@ public class MessageLogger : IMessageLogger private readonly Action _executionStarted; private readonly ILogger _logger; - private readonly Action _messageFailed; - private readonly Action _messageSucceeded; + private readonly Action _messageFailed; + private readonly Action _messageSucceeded; private readonly IMetrics _metrics; private readonly Action _movedToErrorQueue; - private readonly Action _noHandler; + private readonly Action _noHandler; private readonly Action _noRoutes; - private readonly Action _received; - private readonly Action _sent; + private readonly Action _received; + private readonly Action _sent; private readonly Action _undeliverable; public static MessageLogger Empty() @@ -40,10 +40,10 @@ public MessageLogger(ILoggerFactory factory, IMetrics metrics) _metrics = metrics; _logger = factory.CreateLogger("Jasper.Messages"); - _sent = LoggerMessage.Define(LogLevel.Debug, SentEventId, + _sent = LoggerMessage.Define(LogLevel.Debug, SentEventId, "Enqueued for sending {Name}#{Id} to {Destination}"); - _received = LoggerMessage.Define(LogLevel.Debug, ReceivedEventId, + _received = LoggerMessage.Define(LogLevel.Debug, ReceivedEventId, "Received {Name}#{Id} at {Destination} from {ReplyUri}"); _executionStarted = LoggerMessage.Define(LogLevel.Debug, ExecutionStartedEventId, @@ -53,13 +53,13 @@ public MessageLogger(ILoggerFactory factory, IMetrics metrics) "Finished processing {Name}#{Id}"); _messageSucceeded = - LoggerMessage.Define(LogLevel.Information, MessageSucceededEventId, + LoggerMessage.Define(LogLevel.Information, MessageSucceededEventId, "Successfully processed message {Name}#{envelope} from {ReplyUri}"); - _messageFailed = LoggerMessage.Define(LogLevel.Error, MessageFailedEventId, + _messageFailed = LoggerMessage.Define(LogLevel.Error, MessageFailedEventId, "Failed to process message {Name}#{envelope} from {ReplyUri}"); - _noHandler = LoggerMessage.Define(LogLevel.Information, NoHandlerEventId, + _noHandler = LoggerMessage.Define(LogLevel.Information, NoHandlerEventId, "No known handler for {Name}#{Id} from {ReplyUri}"); _noRoutes = LoggerMessage.Define(LogLevel.Information, NoRoutesEventId, @@ -74,13 +74,13 @@ public MessageLogger(ILoggerFactory factory, IMetrics metrics) public virtual void Sent(Envelope envelope) { - _sent(_logger, envelope.GetMessageTypeName(), envelope.Id, envelope.Destination, null); + _sent(_logger, envelope.GetMessageTypeName(), envelope.Id, envelope.Destination?.ToString(), null); } public virtual void Received(Envelope envelope) { - _received(_logger, envelope.GetMessageTypeName(), envelope.Id, envelope.Destination, - envelope.ReplyUri, null); + _received(_logger, envelope.GetMessageTypeName(), envelope.Id, envelope.Destination?.ToString(), + envelope.ReplyUri?.ToString(), null); } public virtual void ExecutionStarted(Envelope envelope) @@ -96,18 +96,18 @@ public virtual void ExecutionFinished(Envelope envelope) public virtual void MessageSucceeded(Envelope envelope) { _metrics.MessageExecuted(envelope); - _messageSucceeded(_logger, envelope.GetMessageTypeName(), envelope.Id, envelope.ReplyUri, null); + _messageSucceeded(_logger, envelope.GetMessageTypeName(), envelope.Id, envelope.ReplyUri?.ToString(), null); } public virtual void MessageFailed(Envelope envelope, Exception ex) { _metrics.MessageExecuted(envelope); - _messageFailed(_logger, envelope.GetMessageTypeName(), envelope.Id, envelope.ReplyUri, ex); + _messageFailed(_logger, envelope.GetMessageTypeName(), envelope.Id, envelope.ReplyUri?.ToString(), ex); } public virtual void NoHandlerFor(Envelope envelope) { - _noHandler(_logger, envelope.GetMessageTypeName(), envelope.Id, envelope.ReplyUri, null); + _noHandler(_logger, envelope.GetMessageTypeName(), envelope.Id, envelope.ReplyUri?.ToString(), null); } public virtual void NoRoutesFor(Envelope envelope) From 951f60e2ddebff7230056a772fafd26c3af71388 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 7 May 2020 17:06:08 -0400 Subject: [PATCH 65/82] Change listener status --- src/Jasper.ConfluentKafka/Internal/ConfluentKafkaListener.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaListener.cs b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaListener.cs index 7df66488a..540e07db1 100644 --- a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaListener.cs +++ b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaListener.cs @@ -1,4 +1,5 @@ using System; +using System.Diagnostics; using System.Threading; using System.Threading.Tasks; using Confluent.Kafka; @@ -43,6 +44,8 @@ public void Start(IReceiverCallback callback) _consumer.Subscribe(new []{ _endpoint.TopicName }); _consumerTask = ConsumeAsync(); + + _logger.ListeningStatusChange(ListeningStatus.Accepting); } private async Task ConsumeAsync() From 08bc4feebf397f45f1dff447062f449d45b3b557 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 7 May 2020 17:07:09 -0400 Subject: [PATCH 66/82] Remove duplicate code --- src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs index a63a1e346..1ef127e0b 100644 --- a/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs +++ b/src/Jasper.ConfluentKafka/Internal/ConfluentKafkaSender.cs @@ -44,9 +44,7 @@ public async Task Ping(CancellationToken cancellationToken) Envelope envelope = Envelope.ForPing(Destination); try { - Message message = _protocol.WriteFromEnvelope(envelope); - - await _publisher.ProduceAsync("jasper-ping", message, cancellationToken); + await Send(envelope); } catch { From 08981e00cb61cab3b8da352c2cd646ac1574709a Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 8 May 2020 09:51:00 -0400 Subject: [PATCH 67/82] Update Kafka NuGet info --- .../Jasper.ConfluentKafka.csproj | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/Jasper.ConfluentKafka/Jasper.ConfluentKafka.csproj b/src/Jasper.ConfluentKafka/Jasper.ConfluentKafka.csproj index 791a43570..68db8cd1b 100644 --- a/src/Jasper.ConfluentKafka/Jasper.ConfluentKafka.csproj +++ b/src/Jasper.ConfluentKafka/Jasper.ConfluentKafka.csproj @@ -1,8 +1,20 @@ + Kafka Transport for Jasper Messaging Systems using Confluent Kafka Client + Jeremy D. Miller, Jarrod Johnson netstandard2.1 portable + Jasper.ConfluentKafka + Jasper.ConfluentKafka + https://avatars2.githubusercontent.com/u/10048186?v=3&s=200 + http://jasperfx.github.io + https://github.com/JasperFX/jasper/blob/master/LICENSE.txt + false + false + false + false + false @@ -14,4 +26,10 @@ + + + CommonAssemblyInfo.cs + + + From e5501a251faccf339aeba52d33081549340db39e Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 8 May 2020 09:52:00 -0400 Subject: [PATCH 68/82] Remove unnecessary dependency --- src/Jasper.ConfluentKafka/Jasper.ConfluentKafka.csproj | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Jasper.ConfluentKafka/Jasper.ConfluentKafka.csproj b/src/Jasper.ConfluentKafka/Jasper.ConfluentKafka.csproj index 68db8cd1b..d3587cd7d 100644 --- a/src/Jasper.ConfluentKafka/Jasper.ConfluentKafka.csproj +++ b/src/Jasper.ConfluentKafka/Jasper.ConfluentKafka.csproj @@ -19,7 +19,6 @@ - From 7850ef1947335200c63bf466c70440658bdf6105 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 8 May 2020 10:32:57 -0400 Subject: [PATCH 69/82] Create Pulsar projects --- Jasper.sln | 30 +++++++++++++++++++ .../Jasper.Pulsar.Tests.csproj | 29 ++++++++++++++++++ src/Jasper.Pulsar/Jasper.Pulsar.csproj | 30 +++++++++++++++++++ 3 files changed, 89 insertions(+) create mode 100644 src/Jasper.Pulsar.Tests/Jasper.Pulsar.Tests.csproj create mode 100644 src/Jasper.Pulsar/Jasper.Pulsar.csproj diff --git a/Jasper.sln b/Jasper.sln index 7added68e..c9add534b 100644 --- a/Jasper.sln +++ b/Jasper.sln @@ -59,6 +59,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Jasper.ConfluentKafka", "sr EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Jasper.ConfluentKafka.Tests", "src\Jasper.ConfluentKafka.Tests\Jasper.ConfluentKafka.Tests.csproj", "{B8F1BCB3-4A8A-4368-85BA-E69EB879BC5A}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Jasper.Pulsar", "src\Jasper.Pulsar\Jasper.Pulsar.csproj", "{BB253930-8225-4737-9BB0-6F89A4073225}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Jasper.Pulsar.Tests", "src\Jasper.Pulsar.Tests\Jasper.Pulsar.Tests.csproj", "{0A6C2CD0-23AF-45AB-A737-9D1D64693717}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -357,6 +361,30 @@ Global {B8F1BCB3-4A8A-4368-85BA-E69EB879BC5A}.Release|x64.Build.0 = Release|Any CPU {B8F1BCB3-4A8A-4368-85BA-E69EB879BC5A}.Release|x86.ActiveCfg = Release|Any CPU {B8F1BCB3-4A8A-4368-85BA-E69EB879BC5A}.Release|x86.Build.0 = Release|Any CPU + {BB253930-8225-4737-9BB0-6F89A4073225}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BB253930-8225-4737-9BB0-6F89A4073225}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BB253930-8225-4737-9BB0-6F89A4073225}.Debug|x64.ActiveCfg = Debug|Any CPU + {BB253930-8225-4737-9BB0-6F89A4073225}.Debug|x64.Build.0 = Debug|Any CPU + {BB253930-8225-4737-9BB0-6F89A4073225}.Debug|x86.ActiveCfg = Debug|Any CPU + {BB253930-8225-4737-9BB0-6F89A4073225}.Debug|x86.Build.0 = Debug|Any CPU + {BB253930-8225-4737-9BB0-6F89A4073225}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BB253930-8225-4737-9BB0-6F89A4073225}.Release|Any CPU.Build.0 = Release|Any CPU + {BB253930-8225-4737-9BB0-6F89A4073225}.Release|x64.ActiveCfg = Release|Any CPU + {BB253930-8225-4737-9BB0-6F89A4073225}.Release|x64.Build.0 = Release|Any CPU + {BB253930-8225-4737-9BB0-6F89A4073225}.Release|x86.ActiveCfg = Release|Any CPU + {BB253930-8225-4737-9BB0-6F89A4073225}.Release|x86.Build.0 = Release|Any CPU + {0A6C2CD0-23AF-45AB-A737-9D1D64693717}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0A6C2CD0-23AF-45AB-A737-9D1D64693717}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0A6C2CD0-23AF-45AB-A737-9D1D64693717}.Debug|x64.ActiveCfg = Debug|Any CPU + {0A6C2CD0-23AF-45AB-A737-9D1D64693717}.Debug|x64.Build.0 = Debug|Any CPU + {0A6C2CD0-23AF-45AB-A737-9D1D64693717}.Debug|x86.ActiveCfg = Debug|Any CPU + {0A6C2CD0-23AF-45AB-A737-9D1D64693717}.Debug|x86.Build.0 = Debug|Any CPU + {0A6C2CD0-23AF-45AB-A737-9D1D64693717}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0A6C2CD0-23AF-45AB-A737-9D1D64693717}.Release|Any CPU.Build.0 = Release|Any CPU + {0A6C2CD0-23AF-45AB-A737-9D1D64693717}.Release|x64.ActiveCfg = Release|Any CPU + {0A6C2CD0-23AF-45AB-A737-9D1D64693717}.Release|x64.Build.0 = Release|Any CPU + {0A6C2CD0-23AF-45AB-A737-9D1D64693717}.Release|x86.ActiveCfg = Release|Any CPU + {0A6C2CD0-23AF-45AB-A737-9D1D64693717}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -385,6 +413,8 @@ Global {6EC1EA66-63C7-4DF6-8DCB-40DFBEA4E07A} = {CA5A0AA5-2CAD-4F42-AF73-980469934F27} {ABBCB70C-9087-4A0C-A3DB-C9BB0DFAAC5A} = {CA5A0AA5-2CAD-4F42-AF73-980469934F27} {B8F1BCB3-4A8A-4368-85BA-E69EB879BC5A} = {CA5A0AA5-2CAD-4F42-AF73-980469934F27} + {BB253930-8225-4737-9BB0-6F89A4073225} = {CA5A0AA5-2CAD-4F42-AF73-980469934F27} + {0A6C2CD0-23AF-45AB-A737-9D1D64693717} = {CA5A0AA5-2CAD-4F42-AF73-980469934F27} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {D92D723F-44EC-4C1E-AAC3-C162FCEAAA08} diff --git a/src/Jasper.Pulsar.Tests/Jasper.Pulsar.Tests.csproj b/src/Jasper.Pulsar.Tests/Jasper.Pulsar.Tests.csproj new file mode 100644 index 000000000..a93188f03 --- /dev/null +++ b/src/Jasper.Pulsar.Tests/Jasper.Pulsar.Tests.csproj @@ -0,0 +1,29 @@ + + + + netcoreapp3.1 + false + + + + + + + + + + + + + + + + + + + + Servers.cs + + + + diff --git a/src/Jasper.Pulsar/Jasper.Pulsar.csproj b/src/Jasper.Pulsar/Jasper.Pulsar.csproj new file mode 100644 index 000000000..96e9f5250 --- /dev/null +++ b/src/Jasper.Pulsar/Jasper.Pulsar.csproj @@ -0,0 +1,30 @@ + + + + Pulsar support for Jasper Messaging Systems + Jeremy D. Miller, Jarrod Johnson + netstandard2.1 + portable + Jasper.Pulsar + Jasper.Pulsar + https://avatars2.githubusercontent.com/u/10048186?v=3&s=200 + http://jasperfx.github.io + https://github.com/JasperFX/jasper/blob/master/LICENSE.txt + false + false + false + false + false + + + + + + + + + CommonAssemblyInfo.cs + + + + From 6329cfd003bd588ed458439f385ccec25adea795 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 8 May 2020 16:57:41 -0400 Subject: [PATCH 70/82] Make internals visible to new Pulsar projects --- src/Jasper/AssemblyAttributes.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Jasper/AssemblyAttributes.cs b/src/Jasper/AssemblyAttributes.cs index 5d879eeac..15bd5049f 100644 --- a/src/Jasper/AssemblyAttributes.cs +++ b/src/Jasper/AssemblyAttributes.cs @@ -1,6 +1,5 @@ using System.Runtime.CompilerServices; using Jasper.Attributes; -using Jasper.Configuration; using Lamar; [assembly: IgnoreAssembly] @@ -18,4 +17,6 @@ [assembly: InternalsVisibleTo("Jasper.Persistence.Database")] [assembly: InternalsVisibleTo("Jasper.Persistence.Marten")] [assembly: InternalsVisibleTo("Jasper.Persistence.EntityFrameworkCore")] +[assembly: InternalsVisibleTo("Jasper.Pulsar")] +[assembly: InternalsVisibleTo("Jasper.Pulsar.Tests")] [assembly: InternalsVisibleTo("StorytellerSpecs")] From 800fd47c2ff4d57d3ee072ac03c39f9afbb5bbfd Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 8 May 2020 16:58:46 -0400 Subject: [PATCH 71/82] Enable transports to support multiple schemas --- src/Jasper.ConfluentKafka/KafkaTransport.cs | 2 +- .../Configuration/TransportCollectionTests.cs | 4 ++-- src/Jasper/Configuration/TransportCollection.cs | 5 ++++- src/Jasper/Transports/ITransport.cs | 4 ++-- src/Jasper/Transports/Local/LocalTransport.cs | 4 ++-- src/Jasper/Transports/TransportBase.cs | 12 ++++++++++-- src/StorytellerSpecs/Stub/StubTransport.cs | 4 ++-- 7 files changed, 23 insertions(+), 12 deletions(-) diff --git a/src/Jasper.ConfluentKafka/KafkaTransport.cs b/src/Jasper.ConfluentKafka/KafkaTransport.cs index f1e4e2bcb..b8110f0a8 100644 --- a/src/Jasper.ConfluentKafka/KafkaTransport.cs +++ b/src/Jasper.ConfluentKafka/KafkaTransport.cs @@ -17,7 +17,7 @@ public class KafkaTransport : TransportBase private readonly Dictionary _endpoints; public KafkaTopicRouter Topics { get; } = new KafkaTopicRouter(); - public KafkaTransport() : base(Protocols.Kafka) + public KafkaTransport() : base(ConfluentKafka.Protocols.Kafka) { _endpoints = new Dictionary(); } diff --git a/src/Jasper.Testing/Configuration/TransportCollectionTests.cs b/src/Jasper.Testing/Configuration/TransportCollectionTests.cs index 87f6be921..8a2a7e00e 100644 --- a/src/Jasper.Testing/Configuration/TransportCollectionTests.cs +++ b/src/Jasper.Testing/Configuration/TransportCollectionTests.cs @@ -23,7 +23,7 @@ public class TransportCollectionTests public void add_transport() { var transport = Substitute.For(); - transport.Protocol.Returns("fake"); + transport.Protocols.Returns(new []{"fake"}); var collection = new TransportCollection(new JasperOptions()) {transport}; @@ -133,7 +133,7 @@ public void Dispose() throw new NotImplementedException(); } - public string Protocol { get; } = "fake"; + public ICollection Protocols { get; } = new []{"fake"}; public ISendingAgent BuildSendingAgent(Uri uri, IMessagingRoot root, CancellationToken cancellation) { throw new NotImplementedException(); diff --git a/src/Jasper/Configuration/TransportCollection.cs b/src/Jasper/Configuration/TransportCollection.cs index e4120ceeb..1d3a2b2f9 100644 --- a/src/Jasper/Configuration/TransportCollection.cs +++ b/src/Jasper/Configuration/TransportCollection.cs @@ -31,7 +31,10 @@ public ITransport TransportForScheme(string scheme) public void Add(ITransport transport) { - _transports.SmartAdd(transport.Protocol, transport); + foreach (var protocol in transport.Protocols) + { + _transports.SmartAdd(protocol, transport); + } } public T Get() where T : ITransport, new() diff --git a/src/Jasper/Transports/ITransport.cs b/src/Jasper/Transports/ITransport.cs index c34f82faf..bac03fcda 100644 --- a/src/Jasper/Transports/ITransport.cs +++ b/src/Jasper/Transports/ITransport.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using Jasper.Configuration; using Jasper.Runtime; @@ -7,7 +7,7 @@ namespace Jasper.Transports { public interface ITransport : IDisposable { - string Protocol { get; } + ICollection Protocols { get; } Endpoint ReplyEndpoint(); diff --git a/src/Jasper/Transports/Local/LocalTransport.cs b/src/Jasper/Transports/Local/LocalTransport.cs index 5edb1dbc1..596f33898 100644 --- a/src/Jasper/Transports/Local/LocalTransport.cs +++ b/src/Jasper/Transports/Local/LocalTransport.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using Baseline; @@ -50,7 +50,7 @@ public void Initialize(IMessagingRoot root) } - public string Protocol { get; } = TransportConstants.Local; + public ICollection Protocols { get; } = new []{ TransportConstants.Local }; void ITransport.StartSenders(IMessagingRoot root, ITransportRuntime runtime) { diff --git a/src/Jasper/Transports/TransportBase.cs b/src/Jasper/Transports/TransportBase.cs index 82bde3d2b..99c546fb0 100644 --- a/src/Jasper/Transports/TransportBase.cs +++ b/src/Jasper/Transports/TransportBase.cs @@ -11,10 +11,18 @@ namespace Jasper.Transports { public TransportBase(string protocol) { - Protocol = protocol; + Protocols.Add(protocol); } - public string Protocol { get; } + public TransportBase(IEnumerable protocols) + { + foreach (string protocol in protocols) + { + Protocols.Add(protocol); + } + } + + public ICollection Protocols { get; } = new List(); public IEnumerable Endpoints() { diff --git a/src/StorytellerSpecs/Stub/StubTransport.cs b/src/StorytellerSpecs/Stub/StubTransport.cs index e89e462d5..fac13474f 100644 --- a/src/StorytellerSpecs/Stub/StubTransport.cs +++ b/src/StorytellerSpecs/Stub/StubTransport.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using Baseline; @@ -67,7 +67,7 @@ public void Dispose() { } - public string Protocol { get; } = "stub"; + public ICollection Protocols { get; } = new []{"stub"}; public void StartSenders(IMessagingRoot root, ITransportRuntime runtime) { From 1d33257bc11b1b38615059dec04b0b62a9578921 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 8 May 2020 16:59:48 -0400 Subject: [PATCH 72/82] Create PulsarTopic for consistency working with Pulsar topics --- src/Jasper.Pulsar/PulsarTopic.cs | 48 ++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 src/Jasper.Pulsar/PulsarTopic.cs diff --git a/src/Jasper.Pulsar/PulsarTopic.cs b/src/Jasper.Pulsar/PulsarTopic.cs new file mode 100644 index 000000000..06fae07dc --- /dev/null +++ b/src/Jasper.Pulsar/PulsarTopic.cs @@ -0,0 +1,48 @@ +using System; +using System.Linq; +using System.Text.RegularExpressions; + +namespace Jasper.Pulsar +{ + public struct PulsarTopic + { + public string Persistence { get; } + public string Tenant { get; } + public string Namespace { get; } + public string TopicName { get; } + + public bool IsForReply { get; } + + private const string PulsarTopicRegex = @"(non-persistent|persistent)://([-A-Za-z0-9]*)/([-A-Za-z0-9]*)/([-A-Za-z0-9]*)(/for-reply)?"; + + private const string InvalidTopicFormatMessage = + "Invalid Pulsar topic. Expecting format of \"{persistent|non-persistent}://tenant/namespace/topic\" (can append \"/for-reply\" for Jasper functionality. It will not be included in communication w/ Pulsar)"; + + public PulsarTopic(Uri topic) : this(topic?.ToString()) + { + + } + + public PulsarTopic(string topic) + { + MatchCollection match = Regex.Matches(topic, PulsarTopicRegex, RegexOptions.Compiled); + + if (!match.Any()) + throw new ArgumentException(InvalidTopicFormatMessage, nameof(topic)); + + Persistence = match[0].Groups[1].Captures[0].Value; + Tenant = match[0].Groups[2].Captures[0].Value; + Namespace = match[0].Groups[3].Captures[0].Value; + TopicName = match[0].Groups[4].Captures[0].Value; + IsForReply = match[0].Groups.Count == 6; + } + + public static implicit operator string(PulsarTopic topic) => topic.ToString(); + public static implicit operator PulsarTopic(string topic) => new PulsarTopic(topic); + public static implicit operator PulsarTopic(Uri topic) => new PulsarTopic(topic); + + public override string ToString() => $"{Persistence}://{Tenant}/{Namespace}/{TopicName}"; + + public Uri ToJasperUri(bool forReply) => new Uri($"{this}{(forReply ? "/for-reply" : string.Empty)}"); + } +} From a92e9e244b7dedd2b88f8ae379fd150af1fbab3e Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 8 May 2020 17:00:00 -0400 Subject: [PATCH 73/82] Initial creation of Pulsar transport and tests --- .../Jasper.Pulsar.Tests.csproj | 2 +- .../PulsarSendingComplianceTests.cs | 98 ++++++++++++++ .../Exceptions/PulsarSenderException.cs | 14 ++ src/Jasper.Pulsar/Internal/PulsarListener.cs | 77 +++++++++++ src/Jasper.Pulsar/Internal/PulsarMessage.cs | 21 +++ .../Internal/PulsarTopicRouter.cs | 30 +++++ .../Internal/PulsarTransportProtocol.cs | 65 +++++++++ src/Jasper.Pulsar/Internal/PulssarSender.cs | 56 ++++++++ src/Jasper.Pulsar/Jasper.Pulsar.csproj | 4 + src/Jasper.Pulsar/PulsarEndpoint.cs | 69 ++++++++++ .../PulsarListenerConfiguration.cs | 12 ++ .../PulsarSubscriberConfiguration.cs | 12 ++ src/Jasper.Pulsar/PulsarTransport.cs | 78 +++++++++++ .../PulsarTransportConfigurationExtensions.cs | 124 ++++++++++++++++++ 14 files changed, 661 insertions(+), 1 deletion(-) create mode 100644 src/Jasper.Pulsar.Tests/PulsarSendingComplianceTests.cs create mode 100644 src/Jasper.Pulsar/Exceptions/PulsarSenderException.cs create mode 100644 src/Jasper.Pulsar/Internal/PulsarListener.cs create mode 100644 src/Jasper.Pulsar/Internal/PulsarMessage.cs create mode 100644 src/Jasper.Pulsar/Internal/PulsarTopicRouter.cs create mode 100644 src/Jasper.Pulsar/Internal/PulsarTransportProtocol.cs create mode 100644 src/Jasper.Pulsar/Internal/PulssarSender.cs create mode 100644 src/Jasper.Pulsar/PulsarEndpoint.cs create mode 100644 src/Jasper.Pulsar/PulsarListenerConfiguration.cs create mode 100644 src/Jasper.Pulsar/PulsarSubscriberConfiguration.cs create mode 100644 src/Jasper.Pulsar/PulsarTransport.cs create mode 100644 src/Jasper.Pulsar/PulsarTransportConfigurationExtensions.cs diff --git a/src/Jasper.Pulsar.Tests/Jasper.Pulsar.Tests.csproj b/src/Jasper.Pulsar.Tests/Jasper.Pulsar.Tests.csproj index a93188f03..71c4ca988 100644 --- a/src/Jasper.Pulsar.Tests/Jasper.Pulsar.Tests.csproj +++ b/src/Jasper.Pulsar.Tests/Jasper.Pulsar.Tests.csproj @@ -14,7 +14,7 @@ - + diff --git a/src/Jasper.Pulsar.Tests/PulsarSendingComplianceTests.cs b/src/Jasper.Pulsar.Tests/PulsarSendingComplianceTests.cs new file mode 100644 index 000000000..fd2f54b3a --- /dev/null +++ b/src/Jasper.Pulsar.Tests/PulsarSendingComplianceTests.cs @@ -0,0 +1,98 @@ +using System; +using System.Threading.Tasks; +using Baseline.Dates; +using DotPulsar; +using DotPulsar.Internal; +using Jasper.Tracking; +using Shouldly; +using TestingSupport.Compliance; +using TestMessages; +using Xunit; + +namespace Jasper.Pulsar.Tests +{ + public class PulsarSendingComplianceTestsShell + { + private static string Server = "pulsar://localhost:6650"; + + public class Sender : JasperOptions + { + public const string Topic = "persistent://public/default/jasper-compliance"; + public static string ReplyTopic = $"{Topic}-reply"; + + public Sender() + { + Endpoints.ConfigurePulsar(new PulsarClientBuilder() + .ExceptionHandler(context => + { + + return new ValueTask(Task.CompletedTask); + }) + .ServiceUrl(new Uri(Server))); + Endpoints.PublishAllMessages().ToPulsarTopic(new ProducerOptions(Topic)); + Endpoints.ListenToPulsarTopic("compliance-tests", ReplyTopic).UseForReplies(); + } + } + + public class FailureSender : JasperOptions + { + public const string Topic = "persistent://public/default/jasper-compliance"; + public FailureSender() + { + Endpoints.ConfigurePulsar(new PulsarClientBuilder().ServiceUrl(new Uri(Server))); + Endpoints.PublishAllMessages().ToPulsarTopic(new ProducerOptions(Topic)); + } + } + + public class Receiver : JasperOptions + { + public Receiver(string topic) + { + Endpoints.ConfigurePulsar(new PulsarClientBuilder().ServiceUrl(new Uri(Server))); + Endpoints.PublishAllMessages().ToPulsarTopic(Sender.ReplyTopic); + Endpoints.ListenToPulsarTopic("receiver", topic); + } + } + + public class PulsarSendingComplianceTests : SendingCompliance + { + public PulsarSendingComplianceTests() : base(new Uri(Sender.Topic)) + { + var sender = new Sender(); + + SenderIs(sender); + + var receiver = new Receiver(Sender.Topic); + + ReceiverIs(receiver); + } + + [Fact] + + public async Task publish_failures_reported_to_caller() + { + theSender = null; + SenderIs(); + + _ = await theSender.TrackActivity(60.Seconds()) + .DoNotAssertOnExceptionsDetected() + .DoNotAssertTimeout() + .ExecuteAndWait(c => + { + Should.Throw(c.Publish(new Message1())); + return Task.CompletedTask; + }); + } + + [Fact] + + public async Task publish_succeeds() + { + _ = await theSender.TrackActivity(60.Seconds()) + .DoNotAssertOnExceptionsDetected() + .DoNotAssertTimeout() + .ExecuteAndWait(c => c.Publish(new Message1())); + } + } + } +} diff --git a/src/Jasper.Pulsar/Exceptions/PulsarSenderException.cs b/src/Jasper.Pulsar/Exceptions/PulsarSenderException.cs new file mode 100644 index 000000000..a79d98db4 --- /dev/null +++ b/src/Jasper.Pulsar/Exceptions/PulsarSenderException.cs @@ -0,0 +1,14 @@ +using System; + +namespace Jasper.Pulsar.Exceptions +{ + public class PulsarSenderException : ApplicationException + { + //public Error Error { get; } + + //public PulsarSenderException(Error error) : base(error.Reason) + //{ + // Error = error; + //} + } +} diff --git a/src/Jasper.Pulsar/Internal/PulsarListener.cs b/src/Jasper.Pulsar/Internal/PulsarListener.cs new file mode 100644 index 000000000..3ae843695 --- /dev/null +++ b/src/Jasper.Pulsar/Internal/PulsarListener.cs @@ -0,0 +1,77 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using DotPulsar; +using DotPulsar.Abstractions; +using Jasper.Logging; +using Jasper.Transports; + +namespace Jasper.Pulsar.Internal +{ + public class PulsarListener : IListener + { + private readonly CancellationToken _cancellation; + private readonly IConsumer _consumer; + private readonly PulsarEndpoint _endpoint; + private readonly ITransportLogger _logger; + private IReceiverCallback _callback; + private readonly ITransportProtocol _protocol; + private Task _consumerTask; + + public PulsarListener(PulsarEndpoint endpoint, ITransportLogger logger, CancellationToken cancellation) + { + _endpoint = endpoint; + _logger = logger; + _cancellation = cancellation; + _protocol = new PulsarTransportProtocol(); + _consumer = endpoint.PulsarClient.CreateConsumer(endpoint.ConsumerOptions); + } + + public void Dispose() + { + _consumer?.DisposeAsync(); + _consumerTask?.Dispose(); + } + + public Uri Address => _endpoint.Uri; + public ListeningStatus Status { get; set; } + + public void Start(IReceiverCallback callback) + { + _callback = callback; + + _consumerTask = ConsumeAsync(); + + _logger.ListeningStatusChange(ListeningStatus.Accepting); + } + + private async Task ConsumeAsync() + { + await foreach (Message message in _consumer.Messages(_cancellation)) + { + Envelope envelope; + + try + { + envelope = _protocol.ReadEnvelope(new PulsarMessage(message.Data, new MessageMetadata())); + } + catch (Exception ex) + { + _logger.LogException(ex, message: $"Error trying to map an incoming Pulsar {_endpoint.Topic} Topic message to an Envelope. See the Dead Letter Queue"); + continue; + } + + try + { + await _callback.Received(Address, new[] {envelope}); + + await _consumer.Acknowledge(message, _cancellation); + } + catch (Exception e) + { + _logger.LogException(e, envelope.Id, "Error trying to receive a message from " + Address); + } + } + } + } +} diff --git a/src/Jasper.Pulsar/Internal/PulsarMessage.cs b/src/Jasper.Pulsar/Internal/PulsarMessage.cs new file mode 100644 index 000000000..3c25a894c --- /dev/null +++ b/src/Jasper.Pulsar/Internal/PulsarMessage.cs @@ -0,0 +1,21 @@ +using System.Buffers; +using DotPulsar; + +namespace Jasper.Pulsar.Internal +{ + internal class PulsarMessage + { + public MessageMetadata Metadata { get; } + public ReadOnlySequence Data { get; } + + public PulsarMessage(ReadOnlySequence data, MessageMetadata metadata) + { + Data = data; + Metadata = metadata; + } + + public PulsarMessage(byte[] data, MessageMetadata metadata) : this(new ReadOnlySequence(data), metadata) + { + } + } +} diff --git a/src/Jasper.Pulsar/Internal/PulsarTopicRouter.cs b/src/Jasper.Pulsar/Internal/PulsarTopicRouter.cs new file mode 100644 index 000000000..914ac1442 --- /dev/null +++ b/src/Jasper.Pulsar/Internal/PulsarTopicRouter.cs @@ -0,0 +1,30 @@ +using System; +using Baseline; +using Jasper.Configuration; +using Jasper.Runtime.Routing; + +namespace Jasper.Pulsar.Internal +{ + public class PulsarTopicRouter : TopicRouter + { + public override Uri BuildUriForTopic(string topic) + { + var endpoint = new PulsarEndpoint + { + IsDurable = true, + Topic = topic + }; + + return endpoint.Uri; + } + + public override PulsarSubscriberConfiguration FindConfigurationForTopic(string topicName, + IEndpoints endpoints) + { + Uri uri = BuildUriForTopic(topicName); + Endpoint endpoint = endpoints.As().GetOrCreateEndpoint(uri); + + return new PulsarSubscriberConfiguration((PulsarEndpoint) endpoint); + } + } +} diff --git a/src/Jasper.Pulsar/Internal/PulsarTransportProtocol.cs b/src/Jasper.Pulsar/Internal/PulsarTransportProtocol.cs new file mode 100644 index 000000000..2487f2ea5 --- /dev/null +++ b/src/Jasper.Pulsar/Internal/PulsarTransportProtocol.cs @@ -0,0 +1,65 @@ +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Linq; +using DotPulsar; +using Jasper.Pulsar.Internal; +using Jasper.Transports; + +namespace Jasper.Pulsar +{ + internal class PulsarTransportProtocol : ITransportProtocol + { + public const string PulsarMessageKeyHeader = "Pulsar.Message.Key"; + public const string PulsarMessageSequenceIdHeader = "Pulsar.Message.SequenceId"; + + private readonly Dictionary> _pulsarMsgPropTypes = new Dictionary>() + { + { PulsarMessageKeyHeader, (metadata, val) => metadata.Key = val?.ToString() }, + { PulsarMessageSequenceIdHeader, (metadata, val) => metadata.SequenceId = val != null ? ulong.Parse(val.ToString()) : default }, + }; + + public PulsarMessage WriteFromEnvelope(Envelope envelope) + { + IDictionary envelopHeaders = new Dictionary(); + envelope.WriteToDictionary(envelopHeaders); + + var metadata = new MessageMetadata(); + + foreach (var header in envelopHeaders.Where(h => !_pulsarMsgPropTypes.Keys.Contains(h.Key))) + { + metadata[header.Key] = header.Value.ToString(); + } + + SetMetaDataFromHeaderValues(metadata, envelopHeaders); + + return new PulsarMessage(envelope.Data, metadata); + } + + private void SetMetaDataFromHeaderValues(MessageMetadata metadata, IDictionary envelopHeaders) + { + foreach (var pulsarMsgPropType in _pulsarMsgPropTypes) + { + SetMetaDataFromHeaderValue(metadata, envelopHeaders, pulsarMsgPropType.Key, pulsarMsgPropType.Value); + } + } + + private void SetMetaDataFromHeaderValue(MessageMetadata metadata, IDictionary envelopHeaders, string propertyName, Action propertySetter) + { + if (envelopHeaders.TryGetValue(propertyName, out object headerValue)) + { + propertySetter(metadata, headerValue); + } + } + + public Envelope ReadEnvelope(PulsarMessage message) + { + var env = new Envelope() + { + Data = message.Data.ToArray() + }; + + return env; + } + } +} diff --git a/src/Jasper.Pulsar/Internal/PulssarSender.cs b/src/Jasper.Pulsar/Internal/PulssarSender.cs new file mode 100644 index 000000000..f860131fd --- /dev/null +++ b/src/Jasper.Pulsar/Internal/PulssarSender.cs @@ -0,0 +1,56 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using DotPulsar.Abstractions; +using Jasper.Transports; +using Jasper.Transports.Sending; + +namespace Jasper.Pulsar.Internal +{ + public class PulsarSender : ISender + { + private readonly ITransportProtocol _protocol; + private readonly IProducer _publisher; + private readonly PulsarEndpoint _endpoint; + public bool SupportsNativeScheduledSend { get; } = false; + public Uri Destination => _endpoint.Uri; + public PulsarSender(PulsarEndpoint endpoint) + { + _endpoint = endpoint; + _publisher = endpoint.PulsarClient.CreateProducer(endpoint.ProducerOptions); + _protocol = new PulsarTransportProtocol(); + } + + public void Dispose() + { + _publisher?.DisposeAsync(); + } + + public async Task Ping(CancellationToken cancellationToken) + { + Envelope envelope = Envelope.ForPing(Destination); + try + { + await Send(envelope); + } + catch + { + return false; + } + + return true; + } + + public async Task Send(Envelope envelope) + { + if (envelope.IsDelayed(DateTime.UtcNow)) + { + throw new Exception("Delayed Message Delivery"); + } + + var message = _protocol.WriteFromEnvelope(envelope); + + _ = await _publisher.Send(message.Metadata, message.Data); + } + } +} diff --git a/src/Jasper.Pulsar/Jasper.Pulsar.csproj b/src/Jasper.Pulsar/Jasper.Pulsar.csproj index 96e9f5250..81d7faa2c 100644 --- a/src/Jasper.Pulsar/Jasper.Pulsar.csproj +++ b/src/Jasper.Pulsar/Jasper.Pulsar.csproj @@ -27,4 +27,8 @@ + + + + diff --git a/src/Jasper.Pulsar/PulsarEndpoint.cs b/src/Jasper.Pulsar/PulsarEndpoint.cs new file mode 100644 index 000000000..b193dc186 --- /dev/null +++ b/src/Jasper.Pulsar/PulsarEndpoint.cs @@ -0,0 +1,69 @@ +using System; +using DotPulsar; +using DotPulsar.Abstractions; +using Jasper.Configuration; +using Jasper.Pulsar.Internal; +using Jasper.Runtime; +using Jasper.Transports.Sending; + +namespace Jasper.Pulsar +{ + public class PulsarEndpoint : Endpoint + { + private PulsarTopic _topic; + public PulsarTopic Topic + { + get => _topic; + set + { + _topic = value; + IsDurable = _topic.Persistence.Equals("persistent"); + } + } + public override Uri Uri => BuildUri(); + public ConsumerOptions ConsumerOptions { get; set; } + public ProducerOptions ProducerOptions { get; set; } + public IPulsarClient PulsarClient { get; set; } + + public PulsarEndpoint() + { + + } + + public PulsarEndpoint(string topic) + { + Topic = topic; + } + + public PulsarEndpoint(Uri uri) : base(uri) + { + Topic = uri; + } + + + private Uri BuildUri(bool forReply = false) + { + return Topic.ToJasperUri(forReply); + } + + public override void Parse(Uri uri) + { + Topic = uri; + } + + protected internal override void StartListening(IMessagingRoot root, ITransportRuntime runtime) + { + if (!IsListener) return; + + var listener = new PulsarListener(this, root.TransportLogger, root.Cancellation); + runtime.AddListener(listener, this); + } + + protected override ISender CreateSender(IMessagingRoot root) + { + return new PulsarSender(this); + } + + public override Uri ReplyUri() => BuildUri(true); + } +} diff --git a/src/Jasper.Pulsar/PulsarListenerConfiguration.cs b/src/Jasper.Pulsar/PulsarListenerConfiguration.cs new file mode 100644 index 000000000..b3d0d54d9 --- /dev/null +++ b/src/Jasper.Pulsar/PulsarListenerConfiguration.cs @@ -0,0 +1,12 @@ +using Jasper.Configuration; +using Jasper.Pulsar; + +namespace Jasper.Pulsar +{ + public class PulsarListenerConfiguration : ListenerConfiguration + { + public PulsarListenerConfiguration(PulsarEndpoint endpoint) : base(endpoint) + { + } + } +} diff --git a/src/Jasper.Pulsar/PulsarSubscriberConfiguration.cs b/src/Jasper.Pulsar/PulsarSubscriberConfiguration.cs new file mode 100644 index 000000000..95851dd98 --- /dev/null +++ b/src/Jasper.Pulsar/PulsarSubscriberConfiguration.cs @@ -0,0 +1,12 @@ +using Jasper.Configuration; + +namespace Jasper.Pulsar +{ + public class PulsarSubscriberConfiguration : SubscriberConfiguration + { + public PulsarSubscriberConfiguration(PulsarEndpoint endpoint) : base(endpoint) + { + } + + } +} diff --git a/src/Jasper.Pulsar/PulsarTransport.cs b/src/Jasper.Pulsar/PulsarTransport.cs new file mode 100644 index 000000000..b41d1fb2c --- /dev/null +++ b/src/Jasper.Pulsar/PulsarTransport.cs @@ -0,0 +1,78 @@ +using System; +using System.Collections.Generic; +using DotPulsar; +using DotPulsar.Abstractions; +using Jasper.Pulsar.Internal; +using Jasper.Transports; + +namespace Jasper.Pulsar +{ + public static class Protocols + { + public static readonly string[] Pulsar = { "persistent", "non-persistent" }; + } + + public class PulsarTransport : TransportBase + { + private readonly Dictionary _endpoints; + + public PulsarTopicRouter Topics { get; } = new PulsarTopicRouter(); + public PulsarTransport() : base(Pulsar.Protocols.Pulsar) + { + _endpoints = new Dictionary(); + } + + public IPulsarClient PulsarClient { get; set; } + + protected override IEnumerable endpoints() => _endpoints.Values; + + protected override PulsarEndpoint findEndpointByUri(Uri uri) + { + if (!_endpoints.ContainsKey(uri)) + { + _endpoints.Add(uri, new PulsarEndpoint(uri) + { + PulsarClient = PulsarClient + }); + } + + return _endpoints[uri]; + } + + public PulsarEndpoint EndpointFor(ProducerOptions producerConifg) => + AddOrUpdateEndpoint(endpoint => + { + endpoint.Topic = producerConifg.Topic; + endpoint.ProducerOptions = producerConifg; + }); + + public PulsarEndpoint EndpointFor(ConsumerOptions consumerConifg) => + AddOrUpdateEndpoint(endpoint => + { + endpoint.Topic = consumerConifg.Topic; + endpoint.ConsumerOptions = consumerConifg; + }); + + PulsarEndpoint AddOrUpdateEndpoint(Action configure) + { + var endpoint = new PulsarEndpoint + { + PulsarClient = PulsarClient + }; + + configure(endpoint); + + if (_endpoints.ContainsKey(endpoint.Uri)) + { + endpoint = _endpoints[endpoint.Uri]; + } + else + { + _endpoints.Add(endpoint.Uri, endpoint); + } + + return endpoint; + } + + } +} diff --git a/src/Jasper.Pulsar/PulsarTransportConfigurationExtensions.cs b/src/Jasper.Pulsar/PulsarTransportConfigurationExtensions.cs new file mode 100644 index 000000000..89c741123 --- /dev/null +++ b/src/Jasper.Pulsar/PulsarTransportConfigurationExtensions.cs @@ -0,0 +1,124 @@ +using System; +using Baseline; +using DotPulsar; +using DotPulsar.Abstractions; +using DotPulsar.Internal; +using Jasper.Configuration; +using Jasper.Pulsar; + +namespace Jasper.Pulsar +{ + public static class PulsarTransportConfigurationExtensions + {/// + /// Quick access to the pulsar Transport within this application. + /// This is for advanced usage + /// + /// + /// + internal static PulsarTransport PulsarTransport(this IEndpoints endpoints) + { + var transports = endpoints.As(); + + var transport = transports.Get(); + + if (transport == null) + { + transport = new PulsarTransport(); + transports.Add(transport); + } + + transports.Subscribers.Fill(transport.Topics); + + return transport; + } + /// + /// Configure connection and authentication information about the Azure Service Bus usage + /// within this Jasper application + /// + /// + /// + public static void ConfigurePulsar(this IEndpoints endpoints, Action configure) + { + var transport = endpoints.PulsarTransport(); + endpoints.As().Subscribers.Fill(transport.Topics); + configure(transport); + } + + /// + /// Configure connection and authentication information about the Azure Service Bus usage + /// within this Jasper application + /// + /// + /// + public static void ConfigurePulsar(this IEndpoints endpoints, IPulsarClient client) + { + endpoints.ConfigurePulsar(_ => { _.PulsarClient = client; }); + } + + + public static void ConfigurePulsar(this IEndpoints endpoints, string pulsarCluster) + { + endpoints.ConfigurePulsar(_ => + { + _.PulsarClient = new PulsarClientBuilder().ServiceUrl(new Uri(pulsarCluster)).Build(); + }); + } + + public static void ConfigurePulsar(this IEndpoints endpoints, Uri pulsarCluster) + { + endpoints.ConfigurePulsar(_ => + { + _.PulsarClient = new PulsarClientBuilder().ServiceUrl(pulsarCluster).Build(); + }); + } + + public static void ConfigurePulsar(this IEndpoints endpoints, IPulsarClientBuilder pulsarClientBuilder) + { + endpoints.ConfigurePulsar(_ => + { + _.PulsarClient = pulsarClientBuilder.Build(); + }); + } + + /// + /// Listen for incoming messages at the designated Pulsar Topic by name + /// + /// + /// + /// + /// + /// + /// + public static PulsarListenerConfiguration ListenToPulsarTopic(this IEndpoints endpoints, string subscription, string topicName) => + ListenToPulsarTopic(endpoints, new ConsumerOptions(subscription, topicName)); + + public static PulsarListenerConfiguration ListenToPulsarTopic(this IEndpoints endpoints, ConsumerOptions consumerConfig) + { + var endpoint = endpoints.PulsarTransport().EndpointFor(consumerConfig); + endpoint.IsListener = true; + return new PulsarListenerConfiguration(endpoint); + } + + /// + /// Publish matching messages to Pulsar Topic using provided Producer Configuration + /// + /// + /// This is used as the topic name when publishing. Can be either a binding key or a queue name or a static topic name if the exchange is topic-based + /// Optional, you only need to supply this if you are using a non-default exchange + /// + public static PulsarSubscriberConfiguration ToPulsarTopic(this IPublishToExpression publishing, string topicName) => ToPulsarTopic(publishing, new ProducerOptions(topicName)); + + public static PulsarSubscriberConfiguration ToPulsarTopic(this IPublishToExpression publishing, ProducerOptions producerOptions) + { + var transports = publishing.As().Parent; + var transport = transports.Get(); + var endpoint = transport.EndpointFor(producerOptions); + + // This is necessary unfortunately to hook up the subscription rules + publishing.To(endpoint.Uri); + + return new PulsarSubscriberConfiguration(endpoint); + } + + } +} From e0cdfd763cc605466ef3990d3863af5d16f508ad Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Sun, 10 May 2020 18:23:20 -0400 Subject: [PATCH 74/82] Get properties from incoming Pulsar message --- src/Jasper.Pulsar/Internal/PulsarListener.cs | 2 +- src/Jasper.Pulsar/Internal/PulsarMessage.cs | 14 +++++++++++++- .../Internal/PulsarTransportProtocol.cs | 15 ++++----------- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/Jasper.Pulsar/Internal/PulsarListener.cs b/src/Jasper.Pulsar/Internal/PulsarListener.cs index 3ae843695..3dfa6d06d 100644 --- a/src/Jasper.Pulsar/Internal/PulsarListener.cs +++ b/src/Jasper.Pulsar/Internal/PulsarListener.cs @@ -53,7 +53,7 @@ private async Task ConsumeAsync() try { - envelope = _protocol.ReadEnvelope(new PulsarMessage(message.Data, new MessageMetadata())); + envelope = _protocol.ReadEnvelope(new PulsarMessage(message.Data, message.Properties)); } catch (Exception ex) { diff --git a/src/Jasper.Pulsar/Internal/PulsarMessage.cs b/src/Jasper.Pulsar/Internal/PulsarMessage.cs index 3c25a894c..5d0dda5df 100644 --- a/src/Jasper.Pulsar/Internal/PulsarMessage.cs +++ b/src/Jasper.Pulsar/Internal/PulsarMessage.cs @@ -1,12 +1,19 @@ using System.Buffers; +using System.Collections.Generic; using DotPulsar; namespace Jasper.Pulsar.Internal { internal class PulsarMessage { - public MessageMetadata Metadata { get; } + public MessageMetadata Metadata { get; } = new MessageMetadata(); public ReadOnlySequence Data { get; } + public IReadOnlyDictionary Properties { get; } = new Dictionary(); + + public PulsarMessage(ReadOnlySequence data) + { + Data = data; + } public PulsarMessage(ReadOnlySequence data, MessageMetadata metadata) { @@ -17,5 +24,10 @@ public PulsarMessage(ReadOnlySequence data, MessageMetadata metadata) public PulsarMessage(byte[] data, MessageMetadata metadata) : this(new ReadOnlySequence(data), metadata) { } + + public PulsarMessage(ReadOnlySequence data, IReadOnlyDictionary properties) : this(data) + { + Properties = properties; + } } } diff --git a/src/Jasper.Pulsar/Internal/PulsarTransportProtocol.cs b/src/Jasper.Pulsar/Internal/PulsarTransportProtocol.cs index 2487f2ea5..496c272ac 100644 --- a/src/Jasper.Pulsar/Internal/PulsarTransportProtocol.cs +++ b/src/Jasper.Pulsar/Internal/PulsarTransportProtocol.cs @@ -46,20 +46,13 @@ private void SetMetaDataFromHeaderValues(MessageMetadata metadata, IDictionary envelopHeaders, string propertyName, Action propertySetter) { - if (envelopHeaders.TryGetValue(propertyName, out object headerValue)) - { - propertySetter(metadata, headerValue); - } + if (envelopHeaders.TryGetValue(propertyName, out object headerValue)) propertySetter(metadata, headerValue); } - public Envelope ReadEnvelope(PulsarMessage message) - { - var env = new Envelope() + public Envelope ReadEnvelope(PulsarMessage message) => new Envelope { - Data = message.Data.ToArray() + Data = message.Data.ToArray(), + Headers = message.Properties.ToDictionary(ks => ks.Key, vs => vs.Value) }; - - return env; - } } } From 56ae6410f894f4a1cae3ca8f57d64670f68bbc6a Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 12 May 2020 16:32:24 -0400 Subject: [PATCH 75/82] Add PulsarEndpointTests --- .../PulsarEndpointTester.cs | 125 ++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 src/Jasper.Pulsar.Tests/PulsarEndpointTester.cs diff --git a/src/Jasper.Pulsar.Tests/PulsarEndpointTester.cs b/src/Jasper.Pulsar.Tests/PulsarEndpointTester.cs new file mode 100644 index 000000000..4789e50f0 --- /dev/null +++ b/src/Jasper.Pulsar.Tests/PulsarEndpointTester.cs @@ -0,0 +1,125 @@ +using System; +using AutoFixture.Xunit2; +using Shouldly; +using Xunit; + +namespace Jasper.Pulsar.Tests +{ + public class PulsarEndpointTester + { + [Fact] + public void parse_non_durable_uri() + { + var endpoint = new PulsarEndpoint(); + endpoint.Parse(new Uri($"{PulsarPersistence.Persistent}://tenant/jasper/key1")); + + endpoint.IsDurable.ShouldBeFalse(); + endpoint.Topic.TopicName.ShouldBe("key1"); + } + + [Theory, AutoData] + public void persistent_pulsar_topic_parts_match(string tenant, string @namespace, string topic) + { + var endpoint = new PulsarEndpoint(); + endpoint.Parse(new Uri($"{PulsarPersistence.Persistent}://{tenant}/{@namespace}/{topic}")); + + endpoint.Topic.Persistence.ShouldBe(PulsarPersistence.Persistent); + endpoint.Topic.Tenant.ShouldBe(tenant); + endpoint.Topic.Namespace.ShouldBe(@namespace); + endpoint.Topic.TopicName.ShouldBe(topic); + } + + [Theory, AutoData] + public void non_persistent_pulsar_topic_parts_match(string tenant, string @namespace, string topic) + { + var endpoint = new PulsarEndpoint(); + endpoint.Parse(new Uri($"{PulsarPersistence.NonPersistent}://{tenant}/{@namespace}/{topic}")); + + endpoint.Topic.Persistence.ShouldBe(PulsarPersistence.NonPersistent); + endpoint.Topic.Tenant.ShouldBe(tenant); + endpoint.Topic.Namespace.ShouldBe(@namespace); + endpoint.Topic.TopicName.ShouldBe(topic); + } + + [Fact] + public void parse_non_durable_persistent_uri() + { + var endpoint = new PulsarEndpoint(); + endpoint.Parse(new Uri($"{PulsarPersistence.Persistent}://tenant/jasper/key1")); + + endpoint.IsDurable.ShouldBeFalse(); + endpoint.Topic.Persistence.ShouldBe(PulsarPersistence.Persistent); + } + + [Fact] + public void parse_non_durable_non_persistent_uri() + { + var endpoint = new PulsarEndpoint(); + endpoint.Parse(new Uri($"{PulsarPersistence.NonPersistent}://tenant/jasper/key1")); + + endpoint.IsDurable.ShouldBeFalse(); + endpoint.Topic.Persistence.ShouldBe(PulsarPersistence.NonPersistent); + } + + [Fact] + public void parse_durable_persistent_uri() + { + var endpoint = new PulsarEndpoint(); + endpoint.Parse(new Uri($"{PulsarPersistence.Persistent}://tenant/jasper/key1/durable")); + + endpoint.IsDurable.ShouldBeTrue(); + endpoint.Topic.Persistence.ShouldBe(PulsarPersistence.Persistent); + } + + [Fact] + public void parse_durable_non_persistent_uri() + { + var endpoint = new PulsarEndpoint(); + endpoint.Parse(new Uri($"{PulsarPersistence.NonPersistent}://tenant/jasper/key1/durable")); + + endpoint.IsDurable.ShouldBeTrue(); + endpoint.Topic.Persistence.ShouldBe(PulsarPersistence.NonPersistent); + } + + [Fact] + public void parse_durable_uri() + { + var endpoint = new PulsarEndpoint(); + endpoint.Parse(new Uri($"{PulsarPersistence.Persistent}://tenant/jasper/key1/durable")); + + endpoint.IsDurable.ShouldBeTrue(); + endpoint.Topic.TopicName.ShouldBe("key1"); + } + + [Fact] + public void build_uri_for_subscription_and_topic() + { + new PulsarEndpoint + { + Topic = $"{PulsarPersistence.Persistent}://tenant/jasper/key1" + } + .Uri.ShouldBe(new Uri($"{PulsarPersistence.Persistent}://tenant/jasper/key1")); + } + + [Fact] + public void generate_reply_uri_for_non_durable() + { + new PulsarEndpoint + { + Topic = $"{PulsarPersistence.Persistent}://tenant/jasper/key1" + } + .ReplyUri().ShouldBe(new Uri($"{PulsarPersistence.Persistent}://tenant/jasper/key1")); + } + + [Fact] + public void generate_reply_uri_for_durable() + { + new PulsarEndpoint + { + Topic = $"{PulsarPersistence.Persistent}://tenant/jasper/key1", + IsDurable = true + }.ReplyUri().ShouldBe(new Uri($"{PulsarPersistence.Persistent}://tenant/jasper/key1/durable")); + } + + } +} From e20bf6de7a577fc76f6793ee17e7579905a4e7a1 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 12 May 2020 16:32:38 -0400 Subject: [PATCH 76/82] Make sure that failed calls return to sender --- .../PulsarSendingComplianceTests.cs | 31 +++++++++++++------ 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/src/Jasper.Pulsar.Tests/PulsarSendingComplianceTests.cs b/src/Jasper.Pulsar.Tests/PulsarSendingComplianceTests.cs index fd2f54b3a..ebe48699f 100644 --- a/src/Jasper.Pulsar.Tests/PulsarSendingComplianceTests.cs +++ b/src/Jasper.Pulsar.Tests/PulsarSendingComplianceTests.cs @@ -4,6 +4,7 @@ using DotPulsar; using DotPulsar.Internal; using Jasper.Tracking; +using Newtonsoft.Json; using Shouldly; using TestingSupport.Compliance; using TestMessages; @@ -23,14 +24,9 @@ public class Sender : JasperOptions public Sender() { Endpoints.ConfigurePulsar(new PulsarClientBuilder() - .ExceptionHandler(context => - { - - return new ValueTask(Task.CompletedTask); - }) .ServiceUrl(new Uri(Server))); Endpoints.PublishAllMessages().ToPulsarTopic(new ProducerOptions(Topic)); - Endpoints.ListenToPulsarTopic("compliance-tests", ReplyTopic).UseForReplies(); + Endpoints.ListenToPulsarTopic("sender", ReplyTopic).UseForReplies(); } } @@ -39,7 +35,7 @@ public class FailureSender : JasperOptions public const string Topic = "persistent://public/default/jasper-compliance"; public FailureSender() { - Endpoints.ConfigurePulsar(new PulsarClientBuilder().ServiceUrl(new Uri(Server))); + Endpoints.ConfigurePulsar(new PulsarClientBuilder().ServiceUrl(new Uri("pulsar://localhost:6651"))); Endpoints.PublishAllMessages().ToPulsarTopic(new ProducerOptions(Topic)); } } @@ -74,12 +70,13 @@ public async Task publish_failures_reported_to_caller() theSender = null; SenderIs(); - _ = await theSender.TrackActivity(60.Seconds()) + _ = await theSender.TrackActivity(10.Seconds()) .DoNotAssertOnExceptionsDetected() .DoNotAssertTimeout() .ExecuteAndWait(c => { - Should.Throw(c.Publish(new Message1())); + var serializationException = Should.Throw(c.Publish(new PoisonEnvelop())); + serializationException.InnerException.ShouldBeOfType(); return Task.CompletedTask; }); } @@ -88,11 +85,25 @@ public async Task publish_failures_reported_to_caller() public async Task publish_succeeds() { - _ = await theSender.TrackActivity(60.Seconds()) + _ = await theSender.TrackActivity(10.Seconds()) .DoNotAssertOnExceptionsDetected() .DoNotAssertTimeout() .ExecuteAndWait(c => c.Publish(new Message1())); } } } + + public class PoisionMessageException : Exception + { + public const string PoisonMessage = "Poison message"; + public PoisionMessageException() : base(PoisonMessage) + { + + } + } + + public class PoisonEnvelop : Envelope + { + public new byte[] Data => throw new PoisionMessageException(); + } } From 63bb8f1ce4c8f47838c84ef1f394d63a6fe9f781 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 12 May 2020 16:33:27 -0400 Subject: [PATCH 77/82] Keep Jasper specifics in Endpoint and out of PulsarTopic --- .../Internal/PulsarTransportProtocol.cs | 9 ++++- src/Jasper.Pulsar/Internal/PulssarSender.cs | 6 ++-- src/Jasper.Pulsar/PulsarEndpoint.cs | 35 ++++++++++--------- src/Jasper.Pulsar/PulsarTopic.cs | 27 +++++++------- 4 files changed, 44 insertions(+), 33 deletions(-) diff --git a/src/Jasper.Pulsar/Internal/PulsarTransportProtocol.cs b/src/Jasper.Pulsar/Internal/PulsarTransportProtocol.cs index 496c272ac..e1db00733 100644 --- a/src/Jasper.Pulsar/Internal/PulsarTransportProtocol.cs +++ b/src/Jasper.Pulsar/Internal/PulsarTransportProtocol.cs @@ -49,10 +49,17 @@ private void SetMetaDataFromHeaderValue(MessageMetadata metadata, IDictionary new Envelope + public Envelope ReadEnvelope(PulsarMessage message) + { + var envelope = new Envelope { Data = message.Data.ToArray(), Headers = message.Properties.ToDictionary(ks => ks.Key, vs => vs.Value) }; + + envelope.ReadPropertiesFromDictionary(message.Properties.ToDictionary(ks => ks.Key, vs => (object)vs.Value)); + + return envelope; + } } } diff --git a/src/Jasper.Pulsar/Internal/PulssarSender.cs b/src/Jasper.Pulsar/Internal/PulssarSender.cs index f860131fd..7371c68a2 100644 --- a/src/Jasper.Pulsar/Internal/PulssarSender.cs +++ b/src/Jasper.Pulsar/Internal/PulssarSender.cs @@ -12,11 +12,13 @@ public class PulsarSender : ISender private readonly ITransportProtocol _protocol; private readonly IProducer _publisher; private readonly PulsarEndpoint _endpoint; + private readonly CancellationToken _cancellationToken; public bool SupportsNativeScheduledSend { get; } = false; public Uri Destination => _endpoint.Uri; - public PulsarSender(PulsarEndpoint endpoint) + public PulsarSender(PulsarEndpoint endpoint, CancellationToken cancellationToken) { _endpoint = endpoint; + _cancellationToken = cancellationToken; _publisher = endpoint.PulsarClient.CreateProducer(endpoint.ProducerOptions); _protocol = new PulsarTransportProtocol(); } @@ -50,7 +52,7 @@ public async Task Send(Envelope envelope) var message = _protocol.WriteFromEnvelope(envelope); - _ = await _publisher.Send(message.Metadata, message.Data); + _ = await _publisher.Send(message.Metadata, message.Data, _cancellationToken); } } } diff --git a/src/Jasper.Pulsar/PulsarEndpoint.cs b/src/Jasper.Pulsar/PulsarEndpoint.cs index b193dc186..f028e4c4d 100644 --- a/src/Jasper.Pulsar/PulsarEndpoint.cs +++ b/src/Jasper.Pulsar/PulsarEndpoint.cs @@ -4,23 +4,16 @@ using Jasper.Configuration; using Jasper.Pulsar.Internal; using Jasper.Runtime; +using Jasper.Transports; using Jasper.Transports.Sending; namespace Jasper.Pulsar { public class PulsarEndpoint : Endpoint { - private PulsarTopic _topic; - public PulsarTopic Topic - { - get => _topic; - set - { - _topic = value; - IsDurable = _topic.Persistence.Equals("persistent"); - } - } - public override Uri Uri => BuildUri(); + public PulsarTopic Topic { get; set; } + + public override Uri Uri => BuildUri(false); public ConsumerOptions ConsumerOptions { get; set; } public ProducerOptions ProducerOptions { get; set; } public IPulsarClient PulsarClient { get; set; } @@ -40,15 +33,23 @@ public PulsarEndpoint(Uri uri) : base(uri) Topic = uri; } - - private Uri BuildUri(bool forReply = false) + public override void Parse(Uri uri) { - return Topic.ToJasperUri(forReply); + IsDurable = uri.ToString().EndsWith(TransportConstants.Durable); + var url = uri.ToString(); + string pulsarTopic = url.Substring(0, url.Length - (IsDurable ? TransportConstants.Durable.Length + 1 : 0)); + + Topic = new PulsarTopic(pulsarTopic); } - public override void Parse(Uri uri) + private Uri BuildUri(bool forReply = false) { - Topic = uri; + if (forReply && IsDurable) + { + return new Uri(Topic + $"/{TransportConstants.Durable}"); + } + + return Topic; } protected internal override void StartListening(IMessagingRoot root, ITransportRuntime runtime) @@ -61,7 +62,7 @@ protected internal override void StartListening(IMessagingRoot root, ITransportR protected override ISender CreateSender(IMessagingRoot root) { - return new PulsarSender(this); + return new PulsarSender(this, root.Cancellation); } public override Uri ReplyUri() => BuildUri(true); diff --git a/src/Jasper.Pulsar/PulsarTopic.cs b/src/Jasper.Pulsar/PulsarTopic.cs index 06fae07dc..708b7d349 100644 --- a/src/Jasper.Pulsar/PulsarTopic.cs +++ b/src/Jasper.Pulsar/PulsarTopic.cs @@ -4,6 +4,12 @@ namespace Jasper.Pulsar { + public static class PulsarPersistence + { + public const string Persistent = "persistent"; + public const string NonPersistent = "non-persistent"; + } + public struct PulsarTopic { public string Persistence { get; } @@ -11,38 +17,33 @@ public struct PulsarTopic public string Namespace { get; } public string TopicName { get; } - public bool IsForReply { get; } - - private const string PulsarTopicRegex = @"(non-persistent|persistent)://([-A-Za-z0-9]*)/([-A-Za-z0-9]*)/([-A-Za-z0-9]*)(/for-reply)?"; + private const string PulsarTopicRegex = @"(non-persistent|persistent)://([-A-Za-z0-9]*)/([-A-Za-z0-9]*)/([-A-Za-z0-9]*)?"; private const string InvalidTopicFormatMessage = - "Invalid Pulsar topic. Expecting format of \"{persistent|non-persistent}://tenant/namespace/topic\" (can append \"/for-reply\" for Jasper functionality. It will not be included in communication w/ Pulsar)"; + "Invalid Pulsar topic. Expecting format of \"{persistent|non-persistent}://tenant/namespace/topic\""; public PulsarTopic(Uri topic) : this(topic?.ToString()) { } - public PulsarTopic(string topic) + public PulsarTopic(string fullyQualifiedTopic) { - MatchCollection match = Regex.Matches(topic, PulsarTopicRegex, RegexOptions.Compiled); + MatchCollection match = Regex.Matches(fullyQualifiedTopic, PulsarTopicRegex, RegexOptions.Compiled); if (!match.Any()) - throw new ArgumentException(InvalidTopicFormatMessage, nameof(topic)); + throw new ArgumentException(InvalidTopicFormatMessage, nameof(fullyQualifiedTopic)); Persistence = match[0].Groups[1].Captures[0].Value; Tenant = match[0].Groups[2].Captures[0].Value; Namespace = match[0].Groups[3].Captures[0].Value; TopicName = match[0].Groups[4].Captures[0].Value; - IsForReply = match[0].Groups.Count == 6; } - public static implicit operator string(PulsarTopic topic) => topic.ToString(); + public static explicit operator string(PulsarTopic topic) => topic.ToString(); + public static implicit operator Uri(PulsarTopic topic) => new Uri(topic.ToString()); public static implicit operator PulsarTopic(string topic) => new PulsarTopic(topic); public static implicit operator PulsarTopic(Uri topic) => new PulsarTopic(topic); - public override string ToString() => $"{Persistence}://{Tenant}/{Namespace}/{TopicName}"; - - public Uri ToJasperUri(bool forReply) => new Uri($"{this}{(forReply ? "/for-reply" : string.Empty)}"); - } + public override string ToString() => $"{Persistence}://{Tenant}/{Namespace}/{TopicName}"; } } From 8312c3577e7e0161e704d0f0bcd708f8403b82da Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 12 May 2020 17:37:12 -0400 Subject: [PATCH 78/82] Cleanup topics and subscribers for Pulsar compliance tests --- .../PulsarSendingComplianceTests.cs | 34 ++++++------------- 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/src/Jasper.Pulsar.Tests/PulsarSendingComplianceTests.cs b/src/Jasper.Pulsar.Tests/PulsarSendingComplianceTests.cs index ebe48699f..41fb29a8b 100644 --- a/src/Jasper.Pulsar.Tests/PulsarSendingComplianceTests.cs +++ b/src/Jasper.Pulsar.Tests/PulsarSendingComplianceTests.cs @@ -18,25 +18,12 @@ public class PulsarSendingComplianceTestsShell public class Sender : JasperOptions { - public const string Topic = "persistent://public/default/jasper-compliance"; - public static string ReplyTopic = $"{Topic}-reply"; - - public Sender() + public Sender(string topic) { Endpoints.ConfigurePulsar(new PulsarClientBuilder() .ServiceUrl(new Uri(Server))); - Endpoints.PublishAllMessages().ToPulsarTopic(new ProducerOptions(Topic)); - Endpoints.ListenToPulsarTopic("sender", ReplyTopic).UseForReplies(); - } - } - - public class FailureSender : JasperOptions - { - public const string Topic = "persistent://public/default/jasper-compliance"; - public FailureSender() - { - Endpoints.ConfigurePulsar(new PulsarClientBuilder().ServiceUrl(new Uri("pulsar://localhost:6651"))); - Endpoints.PublishAllMessages().ToPulsarTopic(new ProducerOptions(Topic)); + Endpoints.PublishAllMessages().ToPulsarTopic(new ProducerOptions(topic)); + Endpoints.ListenToPulsarTopic(Guid.NewGuid().ToString(), topic + "-reply").UseForReplies(); } } @@ -45,20 +32,22 @@ public class Receiver : JasperOptions public Receiver(string topic) { Endpoints.ConfigurePulsar(new PulsarClientBuilder().ServiceUrl(new Uri(Server))); - Endpoints.PublishAllMessages().ToPulsarTopic(Sender.ReplyTopic); - Endpoints.ListenToPulsarTopic("receiver", topic); + Endpoints.PublishAllMessages().ToPulsarTopic(topic + "-reply"); + Endpoints.ListenToPulsarTopic(Guid.NewGuid().ToString(), topic); } } public class PulsarSendingComplianceTests : SendingCompliance { - public PulsarSendingComplianceTests() : base(new Uri(Sender.Topic)) + public static string Topic { get; } = "persistent://public/default/jasper"; + + public PulsarSendingComplianceTests() : base(new Uri(Topic)) { - var sender = new Sender(); + var sender = new Sender(Topic); SenderIs(sender); - var receiver = new Receiver(Sender.Topic); + var receiver = new Receiver(Topic); ReceiverIs(receiver); } @@ -67,9 +56,6 @@ public PulsarSendingComplianceTests() : base(new Uri(Sender.Topic)) public async Task publish_failures_reported_to_caller() { - theSender = null; - SenderIs(); - _ = await theSender.TrackActivity(10.Seconds()) .DoNotAssertOnExceptionsDetected() .DoNotAssertTimeout() From f70c0d28862e84923736a9ee724361ec827a55a9 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 12 May 2020 18:07:21 -0400 Subject: [PATCH 79/82] Add short delay after creating test harness to allow consumer to latch to transport --- src/Jasper.Pulsar.Tests/PulsarSendingComplianceTests.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Jasper.Pulsar.Tests/PulsarSendingComplianceTests.cs b/src/Jasper.Pulsar.Tests/PulsarSendingComplianceTests.cs index 41fb29a8b..8cf1e027c 100644 --- a/src/Jasper.Pulsar.Tests/PulsarSendingComplianceTests.cs +++ b/src/Jasper.Pulsar.Tests/PulsarSendingComplianceTests.cs @@ -1,4 +1,5 @@ using System; +using System.Threading; using System.Threading.Tasks; using Baseline.Dates; using DotPulsar; @@ -41,7 +42,7 @@ public class PulsarSendingComplianceTests : SendingCompliance { public static string Topic { get; } = "persistent://public/default/jasper"; - public PulsarSendingComplianceTests() : base(new Uri(Topic)) + public PulsarSendingComplianceTests() : base(new Uri(Topic), 30.Seconds()) { var sender = new Sender(Topic); @@ -50,6 +51,8 @@ public PulsarSendingComplianceTests() : base(new Uri(Topic)) var receiver = new Receiver(Topic); ReceiverIs(receiver); + + Thread.Sleep(2000); } [Fact] From 97ff5b891458072b6808dc4c8b4fcf777f7fb2c1 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 14 May 2020 13:21:09 -0400 Subject: [PATCH 80/82] Rename ISendingAgent.Forward back to StoreAndForward --- src/Jasper/Envelope.Internals.cs | 2 +- src/Jasper/Transports/Local/DurableLocalSendingAgent.cs | 2 +- src/Jasper/Transports/Local/LightweightLocalSendingAgent.cs | 2 +- src/Jasper/Transports/Sending/ISendingAgent.cs | 2 +- src/Jasper/Transports/Sending/SendingAgent.cs | 2 +- src/StorytellerSpecs/Stub/StubEndpoint.cs | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Jasper/Envelope.Internals.cs b/src/Jasper/Envelope.Internals.cs index fc927a071..fa20aa4f0 100644 --- a/src/Jasper/Envelope.Internals.cs +++ b/src/Jasper/Envelope.Internals.cs @@ -136,7 +136,7 @@ internal Task Send() _enqueued = true; - return Sender.Forward(this); + return Sender.StoreAndForward(this); } internal Task QuickSend() diff --git a/src/Jasper/Transports/Local/DurableLocalSendingAgent.cs b/src/Jasper/Transports/Local/DurableLocalSendingAgent.cs index 888fa6807..5ba564df0 100644 --- a/src/Jasper/Transports/Local/DurableLocalSendingAgent.cs +++ b/src/Jasper/Transports/Local/DurableLocalSendingAgent.cs @@ -52,7 +52,7 @@ public Task EnqueueOutgoing(Envelope envelope) return Enqueue(envelope); } - public async Task Forward(Envelope envelope) + public async Task StoreAndForward(Envelope envelope) { _messageLogger.Sent(envelope); writeMessageData(envelope); diff --git a/src/Jasper/Transports/Local/LightweightLocalSendingAgent.cs b/src/Jasper/Transports/Local/LightweightLocalSendingAgent.cs index 557e871ce..4139a4c89 100644 --- a/src/Jasper/Transports/Local/LightweightLocalSendingAgent.cs +++ b/src/Jasper/Transports/Local/LightweightLocalSendingAgent.cs @@ -41,7 +41,7 @@ public Task EnqueueOutgoing(Envelope envelope) : Enqueue(envelope); } - public Task Forward(Envelope envelope) + public Task StoreAndForward(Envelope envelope) { return EnqueueOutgoing(envelope); } diff --git a/src/Jasper/Transports/Sending/ISendingAgent.cs b/src/Jasper/Transports/Sending/ISendingAgent.cs index 997cc73b9..9429a1a0e 100644 --- a/src/Jasper/Transports/Sending/ISendingAgent.cs +++ b/src/Jasper/Transports/Sending/ISendingAgent.cs @@ -21,7 +21,7 @@ public interface ISendingAgent : IDisposable // This would be called by the EnvelopeSender if invoked // indirectly - Task Forward(Envelope envelope); + Task StoreAndForward(Envelope envelope); Endpoint Endpoint { get; } diff --git a/src/Jasper/Transports/Sending/SendingAgent.cs b/src/Jasper/Transports/Sending/SendingAgent.cs index 3407f0075..838040c26 100644 --- a/src/Jasper/Transports/Sending/SendingAgent.cs +++ b/src/Jasper/Transports/Sending/SendingAgent.cs @@ -62,7 +62,7 @@ public async Task EnqueueOutgoing(Envelope envelope) _messageLogger.Sent(envelope); } - public async Task Forward(Envelope envelope) + public async Task StoreAndForward(Envelope envelope) { setDefaults(envelope); diff --git a/src/StorytellerSpecs/Stub/StubEndpoint.cs b/src/StorytellerSpecs/Stub/StubEndpoint.cs index 24c29b36f..ec7e1630b 100644 --- a/src/StorytellerSpecs/Stub/StubEndpoint.cs +++ b/src/StorytellerSpecs/Stub/StubEndpoint.cs @@ -77,7 +77,7 @@ public Task EnqueueOutgoing(Envelope envelope) return Task.CompletedTask; } - public Task Forward(Envelope envelope) + public Task StoreAndForward(Envelope envelope) { return EnqueueOutgoing(envelope); } From c734f99c157d8ee8bc47c163487e42e5e61fcfd5 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 14 May 2020 13:22:02 -0400 Subject: [PATCH 81/82] Add Pulsar to docker-compose --- docker-compose.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 5f9053c2f..41a79a3ba 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,3 +16,13 @@ services: - "ACCEPT_EULA=Y" - "SA_PASSWORD=P@55w0rd" - "MSSQL_PID=Developer" + standalone: + image: apachepulsar/pulsar + ports: + - "6650:6650" + environment: + - PULSAR_MEM=" -Xms512m -Xmx512m -XX:MaxDirectMemorySize=1g" + command: > + /bin/bash -c + "bin/apply-config-from-env.py conf/standalone.conf + && bin/pulsar standalone" \ No newline at end of file From 7c64cf30447c16777e0443599e6902cfd7f645d9 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 14 May 2020 13:24:07 -0400 Subject: [PATCH 82/82] Rename servcie in docker-compose --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 41a79a3ba..7e42ed9c9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,7 +16,7 @@ services: - "ACCEPT_EULA=Y" - "SA_PASSWORD=P@55w0rd" - "MSSQL_PID=Developer" - standalone: + pulsar: image: apachepulsar/pulsar ports: - "6650:6650"