From c305d9ab6a4f1789ae5b8464237fcc813b40aee9 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Wed, 12 Sep 2018 10:05:00 -0700
Subject: [PATCH 001/220] Implement encoding and decoding of properties

This should help implementing a considerable amount of the MQTT 5
protocol.
---
 lib/tortoise/package.ex                   |  15 ++
 lib/tortoise/package/properties.ex        | 257 ++++++++++++++++++++++
 test/test_helper.exs                      |  67 ++++++
 test/tortoise/package/properties_test.exs |  21 ++
 4 files changed, 360 insertions(+)
 create mode 100644 lib/tortoise/package/properties.ex
 create mode 100644 test/tortoise/package/properties_test.exs

diff --git a/lib/tortoise/package.ex b/lib/tortoise/package.ex
index c8a0f455..e07b09b8 100644
--- a/lib/tortoise/package.ex
+++ b/lib/tortoise/package.ex
@@ -28,6 +28,11 @@ defmodule Tortoise.Package do
     [length_prefix, data]
   end
 
+  @doc false
+  def variable_length(n) do
+    remaining_length(n)
+  end
+
   @doc false
   def variable_length_encode(data) when is_list(data) do
     length_prefix = data |> IO.iodata_length() |> remaining_length()
@@ -40,4 +45,14 @@ defmodule Tortoise.Package do
   defp remaining_length(n) do
     [<<1::1, rem(n, @highbit)::7>>] ++ remaining_length(div(n, @highbit))
   end
+
+  @doc false
+  def drop_length_prefix(payload) do
+    case payload do
+      <<0::1, _::7, r::binary>> -> r
+      <<1::1, _::7, 0::1, _::7, r::binary>> -> r
+      <<1::1, _::7, 1::1, _::7, 0::1, _::7, r::binary>> -> r
+      <<1::1, _::7, 1::1, _::7, 1::1, _::7, 0::1, _::7, r::binary>> -> r
+    end
+  end
 end
diff --git a/lib/tortoise/package/properties.ex b/lib/tortoise/package/properties.ex
new file mode 100644
index 00000000..a2f83d54
--- /dev/null
+++ b/lib/tortoise/package/properties.ex
@@ -0,0 +1,257 @@
+defmodule Tortoise.Package.Properties do
+  @moduledoc false
+
+  alias Tortoise.Package
+
+  import Tortoise.Package, only: [variable_length: 1, length_encode: 1]
+
+  def encode(data) when is_list(data) do
+    data
+    |> Enum.map(&encode_property/1)
+    |> Package.variable_length_encode()
+  end
+
+  # User properties are special; we will allow them to be encoded as
+  # binaries to make the interface a bit cleaner to the end user
+  defp encode_property({key, value}) when is_binary(key) do
+    [0x26, length_encode(key), length_encode(value)]
+  end
+
+  defp encode_property({key, value}) do
+    case key do
+      :payload_format_indicator ->
+        [0x01, <<value::8>>]
+
+      :message_expiry_interval ->
+        [0x02, <<value::integer-size(32)>>]
+
+      :content_type ->
+        [0x03, length_encode(value)]
+
+      :response_topic ->
+        [0x08, length_encode(value)]
+
+      :correlation_data ->
+        [0x09, length_encode(value)]
+
+      :subscription_identifier when is_integer(value) ->
+        [0x0B, variable_length(value)]
+
+      :session_expiry_interval ->
+        [0x11, <<value::integer-size(32)>>]
+
+      :assigned_client_identifier ->
+        [0x12, length_encode(value)]
+
+      :server_keep_alive ->
+        [0x13, <<value::integer-size(16)>>]
+
+      :authentication_method ->
+        [0x15, length_encode(value)]
+
+      :authentication_data when is_binary(value) ->
+        [0x16, length_encode(value)]
+
+      :request_problem_information ->
+        [0x17, <<value::8>>]
+
+      :will_delay_interval when is_integer(value) ->
+        [0x18, <<value::integer-size(32)>>]
+
+      :request_response_information ->
+        [0x19, <<value::8>>]
+
+      :response_information ->
+        [0x1A, length_encode(value)]
+
+      :server_reference ->
+        [0x1C, length_encode(value)]
+
+      :reason_string ->
+        [0x1F, length_encode(value)]
+
+      :receive_maximum ->
+        [0x21, <<value::integer-size(16)>>]
+
+      :topic_alias_maximum ->
+        [0x22, <<value::integer-size(16)>>]
+
+      :topic_alias ->
+        [0x23, <<value::integer-size(16)>>]
+
+      :maximum_qos ->
+        [0x24, <<value::8>>]
+
+      :retain_available ->
+        [0x25, <<value::8>>]
+
+      :maximum_packet_size ->
+        [0x27, <<value::integer-size(32)>>]
+
+      :wildcard_subscription_available ->
+        [0x28, <<value::8>>]
+
+      :subscription_identifier_available ->
+        [0x29, <<value::8>>]
+
+      :shared_subscription_available ->
+        [0x2A, <<value::8>>]
+    end
+  end
+
+  # ---
+  def decode(data) do
+    data
+    |> Package.drop_length_prefix()
+    |> do_decode()
+  end
+
+  defp do_decode(data) do
+    data
+    |> decode_property()
+    |> case do
+      {nil, <<>>} -> []
+      {decoded, rest} -> [decoded] ++ do_decode(rest)
+    end
+  end
+
+  defp decode_property(<<>>) do
+    {nil, <<>>}
+  end
+
+  defp decode_property(<<0x01, value::8, rest::binary>>) do
+    {{:payload_format_indicator, value}, rest}
+  end
+
+  defp decode_property(<<0x02, value::integer-size(32), rest::binary>>) do
+    {{:message_expiry_interval, value}, rest}
+  end
+
+  defp decode_property(<<0x03, length::integer-size(16), rest::binary>>) do
+    <<value::binary-size(length), rest::binary>> = rest
+    {{:content_type, value}, rest}
+  end
+
+  defp decode_property(<<0x08, length::integer-size(16), rest::binary>>) do
+    <<value::binary-size(length), rest::binary>> = rest
+    {{:response_topic, value}, rest}
+  end
+
+  defp decode_property(<<0x09, length::integer-size(16), rest::binary>>) do
+    <<value::binary-size(length), rest::binary>> = rest
+    {{:correlation_data, value}, rest}
+  end
+
+  defp decode_property(<<0x0B, rest::binary>>) do
+    case rest do
+      <<0::1, value::integer-size(7), rest::binary>> ->
+        {{:subscription_identifier, value}, rest}
+
+      <<1::1, a::7, 0::1, b::7, rest::binary>> ->
+        <<value::integer-size(14)>> = <<b::7, a::7>>
+        {{:subscription_identifier, value}, rest}
+
+      <<1::1, a::7, 1::1, b::7, 0::1, c::7, rest::binary>> ->
+        <<value::integer-size(21)>> = <<c::7, b::7, a::7>>
+        {{:subscription_identifier, value}, rest}
+
+      <<1::1, a::7, 1::1, b::7, 1::1, c::7, 0::1, d::7, rest::binary>> ->
+        <<value::integer-size(28)>> = <<d::7, c::7, b::7, a::7>>
+        {{:subscription_identifier, value}, rest}
+    end
+  end
+
+  defp decode_property(<<0x11, value::integer-size(32), rest::binary>>) do
+    {{:session_expiry_interval, value}, rest}
+  end
+
+  defp decode_property(<<0x12, length::integer-size(16), rest::binary>>) do
+    <<value::binary-size(length), rest::binary>> = rest
+    {{:assigned_client_identifier, value}, rest}
+  end
+
+  defp decode_property(<<0x13, value::integer-size(16), rest::binary>>) do
+    {{:server_keep_alive, value}, rest}
+  end
+
+  defp decode_property(<<0x15, length::integer-size(16), rest::binary>>) do
+    <<value::binary-size(length), rest::binary>> = rest
+    {{:authentication_method, value}, rest}
+  end
+
+  defp decode_property(<<0x16, length::integer-size(16), rest::binary>>) do
+    <<value::binary-size(length), rest::binary>> = rest
+    {{:authentication_data, value}, rest}
+  end
+
+  defp decode_property(<<0x17, value::8, rest::binary>>) do
+    {{:request_problem_information, value}, rest}
+  end
+
+  defp decode_property(<<0x18, value::integer-size(32), rest::binary>>) do
+    {{:will_delay_interval, value}, rest}
+  end
+
+  defp decode_property(<<0x19, value::8, rest::binary>>) do
+    {{:request_response_information, value}, rest}
+  end
+
+  defp decode_property(<<0x1A, length::integer-size(16), rest::binary>>) do
+    <<value::binary-size(length), rest::binary>> = rest
+    {{:response_information, value}, rest}
+  end
+
+  defp decode_property(<<0x1C, length::integer-size(16), rest::binary>>) do
+    <<value::binary-size(length), rest::binary>> = rest
+    {{:server_reference, value}, rest}
+  end
+
+  defp decode_property(<<0x1F, length::integer-size(16), rest::binary>>) do
+    <<value::binary-size(length), rest::binary>> = rest
+    {{:reason_string, value}, rest}
+  end
+
+  defp decode_property(<<0x21, value::integer-size(16), rest::binary>>) do
+    {{:receive_maximum, value}, rest}
+  end
+
+  defp decode_property(<<0x22, value::integer-size(16), rest::binary>>) do
+    {{:topic_alias_maximum, value}, rest}
+  end
+
+  defp decode_property(<<0x23, value::integer-size(16), rest::binary>>) do
+    {{:topic_alias, value}, rest}
+  end
+
+  defp decode_property(<<0x24, value::8, rest::binary>>) do
+    {{:maximum_qos, value}, rest}
+  end
+
+  defp decode_property(<<0x25, value::8, rest::binary>>) do
+    {{:retain_available, value}, rest}
+  end
+
+  defp decode_property(<<0x26, rest::binary>>) do
+    <<length::integer-size(16), rest::binary>> = rest
+    <<key::binary-size(length), rest::binary>> = rest
+    <<length::integer-size(16), rest::binary>> = rest
+    <<value::binary-size(length), rest::binary>> = rest
+    {{key, value}, rest}
+  end
+
+  defp decode_property(<<0x27, value::integer-size(32), rest::binary>>) do
+    {{:maximum_packet_size, value}, rest}
+  end
+
+  defp decode_property(<<0x28, value::8, rest::binary>>) do
+    {{:wildcard_subscription_available, value}, rest}
+  end
+
+  defp decode_property(<<0x29, value::8, rest::binary>>) do
+    {{:subscription_identifier_available, value}, rest}
+  end
+
+  defp decode_property(<<0x2A, value::8, rest::binary>>) do
+    {{:shared_subscription_available, value}, rest}
+  end
+end
diff --git a/test/test_helper.exs b/test/test_helper.exs
index efd233ba..57c9f02b 100644
--- a/test/test_helper.exs
+++ b/test/test_helper.exs
@@ -253,6 +253,73 @@ defmodule Tortoise.TestGenerators do
       pubrec
     end
   end
+
+  def gen_properties() do
+    let properties <-
+          oneof([
+            :payload_format_indicator,
+            :message_expiry_interval,
+            :content_type,
+            :response_topic,
+            :correlation_data,
+            :subscription_identifier,
+            :session_expiry_interval,
+            :assigned_client_identifier,
+            :server_keep_alive,
+            :authentication_method,
+            :authentication_data,
+            :request_problem_information,
+            :will_delay_interval,
+            :request_response_information,
+            :response_information,
+            :server_reference,
+            :reason_string,
+            :receive_maximum,
+            :topic_alias_maximum,
+            :topic_alias,
+            :maximum_qos,
+            :retain_available,
+            :user_property,
+            :maximum_packet_size,
+            :wildcard_subscription_available,
+            :subscription_identifier_available,
+            :shared_subscription_available
+          ]) do
+      list(10, lazy(do: gen_property_value(properties)))
+    end
+  end
+
+  def gen_property_value(type) do
+    case type do
+      :payload_format_indicator -> {type, oneof([0, 1])}
+      :message_expiry_interval -> {type, choose(0, 4_294_967_295)}
+      :content_type -> {type, utf8()}
+      :response_topic -> {type, gen_topic()}
+      :correlation_data -> {type, binary()}
+      :subscription_identifier -> {type, choose(1, 268_435_455)}
+      :session_expiry_interval -> {type, choose(1, 268_435_455)}
+      :assigned_client_identifier -> {type, utf8()}
+      :server_keep_alive -> {type, choose(0x0000, 0xFFFF)}
+      :authentication_method -> {type, utf8()}
+      :authentication_data -> {type, binary()}
+      :request_problem_information -> {type, oneof([0, 1])}
+      :will_delay_interval -> {type, choose(0, 4_294_967_295)}
+      :request_response_information -> {type, oneof([0, 1])}
+      :response_information -> {type, utf8()}
+      :server_reference -> {type, utf8()}
+      :reason_string -> {type, utf8()}
+      :receive_maximum -> {type, choose(0x0001, 0xFFFF)}
+      :topic_alias_maximum -> {type, choose(0x0000, 0xFFFF)}
+      :topic_alias -> {type, choose(0x0001, 0xFFFF)}
+      :maximum_qos -> {type, oneof([0, 1])}
+      :retain_available -> {type, oneof([0, 1])}
+      :user_property -> {utf8(), utf8()}
+      :maximum_packet_size -> {type, choose(1, 268_435_455)}
+      :wildcard_subscription_available -> {type, oneof([0, 1])}
+      :subscription_identifier_available -> {type, oneof([0, 1])}
+      :shared_subscription_available -> {type, oneof([0, 1])}
+    end
+  end
 end
 
 # make certs for tests using the SSL transport
diff --git a/test/tortoise/package/properties_test.exs b/test/tortoise/package/properties_test.exs
new file mode 100644
index 00000000..f003c1e5
--- /dev/null
+++ b/test/tortoise/package/properties_test.exs
@@ -0,0 +1,21 @@
+defmodule Tortoise.Package.PropertiesTest do
+  use ExUnit.Case
+  use EQC.ExUnit
+  doctest Tortoise.Package.Properties
+
+  alias Tortoise.Package.Properties
+
+  import Tortoise.TestGenerators, only: [gen_properties: 0]
+
+  property "encoding and decoding properties" do
+    forall properties <- gen_properties() do
+      ensure(
+        properties ==
+          properties
+          |> Properties.encode()
+          |> IO.iodata_to_binary()
+          |> Properties.decode()
+      )
+    end
+  end
+end

From 67ffcfc0f8ccd73ff886e5a6f7a157b5d62ad684 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Wed, 12 Sep 2018 15:06:44 -0700
Subject: [PATCH 002/220] Update the disconnect package to MQTT 5

This still need to be wired up.
---
 lib/tortoise/package/disconnect.ex        | 150 +++++++++++++++++++++-
 test/test_helper.exs                      |  48 +++++++
 test/tortoise/package/disconnect_test.exs |  17 ++-
 3 files changed, 204 insertions(+), 11 deletions(-)

diff --git a/lib/tortoise/package/disconnect.ex b/lib/tortoise/package/disconnect.ex
index a9ee1fc8..5bb84139 100644
--- a/lib/tortoise/package/disconnect.ex
+++ b/lib/tortoise/package/disconnect.ex
@@ -5,18 +5,158 @@ defmodule Tortoise.Package.Disconnect do
 
   alias Tortoise.Package
 
+  @type reason ::
+          :normal_disconnection
+          | :disconnect_with_will_message
+          | :unspecified_error
+          | :malformed_packet
+          | :protocol_error
+          | :implementation_specific_error
+          | :not_authorized
+          | :server_busy
+          | :server_shutting_down
+          | :keep_alive_timeout
+          | :session_taken_over
+          | :topic_filter_invalid
+          | :topic_name_invalid
+          | :receive_maximum_exceeded
+          | :topic_alias_invalid
+          | :packet_too_large
+          | :message_rate_too_high
+          | :quota_exceeded
+          | :administrative_action
+          | :payload_format_invalid
+          | :retain_not_supported
+          | :qos_not_supported
+          | :use_another_server
+          | :server_moved
+          | :shared_subscriptions_not_supported
+          | :connection_rate_exceeded
+          | :maximum_connect_time
+          | :subscription_identifiers_not_supported
+          | :wildcard_subscriptions_not_supported
+
   @opaque t :: %__MODULE__{
-            __META__: Package.Meta.t()
+            __META__: Package.Meta.t(),
+            reason: reason(),
+            # todo, let this live in the properties module
+            properties: [{atom(), String.t()}]
           }
-  defstruct __META__: %Package.Meta{opcode: @opcode, flags: 0}
+  defstruct __META__: %Package.Meta{opcode: @opcode, flags: 0},
+            reason: :normal_disconnection,
+            properties: []
+
+  @spec decode(binary()) :: t
+  def decode(<<@opcode::4, 0::4, 0::8>>) do
+    # If the Remaining Length is less than 1 the value of 0x00 (Normal
+    # disconnection) is used; this makes it compatible with the MQTT
+    # 3.1.1 disconnect package as well.
+    %__MODULE__{reason: coerce_reason_code(0x00)}
+  end
 
-  @spec decode(<<_::16>>) :: t
-  def decode(<<@opcode::4, 0::4, 0>>), do: %__MODULE__{}
+  def decode(<<@opcode::4, 0::4, variable_header::binary>>) do
+    <<reason_code::8, properties::binary>> = drop_length_prefix(variable_header)
+
+    %__MODULE__{
+      reason: coerce_reason_code(reason_code),
+      properties: Package.Properties.decode(properties)
+    }
+  end
+
+  defp coerce_reason_code(reason_code) do
+    case reason_code do
+      0x00 -> :normal_disconnection
+      0x04 -> :disconnect_with_will_message
+      0x80 -> :unspecified_error
+      0x81 -> :malformed_packet
+      0x82 -> :protocol_error
+      0x83 -> :implementation_specific_error
+      0x87 -> :not_authorized
+      0x89 -> :server_busy
+      0x8B -> :server_shutting_down
+      0x8D -> :keep_alive_timeout
+      0x8E -> :session_taken_over
+      0x8F -> :topic_filter_invalid
+      0x90 -> :topic_name_invalid
+      0x93 -> :receive_maximum_exceeded
+      0x94 -> :topic_alias_invalid
+      0x95 -> :packet_too_large
+      0x96 -> :message_rate_too_high
+      0x97 -> :quota_exceeded
+      0x98 -> :administrative_action
+      0x99 -> :payload_format_invalid
+      0x9A -> :retain_not_supported
+      0x9B -> :qos_not_supported
+      0x9C -> :use_another_server
+      0x9D -> :server_moved
+      0x9E -> :shared_subscriptions_not_supported
+      0x9F -> :connection_rate_exceeded
+      0xA0 -> :maximum_connect_time
+      0xA1 -> :subscription_identifiers_not_supported
+      0xA2 -> :wildcard_subscriptions_not_supported
+    end
+  end
+
+  defp drop_length_prefix(payload) do
+    case payload do
+      <<0::1, _::7, r::binary>> -> r
+      <<1::1, _::7, 0::1, _::7, r::binary>> -> r
+      <<1::1, _::7, 1::1, _::7, 0::1, _::7, r::binary>> -> r
+      <<1::1, _::7, 1::1, _::7, 1::1, _::7, 0::1, _::7, r::binary>> -> r
+    end
+  end
 
   # Protocols ----------------------------------------------------------
   defimpl Tortoise.Encodable do
-    def encode(%Package.Disconnect{} = t) do
+    def encode(%Package.Disconnect{reason: :normal_disconnection, properties: []} = t) do
+      # if reason is 0x00 and properties are empty we are allowed to
+      # just return a zero; this will also make the disconnect package
+      # compatible with the MQTT 3.1.1 disconnect package.
       [Package.Meta.encode(t.__META__), 0]
     end
+
+    def encode(%Package.Disconnect{} = t) do
+      [
+        Package.Meta.encode(t.__META__),
+        Package.variable_length_encode([
+          <<to_reason_code(t.reason)::8>>,
+          Package.Properties.encode(t.properties)
+        ])
+      ]
+    end
+
+    defp to_reason_code(reason) do
+      case reason do
+        :normal_disconnection -> 0x00
+        :disconnect_with_will_message -> 0x04
+        :unspecified_error -> 0x80
+        :malformed_packet -> 0x81
+        :protocol_error -> 0x82
+        :implementation_specific_error -> 0x83
+        :not_authorized -> 0x87
+        :server_busy -> 0x89
+        :server_shutting_down -> 0x8B
+        :keep_alive_timeout -> 0x8D
+        :session_taken_over -> 0x8E
+        :topic_filter_invalid -> 0x8F
+        :topic_name_invalid -> 0x90
+        :receive_maximum_exceeded -> 0x93
+        :topic_alias_invalid -> 0x94
+        :packet_too_large -> 0x95
+        :message_rate_too_high -> 0x96
+        :quota_exceeded -> 0x97
+        :administrative_action -> 0x98
+        :payload_format_invalid -> 0x99
+        :retain_not_supported -> 0x9A
+        :qos_not_supported -> 0x9B
+        :use_another_server -> 0x9C
+        :server_moved -> 0x9D
+        :shared_subscriptions_not_supported -> 0x9E
+        :connection_rate_exceeded -> 0x9F
+        :maximum_connect_time -> 0xA0
+        :subscription_identifiers_not_supported -> 0xA1
+        :wildcard_subscriptions_not_supported -> 0xA2
+      end
+    end
   end
 end
diff --git a/test/test_helper.exs b/test/test_helper.exs
index 57c9f02b..a35cbd3f 100644
--- a/test/test_helper.exs
+++ b/test/test_helper.exs
@@ -254,6 +254,54 @@ defmodule Tortoise.TestGenerators do
     end
   end
 
+  def gen_disconnect() do
+    let disconnect <-
+          %Package.Disconnect{
+            reason:
+              oneof([
+                :normal_disconnection,
+                :disconnect_with_will_message,
+                :unspecified_error,
+                :malformed_packet,
+                :protocol_error,
+                :implementation_specific_error,
+                :not_authorized,
+                :server_busy,
+                :server_shutting_down,
+                :keep_alive_timeout,
+                :session_taken_over,
+                :topic_filter_invalid,
+                :topic_name_invalid,
+                :receive_maximum_exceeded,
+                :topic_alias_invalid,
+                :packet_too_large,
+                :message_rate_too_high,
+                :quota_exceeded,
+                :administrative_action,
+                :payload_format_invalid,
+                :retain_not_supported,
+                :qos_not_supported,
+                :use_another_server,
+                :server_moved,
+                :shared_subscriptions_not_supported,
+                :connection_rate_exceeded,
+                :maximum_connect_time,
+                :subscription_identifiers_not_supported,
+                :wildcard_subscriptions_not_supported
+              ])
+          } do
+      %Package.Disconnect{disconnect | properties: gen_properties(disconnect)}
+    end
+  end
+
+  def gen_properties(%Package.Disconnect{reason: :normal_disconnection}) do
+    []
+  end
+
+  def gen_properties(%{}) do
+    []
+  end
+
   def gen_properties() do
     let properties <-
           oneof([
diff --git a/test/tortoise/package/disconnect_test.exs b/test/tortoise/package/disconnect_test.exs
index 9e6e34b0..620a19f4 100644
--- a/test/tortoise/package/disconnect_test.exs
+++ b/test/tortoise/package/disconnect_test.exs
@@ -1,15 +1,20 @@
 defmodule Tortoise.Package.DisconnectTest do
   use ExUnit.Case
+  use EQC.ExUnit
   doctest Tortoise.Package.Disconnect
 
   alias Tortoise.Package
 
-  test "encoding and decoding disconnect messages" do
-    disconnect = %Package.Disconnect{}
+  import Tortoise.TestGenerators, only: [gen_disconnect: 0]
 
-    assert ^disconnect =
-             disconnect
-             |> Package.encode()
-             |> Package.decode()
+  property "encoding and decoding disconnect messages" do
+    forall disconnect <- gen_disconnect() do
+      ensure(
+        disconnect ==
+          disconnect
+          |> Package.encode()
+          |> Package.decode()
+      )
+    end
   end
 end

From a0e31b6a93cfda4bcd06fdcb351bbe178dbf58a6 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Thu, 13 Sep 2018 10:36:49 -0700
Subject: [PATCH 003/220] Add the auth package

---
 lib/tortoise/decodable.ex           |  4 +-
 lib/tortoise/package.ex             |  1 +
 lib/tortoise/package/auth.ex        | 68 +++++++++++++++++++++++++++++
 test/test_helper.exs                |  9 ++++
 test/tortoise/package/auth_test.exs | 20 +++++++++
 5 files changed, 101 insertions(+), 1 deletion(-)
 create mode 100644 lib/tortoise/package/auth.ex
 create mode 100644 test/tortoise/package/auth_test.exs

diff --git a/lib/tortoise/decodable.ex b/lib/tortoise/decodable.ex
index aa99a3e9..f829c997 100644
--- a/lib/tortoise/decodable.ex
+++ b/lib/tortoise/decodable.ex
@@ -19,7 +19,8 @@ defimpl Tortoise.Decodable, for: BitString do
     Unsuback,
     Pingreq,
     Pingresp,
-    Disconnect
+    Disconnect,
+    Auth
   }
 
   def decode(<<1::4, _::4, _::binary>> = data), do: Connect.decode(data)
@@ -36,6 +37,7 @@ defimpl Tortoise.Decodable, for: BitString do
   def decode(<<12::4, _::4, _::binary>> = data), do: Pingreq.decode(data)
   def decode(<<13::4, _::4, _::binary>> = data), do: Pingresp.decode(data)
   def decode(<<14::4, _::4, _::binary>> = data), do: Disconnect.decode(data)
+  def decode(<<15::4, _::4, _::binary>> = data), do: Auth.decode(data)
 end
 
 defimpl Tortoise.Decodable, for: List do
diff --git a/lib/tortoise/package.ex b/lib/tortoise/package.ex
index e07b09b8..756ce0b6 100644
--- a/lib/tortoise/package.ex
+++ b/lib/tortoise/package.ex
@@ -18,6 +18,7 @@ defmodule Tortoise.Package do
             | Package.Pingreq.t()
             | Package.Pingresp.t()
             | Package.Disconnect.t()
+            | Package.Auth.t()
 
   defdelegate encode(data), to: Tortoise.Encodable
   defdelegate decode(data), to: Tortoise.Decodable
diff --git a/lib/tortoise/package/auth.ex b/lib/tortoise/package/auth.ex
new file mode 100644
index 00000000..32246871
--- /dev/null
+++ b/lib/tortoise/package/auth.ex
@@ -0,0 +1,68 @@
+defmodule Tortoise.Package.Auth do
+  @moduledoc false
+
+  @opcode 15
+
+  # @allowed_properties [:authentication_method, :authentication_data, :reason_string, :user_property]
+
+  alias Tortoise.Package
+
+  @type reason :: :success | :continue_authentication | :re_authenticate
+
+  @opaque t :: %__MODULE__{
+            __META__: Package.Meta.t(),
+            reason: reason(),
+            properties: [{any(), any()}]
+          }
+  @enforce_keys [:reason]
+  defstruct __META__: %Package.Meta{opcode: @opcode, flags: 0},
+            reason: nil,
+            properties: []
+
+  @spec decode(binary()) :: t
+  def decode(<<@opcode::4, 0::4, 0>>) do
+    %__MODULE__{reason: coerce_reason_code(0x00)}
+  end
+
+  def decode(<<@opcode::4, 0::4, variable_header::binary>>) do
+    <<reason_code::8, properties::binary>> = Package.drop_length_prefix(variable_header)
+
+    %__MODULE__{
+      reason: coerce_reason_code(reason_code),
+      properties: Package.Properties.decode(properties)
+    }
+  end
+
+  defp coerce_reason_code(reason_code) do
+    case reason_code do
+      0x00 -> :success
+      0x18 -> :continue_authentication
+      0x19 -> :re_authenticate
+    end
+  end
+
+  defimpl Tortoise.Encodable do
+    def encode(%Package.Auth{reason: :success, properties: []} = t) do
+      [Package.Meta.encode(t.__META__), 0]
+    end
+
+    def encode(%Package.Auth{reason: reason} = t)
+        when reason in [:success, :continue_authentication, :re_authenticate] do
+      [
+        Package.Meta.encode(t.__META__),
+        Package.variable_length_encode([
+          <<to_reason_code(t.reason)::8>>,
+          Package.Properties.encode(t.properties)
+        ])
+      ]
+    end
+
+    defp to_reason_code(reason) do
+      case reason do
+        :success -> 0x00
+        :continue_authentication -> 0x18
+        :re_authenticate -> 0x19
+      end
+    end
+  end
+end
diff --git a/test/test_helper.exs b/test/test_helper.exs
index a35cbd3f..6e14fe04 100644
--- a/test/test_helper.exs
+++ b/test/test_helper.exs
@@ -294,6 +294,15 @@ defmodule Tortoise.TestGenerators do
     end
   end
 
+  def gen_auth() do
+    let auth <-
+          %Package.Auth{
+            reason: oneof([:success, :continue_authentication, :re_authenticate])
+          } do
+      %Package.Auth{auth | properties: gen_properties(auth)}
+    end
+  end
+
   def gen_properties(%Package.Disconnect{reason: :normal_disconnection}) do
     []
   end
diff --git a/test/tortoise/package/auth_test.exs b/test/tortoise/package/auth_test.exs
new file mode 100644
index 00000000..907cf487
--- /dev/null
+++ b/test/tortoise/package/auth_test.exs
@@ -0,0 +1,20 @@
+defmodule Tortoise.Package.AuthTest do
+  use ExUnit.Case
+  use EQC.ExUnit
+  doctest Tortoise.Package.Auth
+
+  alias Tortoise.Package
+
+  import Tortoise.TestGenerators, only: [gen_auth: 0]
+
+  property "encoding and decoding auth messages" do
+    forall auth <- gen_auth() do
+      ensure(
+        auth ==
+          auth
+          |> Package.encode()
+          |> Package.decode()
+      )
+    end
+  end
+end

From 82d037633743711bc7002c31b26900ef4a181b65 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Thu, 13 Sep 2018 10:42:24 -0700
Subject: [PATCH 004/220] Added a note about the allowed properties for each of
 the packages

---
 lib/tortoise/package/connack.ex     | 2 ++
 lib/tortoise/package/connect.ex     | 2 ++
 lib/tortoise/package/disconnect.ex  | 2 ++
 lib/tortoise/package/puback.ex      | 2 ++
 lib/tortoise/package/pubcomp.ex     | 2 ++
 lib/tortoise/package/pubrec.ex      | 2 ++
 lib/tortoise/package/pubrel.ex      | 2 ++
 lib/tortoise/package/suback.ex      | 2 ++
 lib/tortoise/package/subscribe.ex   | 2 ++
 lib/tortoise/package/unsuback.ex    | 2 ++
 lib/tortoise/package/unsubscribe.ex | 2 ++
 11 files changed, 22 insertions(+)

diff --git a/lib/tortoise/package/connack.ex b/lib/tortoise/package/connack.ex
index 5fbd8f95..d8180986 100644
--- a/lib/tortoise/package/connack.ex
+++ b/lib/tortoise/package/connack.ex
@@ -3,6 +3,8 @@ defmodule Tortoise.Package.Connack do
 
   @opcode 2
 
+  # @allowed_properties [:assigned_client_identifier, :authentication_data, :authentication_method, :maximum_packet_size, :maximum_qos, :reason_string, :receive_maximum, :response_information, :retain_available, :server_keep_alive, :server_reference, :session_expiry_interval, :shared_subscription_available, :subscription_identifier_available, :topic_alias_maximum, :user_property, :wildcard_subscription_available]
+
   alias Tortoise.Package
 
   @type status :: :accepted | {:refused, refusal_reasons()}
diff --git a/lib/tortoise/package/connect.ex b/lib/tortoise/package/connect.ex
index 561ee596..7dd1e7b7 100644
--- a/lib/tortoise/package/connect.ex
+++ b/lib/tortoise/package/connect.ex
@@ -3,6 +3,8 @@ defmodule Tortoise.Package.Connect do
 
   @opcode 1
 
+  # @allowed_properties [:authentication_data, :authentication_method, :maximum_packet_size, :receive_maximum, :request_problem_information, :request_response_information, :session_expiry_interval, :topic_alias_maximum, :user_property]
+
   alias Tortoise.Package
 
   @opaque t :: %__MODULE__{
diff --git a/lib/tortoise/package/disconnect.ex b/lib/tortoise/package/disconnect.ex
index 5bb84139..5520b117 100644
--- a/lib/tortoise/package/disconnect.ex
+++ b/lib/tortoise/package/disconnect.ex
@@ -3,6 +3,8 @@ defmodule Tortoise.Package.Disconnect do
 
   @opcode 14
 
+  # @allowed_properties [:reason_string, :server_reference, :session_expiry_interval, :user_property]
+
   alias Tortoise.Package
 
   @type reason ::
diff --git a/lib/tortoise/package/puback.ex b/lib/tortoise/package/puback.ex
index 77a11f7e..5a77530b 100644
--- a/lib/tortoise/package/puback.ex
+++ b/lib/tortoise/package/puback.ex
@@ -3,6 +3,8 @@ defmodule Tortoise.Package.Puback do
 
   @opcode 4
 
+  # @allowed_properties [:reason_string, :user_property]
+
   alias Tortoise.Package
 
   @opaque t :: %__MODULE__{
diff --git a/lib/tortoise/package/pubcomp.ex b/lib/tortoise/package/pubcomp.ex
index 6cc8ea7a..30ae3b7d 100644
--- a/lib/tortoise/package/pubcomp.ex
+++ b/lib/tortoise/package/pubcomp.ex
@@ -3,6 +3,8 @@ defmodule Tortoise.Package.Pubcomp do
 
   @opcode 7
 
+  @allowed_properties [:reason_string, :user_property]
+
   alias Tortoise.Package
 
   @opaque t :: %__MODULE__{
diff --git a/lib/tortoise/package/pubrec.ex b/lib/tortoise/package/pubrec.ex
index 5a011630..f1adb77f 100644
--- a/lib/tortoise/package/pubrec.ex
+++ b/lib/tortoise/package/pubrec.ex
@@ -3,6 +3,8 @@ defmodule Tortoise.Package.Pubrec do
 
   @opcode 5
 
+  # @allowed_properties [:reason_string, :user_property]
+
   alias Tortoise.Package
 
   @opaque t :: %__MODULE__{
diff --git a/lib/tortoise/package/pubrel.ex b/lib/tortoise/package/pubrel.ex
index 72a05fcc..71751beb 100644
--- a/lib/tortoise/package/pubrel.ex
+++ b/lib/tortoise/package/pubrel.ex
@@ -3,6 +3,8 @@ defmodule Tortoise.Package.Pubrel do
 
   @opcode 6
 
+  # @allowed_properties [:reason_string, :user_property]
+
   alias Tortoise.Package
 
   @opaque t :: %__MODULE__{
diff --git a/lib/tortoise/package/suback.ex b/lib/tortoise/package/suback.ex
index b70757a2..d797574c 100644
--- a/lib/tortoise/package/suback.ex
+++ b/lib/tortoise/package/suback.ex
@@ -3,6 +3,8 @@ defmodule Tortoise.Package.Suback do
 
   @opcode 9
 
+  # @allowed_properties [:reason_string, :user_property]
+
   alias Tortoise.Package
 
   @type qos :: 0 | 1 | 2
diff --git a/lib/tortoise/package/subscribe.ex b/lib/tortoise/package/subscribe.ex
index ab326469..5fee1025 100644
--- a/lib/tortoise/package/subscribe.ex
+++ b/lib/tortoise/package/subscribe.ex
@@ -3,6 +3,8 @@ defmodule Tortoise.Package.Subscribe do
 
   @opcode 8
 
+  # @allowed_properties [:subscription_identifier, :user_property]
+
   alias Tortoise.Package
 
   @type qos :: 0 | 1 | 2
diff --git a/lib/tortoise/package/unsuback.ex b/lib/tortoise/package/unsuback.ex
index 2d1b2912..200386f2 100644
--- a/lib/tortoise/package/unsuback.ex
+++ b/lib/tortoise/package/unsuback.ex
@@ -3,6 +3,8 @@ defmodule Tortoise.Package.Unsuback do
 
   @opcode 11
 
+  # @allowed_properties [:reason_string, :user_property]
+
   alias Tortoise.Package
 
   @opaque t :: %__MODULE__{
diff --git a/lib/tortoise/package/unsubscribe.ex b/lib/tortoise/package/unsubscribe.ex
index 58c408ef..57d8b6fc 100644
--- a/lib/tortoise/package/unsubscribe.ex
+++ b/lib/tortoise/package/unsubscribe.ex
@@ -3,6 +3,8 @@ defmodule Tortoise.Package.Unsubscribe do
 
   @opcode 10
 
+  # @allowed_properties [:user_property]
+
   alias Tortoise.Package
 
   @type topic :: binary()

From 784b692198de7bc8241f30ddad1ba3caf862fe3d Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Thu, 13 Sep 2018 16:07:46 -0700
Subject: [PATCH 005/220] Encode and decode properties in publish packages

---
 lib/tortoise/package.ex         | 27 +++++++++++++
 lib/tortoise/package/publish.ex | 32 ++++++++++------
 test/test_helper.exs            | 68 ++++++++++++++++++---------------
 3 files changed, 85 insertions(+), 42 deletions(-)

diff --git a/lib/tortoise/package.ex b/lib/tortoise/package.ex
index 756ce0b6..bbded519 100644
--- a/lib/tortoise/package.ex
+++ b/lib/tortoise/package.ex
@@ -56,4 +56,31 @@ defmodule Tortoise.Package do
       <<1::1, _::7, 1::1, _::7, 1::1, _::7, 0::1, _::7, r::binary>> -> r
     end
   end
+
+  def parse_variable_length(data) do
+    case data do
+      <<0::1, length::integer-size(7), _::binary>> ->
+        length = length + 1
+        <<properties::binary-size(length), rest::binary>> = data
+        {properties, rest}
+
+      <<1::1, a::7, 0::1, b::7, _::binary>> ->
+        <<length::integer-size(14)>> = <<b::7, a::7>>
+        length = length + 2
+        <<properties::binary-size(length), rest::binary>> = data
+        {properties, rest}
+
+      <<1::1, a::7, 1::1, b::7, 0::1, c::7, _::binary>> ->
+        <<length::integer-size(21)>> = <<c::7, b::7, a::7>>
+        length = length + 3
+        <<properties::binary-size(length), rest::binary>> = data
+        {properties, rest}
+
+      <<1::1, a::7, 1::1, b::7, 1::1, c::7, 0::1, d::7, _::binary>> ->
+        <<length::integer-size(28)>> = <<d::7, c::7, b::7, a::7>>
+        length = length + 4
+        <<properties::binary-size(length), rest::binary>> = data
+        {properties, rest}
+    end
+  end
 end
diff --git a/lib/tortoise/package/publish.ex b/lib/tortoise/package/publish.ex
index c31f4928..613af271 100644
--- a/lib/tortoise/package/publish.ex
+++ b/lib/tortoise/package/publish.ex
@@ -12,7 +12,8 @@ defmodule Tortoise.Package.Publish do
           payload: Tortoise.payload(),
           identifier: Tortoise.package_identifier(),
           dup: boolean(),
-          retain: boolean()
+          retain: boolean(),
+          properties: [{any(), any()}]
         }
   defstruct __META__: %Package.Meta{opcode: @opcode, flags: 0},
             identifier: nil,
@@ -20,12 +21,13 @@ defmodule Tortoise.Package.Publish do
             payload: nil,
             qos: 0,
             dup: false,
-            retain: false
+            retain: false,
+            properties: []
 
   @spec decode(binary()) :: t
   def decode(<<@opcode::4, 0::1, 0::2, retain::1, length_prefixed_payload::binary>>) do
     payload = drop_length_prefix(length_prefixed_payload)
-    {topic, payload} = decode_message(payload)
+    {topic, properties, payload} = decode_message(payload)
 
     %__MODULE__{
       qos: 0,
@@ -33,7 +35,8 @@ defmodule Tortoise.Package.Publish do
       dup: false,
       retain: retain == 1,
       topic: topic,
-      payload: payload
+      payload: payload,
+      properties: properties
     }
   end
 
@@ -41,7 +44,7 @@ defmodule Tortoise.Package.Publish do
         <<@opcode::4, dup::1, qos::integer-size(2), retain::1, length_prefixed_payload::binary>>
       ) do
     payload = drop_length_prefix(length_prefixed_payload)
-    {topic, identifier, payload} = decode_message_with_id(payload)
+    {topic, identifier, properties, payload} = decode_message_with_id(payload)
 
     %__MODULE__{
       qos: qos,
@@ -49,7 +52,8 @@ defmodule Tortoise.Package.Publish do
       dup: dup == 1,
       retain: retain == 1,
       topic: topic,
-      payload: payload
+      payload: payload,
+      properties: properties
     }
   end
 
@@ -62,14 +66,16 @@ defmodule Tortoise.Package.Publish do
     end
   end
 
-  defp decode_message(<<topic_length::big-integer-size(16), msg::binary>>) do
-    <<topic::binary-size(topic_length), payload::binary>> = msg
-    {topic, nullify(payload)}
+  defp decode_message(<<topic_length::big-integer-size(16), package::binary>>) do
+    <<topic::binary-size(topic_length), rest::binary>> = package
+    {properties, payload} = Package.parse_variable_length(rest)
+    {topic, Package.Properties.decode(properties), nullify(payload)}
   end
 
-  defp decode_message_with_id(<<topic_length::big-integer-size(16), msg::binary>>) do
-    <<topic::binary-size(topic_length), identifier::big-integer-size(16), payload::binary>> = msg
-    {topic, identifier, nullify(payload)}
+  defp decode_message_with_id(<<topic_length::big-integer-size(16), package::binary>>) do
+    <<topic::binary-size(topic_length), identifier::big-integer-size(16), rest::binary>> = package
+    {properties, payload} = Package.parse_variable_length(rest)
+    {topic, identifier, Package.Properties.decode(properties), nullify(payload)}
   end
 
   defp nullify(""), do: nil
@@ -82,6 +88,7 @@ defmodule Tortoise.Package.Publish do
         Package.Meta.encode(%{t.__META__ | flags: encode_flags(t)}),
         Package.variable_length_encode([
           Package.length_encode(t.topic),
+          Package.Properties.encode(t.properties),
           encode_payload(t)
         ])
       ]
@@ -94,6 +101,7 @@ defmodule Tortoise.Package.Publish do
         Package.variable_length_encode([
           Package.length_encode(t.topic),
           <<identifier::big-integer-size(16)>>,
+          Package.Properties.encode(t.properties),
           encode_payload(t)
         ])
       ]
diff --git a/test/test_helper.exs b/test/test_helper.exs
index 6e14fe04..a80c3d69 100644
--- a/test/test_helper.exs
+++ b/test/test_helper.exs
@@ -163,6 +163,7 @@ defmodule Tortoise.TestGenerators do
           payload: oneof([non_empty(binary()), nil]),
           retain: bool()
       }
+      |> gen_properties()
     end
   end
 
@@ -303,6 +304,10 @@ defmodule Tortoise.TestGenerators do
     end
   end
 
+  def gen_properties(%Package.Publish{} = publish) do
+    %Package.Publish{publish | properties: gen_properties()}
+  end
+
   def gen_properties(%Package.Disconnect{reason: :normal_disconnection}) do
     []
   end
@@ -313,36 +318,39 @@ defmodule Tortoise.TestGenerators do
 
   def gen_properties() do
     let properties <-
-          oneof([
-            :payload_format_indicator,
-            :message_expiry_interval,
-            :content_type,
-            :response_topic,
-            :correlation_data,
-            :subscription_identifier,
-            :session_expiry_interval,
-            :assigned_client_identifier,
-            :server_keep_alive,
-            :authentication_method,
-            :authentication_data,
-            :request_problem_information,
-            :will_delay_interval,
-            :request_response_information,
-            :response_information,
-            :server_reference,
-            :reason_string,
-            :receive_maximum,
-            :topic_alias_maximum,
-            :topic_alias,
-            :maximum_qos,
-            :retain_available,
-            :user_property,
-            :maximum_packet_size,
-            :wildcard_subscription_available,
-            :subscription_identifier_available,
-            :shared_subscription_available
-          ]) do
-      list(10, lazy(do: gen_property_value(properties)))
+          list(
+            5,
+            oneof([
+              :payload_format_indicator,
+              :message_expiry_interval,
+              :content_type,
+              :response_topic,
+              :correlation_data,
+              :subscription_identifier,
+              :session_expiry_interval,
+              :assigned_client_identifier,
+              :server_keep_alive,
+              :authentication_method,
+              :authentication_data,
+              :request_problem_information,
+              :will_delay_interval,
+              :request_response_information,
+              :response_information,
+              :server_reference,
+              :reason_string,
+              :receive_maximum,
+              :topic_alias_maximum,
+              :topic_alias,
+              :maximum_qos,
+              :retain_available,
+              :user_property,
+              :maximum_packet_size,
+              :wildcard_subscription_available,
+              :subscription_identifier_available,
+              :shared_subscription_available
+            ])
+          ) do
+      Enum.map(properties, &gen_property_value/1)
     end
   end
 

From 16511dd3d19f05a49929231f0029e2f15989508f Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Thu, 13 Sep 2018 18:01:43 -0700
Subject: [PATCH 006/220] Prepare making non allowed properties protocol
 violations in publish

---
 lib/tortoise/package/publish.ex | 33 +++++++++++++++++++++++++++++++--
 test/test_helper.exs            | 25 ++++++++++++++++++++-----
 2 files changed, 51 insertions(+), 7 deletions(-)

diff --git a/lib/tortoise/package/publish.ex b/lib/tortoise/package/publish.ex
index 613af271..330f992b 100644
--- a/lib/tortoise/package/publish.ex
+++ b/lib/tortoise/package/publish.ex
@@ -3,6 +3,17 @@ defmodule Tortoise.Package.Publish do
 
   @opcode 3
 
+  @allowed_properties [
+    :payload_format_indicator,
+    :message_expiry_interval,
+    :topic_alias,
+    :response_topic,
+    :correlation_data,
+    :user_property,
+    :subscription_identifier,
+    :content_type
+  ]
+
   alias Tortoise.Package
 
   @type t :: %__MODULE__{
@@ -69,13 +80,31 @@ defmodule Tortoise.Package.Publish do
   defp decode_message(<<topic_length::big-integer-size(16), package::binary>>) do
     <<topic::binary-size(topic_length), rest::binary>> = package
     {properties, payload} = Package.parse_variable_length(rest)
-    {topic, Package.Properties.decode(properties), nullify(payload)}
+    properties = Package.Properties.decode(properties)
+
+    case Keyword.split(properties, @allowed_properties) do
+      {^properties, []} ->
+        {topic, properties, nullify(payload)}
+
+      {_, _violations} ->
+        # todo !
+        {topic, properties, nullify(payload)}
+    end
   end
 
   defp decode_message_with_id(<<topic_length::big-integer-size(16), package::binary>>) do
     <<topic::binary-size(topic_length), identifier::big-integer-size(16), rest::binary>> = package
     {properties, payload} = Package.parse_variable_length(rest)
-    {topic, identifier, Package.Properties.decode(properties), nullify(payload)}
+    properties = Package.Properties.decode(properties)
+
+    case Keyword.split(properties, @allowed_properties) do
+      {^properties, []} ->
+        {topic, identifier, properties, nullify(payload)}
+
+      {_, _violations} ->
+        # todo !
+        {topic, identifier, properties, nullify(payload)}
+    end
   end
 
   defp nullify(""), do: nil
diff --git a/test/test_helper.exs b/test/test_helper.exs
index a80c3d69..952d280e 100644
--- a/test/test_helper.exs
+++ b/test/test_helper.exs
@@ -163,7 +163,7 @@ defmodule Tortoise.TestGenerators do
           payload: oneof([non_empty(binary()), nil]),
           retain: bool()
       }
-      |> gen_properties()
+      |> gen_publish_properties()
     end
   end
 
@@ -179,6 +179,25 @@ defmodule Tortoise.TestGenerators do
     }
   end
 
+  defp gen_publish_properties(%Package.Publish{} = publish) do
+    allowed_properties = [
+      :payload_format_indicator,
+      :message_expiry_interval,
+      :topic_alias,
+      :response_topic,
+      :correlation_data,
+      :user_property,
+      :subscription_identifier,
+      :content_type
+    ]
+
+    let properties <- list(5, oneof(allowed_properties)) do
+      # @todo only user_properties and subscription_identifiers are allowed multiple times
+      properties = Enum.map(properties, &gen_property_value/1)
+      %Package.Publish{publish | properties: properties}
+    end
+  end
+
   @doc """
   Generate a valid subscribe message.
 
@@ -304,10 +323,6 @@ defmodule Tortoise.TestGenerators do
     end
   end
 
-  def gen_properties(%Package.Publish{} = publish) do
-    %Package.Publish{publish | properties: gen_properties()}
-  end
-
   def gen_properties(%Package.Disconnect{reason: :normal_disconnection}) do
     []
   end

From edb635be1c352e9613d7d8da5db59d9a0ccb5c71 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Wed, 3 Oct 2018 21:21:14 +0200
Subject: [PATCH 007/220] WIP restructure of the entire application to support
 MQTT 5

The controller has been merged with the connection process as we now
need to support auth packages; thus we cannot just receive the fixed
number of bytes when we await the connack message. The receiver will
have to be booted earlier in the process, so that has been rewritten.

The connection process is now a gen_statem; and it will handle the
user specified callback module in the future.

This is far from done. Some of the protocol logic has been
implemented, but it is far from a MQTT client right now.
---
 lib/tortoise/connection.ex                   | 595 ++++++++++++-------
 lib/tortoise/connection/inflight.ex          |  20 +-
 lib/tortoise/connection/receiver.ex          |  85 ++-
 lib/tortoise/connection/supervisor.ex        |   7 +-
 lib/tortoise/transport.ex                    |   5 +
 test/support/scripted_mqtt_server.exs        |   5 +
 test/support/scripted_transport.exs          |  29 +-
 test/support/test_tcp_tunnel.exs             |   7 +-
 test/test_helper.exs                         | 119 +++-
 test/tortoise/connection/controller_test.exs |  69 ++-
 test/tortoise/connection/inflight_test.exs   |   8 +-
 test/tortoise/connection/receiver_test.exs   |  42 +-
 test/tortoise/connection_test.exs            | 293 +++++++--
 13 files changed, 917 insertions(+), 367 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 6ae6d808..ab23b7e4 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -5,16 +5,29 @@ defmodule Tortoise.Connection do
   Todo.
   """
 
-  use GenServer
+  use GenStateMachine
 
   require Logger
 
-  defstruct [:client_id, :connect, :server, :status, :backoff, :subscriptions, :keep_alive, :opts]
+  defstruct [
+    :client_id,
+    :connect,
+    :server,
+    :backoff,
+    :subscriptions,
+    :keep_alive,
+    :opts,
+    :pending_refs,
+    :connection,
+    :ping,
+    :handler
+  ]
+
   alias __MODULE__, as: State
 
-  alias Tortoise.{Transport, Connection, Package, Events}
-  alias Tortoise.Connection.{Inflight, Controller, Receiver, Backoff}
-  alias Tortoise.Package.{Connect, Connack}
+  alias Tortoise.{Handler, Transport, Package, Events}
+  alias Tortoise.Connection.{Inflight, Backoff}
+  alias Tortoise.Package.Connect
 
   @doc """
   Start a connection process and link it to the current process.
@@ -44,7 +57,7 @@ defmodule Tortoise.Connection do
       keep_alive: Keyword.get(connection_opts, :keep_alive, 60),
       will: Keyword.get(connection_opts, :will),
       # if we re-spawn from here it means our state is gone
-      clean_session: true
+      clean_start: true
     }
 
     backoff = Keyword.get(connection_opts, :backoff, [])
@@ -62,10 +75,18 @@ defmodule Tortoise.Connection do
       end
 
     # @todo, validate that the handler is valid
-    connection_opts = Keyword.take(connection_opts, [:client_id, :handler])
-    initial = {server, connect, backoff, subscriptions, connection_opts}
+    handler =
+      opts
+      |> Keyword.get(:handler, %Handler{module: Handler.Default, initial_args: []})
+      |> Handler.new()
+
+    connection_opts = [
+      {:transport, server} | Keyword.take(connection_opts, [:client_id])
+    ]
+
+    initial = {server, connect, backoff, subscriptions, handler, connection_opts}
     opts = Keyword.merge(opts, name: via_name(client_id))
-    GenServer.start_link(__MODULE__, initial, opts)
+    GenStateMachine.start_link(__MODULE__, initial, opts)
   end
 
   @doc false
@@ -99,7 +120,7 @@ defmodule Tortoise.Connection do
   """
   @spec disconnect(Tortoise.client_id()) :: :ok
   def disconnect(client_id) do
-    GenServer.call(via_name(client_id), :disconnect)
+    GenStateMachine.call(via_name(client_id), :disconnect)
   end
 
   @doc """
@@ -110,7 +131,7 @@ defmodule Tortoise.Connection do
   """
   @spec subscriptions(Tortoise.client_id()) :: Tortoise.Package.Subscribe.t()
   def subscriptions(client_id) do
-    GenServer.call(via_name(client_id), :subscriptions)
+    GenStateMachine.call(via_name(client_id), :subscriptions)
   end
 
   @doc """
@@ -146,7 +167,7 @@ defmodule Tortoise.Connection do
     caller = {_, ref} = {self(), make_ref()}
     {identifier, opts} = Keyword.pop_first(opts, :identifier, nil)
     subscribe = Enum.into(topics, %Package.Subscribe{identifier: identifier})
-    GenServer.cast(via_name(client_id), {:subscribe, caller, subscribe, opts})
+    GenStateMachine.cast(via_name(client_id), {:subscribe, caller, subscribe, opts})
     {:ok, ref}
   end
 
@@ -233,7 +254,7 @@ defmodule Tortoise.Connection do
     caller = {_, ref} = {self(), make_ref()}
     {identifier, opts} = Keyword.pop_first(opts, :identifier, nil)
     unsubscribe = %Package.Unsubscribe{identifier: identifier, topics: topics}
-    GenServer.cast(via_name(client_id), {:unsubscribe, caller, unsubscribe, opts})
+    GenStateMachine.cast(via_name(client_id), {:unsubscribe, caller, unsubscribe, opts})
     {:ok, ref}
   end
 
@@ -290,7 +311,11 @@ defmodule Tortoise.Connection do
   PubSub.
   """
   @spec ping(Tortoise.client_id()) :: {:ok, reference()}
-  defdelegate ping(client_id), to: Tortoise.Connection.Controller
+  def ping(client_id) do
+    ref = make_ref()
+    :ok = GenStateMachine.cast(via_name(client_id), {:ping, {self(), ref}})
+    {:ok, ref}
+  end
 
   @doc """
   Ping the server and await the ping latency reply.
@@ -346,285 +371,411 @@ defmodule Tortoise.Connection do
     unless active?, do: Events.unregister(client_id, :connection)
   end
 
+  @doc false
+  @spec subscribe_all(Tortoise.client_id()) :: :ok
+  def subscribe_all(client_id) do
+    GenStateMachine.cast(via_name(client_id), :subscribe_all)
+  end
+
   # Callbacks
   @impl true
-  def init(
-        {transport, %Connect{client_id: client_id} = connect, backoff_opts, subscriptions, opts}
-      ) do
-    state = %State{
-      client_id: client_id,
+  def init({transport, connect, backoff_opts, subscriptions, handler, opts}) do
+    data = %State{
+      client_id: connect.client_id,
       server: transport,
       connect: connect,
       backoff: Backoff.new(backoff_opts),
       subscriptions: subscriptions,
       opts: opts,
-      status: :down
+      pending_refs: %{},
+      ping: :queue.new(),
+      handler: handler
     }
 
-    Tortoise.Registry.put_meta(via_name(client_id), :connecting)
-    Tortoise.Events.register(client_id, :status)
+    case Handler.execute(handler, :init) do
+      {:ok, %Handler{} = updated_handler} ->
+        {:ok, _pid} = Tortoise.Events.register(data.client_id, :status)
+
+        next_events = [{:next_event, :internal, :connect}]
+        updated_data = %State{data | handler: updated_handler}
+        {:ok, :connecting, updated_data, next_events}
 
-    # eventually, switch to handle_continue
-    send(self(), :connect)
-    {:ok, state}
+        # todo, handle ignore from handler:init/1
+    end
   end
 
   @impl true
-  def terminate(_reason, state) do
-    :ok = Tortoise.Registry.delete_meta(via_name(state.connect.client_id))
-    :ok = Events.dispatch(state.client_id, :status, :terminated)
+  def terminate(reason, _state, %State{handler: handler}) do
+    _ignored = Handler.execute(handler, {:terminate, reason})
     :ok
   end
 
   @impl true
-  def handle_info(:connect, state) do
-    # make sure we will not fall for a keep alive timeout while we reconnect
-    state = cancel_keep_alive(state)
-
-    with {%Connack{status: :accepted} = connack, socket} <-
-           do_connect(state.server, state.connect),
-         {:ok, state} = init_connection(socket, state) do
-      # we are connected; reset backoff state, etc
-      state =
-        %State{state | backoff: Backoff.reset(state.backoff)}
-        |> update_connection_status(:up)
-        |> reset_keep_alive()
-
-      case connack do
-        %Connack{session_present: true} ->
-          {:noreply, state}
-
-        %Connack{session_present: false} ->
-          :ok = Inflight.reset(state.client_id)
-          unless Enum.empty?(state.subscriptions), do: send(self(), :subscribe)
-          {:noreply, state}
-      end
-    else
-      %Connack{status: {:refused, reason}} ->
-        {:stop, {:connection_failed, reason}, state}
+  def handle_event(:info, {:incoming, package}, _, _data) when is_binary(package) do
+    next_actions = [{:next_event, :internal, {:received, Package.decode(package)}}]
+    {:keep_state_and_data, next_actions}
+  end
+
+  # connection acknowledgement
+  def handle_event(
+        :internal,
+        {:received, %Package.Connack{reason: :success} = connack},
+        # :handshake,
+        :connecting,
+        %State{server: %Transport{type: transport}, connection: {transport, socket}} = data
+      ) do
+    connection = {transport, socket}
+    :ok = Tortoise.Registry.put_meta(via_name(data.client_id), connection)
+    :ok = Events.dispatch(data.client_id, :connection, connection)
+    :ok = Events.dispatch(data.client_id, :status, :connected)
 
-      {:error, reason} ->
-        {timeout, state} = Map.get_and_update(state, :backoff, &Backoff.next/1)
+    case connack do
+      %Package.Connack{session_present: true} ->
+        {:next_state, :connected, data}
 
-        case categorize_error(reason) do
-          :connectivity ->
-            Process.send_after(self(), :connect, timeout)
-            {:noreply, state}
+      %Package.Connack{session_present: false} ->
+        caller = {self(), make_ref()}
 
-          :other ->
-            {:stop, reason, state}
-        end
+        next_actions = [
+          {:next_event, :cast, {:subscribe, caller, data.subscriptions, []}}
+        ]
+
+        {:next_state, :connected, data, next_actions}
     end
   end
 
-  def handle_info(:subscribe, %State{subscriptions: subscriptions} = state) do
-    client_id = state.connect.client_id
-
-    case Enum.empty?(subscriptions) do
-      true ->
-        # nothing to subscribe to, just continue
-        {:noreply, state}
+  def handle_event(
+        :internal,
+        {:received, %Package.Connack{reason: {:refused, reason}}},
+        # :handshake,
+        :connecting,
+        %State{} = data
+      ) do
+    {:stop, {:connection_failed, reason}, data}
+  end
 
-      false ->
-        # subscribe to the predefined topics
-        case Inflight.track_sync(client_id, {:outgoing, subscriptions}, 5000) do
-          {:error, :timeout} ->
-            {:stop, :subscription_timeout, state}
+  def handle_event(
+        :internal,
+        {:received, package},
+        # :handshake,
+        :connecting,
+        %State{} = data
+      ) do
+    reason = %{expected: [Package.Connack, Package.Auth], got: package}
+    {:stop, {:protocol_violation, reason}, data}
+  end
 
-          result ->
-            case handle_suback_result(result, state) do
-              {:ok, updated_state} ->
-                {:noreply, updated_state}
+  # publish packages
+  def handle_event(
+        :internal,
+        {:received, %Package.Publish{qos: 0, dup: false} = publish},
+        _,
+        %State{handler: handler} = data
+      ) do
+    case Handler.execute(handler, {:publish, publish}) do
+      {:ok, updated_handler} ->
+        {:keep_state, %State{data | handler: updated_handler}}
 
-              {:error, reasons} ->
-                error = {:unable_to_subscribe, reasons}
-                {:stop, error, state}
-            end
-        end
+        # handle stop
     end
   end
 
-  def handle_info(:ping, %State{} = state) do
-    case Controller.ping_sync(state.connect.client_id, 5000) do
-      {:ok, round_trip_time} ->
-        Events.dispatch(state.connect.client_id, :ping_response, round_trip_time)
-        state = reset_keep_alive(state)
-        {:noreply, state}
+  # incoming publish QoS=1 ---------------------------------------------
+  def handle_event(
+        :internal,
+        {:received, %Package.Publish{qos: 1} = publish},
+        :connected,
+        %State{client_id: client_id, handler: handler} = data
+      ) do
+    :ok = Inflight.track(client_id, {:incoming, publish})
 
-      {:error, :timeout} ->
-        {:stop, :ping_timeout, state}
+    case Handler.execute(handler, {:publish, publish}) do
+      {:ok, updated_handler} ->
+        {:keep_state, %State{data | handler: updated_handler}}
     end
   end
 
-  # dropping connection
-  def handle_info({transport, _socket}, state) when transport in [:tcp_closed, :ssl_closed] do
-    Logger.error("Socket closed before we handed it to the receiver")
-    # communicate that we are down
-    :ok = Events.dispatch(state.client_id, :status, :down)
-    {:noreply, state}
+  # outgoing publish QoS=1 ---------------------------------------------
+  def handle_event(
+        :internal,
+        {:received, %Package.Puback{} = puback},
+        _,
+        %State{client_id: client_id}
+      ) do
+    :ok = Inflight.update(client_id, {:received, puback})
+    :keep_state_and_data
   end
 
-  # react to connection status change events
-  def handle_info(
-        {{Tortoise, client_id}, :status, status},
-        %{client_id: client_id, status: current} = state
+  # incoming publish QoS=2 ---------------------------------------------
+  def handle_event(
+        :internal,
+        {:received, %Package.Publish{qos: 2} = publish},
+        :connected,
+        %State{client_id: client_id}
       ) do
-    case status do
-      ^current ->
-        {:noreply, state}
-
-      :up ->
-        {:noreply, %State{state | status: status}}
+    :ok = Inflight.track(client_id, {:incoming, publish})
+    :keep_state_and_data
+  end
 
-      :down ->
-        send(self(), :connect)
-        {:noreply, %State{state | status: status}}
+  def handle_event(
+        :internal,
+        {:received, %Package.Pubrel{} = pubrel},
+        :connected,
+        %State{client_id: client_id}
+      ) do
+    :ok = Inflight.update(client_id, {:received, pubrel})
+    :keep_state_and_data
+  end
+
+  # an incoming publish with QoS=2 will get parked in the inflight
+  # manager process, which will onward it to the controller, making
+  # sure we will only dispatch it once to the publish-handler.
+  def handle_event(
+        :info,
+        {{Inflight, client_id}, %Package.Publish{qos: 2} = publish},
+        _,
+        %State{client_id: client_id, handler: handler} = data
+      ) do
+    case Handler.execute(handler, {:publish, publish}) do
+      {:ok, updated_handler} ->
+        {:keep_state, %State{data | handler: updated_handler}}
     end
   end
 
-  @impl true
-  def handle_call(:subscriptions, _from, state) do
-    {:reply, state.subscriptions, state}
+  # outgoing publish QoS=2 ---------------------------------------------
+  def handle_event(
+        :internal,
+        {:received, %Package.Pubrec{} = pubrec},
+        :connected,
+        %State{client_id: client_id}
+      ) do
+    :ok = Inflight.update(client_id, {:received, pubrec})
+    :keep_state_and_data
   end
 
-  def handle_call(:disconnect, from, state) do
-    :ok = Events.dispatch(state.client_id, :status, :terminating)
-    :ok = Inflight.drain(state.client_id)
-    :ok = Controller.stop(state.client_id)
-    :ok = GenServer.reply(from, :ok)
-    {:stop, :shutdown, state}
+  def handle_event(
+        :internal,
+        {:received, %Package.Pubcomp{} = pubcomp},
+        :connected,
+        %State{client_id: client_id}
+      ) do
+    :ok = Inflight.update(client_id, {:received, pubcomp})
+    :keep_state_and_data
   end
 
-  @impl true
-  def handle_cast({:subscribe, {caller_pid, ref}, subscribe, opts}, state) do
-    client_id = state.connect.client_id
-    timeout = Keyword.get(opts, :timeout, 5000)
+  # subscription logic
+  def handle_event(
+        :cast,
+        {:subscribe, caller, subscribe, _opts},
+        :connected,
+        %State{client_id: client_id} = data
+      ) do
+    unless Enum.empty?(subscribe) do
+      {:ok, ref} = Inflight.track(client_id, {:outgoing, subscribe})
+      pending = Map.put_new(data.pending_refs, ref, caller)
 
-    case Inflight.track_sync(client_id, {:outgoing, subscribe}, timeout) do
-      {:error, :timeout} = error ->
-        send(caller_pid, {{Tortoise, client_id}, ref, error})
-        {:noreply, state}
-
-      result ->
-        case handle_suback_result(result, state) do
-          {:ok, updated_state} ->
-            send(caller_pid, {{Tortoise, client_id}, ref, :ok})
-            {:noreply, updated_state}
-
-          {:error, reasons} ->
-            error = {:unable_to_subscribe, reasons}
-            send(caller_pid, {{Tortoise, client_id}, ref, {:error, reasons}})
-            {:stop, error, state}
-        end
+      {:keep_state, %State{data | pending_refs: pending}}
+    else
+      :keep_state_and_data
     end
   end
 
-  def handle_cast({:unsubscribe, {caller_pid, ref}, unsubscribe, opts}, state) do
-    client_id = state.connect.client_id
-    timeout = Keyword.get(opts, :timeout, 5000)
+  def handle_event(:cast, {:subscribe, _, _, _}, _state_name, _data) do
+    {:keep_state_and_data, [:postpone]}
+  end
 
-    case Inflight.track_sync(client_id, {:outgoing, unsubscribe}, timeout) do
-      {:error, :timeout} = error ->
-        send(caller_pid, {{Tortoise, client_id}, ref, error})
-        {:noreply, state}
+  def handle_event(
+        :internal,
+        {:received, %Package.Suback{} = suback},
+        :connected,
+        data
+      ) do
+    :ok = Inflight.update(data.client_id, {:received, suback})
+    :keep_state_and_data
+  end
 
-      unsubbed ->
-        topics = Keyword.drop(state.subscriptions.topics, unsubbed)
-        subscriptions = %Package.Subscribe{state.subscriptions | topics: topics}
-        send(caller_pid, {{Tortoise, client_id}, ref, :ok})
-        {:noreply, %State{state | subscriptions: subscriptions}}
+  def handle_event(
+        :info,
+        {{Tortoise, client_id}, {Package.Subscribe, ref}, result},
+        _current_state,
+        %State{client_id: client_id, pending_refs: %{} = pending} = data
+      ) do
+    case Map.pop(pending, ref) do
+      {{pid, msg_ref}, updated_pending} when is_pid(pid) and is_reference(msg_ref) ->
+        unless pid == self(), do: send(pid, {{Tortoise, client_id}, msg_ref, :ok})
+        subscriptions = Enum.into(result[:ok] ++ result[:warn], data.subscriptions)
+        {:keep_state, %State{data | subscriptions: subscriptions, pending_refs: updated_pending}}
     end
   end
 
-  # Helpers
-  defp handle_suback_result(%{:error => []} = results, %State{} = state) do
-    subscriptions = Enum.into(results[:ok], state.subscriptions)
-    {:ok, %State{state | subscriptions: subscriptions}}
-  end
+  def handle_event(:cast, {:unsubscribe, caller, unsubscribe, opts}, :connected, data) do
+    client_id = data.client_id
+    _timeout = Keyword.get(opts, :timeout, 5000)
 
-  defp handle_suback_result(%{:error => errors}, %State{}) do
-    {:error, errors}
-  end
+    {:ok, ref} = Inflight.track(client_id, {:outgoing, unsubscribe})
+    pending = Map.put_new(data.pending_refs, ref, caller)
 
-  defp reset_keep_alive(%State{keep_alive: nil} = state) do
-    ref = Process.send_after(self(), :ping, state.connect.keep_alive * 1000)
-    %State{state | keep_alive: ref}
+    {:keep_state, %State{data | pending_refs: pending}}
   end
 
-  defp reset_keep_alive(%State{keep_alive: previous_ref} = state) do
-    # Cancel the previous timer, just in case one was already set
-    _ = Process.cancel_timer(previous_ref)
-    ref = Process.send_after(self(), :ping, state.connect.keep_alive * 1000)
-    %State{state | keep_alive: ref}
+  def handle_event(
+        :internal,
+        {:received, %Package.Unsuback{} = unsuback},
+        :connected,
+        data
+      ) do
+    :ok = Inflight.update(data.client_id, {:received, unsuback})
+    :keep_state_and_data
   end
 
-  defp cancel_keep_alive(%State{keep_alive: nil} = state) do
-    state
+  def handle_event(
+        :info,
+        {{Tortoise, client_id}, {Package.Unsubscribe, ref}, unsubbed},
+        _current_state,
+        %State{client_id: client_id, pending_refs: %{} = pending} = data
+      ) do
+    case Map.pop(pending, ref) do
+      {{pid, msg_ref}, updated_pending} when is_pid(pid) and is_reference(msg_ref) ->
+        topics = Keyword.drop(data.subscriptions.topics, unsubbed)
+        subscriptions = %Package.Subscribe{data.subscriptions | topics: topics}
+        unless pid == self(), do: send(pid, {{Tortoise, client_id}, msg_ref, :ok})
+        {:keep_state, %State{data | pending_refs: updated_pending, subscriptions: subscriptions}}
+    end
   end
 
-  defp cancel_keep_alive(%State{keep_alive: keep_alive_ref} = state) do
-    _ = Process.cancel_timer(keep_alive_ref)
-    %State{state | keep_alive: nil}
+  def handle_event({:call, from}, :subscriptions, _, %State{subscriptions: subscriptions}) do
+    next_actions = [{:reply, from, subscriptions}]
+    {:keep_state_and_data, next_actions}
   end
 
-  # dispatch connection status if the connection status change
-  defp update_connection_status(%State{status: same} = state, same) do
-    state
+  # connection logic ===================================================
+  def handle_event(
+        :internal,
+        :connect,
+        :connecting,
+        %State{connect: connect, backoff: backoff} = data
+      ) do
+    :ok = Tortoise.Registry.put_meta(via_name(data.client_id), :connecting)
+
+    with :ok = start_connection_supervisor([{:parent, self()} | data.opts]),
+         {:ok, {transport, socket}} <- Tortoise.Connection.Receiver.connect(data.client_id),
+         :ok = transport.send(socket, Package.encode(data.connect)) do
+      new_data = %State{
+        data
+        | connect: %Connect{connect | clean_start: false},
+          connection: {transport, socket}
+      }
+
+      # {:next_state, :handshake, new_state}
+      {:keep_state, new_data}
+    else
+      {:error, {:stop, reason}} ->
+        {:stop, reason, data}
+
+      {:error, {:retry, _reason}} ->
+        # {timeout, updated_data} = Map.get_and_update(data, :backoff, &Backoff.next/1)
+        next_actions = [{:next_event, :internal, :connect}]
+        {:keep_state, data, next_actions}
+    end
   end
 
-  defp update_connection_status(%State{} = state, status) do
-    :ok = Events.dispatch(state.connect.client_id, :status, status)
-    %State{state | status: status}
+  # system state changes; we need to react to connection down messages
+  def handle_event(
+        :info,
+        {{Tortoise, client_id}, :status, current_state},
+        current_state,
+        %State{} = data
+      ) do
+    :keep_state_and_data
   end
 
-  defp do_connect(server, %Connect{} = connect) do
-    %Transport{type: transport, host: host, port: port, opts: opts} = server
+  def handle_event(
+        :info,
+        {{Tortoise, client_id}, :status, status},
+        _current_state,
+        %State{handler: handler, client_id: client_id} = data
+      ) do
+    case status do
+      :down ->
+        next_actions = [{:next_event, :internal, :connect}]
 
-    with {:ok, socket} <- transport.connect(host, port, opts, 10000),
-         :ok = transport.send(socket, Package.encode(connect)),
-         {:ok, packet} <- transport.recv(socket, 4, 5000) do
-      try do
-        case Package.decode(packet) do
-          %Connack{status: :accepted} = connack ->
-            {connack, socket}
+        case Handler.execute(handler, {:connection, status}) do
+          {:ok, updated_handler} ->
+            updated_data = %State{data | handler: updated_handler}
+            {:next_state, :connecting, updated_data, next_actions}
+        end
 
-          %Connack{status: {:refused, _reason}} = connack ->
-            connack
+      _otherwise ->
+        case Handler.execute(handler, {:connection, status}) do
+          {:ok, updated_handler} ->
+            {:keep_state, %State{data | handler: updated_handler}}
         end
-      catch
-        :error, {:badmatch, _unexpected} ->
-          violation = %{expected: Connect, got: packet}
-          {:error, {:protocol_violation, violation}}
-      end
-    else
-      {:error, :econnrefused} ->
-        {:error, {:connection_refused, host, port}}
+    end
+  end
 
-      {:error, :nxdomain} ->
-        {:error, {:nxdomain, host, port}}
+  # disconnect protocol messages ---------------------------------------
+  def handle_event(
+        {:call, from},
+        :disconnect,
+        :connected,
+        %State{client_id: client_id} = data
+      ) do
+    :ok = Events.dispatch(client_id, :status, :terminating)
 
-      {:error, {:options, {:cacertfile, []}}} ->
-        {:error, :no_cacartfile_specified}
+    :ok = Inflight.drain(client_id)
 
-      {:error, :closed} ->
-        {:error, :server_closed_connection}
-    end
+    {:stop_and_reply, :shutdown, [{:reply, from, :ok}], data}
   end
 
-  defp init_connection(socket, %State{opts: opts, server: transport, connect: connect} = state) do
-    connection = {transport.type, socket}
-    :ok = start_connection_supervisor(opts)
-    :ok = Receiver.handle_socket(connect.client_id, connection)
-    :ok = Tortoise.Registry.put_meta(via_name(connect.client_id), connection)
-    :ok = Events.dispatch(connect.client_id, :connection, connection)
+  def handle_event(
+        {:call, _from},
+        :disconnect,
+        _,
+        %State{}
+      ) do
+    {:keep_state_and_data, [:postpone]}
+  end
 
-    # set clean session to false for future reconnect attempts
-    connect = %Connect{connect | clean_session: false}
-    {:ok, %State{state | connect: connect}}
+  # ping handling ------------------------------------------------------
+  def handle_event(
+        :cast,
+        {:ping, caller},
+        :connected,
+        %State{connection: {transport, socket}} = data
+      ) do
+    time = System.monotonic_time(:microsecond)
+    apply(transport, :send, [socket, Package.encode(%Package.Pingreq{})])
+    ping = :queue.in({caller, time}, data.ping)
+    {:keep_state, %State{data | ping: ping}}
+  end
+
+  # def handle_event(:cast, {:ping, _}, _, %State{}) do
+  #   {:keep_state_and_data, [:postpone]}
+  # end
+
+  def handle_event(
+        :internal,
+        {:received, %Package.Pingresp{}},
+        :connected,
+        %State{ping: ping} = data
+      ) do
+    {{:value, {{caller, ref}, start_time}}, ping} = :queue.out(ping)
+    round_trip_time = System.monotonic_time(:microsecond) - start_time
+    send(caller, {Tortoise, {:ping_response, ref, round_trip_time}})
+    {:keep_state, %State{data | ping: ping}}
+  end
+
+  def handle_event(
+        :internal,
+        {:received, package},
+        _current_state,
+        %State{} = data
+      ) do
+    {:stop, {:protocol_violation, {:unexpected_package, package}}, data}
   end
 
   defp start_connection_supervisor(opts) do
-    case Connection.Supervisor.start_link(opts) do
+    case Tortoise.Connection.Supervisor.start_link(opts) do
       {:ok, _pid} ->
         :ok
 
@@ -632,20 +783,4 @@ defmodule Tortoise.Connection do
         :ok
     end
   end
-
-  defp categorize_error({:nxdomain, _host, _port}) do
-    :connectivity
-  end
-
-  defp categorize_error({:connection_refused, _host, _port}) do
-    :connectivity
-  end
-
-  defp categorize_error(:server_closed_connection) do
-    :connectivity
-  end
-
-  defp categorize_error(_other) do
-    :other
-  end
 end
diff --git a/lib/tortoise/connection/inflight.ex b/lib/tortoise/connection/inflight.ex
index eabc5a25..37bab254 100644
--- a/lib/tortoise/connection/inflight.ex
+++ b/lib/tortoise/connection/inflight.ex
@@ -7,14 +7,15 @@ defmodule Tortoise.Connection.Inflight do
 
   use GenStateMachine
 
-  @enforce_keys [:client_id]
-  defstruct client_id: nil, pending: %{}, order: []
+  @enforce_keys [:client_id, :parent]
+  defstruct client_id: nil, parent: nil, pending: %{}, order: []
 
   alias __MODULE__, as: State
 
   # Client API
   def start_link(opts) do
     client_id = Keyword.fetch!(opts, :client_id)
+
     GenStateMachine.start_link(__MODULE__, opts, name: via_name(client_id))
   end
 
@@ -81,12 +82,15 @@ defmodule Tortoise.Connection.Inflight do
   @impl true
   def init(opts) do
     client_id = Keyword.fetch!(opts, :client_id)
-    initial_data = %State{client_id: client_id}
+    parent_pid = Keyword.fetch!(opts, :parent)
+    initial_data = %State{client_id: client_id, parent: parent_pid}
 
     next_actions = [
       {:next_event, :internal, :post_init}
     ]
 
+    {:ok, _} = Tortoise.Events.register(client_id, :status)
+
     {:ok, :disconnected, initial_data, next_actions}
   end
 
@@ -94,7 +98,6 @@ defmodule Tortoise.Connection.Inflight do
   def handle_event(:internal, :post_init, :disconnected, data) do
     case Connection.connection(data.client_id, active: true) do
       {:ok, {_transport, _socket} = connection} ->
-        {:ok, _} = Tortoise.Events.register(data.client_id, :status)
         {:next_state, {:connected, connection}, data}
 
       {:error, :timeout} ->
@@ -267,9 +270,9 @@ defmodule Tortoise.Connection.Inflight do
         :internal,
         {:onward_publish, %Package.Publish{qos: 2} = publish},
         _,
-        %State{} = data
+        %State{client_id: client_id, parent: parent_pid} = data
       ) do
-    :ok = Controller.handle_onward(data.client_id, publish)
+    send(parent_pid, {{__MODULE__, client_id}, publish})
     :keep_state_and_data
   end
 
@@ -307,13 +310,13 @@ defmodule Tortoise.Connection.Inflight do
 
   def handle_event(
         :internal,
-        {:execute, %Track{pending: [[{:respond, caller}, _] | _]} = track},
+        {:execute, %Track{pending: [[{:respond, {pid, ref}}, _] | _]} = track},
         _state,
         %State{client_id: client_id} = data
       ) do
     case Track.result(track) do
       {:ok, result} ->
-        :ok = Controller.handle_result(client_id, {caller, track.type, result})
+        send(pid, {{Tortoise, client_id}, {track.type, ref}, result})
         {:keep_state, handle_next(track, data)}
     end
   end
@@ -330,6 +333,7 @@ defmodule Tortoise.Connection.Inflight do
       :ok ->
         :ok = transport.close(socket)
         reply = {:reply, from, :ok}
+
         {:next_state, :draining, data, reply}
     end
   end
diff --git a/lib/tortoise/connection/receiver.ex b/lib/tortoise/connection/receiver.ex
index ecdd298f..73591703 100644
--- a/lib/tortoise/connection/receiver.ex
+++ b/lib/tortoise/connection/receiver.ex
@@ -3,16 +3,19 @@ defmodule Tortoise.Connection.Receiver do
 
   use GenStateMachine
 
-  alias Tortoise.Connection.Controller
-  alias Tortoise.Events
+  alias Tortoise.{Events, Transport}
 
-  defstruct client_id: nil, transport: nil, socket: nil, buffer: <<>>
+  defstruct client_id: nil, transport: nil, socket: nil, buffer: <<>>, parent: nil
   alias __MODULE__, as: State
 
   def start_link(opts) do
     client_id = Keyword.fetch!(opts, :client_id)
 
-    data = %State{client_id: client_id}
+    data = %State{
+      client_id: client_id,
+      transport: Keyword.fetch!(opts, :transport),
+      parent: Keyword.fetch!(opts, :parent)
+    }
 
     GenStateMachine.start_link(__MODULE__, data, name: via_name(client_id))
   end
@@ -31,6 +34,10 @@ defmodule Tortoise.Connection.Receiver do
     }
   end
 
+  def connect(client_id) do
+    GenStateMachine.call(via_name(client_id), :connect)
+  end
+
   def handle_socket(client_id, {transport, socket}) do
     {:ok, pid} = GenStateMachine.call(via_name(client_id), {:handle_socket, transport, socket})
 
@@ -49,10 +56,14 @@ defmodule Tortoise.Connection.Receiver do
     {:ok, :disconnected, data}
   end
 
+  def terminate(_reason, _state) do
+    :ok
+  end
+
   @impl true
   # receiving data on the network connection
   def handle_event(:info, {transport, socket, tcp_data}, _, %{socket: socket} = data)
-      when transport in [:tcp, :ssl] do
+      when transport in [:tcp, :ssl, ScriptedTransport] do
     next_actions = [
       {:next_event, :internal, :activate_socket},
       {:next_event, :internal, :consume_buffer}
@@ -70,7 +81,6 @@ defmodule Tortoise.Connection.Receiver do
   def handle_event(:info, {transport, socket}, _state, %{socket: socket} = data)
       when transport in [:tcp_closed, :ssl_closed] do
     # should we empty the buffer?
-
     # communicate to the world that we have dropped the connection
     :ok = Events.dispatch(data.client_id, :status, :down)
     {:next_state, :disconnected, %{data | socket: nil}}
@@ -81,8 +91,13 @@ defmodule Tortoise.Connection.Receiver do
     {:stop, :no_transport}
   end
 
-  def handle_event(:internal, :activate_socket, _state_name, data) do
-    case data.transport.setopts(data.socket, active: :once) do
+  def handle_event(
+        :internal,
+        :activate_socket,
+        _state_name,
+        %State{transport: %Transport{type: transport}} = data
+      ) do
+    case transport.setopts(data.socket, active: :once) do
       :ok ->
         :keep_state_and_data
 
@@ -93,7 +108,7 @@ defmodule Tortoise.Connection.Receiver do
   end
 
   # consume buffer
-  def handle_event(:internal, :consume_buffer, _state_name, %{buffer: <<>>}) do
+  def handle_event(:internal, :consume_buffer, state_name, %{buffer: <<>>}) do
     :keep_state_and_data
   end
 
@@ -146,7 +161,7 @@ defmodule Tortoise.Connection.Receiver do
   end
 
   def handle_event(:internal, {:emit, package}, _, data) do
-    :ok = Controller.handle_incoming(data.client_id, package)
+    send(data.parent, {:incoming, package})
     :keep_state_and_data
   end
 
@@ -165,6 +180,56 @@ defmodule Tortoise.Connection.Receiver do
     {:next_state, new_state, new_data, next_actions}
   end
 
+  def handle_event({:call, from}, {:handle_socket, _transport, _socket}, current_state, data) do
+    next_actions = [{:reply, from, {:error, :not_ready}}]
+    reason = {:got_socket_in_wrong_state, current_state}
+    {:stop_and_reply, reason, next_actions, data}
+  end
+
+  # connect
+  def handle_event(
+        {:call, from},
+        :connect,
+        :disconnected,
+        %State{
+          transport: %Transport{type: transport, host: host, port: port, opts: opts}
+        } = data
+      ) do
+    Events.dispatch(data.client_id, :status, :connecting)
+
+    case transport.connect(host, port, opts, 10000) do
+      {:ok, socket} ->
+        new_state = {:connected, :receiving_fixed_header}
+
+        next_actions = [
+          {:reply, from, {:ok, {transport, socket}}},
+          {:next_event, :internal, :activate_socket},
+          {:next_event, :internal, :consume_buffer}
+        ]
+
+        # better reset the buffer
+        new_data = %State{data | socket: socket, buffer: <<>>}
+        {:next_state, new_state, new_data, next_actions}
+
+      {:error, reason} ->
+        next_actions = [{:reply, from, {:error, connection_error(reason)}}]
+        {:next_state, :disconnected, data, next_actions}
+    end
+  end
+
+  defp connection_error(reason) do
+    case reason do
+      {:options, {:cacertfile, []}} ->
+        {:stop, :no_cacartfile_specified}
+
+      :nxdomain ->
+        {:retry, :nxdomain}
+
+      :econnrefused ->
+        {:retry, :econnrefused}
+    end
+  end
+
   defp parse_fixed_header(<<_::8, 0::1, length::7, _::binary>>) do
     {:ok, length + 2}
   end
diff --git a/lib/tortoise/connection/supervisor.ex b/lib/tortoise/connection/supervisor.ex
index d52cdd23..730059ad 100644
--- a/lib/tortoise/connection/supervisor.ex
+++ b/lib/tortoise/connection/supervisor.ex
@@ -3,7 +3,7 @@ defmodule Tortoise.Connection.Supervisor do
 
   use Supervisor
 
-  alias Tortoise.Connection.{Receiver, Controller, Inflight}
+  alias Tortoise.Connection.{Receiver, Inflight}
 
   def start_link(opts) do
     client_id = Keyword.fetch!(opts, :client_id)
@@ -17,9 +17,8 @@ defmodule Tortoise.Connection.Supervisor do
   @impl true
   def init(opts) do
     children = [
-      {Inflight, Keyword.take(opts, [:client_id])},
-      {Receiver, Keyword.take(opts, [:client_id])},
-      {Controller, Keyword.take(opts, [:client_id, :handler])}
+      {Inflight, Keyword.take(opts, [:client_id, :parent])},
+      {Receiver, Keyword.take(opts, [:client_id, :transport, :parent])}
     ]
 
     Supervisor.init(children, strategy: :rest_for_one)
diff --git a/lib/tortoise/transport.ex b/lib/tortoise/transport.ex
index be09dc2b..59c124e0 100644
--- a/lib/tortoise/transport.ex
+++ b/lib/tortoise/transport.ex
@@ -68,4 +68,9 @@ defmodule Tortoise.Transport do
   @callback shutdown(socket(), :read | :write | :read_write) :: :ok | {:error, atom()}
 
   @callback close(socket()) :: :ok
+
+  # todo
+  @callback format_error(term()) :: term()
+
+  @optional_callbacks format_error: 1
 end
diff --git a/test/support/scripted_mqtt_server.exs b/test/support/scripted_mqtt_server.exs
index 14161298..d13b07d8 100644
--- a/test/support/scripted_mqtt_server.exs
+++ b/test/support/scripted_mqtt_server.exs
@@ -58,6 +58,11 @@ defmodule Tortoise.Integration.ScriptedMqttServer do
     end
   end
 
+  def handle_call({:enact, script}, {pid, _} = caller, %State{client_pid: pid} = state) do
+    GenServer.reply(caller, {:ok, state.server_info})
+    next_action(%State{state | script: state.script ++ script})
+  end
+
   def handle_call({:enact, script}, {pid, _} = caller, state) do
     GenServer.reply(caller, {:ok, state.server_info})
     {:ok, client} = state.transport.accept(state.server_socket, 200)
diff --git a/test/support/scripted_transport.exs b/test/support/scripted_transport.exs
index 9b547837..028cc649 100644
--- a/test/support/scripted_transport.exs
+++ b/test/support/scripted_transport.exs
@@ -188,7 +188,14 @@ defmodule Tortoise.Integration.ScriptedTransport do
   end
 
   def handle_call({:connect, opts, _timeout}, {client_pid, _ref}, %State{client: nil} = state) do
-    state = %State{state | client: client_pid, status: :open, opts: opts}
+    state = %State{
+      state
+      | client: client_pid,
+        status: :open,
+        opts: opts,
+        controlling_process: client_pid
+    }
+
     Kernel.send(state.test_process, {__MODULE__, :connected})
     {:reply, {:ok, self()}, setup_next(state)}
   end
@@ -268,10 +275,26 @@ defmodule Tortoise.Integration.ScriptedTransport do
     state
   end
 
-  defp setup_next(%State{script: [{:dispatch, package} | remaining]} = state) do
+  defp setup_next(%State{script: [{:dispatch, package} | remaining], opts: opts} = state) do
     data = IO.iodata_to_binary(Tortoise.Package.encode(package))
     buffer = state.buffer <> data
-    %State{state | script: remaining, buffer: buffer}
+
+    case Keyword.pop(opts, :active, false) do
+      {false, opts} ->
+        opts = [{:active, false} | opts]
+        %State{state | opts: opts, script: remaining, buffer: buffer}
+
+      {true, opts} ->
+        opts = [{:active, true} | opts]
+        Kernel.send(state.controlling_process, {ScriptedTransport, self(), buffer})
+        %State{state | opts: opts, script: remaining, buffer: <<>>}
+
+      {:once, opts} ->
+        opts = [{:active, false} | opts]
+        Kernel.send(state.controlling_process, {ScriptedTransport, self(), buffer})
+        %State{state | opts: opts, script: remaining, buffer: <<>>}
+    end
+    |> setup_next()
   end
 
   defp setup_next(%State{script: [{:expect, _} | _]} = state) do
diff --git a/test/support/test_tcp_tunnel.exs b/test/support/test_tcp_tunnel.exs
index 93ed8173..a25b35bf 100644
--- a/test/support/test_tcp_tunnel.exs
+++ b/test/support/test_tcp_tunnel.exs
@@ -4,7 +4,7 @@ defmodule Tortoise.Integration.TestTCPTunnel do
   send to the client_socket and assert on the received data on the
   server_socket.
 
-  This work for our Transmitter-module which is handled a TCP-socket
+  This work for our Inflight-module which is handled a TCP-socket
   from the Receiver.
   """
   use GenServer
@@ -30,6 +30,11 @@ defmodule Tortoise.Integration.TestTCPTunnel do
     end
   end
 
+  def new(transport) do
+    {ref, {ip, port}} = GenServer.call(__MODULE__, :create)
+    {:ok, ref, Tortoise.Transport.new({transport, [host: ip, port: port]})}
+  end
+
   # Server callbacks
   def init(state) do
     {:ok, socket} = :gen_tcp.listen(0, [:binary, active: false])
diff --git a/test/test_helper.exs b/test/test_helper.exs
index 952d280e..ec05c4a5 100644
--- a/test/test_helper.exs
+++ b/test/test_helper.exs
@@ -101,7 +101,8 @@ defmodule Tortoise.TestGenerators do
               topic: gen_topic(),
               payload: oneof([non_empty(binary()), nil]),
               qos: gen_qos(),
-              retain: bool()
+              retain: bool(),
+              properties: [receive_maximum: 201]
             }
           ]) do
       # zero byte client id is allowed, but clean session should be set to true
@@ -118,9 +119,10 @@ defmodule Tortoise.TestGenerators do
             client_id: binary(),
             user_name: oneof([nil, utf8()]),
             password: oneof([nil, utf8()]),
-            clean_session: bool(),
+            clean_start: bool(),
             keep_alive: choose(0, 65535),
-            will: will
+            will: will,
+            properties: [receive_maximum: 201]
           } do
         connect
       end
@@ -133,14 +135,30 @@ defmodule Tortoise.TestGenerators do
   def gen_connack() do
     let connack <- %Package.Connack{
           session_present: bool(),
-          status:
+          reason:
             oneof([
-              :accepted,
-              {:refused, :unacceptable_protocol_version},
-              {:refused, :identifier_rejected},
-              {:refused, :server_unavailable},
+              :success,
+              {:refused, :unspecified_error},
+              {:refused, :malformed_packet},
+              {:refused, :protocol_error},
+              {:refused, :implementation_specific_error},
+              {:refused, :unsupported_protocol_version},
+              {:refused, :client_identifier_not_valid},
               {:refused, :bad_user_name_or_password},
-              {:refused, :not_authorized}
+              {:refused, :not_authorized},
+              {:refused, :server_unavailable},
+              {:refused, :server_busy},
+              {:refused, :banned},
+              {:refused, :bad_authentication_method},
+              {:refused, :topic_name_invalid},
+              {:refused, :packet_too_large},
+              {:refused, :quota_exceeded},
+              {:refused, :payload_format_invalid},
+              {:refused, :retain_not_supported},
+              {:refused, :qos_not_supported},
+              {:refused, :use_another_server},
+              {:refused, :server_moved},
+              {:refused, :connection_rate_exceeded}
             ])
         } do
       connack
@@ -207,16 +225,52 @@ defmodule Tortoise.TestGenerators do
   def gen_subscribe() do
     let subscribe <- %Package.Subscribe{
           identifier: gen_identifier(),
-          topics: non_empty(list({gen_topic_filter(), gen_qos()}))
+          topics: non_empty(list({gen_topic_filter(), gen_subscribe_opts()})),
+          # todo, add properties
+          properties: []
         } do
       subscribe
     end
   end
 
+  # @todo improve this generator
+  def gen_subscribe_opts() do
+    let {qos, no_local, retain_as_published, retain_handling} <-
+          {gen_qos(), bool(), bool(), choose(0, 3)} do
+      [
+        qos: qos,
+        no_local: no_local,
+        retain_as_published: retain_as_published,
+        retain_handling: retain_handling
+      ]
+    end
+  end
+
   def gen_suback() do
     let suback <- %Package.Suback{
           identifier: choose(0x0001, 0xFFFF),
-          acks: non_empty(list(oneof([{:ok, gen_qos()}, {:error, :access_denied}])))
+          acks:
+            non_empty(
+              list(
+                oneof([
+                  {:ok, gen_qos()},
+                  {:error,
+                   oneof([
+                     :unspecified_error,
+                     :implementation_specific_error,
+                     :not_authorized,
+                     :topic_filter_invalid,
+                     :packet_identifier_in_use,
+                     :quota_exceeded,
+                     :shared_subscriptions_not_supported,
+                     :subscription_identifiers_not_supported,
+                     :wildcard_subscriptions_not_supported
+                   ])}
+                ])
+              )
+            ),
+          # todo, add generators for [:reason_string, :user_property]
+          properties: []
         } do
       suback
     end
@@ -228,7 +282,8 @@ defmodule Tortoise.TestGenerators do
   def gen_unsubscribe() do
     let unsubscribe <- %Package.Unsubscribe{
           identifier: gen_identifier(),
-          topics: non_empty(list(gen_topic_filter()))
+          topics: non_empty(list(gen_topic_filter())),
+          properties: []
         } do
       unsubscribe
     end
@@ -236,15 +291,37 @@ defmodule Tortoise.TestGenerators do
 
   def gen_unsuback() do
     let unsuback <- %Package.Unsuback{
-          identifier: gen_identifier()
+          identifier: gen_identifier(),
+          results:
+            non_empty(
+              list(
+                oneof([
+                  :success,
+                  {:error,
+                   oneof([
+                     :no_subscription_existed,
+                     :unspecified_error,
+                     :implementation_specific_error,
+                     :not_authorized,
+                     :topic_filter_invalid,
+                     :packet_identifier_in_use
+                   ])}
+                ])
+              )
+            ),
+          # todo, generate :reason_string and :user_property
+          properties: []
         } do
       unsuback
     end
   end
 
   def gen_puback() do
+    # todo, make this generator generate properties and other reasons
     let puback <- %Package.Puback{
-          identifier: gen_identifier()
+          identifier: gen_identifier(),
+          reason: :success,
+          properties: []
         } do
       puback
     end
@@ -252,23 +329,31 @@ defmodule Tortoise.TestGenerators do
 
   def gen_pubcomp() do
     let pubcomp <- %Package.Pubcomp{
-          identifier: gen_identifier()
+          identifier: gen_identifier(),
+          reason: {:refused, :packet_identifier_not_found},
+          properties: []
         } do
       pubcomp
     end
   end
 
   def gen_pubrel() do
+    # todo, improve this generator
     let pubrel <- %Package.Pubrel{
-          identifier: gen_identifier()
+          identifier: gen_identifier(),
+          reason: :success,
+          properties: []
         } do
       pubrel
     end
   end
 
   def gen_pubrec() do
+    # todo, improve this generator
     let pubrec <- %Package.Pubrec{
-          identifier: gen_identifier()
+          identifier: gen_identifier(),
+          reason: :success,
+          properties: []
         } do
       pubrec
     end
diff --git a/test/tortoise/connection/controller_test.exs b/test/tortoise/connection/controller_test.exs
index 76eba59e..48fe30f1 100644
--- a/test/tortoise/connection/controller_test.exs
+++ b/test/tortoise/connection/controller_test.exs
@@ -147,28 +147,30 @@ defmodule Tortoise.Connection.ControllerTest do
   describe "Connection Control Packets" do
     setup [:setup_controller]
 
-    test "receiving a connect from the server is a protocol violation",
-         %{controller_pid: pid} = context do
-      Process.flag(:trap_exit, true)
-      # receiving a connect from the server is a protocol violation
-      connect = %Package.Connect{client_id: "foo"}
-      Controller.handle_incoming(context.client_id, connect)
-
-      assert_receive {:EXIT, ^pid,
-                      {:protocol_violation, {:unexpected_package_from_remote, ^connect}}}
-    end
-
-    test "receiving a connack at this point is a protocol violation",
-         %{controller_pid: pid} = context do
-      Process.flag(:trap_exit, true)
-      # receiving a connack from the server *after* the connection has
-      # been acknowledged is a protocol violation
-      connack = %Package.Connack{status: :accepted}
-      Controller.handle_incoming(context.client_id, connack)
-
-      assert_receive {:EXIT, ^pid,
-                      {:protocol_violation, {:unexpected_package_from_remote, ^connack}}}
-    end
+    # done
+    # test "receiving a connect from the server is a protocol violation",
+    #      %{controller_pid: pid} = context do
+    #   Process.flag(:trap_exit, true)
+    #   # receiving a connect from the server is a protocol violation
+    #   connect = %Package.Connect{client_id: "foo"}
+    #   Controller.handle_incoming(context.client_id, connect)
+
+    #   assert_receive {:EXIT, ^pid,
+    #                   {:protocol_violation, {:unexpected_package_from_remote, ^connect}}}
+    # end
+
+    # done
+    # test "receiving a connack at this point is a protocol violation",
+    #      %{controller_pid: pid} = context do
+    #   Process.flag(:trap_exit, true)
+    #   # receiving a connack from the server *after* the connection has
+    #   # been acknowledged is a protocol violation
+    #   connack = %Package.Connack{reason: :success}
+    #   Controller.handle_incoming(context.client_id, connack)
+
+    #   assert_receive {:EXIT, ^pid,
+    #                   {:protocol_violation, {:unexpected_package_from_remote, ^connack}}}
+    # end
 
     test "receiving a disconnect from the server is a protocol violation",
          %{controller_pid: pid} = context do
@@ -214,15 +216,16 @@ defmodule Tortoise.Connection.ControllerTest do
       assert_receive {:ping_result, _time}
     end
 
-    test "receiving a ping request", %{controller_pid: pid} = context do
-      Process.flag(:trap_exit, true)
-      # receiving a ping request from the server is a protocol violation
-      pingreq = %Package.Pingreq{}
-      Controller.handle_incoming(context.client_id, pingreq)
+    # done
+    # test "receiving a ping request", %{controller_pid: pid} = context do
+    #   Process.flag(:trap_exit, true)
+    #   # receiving a ping request from the server is a protocol violation
+    #   pingreq = %Package.Pingreq{}
+    #   Controller.handle_incoming(context.client_id, pingreq)
 
-      assert_receive {:EXIT, ^pid,
-                      {:protocol_violation, {:unexpected_package_from_remote, ^pingreq}}}
-    end
+    #   assert_receive {:EXIT, ^pid,
+    #                   {:protocol_violation, {:unexpected_package_from_remote, ^pingreq}}}
+    # end
 
     test "ping request reports are sent in the correct order", context do
       # send two ping requests to the server
@@ -289,7 +292,7 @@ defmodule Tortoise.Connection.ControllerTest do
       # the server will send back an ack message
       Controller.handle_incoming(client_id, %Package.Puback{identifier: 1})
       # the caller should get a message in its mailbox
-      assert_receive {{Tortoise, ^client_id}, ^ref, :ok}
+      assert_receive {{Tortoise, ^client_id}, {Package.Publish, ^ref}, :ok}
     end
 
     test "outgoing publish with qos 1 sync call", context do
@@ -383,7 +386,7 @@ defmodule Tortoise.Connection.ControllerTest do
       # receive pubcomp
       Controller.handle_incoming(client_id, %Package.Pubcomp{identifier: 1})
       # the caller should get a message in its mailbox
-      assert_receive {{Tortoise, ^client_id}, ^ref, :ok}
+      assert_receive {{Tortoise, ^client_id}, {Package.Publish, ^ref}, :ok}
     end
   end
 
@@ -408,7 +411,7 @@ defmodule Tortoise.Connection.ControllerTest do
       # the server will send back a subscription acknowledgement message
       :ok = Controller.handle_incoming(client_id, suback)
 
-      assert_receive {{Tortoise, ^client_id}, ^ref, _}
+      assert_receive {{Tortoise, ^client_id}, {Package.Subscribe, ^ref}, _}
       # the client callback module should get the subscribe notifications in order
       assert_receive %TestHandler{subscriptions: [{"foo", :ok}]}
       assert_receive %TestHandler{subscriptions: [{"bar", :ok} | _]}
diff --git a/test/tortoise/connection/inflight_test.exs b/test/tortoise/connection/inflight_test.exs
index 7ff04a1f..af53b4c5 100644
--- a/test/tortoise/connection/inflight_test.exs
+++ b/test/tortoise/connection/inflight_test.exs
@@ -69,7 +69,7 @@ defmodule Tortoise.Connection.InflightTest do
       Inflight.update(client_id, {:received, %Package.Puback{identifier: 1}})
 
       # the calling process should get a result response
-      assert_receive {{Tortoise, ^client_id}, ^ref, :ok}
+      assert_receive {{Tortoise, ^client_id}, {Package.Publish, ^ref}, :ok}
     end
   end
 
@@ -125,7 +125,7 @@ defmodule Tortoise.Connection.InflightTest do
 
       # When we receive the pubcomp message we should respond the caller
       Inflight.update(client_id, {:received, %Package.Pubcomp{identifier: 1}})
-      assert_receive {{Tortoise, ^client_id}, ^ref, :ok}
+      assert_receive {{Tortoise, ^client_id}, {Package.Publish, ^ref}, :ok}
     end
   end
 
@@ -157,7 +157,7 @@ defmodule Tortoise.Connection.InflightTest do
 
       Inflight.update(client_id, {:received, suback})
 
-      assert_receive {{Tortoise, ^client_id}, ^ref, _}
+      assert_receive {{Tortoise, ^client_id}, {Package.Subscribe, ^ref}, _}
     end
   end
 
@@ -184,7 +184,7 @@ defmodule Tortoise.Connection.InflightTest do
       # when receiving the suback we should respond to the caller
       Inflight.update(client_id, {:received, %Package.Unsuback{identifier: 1}})
 
-      assert_receive {{Tortoise, ^client_id}, ^ref, _}
+      assert_receive {{Tortoise, ^client_id}, {Package.Unsubscribe, ^ref}, _}
     end
   end
 
diff --git a/test/tortoise/connection/receiver_test.exs b/test/tortoise/connection/receiver_test.exs
index c45f42b2..60af9702 100644
--- a/test/tortoise/connection/receiver_test.exs
+++ b/test/tortoise/connection/receiver_test.exs
@@ -1,30 +1,35 @@
 defmodule Tortoise.Connection.ReceiverTest do
   use ExUnit.Case
-  # use EQC.ExUnit
-  doctest Tortoise.Connection.Controller
+
+  doctest Tortoise.Connection.Receiver
 
   alias Tortoise.Package
-  alias Tortoise.Connection.{Receiver, Controller}
+  alias Tortoise.Connection.Receiver
+  alias Tortoise.Integration.TestTCPTunnel
 
   setup context do
     {:ok, %{client_id: context.test}}
   end
 
   def setup_receiver(context) do
-    opts = [client_id: context.client_id]
-    {:ok, client_socket, server_socket} = Tortoise.Integration.TestTCPTunnel.new()
+    {:ok, ref, transport} = TestTCPTunnel.new(Tortoise.Transport.Tcp)
+    opts = [client_id: context.client_id, transport: transport, parent: self()]
     {:ok, receiver_pid} = Receiver.start_link(opts)
-    :ok = Receiver.handle_socket(context.client_id, {Tortoise.Transport.Tcp, client_socket})
-    {:ok, %{receiver_pid: receiver_pid, client: client_socket, server: server_socket}}
+    {:ok, %{connection_ref: ref, transport: transport, receiver_pid: receiver_pid}}
+  end
+
+  def setup_connection(%{connection_ref: ref} = context) when is_reference(ref) do
+    {:ok, _connection} = Receiver.connect(context.client_id)
+    assert_receive {:server_socket, ^ref, server_socket}
+    {:ok, Map.put(context, :server, server_socket)}
   end
 
-  def setup_controller(context) do
-    Registry.register(Tortoise.Registry, {Controller, context.client_id}, self())
-    :ok
+  def setup_connection(_) do
+    raise "run `:setup_receiver/1` before `:setup_connection/1` in the test setup"
   end
 
   describe "receiving" do
-    setup [:setup_receiver, :setup_controller]
+    setup [:setup_receiver, :setup_connection]
 
     # when a message reached a certain size an issue where the message
     # got chunked on the connection lead to a crash in the
@@ -38,18 +43,18 @@ defmodule Tortoise.Connection.ReceiverTest do
 
       :ok = :gen_tcp.send(context.server, Package.encode(package))
 
-      assert_receive {:"$gen_cast", {:incoming, data}}
+      assert_receive {:incoming, data}
       assert ^package = Package.decode(data)
     end
 
     test "receive a larger message of about 5000 bytes", context do
-      # payload = :crypto.strong_rand_bytes(268_435_446)
+      # payload = :crypto.strong_rand_bytes(268_435_146)
       payload = :crypto.strong_rand_bytes(5000)
       package = %Package.Publish{topic: "foo/bar", payload: payload}
 
       :ok = :gen_tcp.send(context.server, Package.encode(package))
 
-      assert_receive {:"$gen_cast", {:incoming, data}}, 10000
+      assert_receive {:incoming, data}, 10_000
       assert ^package = Package.decode(data)
     end
 
@@ -62,21 +67,20 @@ defmodule Tortoise.Connection.ReceiverTest do
       :ok = :gen_tcp.send(context.server, <<0b11010000>>)
       refute_receive {:EXIT, ^receiver_pid, {:protocol_violation, :invalid_header_length}}, 400
       :ok = :gen_tcp.send(context.server, <<0>>)
-      assert_receive {:"$gen_cast", {:incoming, data}}, 10000
+      assert_receive {:incoming, data}, 10000
       assert %Package.Pingresp{} = Package.decode(data)
     end
   end
 
   describe "invalid packages" do
-    setup [:setup_receiver]
+    setup [:setup_receiver, :setup_connection]
 
-    test "invalid header length", context do
+    test "invalid header length", %{receiver_pid: receiver_pid} = context do
       Process.flag(:trap_exit, true)
-      receiver_pid = context.receiver_pid
       # send too many bytes into the receiver, the header parser
       # should throw a protocol violation on this
       :ok = :gen_tcp.send(context.server, <<1, 255, 255, 255, 255, 0>>)
-      assert_receive {:EXIT, ^receiver_pid, {:protocol_violation, :invalid_header_length}}
+      assert_receive {:EXIT, ^receiver_pid, {:protocol_violation, :invalid_header_length}}, 5000
     end
   end
 end
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index 52cabb71..28d32994 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -5,9 +5,9 @@ defmodule Tortoise.ConnectionTest do
   use ExUnit.Case, async: true
   doctest Tortoise.Connection
 
-  alias Tortoise.Integration.ScriptedMqttServer
-  alias Tortoise.Integration.ScriptedTransport
+  alias Tortoise.Integration.{ScriptedMqttServer, ScriptedTransport}
   alias Tortoise.Connection
+  alias Tortoise.Connection.Inflight
   alias Tortoise.Package
 
   setup context do
@@ -46,14 +46,39 @@ defmodule Tortoise.ConnectionTest do
      }}
   end
 
+  def setup_connection_and_perform_handshake(%{
+        client_id: client_id,
+        scripted_mqtt_server: scripted_mqtt_server
+      }) do
+    script = [
+      {:receive, %Package.Connect{client_id: client_id}},
+      {:send, %Package.Connack{reason: :success, session_present: false}}
+    ]
+
+    {:ok, {ip, port}} = ScriptedMqttServer.enact(scripted_mqtt_server, script)
+
+    opts = [
+      client_id: client_id,
+      server: {Tortoise.Transport.Tcp, [host: ip, port: port]},
+      handler: {Tortoise.Handler.Default, []}
+    ]
+
+    assert {:ok, connection_pid} = Connection.start_link(opts)
+
+    assert_receive {ScriptedMqttServer, {:received, %Package.Connect{}}}
+    assert_receive {ScriptedMqttServer, :completed}
+
+    {:ok, %{connection_pid: connection_pid}}
+  end
+
   describe "successful connect" do
     setup [:setup_scripted_mqtt_server]
 
     test "without present state", context do
       client_id = context.client_id
 
-      connect = %Package.Connect{client_id: client_id, clean_session: true}
-      expected_connack = %Package.Connack{status: :accepted, session_present: false}
+      connect = %Package.Connect{client_id: client_id, clean_start: true}
+      expected_connack = %Package.Connack{reason: :success, session_present: false}
 
       script = [{:receive, connect}, {:send, expected_connack}]
 
@@ -73,15 +98,15 @@ defmodule Tortoise.ConnectionTest do
     test "reconnect with present state", context do
       client_id = context.client_id
 
-      connect = %Package.Connect{client_id: client_id, clean_session: true}
-      reconnect = %Package.Connect{connect | clean_session: false}
+      connect = %Package.Connect{client_id: client_id, clean_start: true}
+      reconnect = %Package.Connect{connect | clean_start: false}
 
       script = [
         {:receive, connect},
-        {:send, %Package.Connack{status: :accepted, session_present: false}},
+        {:send, %Package.Connack{reason: :success, session_present: false}},
         :disconnect,
         {:receive, reconnect},
-        {:send, %Package.Connack{status: :accepted, session_present: true}}
+        {:send, %Package.Connack{reason: :success, session_present: true}}
       ]
 
       {:ok, {ip, port}} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
@@ -102,7 +127,7 @@ defmodule Tortoise.ConnectionTest do
   describe "unsuccessful connect" do
     setup [:setup_scripted_mqtt_server]
 
-    test "unacceptable protocol version", context do
+    test "unsupported protocol version", context do
       Process.flag(:trap_exit, true)
       client_id = context.client_id
 
@@ -110,7 +135,7 @@ defmodule Tortoise.ConnectionTest do
 
       script = [
         {:receive, connect},
-        {:send, %Package.Connack{status: {:refused, :unacceptable_protocol_version}}}
+        {:send, %Package.Connack{reason: {:refused, :unsupported_protocol_version}}}
       ]
 
       true = Process.unlink(context.scripted_mqtt_server)
@@ -126,15 +151,15 @@ defmodule Tortoise.ConnectionTest do
 
       assert_receive {ScriptedMqttServer, {:received, ^connect}}
       assert_receive {ScriptedMqttServer, :completed}
-      assert_receive {:EXIT, ^pid, {:connection_failed, :unacceptable_protocol_version}}
+      assert_receive {:EXIT, ^pid, {:connection_failed, :unsupported_protocol_version}}
     end
 
-    test "identifier rejected", context do
+    test "reject client identifier", context do
       Process.flag(:trap_exit, true)
       client_id = context.client_id
 
       connect = %Package.Connect{client_id: client_id}
-      expected_connack = %Package.Connack{status: {:refused, :identifier_rejected}}
+      expected_connack = %Package.Connack{reason: {:refused, :client_identifier_not_valid}}
 
       script = [{:receive, connect}, {:send, expected_connack}]
       {:ok, {ip, port}} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
@@ -148,7 +173,7 @@ defmodule Tortoise.ConnectionTest do
       assert {:ok, pid} = Connection.start_link(opts)
       assert_receive {ScriptedMqttServer, {:received, ^connect}}
       assert_receive {ScriptedMqttServer, :completed}
-      assert_receive {:EXIT, ^pid, {:connection_failed, :identifier_rejected}}
+      assert_receive {:EXIT, ^pid, {:connection_failed, :client_identifier_not_valid}}
     end
 
     test "server unavailable", context do
@@ -156,7 +181,7 @@ defmodule Tortoise.ConnectionTest do
       client_id = context.client_id
 
       connect = %Package.Connect{client_id: client_id}
-      expected_connack = %Package.Connack{status: {:refused, :server_unavailable}}
+      expected_connack = %Package.Connack{reason: {:refused, :server_unavailable}}
 
       script = [{:receive, connect}, {:send, expected_connack}]
       {:ok, {ip, port}} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
@@ -178,8 +203,7 @@ defmodule Tortoise.ConnectionTest do
       client_id = context.client_id
 
       connect = %Package.Connect{client_id: client_id}
-      expected_connack = %Package.Connack{status: {:refused, :bad_user_name_or_password}}
-
+      expected_connack = %Package.Connack{reason: {:refused, :bad_user_name_or_password}}
       script = [{:receive, connect}, {:send, expected_connack}]
       {:ok, {ip, port}} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
 
@@ -200,7 +224,7 @@ defmodule Tortoise.ConnectionTest do
       client_id = context.client_id
 
       connect = %Package.Connect{client_id: client_id}
-      expected_connack = %Package.Connack{status: {:refused, :not_authorized}}
+      expected_connack = %Package.Connack{reason: {:refused, :not_authorized}}
 
       script = [{:receive, connect}, {:send, expected_connack}]
       {:ok, {ip, port}} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
@@ -225,14 +249,14 @@ defmodule Tortoise.ConnectionTest do
     test "successful subscription", context do
       client_id = context.client_id
 
-      connect = %Package.Connect{client_id: client_id, clean_session: true}
+      connect = %Package.Connect{client_id: client_id, clean_start: true}
       subscription_foo = Enum.into([{"foo", 0}], %Package.Subscribe{identifier: 1})
       subscription_bar = Enum.into([{"bar", 1}], %Package.Subscribe{identifier: 2})
       subscription_baz = Enum.into([{"baz", 2}], %Package.Subscribe{identifier: 3})
 
       script = [
         {:receive, connect},
-        {:send, %Package.Connack{status: :accepted, session_present: false}},
+        {:send, %Package.Connack{reason: :success, session_present: false}},
         # subscribe to foo with qos 0
         {:receive, subscription_foo},
         {:send, %Package.Suback{identifier: 1, acks: [{:ok, 0}]}},
@@ -283,13 +307,13 @@ defmodule Tortoise.ConnectionTest do
     test "successful unsubscribe", context do
       client_id = context.client_id
 
-      connect = %Package.Connect{client_id: client_id, clean_session: true}
+      connect = %Package.Connect{client_id: client_id, clean_start: true}
       unsubscribe_foo = %Package.Unsubscribe{identifier: 2, topics: ["foo"]}
       unsubscribe_bar = %Package.Unsubscribe{identifier: 3, topics: ["bar"]}
 
       script = [
         {:receive, connect},
-        {:send, %Package.Connack{status: :accepted, session_present: false}},
+        {:send, %Package.Connack{reason: :success, session_present: false}},
         {:receive, %Package.Subscribe{topics: [{"foo", 0}, {"bar", 2}], identifier: 1}},
         {:send, %Package.Suback{acks: [ok: 0, ok: 2], identifier: 1}},
         # unsubscribe foo
@@ -339,8 +363,8 @@ defmodule Tortoise.ConnectionTest do
     test "successful connect", context do
       client_id = context.client_id
 
-      connect = %Package.Connect{client_id: client_id, clean_session: true}
-      expected_connack = %Package.Connack{status: :accepted, session_present: false}
+      connect = %Package.Connect{client_id: client_id, clean_start: true}
+      expected_connack = %Package.Connack{reason: :success, session_present: false}
 
       script = [{:receive, connect}, {:send, expected_connack}]
       {:ok, {ip, port}} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
@@ -369,8 +393,8 @@ defmodule Tortoise.ConnectionTest do
     test "successful connect (no certificate verification)", context do
       client_id = context.client_id
 
-      connect = %Package.Connect{client_id: client_id, clean_session: true}
-      expected_connack = %Package.Connack{status: :accepted, session_present: false}
+      connect = %Package.Connect{client_id: client_id, clean_start: true}
+      expected_connack = %Package.Connack{reason: :success, session_present: false}
 
       script = [{:receive, connect}, {:send, expected_connack}]
       {:ok, {ip, port}} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
@@ -424,8 +448,8 @@ defmodule Tortoise.ConnectionTest do
     test "nxdomain", context do
       client_id = context.client_id
 
-      connect = %Package.Connect{client_id: client_id, clean_session: true}
-      expected_connack = %Package.Connack{status: :accepted, session_present: false}
+      connect = %Package.Connect{client_id: client_id, clean_start: true}
+      expected_connack = %Package.Connack{reason: :success, session_present: false}
       refusal = {:error, :nxdomain}
 
       {:ok, _} =
@@ -462,8 +486,8 @@ defmodule Tortoise.ConnectionTest do
       Process.flag(:trap_exit, true)
       client_id = context.client_id
 
-      connect = %Package.Connect{client_id: client_id, clean_session: true}
-      expected_connack = %Package.Connack{status: :accepted, session_present: false}
+      connect = %Package.Connect{client_id: client_id, clean_start: true}
+      expected_connack = %Package.Connack{reason: :success, session_present: false}
       refusal = {:error, :econnrefused}
 
       {:ok, _pid} =
@@ -478,7 +502,7 @@ defmodule Tortoise.ConnectionTest do
             {:refute_connection, refusal},
             {:refute_connection, refusal},
             # finally start accepting connections again
-            {:expect, %Package.Connect{connect | clean_session: false}},
+            {:expect, %Package.Connect{connect | clean_start: false}},
             {:dispatch, expected_connack}
           ]
         )
@@ -503,7 +527,7 @@ defmodule Tortoise.ConnectionTest do
       Process.flag(:trap_exit, true)
       client_id = context.client_id
 
-      connect = %Package.Connect{client_id: client_id, clean_session: true}
+      connect = %Package.Connect{client_id: client_id, clean_start: true}
 
       {:ok, _pid} =
         ScriptedTransport.start_link(
@@ -523,8 +547,9 @@ defmodule Tortoise.ConnectionTest do
 
       assert_receive {ScriptedTransport, :connected}
       assert_receive {ScriptedTransport, {:received, %Package.Connect{}}}
+
       assert_receive {:EXIT, ^pid, {:protocol_violation, violation}}
-      assert %{expected: Tortoise.Package.Connect, got: _} = violation
+      assert %{expected: [Tortoise.Package.Connack, Tortoise.Package.Auth], got: _} = violation
       assert_receive {ScriptedTransport, :completed}
     end
   end
@@ -539,8 +564,8 @@ defmodule Tortoise.ConnectionTest do
     test "receive a socket from a connection", context do
       client_id = context.client_id
 
-      connect = %Package.Connect{client_id: client_id, clean_session: true}
-      expected_connack = %Package.Connack{status: :accepted, session_present: false}
+      connect = %Package.Connect{client_id: client_id, clean_start: true}
+      expected_connack = %Package.Connack{reason: :success, session_present: false}
 
       script = [{:receive, connect}, {:send, expected_connack}]
 
@@ -564,7 +589,7 @@ defmodule Tortoise.ConnectionTest do
     test "timeout on a socket from a connection", context do
       client_id = context.client_id
 
-      connect = %Package.Connect{client_id: client_id, clean_session: true}
+      connect = %Package.Connect{client_id: client_id, clean_start: true}
 
       script = [{:receive, connect}, :pause]
 
@@ -595,7 +620,7 @@ defmodule Tortoise.ConnectionTest do
       client_id = context.client_id
 
       connect = %Package.Connect{client_id: client_id}
-      expected_connack = %Package.Connack{status: :accepted, session_present: false}
+      expected_connack = %Package.Connack{reason: :success, session_present: false}
       disconnect = %Package.Disconnect{}
 
       script = [{:receive, connect}, {:send, expected_connack}, {:receive, disconnect}]
@@ -610,12 +635,204 @@ defmodule Tortoise.ConnectionTest do
 
       assert {:ok, pid} = Connection.start_link(opts)
       assert_receive {ScriptedMqttServer, {:received, ^connect}}
-
       assert :ok = Tortoise.Connection.disconnect(client_id)
+
       assert_receive {ScriptedMqttServer, {:received, ^disconnect}}
       assert_receive {:EXIT, ^pid, :shutdown}
 
       assert_receive {ScriptedMqttServer, :completed}
     end
   end
+
+  describe "ping" do
+    setup [:setup_scripted_mqtt_server]
+
+    test "send pingreq and receive a pingresp", context do
+      Process.flag(:trap_exit, true)
+      client_id = context.client_id
+
+      connect = %Package.Connect{client_id: client_id}
+      expected_connack = %Package.Connack{reason: :success, session_present: false}
+      ping_request = %Package.Pingreq{}
+      expected_pingresp = %Package.Pingresp{}
+
+      script = [
+        {:receive, connect},
+        {:send, expected_connack},
+        {:receive, ping_request},
+        {:send, expected_pingresp}
+      ]
+
+      {:ok, {ip, port}} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
+
+      opts = [
+        client_id: client_id,
+        server: {Tortoise.Transport.Tcp, [host: ip, port: port]},
+        handler: {Tortoise.Handler.Default, []}
+      ]
+
+      {:ok, _pid} = Tortoise.Events.register(client_id, :status)
+
+      assert {:ok, pid} = Connection.start_link(opts)
+      assert_receive {ScriptedMqttServer, {:received, ^connect}}
+
+      # assure that we are connected
+      assert_receive {{Tortoise, ^client_id}, :status, :connected}
+
+      {:ok, ref} = Connection.ping(client_id)
+      assert_receive {ScriptedMqttServer, {:received, ^ping_request}}
+
+      assert_receive {Tortoise, {:ping_response, ^ref, _}}
+      assert_receive {ScriptedMqttServer, :completed}
+    end
+  end
+
+  describe "Protocol violations" do
+    setup [:setup_scripted_mqtt_server, :setup_connection_and_perform_handshake]
+
+    test "Receiving a connect from the server is a protocol violation", context do
+      Process.flag(:trap_exit, true)
+      unexpected_connect = %Package.Connect{client_id: "foo"}
+      script = [{:send, unexpected_connect}]
+
+      {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
+      pid = context.connection_pid
+
+      assert_receive {:EXIT, ^pid,
+                      {:protocol_violation, {:unexpected_package, ^unexpected_connect}}}
+    end
+
+    test "Receiving a connack after the handshake is a protocol violation", context do
+      Process.flag(:trap_exit, true)
+      unexpected_connack = %Package.Connack{reason: :success}
+      script = [{:send, unexpected_connack}]
+
+      {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
+      pid = context.connection_pid
+
+      assert_receive {:EXIT, ^pid,
+                      {:protocol_violation, {:unexpected_package, ^unexpected_connack}}}
+    end
+
+    test "Receiving a ping request from the server is a protocol violation", context do
+      Process.flag(:trap_exit, true)
+      unexpected_pingreq = %Package.Pingreq{}
+      script = [{:send, unexpected_pingreq}]
+
+      {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
+      pid = context.connection_pid
+
+      assert_receive {:EXIT, ^pid,
+                      {:protocol_violation, {:unexpected_package, ^unexpected_pingreq}}}
+    end
+  end
+
+  describe "Publish with QoS=0" do
+    setup [:setup_scripted_mqtt_server, :setup_connection_and_perform_handshake]
+
+    test "Receiving a publish", context do
+      Process.flag(:trap_exit, true)
+      publish = %Package.Publish{topic: "foo/bar", qos: 0}
+
+      script = [{:send, publish}]
+
+      {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
+      pid = context.connection_pid
+
+      refute_receive {:EXIT, ^pid, {:protocol_violation, {:unexpected_package, ^publish}}}
+      assert_receive {ScriptedMqttServer, :completed}
+    end
+  end
+
+  describe "Publish with QoS=1" do
+    setup [:setup_scripted_mqtt_server, :setup_connection_and_perform_handshake]
+
+    test "incoming publish with QoS=1", context do
+      Process.flag(:trap_exit, true)
+      publish = %Package.Publish{identifier: 1, topic: "foo/bar", qos: 1}
+      expected_puback = %Package.Puback{identifier: 1}
+
+      script = [
+        {:send, publish},
+        {:receive, expected_puback}
+      ]
+
+      {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
+      assert_receive {ScriptedMqttServer, :completed}
+    end
+
+    test "outgoing publish with QoS=1", context do
+      Process.flag(:trap_exit, true)
+      publish = %Package.Publish{identifier: 1, topic: "foo/bar", qos: 1}
+      puback = %Package.Puback{identifier: 1}
+
+      script = [
+        {:receive, publish},
+        {:send, puback}
+      ]
+
+      {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
+      pid = context.connection_pid
+
+      client_id = context.client_id
+      assert {:ok, ref} = Inflight.track(client_id, {:outgoing, publish})
+
+      refute_receive {:EXIT, ^pid, {:protocol_violation, {:unexpected_package, _}}}
+      assert_receive {ScriptedMqttServer, {:received, ^publish}}
+      assert_receive {ScriptedMqttServer, :completed}
+      assert_receive {{Tortoise, ^client_id}, {Package.Publish, ^ref}, :ok}
+    end
+  end
+
+  describe "Publish with QoS=2" do
+    setup [:setup_scripted_mqtt_server, :setup_connection_and_perform_handshake]
+
+    test "incoming publish with QoS=2", context do
+      Process.flag(:trap_exit, true)
+      publish = %Package.Publish{identifier: 1, topic: "foo/bar", qos: 2}
+      expected_pubrec = %Package.Pubrec{identifier: 1}
+      pubrel = %Package.Pubrel{identifier: 1}
+      expected_pubcomp = %Package.Pubcomp{identifier: 1}
+
+      script = [
+        {:send, publish},
+        {:receive, expected_pubrec},
+        {:send, pubrel},
+        {:receive, expected_pubcomp}
+      ]
+
+      {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
+      pid = context.connection_pid
+
+      refute_receive {:EXIT, ^pid, {:protocol_violation, {:unexpected_package, _}}}
+      assert_receive {ScriptedMqttServer, :completed}
+    end
+
+    test "outgoing publish with QoS=2", context do
+      Process.flag(:trap_exit, true)
+      publish = %Package.Publish{identifier: 1, topic: "foo/bar", qos: 2}
+      pubrec = %Package.Pubrec{identifier: 1}
+      pubrel = %Package.Pubrel{identifier: 1}
+      pubcomp = %Package.Pubcomp{identifier: 1}
+
+      script = [
+        {:receive, publish},
+        {:send, pubrec},
+        {:receive, pubrel},
+        {:send, pubcomp}
+      ]
+
+      {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
+      pid = context.connection_pid
+
+      client_id = context.client_id
+      assert {:ok, ref} = Inflight.track(client_id, {:outgoing, publish})
+
+      refute_receive {:EXIT, ^pid, {:protocol_violation, {:unexpected_package, _}}}
+      assert_receive {ScriptedMqttServer, {:received, ^publish}}
+      assert_receive {ScriptedMqttServer, {:received, ^pubrel}}
+      assert_receive {ScriptedMqttServer, :completed}
+      assert_receive {{Tortoise, ^client_id}, {Package.Publish, ^ref}, :ok}
+    end
+  end
 end

From 4627abe88627a570f55c1b47ecc76bd059535cbd Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Wed, 3 Oct 2018 21:25:04 +0200
Subject: [PATCH 008/220] Update the protocol packages and their
 encoders/decoders to MQTT 5

---
 lib/tortoise/package/connack.ex         | 112 +++++++++++++-----
 lib/tortoise/package/connect.ex         | 146 +++++++++++++++++-------
 lib/tortoise/package/disconnect.ex      |   6 +-
 lib/tortoise/package/properties.ex      |   2 +
 lib/tortoise/package/puback.ex          |  83 ++++++++++++--
 lib/tortoise/package/pubcomp.ex         |  50 +++++++-
 lib/tortoise/package/pubrec.ex          |  86 +++++++++++++-
 lib/tortoise/package/pubrel.ex          |  50 +++++++-
 lib/tortoise/package/suback.ex          |  77 +++++++++++--
 lib/tortoise/package/subscribe.ex       |  69 ++++++++---
 lib/tortoise/package/unsuback.ex        |  86 ++++++++++++--
 lib/tortoise/package/unsubscribe.ex     |  23 ++--
 test/tortoise/package/puback_test.exs   |  16 +++
 test/tortoise/package/pubcomp_test.exs  |  16 +++
 test/tortoise/package/pubrec_test.exs   |  16 +++
 test/tortoise/package/pubrel_test.exs   |  16 +++
 test/tortoise/package/unsuback_test.exs |  16 +++
 17 files changed, 738 insertions(+), 132 deletions(-)

diff --git a/lib/tortoise/package/connack.ex b/lib/tortoise/package/connack.ex
index d8180986..f0ac54f6 100644
--- a/lib/tortoise/package/connack.ex
+++ b/lib/tortoise/package/connack.ex
@@ -7,61 +7,117 @@ defmodule Tortoise.Package.Connack do
 
   alias Tortoise.Package
 
-  @type status :: :accepted | {:refused, refusal_reasons()}
+  @type reason :: :success | {:refused, refusal_reasons()}
   @type refusal_reasons ::
-          :unacceptable_protocol_version
-          | :identifier_rejected
-          | :server_unavailable
+          :unspecified_error
+          | :malformed_packet
+          | :protocol_error
+          | :implementation_specific_error
+          | :unsupported_protocol_version
+          | :client_identifier_not_valid
           | :bad_user_name_or_password
           | :not_authorized
+          | :server_unavailable
+          | :server_busy
+          | :banned
+          | :bad_authentication_method
+          | :topic_name_invalid
+          | :packet_too_large
+          | :quota_exceeded
+          | :payload_format_invalid
+          | :retain_not_supported
+          | :qos_not_supported
+          | :use_another_server
+          | :server_moved
+          | :connection_rate_exceeded
 
   @opaque t :: %__MODULE__{
             __META__: Package.Meta.t(),
             session_present: boolean(),
-            status: status() | nil
+            reason: reason(),
+            properties: [{any(), any()}]
           }
-  @enforce_keys [:status]
+  @enforce_keys [:reason]
   defstruct __META__: %Package.Meta{opcode: @opcode, flags: 0},
             session_present: false,
-            status: nil
+            reason: :success,
+            properties: []
+
+  @spec decode(binary()) :: t
+  def decode(<<@opcode::4, 0::4, variable_header::binary>>) do
+    <<0::7, session_present::1, reason_code::8, properties::binary>> =
+      Package.drop_length_prefix(variable_header)
 
-  @spec decode(<<_::32>>) :: t
-  def decode(<<@opcode::4, 0::4, 2, 0::7, session_present::1, return_code::8>>) do
     %__MODULE__{
       session_present: session_present == 1,
-      status: coerce_return_code(return_code)
+      reason: coerce_reason_code(reason_code),
+      properties: Package.Properties.decode(properties)
     }
   end
 
-  defp coerce_return_code(return_code) do
-    case return_code do
-      0x00 -> :accepted
-      0x01 -> {:refused, :unacceptable_protocol_version}
-      0x02 -> {:refused, :identifier_rejected}
-      0x03 -> {:refused, :server_unavailable}
-      0x04 -> {:refused, :bad_user_name_or_password}
-      0x05 -> {:refused, :not_authorized}
+  defp coerce_reason_code(reason_code) do
+    case reason_code do
+      0x00 -> :success
+      0x80 -> {:refused, :unspecified_error}
+      0x81 -> {:refused, :malformed_packet}
+      0x82 -> {:refused, :protocol_error}
+      0x83 -> {:refused, :implementation_specific_error}
+      0x84 -> {:refused, :unsupported_protocol_version}
+      0x85 -> {:refused, :client_identifier_not_valid}
+      0x86 -> {:refused, :bad_user_name_or_password}
+      0x87 -> {:refused, :not_authorized}
+      0x88 -> {:refused, :server_unavailable}
+      0x89 -> {:refused, :server_busy}
+      0x8A -> {:refused, :banned}
+      0x8C -> {:refused, :bad_authentication_method}
+      0x90 -> {:refused, :topic_name_invalid}
+      0x95 -> {:refused, :packet_too_large}
+      0x97 -> {:refused, :quota_exceeded}
+      0x99 -> {:refused, :payload_format_invalid}
+      0x9A -> {:refused, :retain_not_supported}
+      0x9B -> {:refused, :qos_not_supported}
+      0x9C -> {:refused, :use_another_server}
+      0x9D -> {:refused, :server_moved}
+      0x9F -> {:refused, :connection_rate_exceeded}
     end
   end
 
   defimpl Tortoise.Encodable do
-    def encode(%Package.Connack{session_present: session_present, status: status} = t)
-        when status != nil do
+    def encode(%Package.Connack{} = t) do
       [
         Package.Meta.encode(t.__META__),
-        <<2, 0::7, flag(session_present)::1, to_return_code(status)::8>>
+        Package.variable_length_encode([
+          <<0::7, flag(t.session_present)::1, to_reason_code(t.reason)::8>>,
+          Package.Properties.encode(t.properties)
+        ])
       ]
     end
 
-    defp to_return_code(:accepted), do: 0x00
+    defp to_reason_code(:success), do: 0x00
 
-    defp to_return_code({:refused, reason}) do
+    defp to_reason_code({:refused, reason}) do
       case reason do
-        :unacceptable_protocol_version -> 0x01
-        :identifier_rejected -> 0x02
-        :server_unavailable -> 0x03
-        :bad_user_name_or_password -> 0x04
-        :not_authorized -> 0x05
+        :unspecified_error -> 0x80
+        :malformed_packet -> 0x81
+        :protocol_error -> 0x82
+        :implementation_specific_error -> 0x83
+        :unsupported_protocol_version -> 0x84
+        :client_identifier_not_valid -> 0x85
+        :bad_user_name_or_password -> 0x86
+        :not_authorized -> 0x87
+        :server_unavailable -> 0x88
+        :server_busy -> 0x89
+        :banned -> 0x8A
+        :bad_authentication_method -> 0x8C
+        :topic_name_invalid -> 0x90
+        :packet_too_large -> 0x95
+        :quota_exceeded -> 0x97
+        :payload_format_invalid -> 0x99
+        :retain_not_supported -> 0x9A
+        :qos_not_supported -> 0x9B
+        :use_another_server -> 0x9C
+        :server_moved -> 0x9D
+        :connection_rate_exceeded -> 0x9F
       end
     end
 
diff --git a/lib/tortoise/package/connect.ex b/lib/tortoise/package/connect.ex
index 7dd1e7b7..d012c378 100644
--- a/lib/tortoise/package/connect.ex
+++ b/lib/tortoise/package/connect.ex
@@ -3,7 +3,17 @@ defmodule Tortoise.Package.Connect do
 
   @opcode 1
 
-  # @allowed_properties [:authentication_data, :authentication_method, :maximum_packet_size, :receive_maximum, :request_problem_information, :request_response_information, :session_expiry_interval, :topic_alias_maximum, :user_property]
+  @allowed_properties [
+    :authentication_data,
+    :authentication_method,
+    :maximum_packet_size,
+    :receive_maximum,
+    :request_problem_information,
+    :request_response_information,
+    :session_expiry_interval,
+    :topic_alias_maximum,
+    :user_property
+  ]
 
   alias Tortoise.Package
 
@@ -13,58 +23,102 @@ defmodule Tortoise.Package.Connect do
             protocol_version: non_neg_integer(),
             user_name: binary() | nil,
             password: binary() | nil,
-            clean_session: boolean(),
+            clean_start: boolean(),
             keep_alive: non_neg_integer(),
             client_id: Tortoise.client_id(),
-            will: Package.Publish.t() | nil
+            will: Package.Publish.t() | nil,
+            properties: [{any(), any()}]
           }
   @enforce_keys [:client_id]
   defstruct __META__: %Package.Meta{opcode: @opcode},
             protocol: "MQTT",
-            protocol_version: 0b00000100,
+            protocol_version: 0b00000101,
             user_name: nil,
             password: nil,
-            clean_session: true,
+            clean_start: true,
             keep_alive: 60,
             client_id: nil,
-            will: nil
+            will: nil,
+            properties: []
 
   @spec decode(binary()) :: t
   def decode(<<@opcode::4, 0::4, variable::binary>>) do
-    <<4::big-integer-size(16), "MQTT", 4::8, user_name::1, password::1, will_retain::1,
-      will_qos::2, will::1, clean_session::1, 0::1, keep_alive::big-integer-size(16),
-      package::binary>> = drop_length(variable)
-
-    options =
-      [
-        client_id: 1,
-        will_topic: will,
-        will_payload: will,
-        user_name: user_name,
-        password: password
-      ]
-      |> Enum.filter(fn {_, present} -> present == 1 end)
-      |> Enum.map(fn {value, 1} -> value end)
-      |> Enum.zip(decode_length_prefixed(package))
+    <<
+      4::big-integer-size(16),
+      "MQTT",
+      5::8,
+      user_name::1,
+      password::1,
+      will_retain::1,
+      will_qos::2,
+      will::1,
+      clean_start::1,
+      0::1,
+      keep_alive::big-integer-size(16),
+      rest::binary
+    >> = drop_length(variable)
+
+    {properties, package} = Package.parse_variable_length(rest)
+    properties = Package.Properties.decode(properties)
+
+    payload =
+      decode_payload(
+        [
+          client_id: true,
+          will_properties: will == 1,
+          will_topic: will == 1,
+          will_payload: will == 1,
+          user_name: user_name == 1,
+          password: password == 1
+        ],
+        package
+      )
 
     %__MODULE__{
-      client_id: options[:client_id],
-      user_name: options[:user_name],
-      password: options[:password],
+      client_id: payload[:client_id],
+      user_name: payload[:user_name],
+      password: payload[:password],
       will:
         if will == 1 do
           %Package.Publish{
-            topic: options[:will_topic],
-            payload: nullify(options[:will_payload]),
+            topic: payload[:will_topic],
+            payload: nullify(payload[:will_payload]),
             qos: will_qos,
-            retain: will_retain == 1
+            retain: will_retain == 1,
+            properties: payload[:will_properties]
           }
         end,
-      clean_session: clean_session == 1,
-      keep_alive: keep_alive
+      clean_start: clean_start == 1,
+      keep_alive: keep_alive,
+      properties: properties
     }
   end
 
+  defp decode_payload([], <<>>) do
+    []
+  end
+
+  defp decode_payload([{_ignored, false} | remaining_fields], payload) do
+    decode_payload(remaining_fields, payload)
+  end
+
+  defp decode_payload(
+         [{:will_properties, true} | remaining_fields],
+         payload
+       ) do
+    {properties, rest} = Package.parse_variable_length(payload)
+    value = Package.Properties.decode(properties)
+    [{:will_properties, value}] ++ decode_payload(remaining_fields, rest)
+  end
+
+  defp decode_payload(
+         [{field, true} | remaining_fields],
+         <<length::big-integer-size(16), payload::binary>>
+       ) do
+    <<value::binary-size(length), rest::binary>> = payload
+    [{field, value}] ++ decode_payload(remaining_fields, rest)
+  end
+
   defp nullify(""), do: nil
   defp nullify(payload), do: payload
 
@@ -77,13 +131,6 @@ defmodule Tortoise.Package.Connect do
     end
   end
 
-  defp decode_length_prefixed(<<>>), do: []
-
-  defp decode_length_prefixed(<<length::big-integer-size(16), payload::binary>>) do
-    <<item::binary-size(length), rest::binary>> = payload
-    [item] ++ decode_length_prefixed(rest)
-  end
-
   defimpl Tortoise.Encodable do
     def encode(%Package.Connect{client_id: client_id} = t)
         when is_binary(client_id) do
@@ -93,6 +140,7 @@ defmodule Tortoise.Package.Connect do
           protocol_header(t),
           connection_flags(t),
           keep_alive(t),
+          Package.Properties.encode(t.properties),
           payload(t)
         ])
       ]
@@ -117,7 +165,7 @@ defmodule Tortoise.Package.Connect do
         0::integer-size(2),
         # will flag
         flag(0)::integer-size(1),
-        flag(f.clean_session)::integer-size(1),
+        flag(f.clean_start)::integer-size(1),
         # reserved bit
         0::1
       >>
@@ -130,7 +178,7 @@ defmodule Tortoise.Package.Connect do
         flag(f.will.retain)::integer-size(1),
         f.will.qos::integer-size(2),
         flag(f.will.topic)::integer-size(1),
-        flag(f.clean_session)::integer-size(1),
+        flag(f.clean_start)::integer-size(1),
         # reserved bit
         0::1
       >>
@@ -147,11 +195,25 @@ defmodule Tortoise.Package.Connect do
     end
 
     defp payload(f) do
-      will_payload = encode_payload(f.will.payload)
+      options = [
+        f.client_id,
+        f.will.properties,
+        f.will.topic,
+        encode_payload(f.will.payload),
+        f.user_name,
+        f.password
+      ]
 
-      [f.client_id, f.will.topic, will_payload, f.user_name, f.password]
-      |> Enum.filter(&is_binary/1)
-      |> Enum.map(&Package.length_encode/1)
+      for data <- options,
+          data != nil do
+        case data do
+          data when is_binary(data) ->
+            Package.length_encode(data)
+
+          data when is_list(data) ->
+            Package.Properties.encode(data)
+        end
+      end
     end
 
     defp encode_payload(nil), do: ""
diff --git a/lib/tortoise/package/disconnect.ex b/lib/tortoise/package/disconnect.ex
index 5520b117..6509669d 100644
--- a/lib/tortoise/package/disconnect.ex
+++ b/lib/tortoise/package/disconnect.ex
@@ -51,8 +51,7 @@ defmodule Tortoise.Package.Disconnect do
   @spec decode(binary()) :: t
   def decode(<<@opcode::4, 0::4, 0::8>>) do
     # If the Remaining Length is less than 1 the value of 0x00 (Normal
-    # disconnection) is used; this makes it compatible with the MQTT
-    # 3.1.1 disconnect package as well.
+    # disconnection) is used
     %__MODULE__{reason: coerce_reason_code(0x00)}
   end
 
@@ -111,9 +110,6 @@ defmodule Tortoise.Package.Disconnect do
   # Protocols ----------------------------------------------------------
   defimpl Tortoise.Encodable do
     def encode(%Package.Disconnect{reason: :normal_disconnection, properties: []} = t) do
-      # if reason is 0x00 and properties are empty we are allowed to
-      # just return a zero; this will also make the disconnect package
-      # compatible with the MQTT 3.1.1 disconnect package.
       [Package.Meta.encode(t.__META__), 0]
     end
 
diff --git a/lib/tortoise/package/properties.ex b/lib/tortoise/package/properties.ex
index a2f83d54..910c32d2 100644
--- a/lib/tortoise/package/properties.ex
+++ b/lib/tortoise/package/properties.ex
@@ -13,6 +13,8 @@ defmodule Tortoise.Package.Properties do
 
   # User properties are special; we will allow them to be encoded as
   # binaries to make the interface a bit cleaner to the end user
+  #
+  # Todo, revert this decision
   defp encode_property({key, value}) when is_binary(key) do
     [0x26, length_encode(key), length_encode(value)]
   end
diff --git a/lib/tortoise/package/puback.ex b/lib/tortoise/package/puback.ex
index 5a77530b..e351b68c 100644
--- a/lib/tortoise/package/puback.ex
+++ b/lib/tortoise/package/puback.ex
@@ -7,25 +7,94 @@ defmodule Tortoise.Package.Puback do
 
   alias Tortoise.Package
 
+  @type reason :: :success | {:refused, refusal_reasons()}
+  @type refusal_reasons :: :test
+
   @opaque t :: %__MODULE__{
             __META__: Package.Meta.t(),
-            identifier: Tortoise.package_identifier()
+            identifier: Tortoise.package_identifier(),
+            reason: reason(),
+            properties: [{any(), any()}]
           }
   @enforce_keys [:identifier]
   defstruct __META__: %Package.Meta{opcode: @opcode, flags: 0b0000},
-            identifier: nil
+            identifier: nil,
+            reason: :success,
+            properties: []
+
+  @spec decode(binary()) :: t
+  def decode(<<@opcode::4, 0::4, 2, identifier::big-integer-size(16)>>) do
+    %__MODULE__{
+      identifier: identifier,
+      reason: :success,
+      properties: []
+    }
+  end
+
+  def decode(<<@opcode::4, 0::4, variable_header::binary>>) do
+    <<identifier::big-integer-size(16), reason_code::8, properties::binary>> =
+      Package.drop_length_prefix(variable_header)
 
-  @spec decode(<<_::32>>) :: t
-  def decode(<<@opcode::4, 0::4, 2, identifier::big-integer-size(16)>>)
-      when identifier in 0x0001..0xFFFF do
-    %__MODULE__{identifier: identifier}
+    %__MODULE__{
+      identifier: identifier,
+      reason: coerce_reason_code(reason_code),
+      properties: Package.Properties.decode(properties)
+    }
+  end
+
+  defp coerce_reason_code(reason_code) do
+    case reason_code do
+      0x00 -> :success
+      0x10 -> {:refused, :no_matching_subscribers}
+      0x80 -> {:refused, :unspecified_error}
+      0x83 -> {:refused, :implementation_specific_error}
+      0x87 -> {:refused, :not_authorized}
+      0x90 -> {:refused, :topic_Name_invalid}
+      0x91 -> {:refused, :packet_identifier_in_use}
+      0x97 -> {:refused, :quota_exceeded}
+      0x99 -> {:refused, :payload_format_invalid}
+    end
   end
 
   # Protocols ----------------------------------------------------------
   defimpl Tortoise.Encodable do
-    def encode(%Package.Puback{identifier: identifier} = t)
+    def encode(
+          %Package.Puback{
+            identifier: identifier,
+            reason: :success,
+            properties: []
+          } = t
+        )
         when identifier in 0x0001..0xFFFF do
+      # The Reason Code and Property Length can be omitted if the
+      # Reason Code is 0x00 (Success) and there are no Properties
       [Package.Meta.encode(t.__META__), <<2, identifier::big-integer-size(16)>>]
     end
+
+    def encode(%Package.Puback{identifier: identifier} = t)
+        when identifier in 0x0001..0xFFFF do
+      [
+        Package.Meta.encode(t.__META__),
+        Package.variable_length_encode([
+          <<identifier::big-integer-size(16), to_reason_code(t.reason)::8>>,
+          Package.Properties.encode(t.properties)
+        ])
+      ]
+    end
+
+    defp to_reason_code(:success), do: 0x00
+
+    defp to_reason_code({:refused, reason}) do
+      case reason do
+        :no_matching_subscribers -> 0x10
+        :unspecified_error -> 0x80
+        :implementation_specific_error -> 0x83
+        :not_authorized -> 0x87
+        :topic_Name_invalid -> 0x90
+        :packet_identifier_in_use -> 0x91
+        :quota_exceeded -> 0x97
+        :payload_format_invalid -> 0x99
+      end
+    end
   end
 end
diff --git a/lib/tortoise/package/pubcomp.ex b/lib/tortoise/package/pubcomp.ex
index 30ae3b7d..94273471 100644
--- a/lib/tortoise/package/pubcomp.ex
+++ b/lib/tortoise/package/pubcomp.ex
@@ -9,22 +9,64 @@ defmodule Tortoise.Package.Pubcomp do
 
   @opaque t :: %__MODULE__{
             __META__: Package.Meta.t(),
-            identifier: Tortoise.package_identifier()
+            identifier: Tortoise.package_identifier(),
+            reason: :success,
+            properties: []
           }
   defstruct __META__: %Package.Meta{opcode: @opcode, flags: 0},
-            identifier: nil
+            identifier: nil,
+            reason: :success,
+            properties: []
 
-  @spec decode(<<_::32>>) :: t
+  @spec decode(binary()) :: t
   def decode(<<@opcode::4, 0::4, 2, identifier::big-integer-size(16)>>)
       when identifier in 0x0001..0xFFFF do
     %__MODULE__{identifier: identifier}
   end
 
+  def decode(<<@opcode::4, 0::4, variable_header::binary>>) do
+    <<identifier::big-integer-size(16), reason_code::8, properties::binary>> =
+      Package.drop_length_prefix(variable_header)
+
+    %__MODULE__{
+      identifier: identifier,
+      reason: coerce_reason_code(reason_code),
+      properties: Package.Properties.decode(properties)
+    }
+  end
+
+  defp coerce_reason_code(reason_code) do
+    case reason_code do
+      0x00 -> :success
+      0x92 -> {:refused, :packet_identifier_not_found}
+    end
+  end
+
   # Protocols ----------------------------------------------------------
   defimpl Tortoise.Encodable do
-    def encode(%Package.Pubcomp{identifier: identifier} = t)
+    def encode(
+          %Package.Pubcomp{
+            identifier: identifier,
+            reason: :success,
+            properties: []
+          } = t
+        )
         when identifier in 0x0001..0xFFFF do
       [Package.Meta.encode(t.__META__), <<2, t.identifier::big-integer-size(16)>>]
     end
+
+    def encode(%Package.Pubcomp{identifier: identifier} = t)
+        when identifier in 0x0001..0xFFFF do
+      [
+        Package.Meta.encode(t.__META__),
+        Package.variable_length_encode([
+          <<identifier::big-integer-size(16), to_reason_code(t.reason)::8>>,
+          Package.Properties.encode(t.properties)
+        ])
+      ]
+    end
+
+    defp to_reason_code(:success), do: 0x00
+    defp to_reason_code({:refused, :packet_identifier_not_found}), do: 0x92
   end
 end
diff --git a/lib/tortoise/package/pubrec.ex b/lib/tortoise/package/pubrec.ex
index f1adb77f..29fc23c0 100644
--- a/lib/tortoise/package/pubrec.ex
+++ b/lib/tortoise/package/pubrec.ex
@@ -7,24 +7,98 @@ defmodule Tortoise.Package.Pubrec do
 
   alias Tortoise.Package
 
+  @type reason :: :success | {:refused, refusal_reasons()}
+  @type refusal_reasons ::
+          :no_matching_subscribers
+          | :unspecified_error
+          | :implementation_specific_error
+          | :not_authorized
+          | :topic_Name_invalid
+          | :packet_identifier_in_use
+          | :quota_exceeded
+          | :payload_format_invalid
+
   @opaque t :: %__MODULE__{
             __META__: Package.Meta.t(),
-            identifier: Tortoise.package_identifier()
+            identifier: Tortoise.package_identifier(),
+            reason: reason(),
+            properties: [{any(), any()}]
           }
-  defstruct __META__: %Package.Meta{opcode: @opcode, flags: 0b000},
-            identifier: nil
+  defstruct __META__: %Package.Meta{opcode: @opcode, flags: 0b0000},
+            identifier: nil,
+            reason: :success,
+            properties: []
 
-  @spec decode(<<_::32>>) :: t
+  @spec decode(binary()) :: t
   def decode(<<@opcode::4, 0::4, 2, identifier::big-integer-size(16)>>)
       when identifier in 0x0001..0xFFFF do
-    %__MODULE__{identifier: identifier}
+    %__MODULE__{identifier: identifier, reason: :success, properties: []}
+  end
+
+  def decode(<<@opcode::4, 0::4, variable_header::binary>>) do
+    <<identifier::big-integer-size(16), reason_code::8, properties::binary>> =
+      Package.drop_length_prefix(variable_header)
+
+    %__MODULE__{
+      identifier: identifier,
+      reason: coerce_reason_code(reason_code),
+      properties: Package.Properties.decode(properties)
+    }
+  end
+
+  defp coerce_reason_code(reason_code) do
+    case reason_code do
+      0x00 -> :success
+      0x10 -> :no_matching_subscribers
+      0x80 -> :unspecified_error
+      0x83 -> :implementation_specific_error
+      0x87 -> :not_authorized
+      0x90 -> :topic_Name_invalid
+      0x91 -> :packet_identifier_in_use
+      0x97 -> :quota_exceeded
+      0x99 -> :payload_format_invalid
+    end
   end
 
   # Protocols ----------------------------------------------------------
   defimpl Tortoise.Encodable do
+    def encode(
+          %Package.Pubrec{
+            identifier: identifier,
+            reason: :success,
+            properties: []
+          } = t
+        )
+        when identifier in 0x0001..0xFFFF do
+      # The Reason Code and Property Length can be omitted if the
+      # Reason Code is 0x00 (Success) and there are no Properties
+      [Package.Meta.encode(t.__META__), <<2, identifier::big-integer-size(16)>>]
+    end
+
     def encode(%Package.Pubrec{identifier: identifier} = t)
         when identifier in 0x0001..0xFFFF do
-      [Package.Meta.encode(t.__META__), <<2, t.identifier::big-integer-size(16)>>]
+      [
+        Package.Meta.encode(t.__META__),
+        Package.variable_length_encode([
+          <<identifier::big-integer-size(16), to_reason_code(t.reason)::8>>,
+          Package.Properties.encode(t.properties)
+        ])
+      ]
+    end
+
+    defp to_reason_code(:success), do: 0x00
+
+    defp to_reason_code({:refused, reason}) do
+      case reason do
+        :no_matching_subscribers -> 0x10
+        :unspecified_error -> 0x80
+        :implementation_specific_error -> 0x83
+        :not_authorized -> 0x87
+        :topic_Name_invalid -> 0x90
+        :packet_identifier_in_use -> 0x91
+        :quota_exceeded -> 0x97
+        :payload_format_invalid -> 0x99
+      end
     end
   end
 end
diff --git a/lib/tortoise/package/pubrel.ex b/lib/tortoise/package/pubrel.ex
index 71751beb..fdd1123b 100644
--- a/lib/tortoise/package/pubrel.ex
+++ b/lib/tortoise/package/pubrel.ex
@@ -9,11 +9,15 @@ defmodule Tortoise.Package.Pubrel do
 
   @opaque t :: %__MODULE__{
             __META__: Package.Meta.t(),
-            identifier: Tortoise.package_identifier()
+            identifier: Tortoise.package_identifier(),
+            reason: :success | {:refused, :packet_identifier_not_found},
+            properties: []
           }
   @enforce_keys [:identifier]
   defstruct __META__: %Package.Meta{opcode: @opcode, flags: 0b0010},
-            identifier: nil
+            identifier: nil,
+            reason: :success,
+            properties: []
 
   @spec decode(<<_::32>>) :: t
   def decode(<<@opcode::4, 2::4, 2, identifier::big-integer-size(16)>>)
@@ -21,11 +25,51 @@ defmodule Tortoise.Package.Pubrel do
     %__MODULE__{identifier: identifier}
   end
 
+  def decode(<<@opcode::4, 2::4, variable_header::binary>>) do
+    <<identifier::big-integer-size(16), reason_code::8, properties::binary>> =
+      Package.drop_length_prefix(variable_header)
+
+    %__MODULE__{
+      identifier: identifier,
+      reason: coerce_reason_code(reason_code),
+      properties: Package.Properties.decode(properties)
+    }
+  end
+
+  defp coerce_reason_code(reason_code) do
+    case reason_code do
+      0x00 -> :success
+      0x92 -> {:refused, :packet_identifier_not_found}
+    end
+  end
+
   # Protocols ----------------------------------------------------------
   defimpl Tortoise.Encodable do
+    def encode(
+          %Package.Pubrel{
+            identifier: identifier,
+            reason: :success,
+            properties: []
+          } = t
+        )
+        when identifier in 0x0001..0xFFFF do
+      # The Reason Code and Property Length can be omitted if the
+      # Reason Code is 0x00 (Success) and there are no Properties
+      [Package.Meta.encode(t.__META__), <<2, identifier::big-integer-size(16)>>]
+    end
+
     def encode(%Package.Pubrel{identifier: identifier} = t)
         when identifier in 0x0001..0xFFFF do
-      [Package.Meta.encode(t.__META__), <<2, t.identifier::big-integer-size(16)>>]
+      [
+        Package.Meta.encode(t.__META__),
+        Package.variable_length_encode([
+          <<identifier::big-integer-size(16), to_reason_code(t.reason)::8>>,
+          Package.Properties.encode(t.properties)
+        ])
+      ]
     end
+
+    defp to_reason_code(:success), do: 0x00
+    defp to_reason_code({:refused, :packet_identifier_not_found}), do: 0x92
   end
 end
diff --git a/lib/tortoise/package/suback.ex b/lib/tortoise/package/suback.ex
index d797574c..82cffbc3 100644
--- a/lib/tortoise/package/suback.ex
+++ b/lib/tortoise/package/suback.ex
@@ -8,28 +8,46 @@ defmodule Tortoise.Package.Suback do
   alias Tortoise.Package
 
   @type qos :: 0 | 1 | 2
-  @type ack_result :: {:ok, qos} | {:error, :access_denied}
+  @type refusal_reason ::
+          :unspecified_error
+          | :implementation_specific_error
+          | :not_authorized
+          | :topic_filter_invalid
+          | :packet_identifier_in_use
+          | :quota_exceeded
+          | :shared_subscriptions_not_supported
+          | :subscription_identifiers_not_supported
+          | :wildcard_subscriptions_not_supported
+
+  @type ack_result :: {:ok, qos} | {:error, refusal_reason()}
 
   @opaque t :: %__MODULE__{
             __META__: Package.Meta.t(),
             identifier: Tortoise.package_identifier(),
-            acks: [ack_result]
+            acks: [ack_result],
+            properties: [{any(), any()}]
           }
   @enforce_keys [:identifier]
   defstruct __META__: %Package.Meta{opcode: @opcode, flags: 0},
             identifier: nil,
-            acks: []
+            acks: [],
+            properties: []
 
   @spec decode(binary()) :: t
   def decode(<<@opcode::4, 0::4, payload::binary>>) do
     with payload <- drop_length(payload),
-         <<identifier::big-integer-size(16), acks::binary>> <- payload do
+         <<identifier::big-integer-size(16), rest::binary>> <- payload,
+         {properties, acks} = Package.parse_variable_length(rest) do
       case return_codes_to_list(acks) do
         [] ->
           {:error, {:protocol_violation, :empty_subscription_ack}}
 
         sub_acks ->
-          %__MODULE__{identifier: identifier, acks: sub_acks}
+          %__MODULE__{
+            identifier: identifier,
+            acks: sub_acks,
+            properties: Package.Properties.decode(properties)
+          }
       end
     end
   end
@@ -45,11 +63,41 @@ defmodule Tortoise.Package.Suback do
 
   defp return_codes_to_list(<<>>), do: []
 
-  defp return_codes_to_list(<<0x80::integer, acks::binary>>),
-    do: [{:error, :access_denied}] ++ return_codes_to_list(acks)
+  defp return_codes_to_list(<<code::8, rest::binary>>) do
+    [
+      case code do
+        maximum_qos when code in 0x00..0x02 ->
+          {:ok, maximum_qos}
+
+        0x80 ->
+          {:error, :unspecified_error}
+
+        0x83 ->
+          {:error, :implementation_specific_error}
+
+        0x87 ->
+          {:error, :not_authorized}
+
+        0x8F ->
+          {:error, :topic_filter_invalid}
 
-  defp return_codes_to_list(<<ack::integer, acks::binary>>) when ack in 0x00..0x02,
-    do: [{:ok, ack}] ++ return_codes_to_list(acks)
+        0x91 ->
+          {:error, :packet_identifier_in_use}
+
+        0x97 ->
+          {:error, :quota_exceeded}
+
+        0x9E ->
+          {:error, :shared_subscriptions_not_supported}
+
+        0xA1 ->
+          {:error, :subscription_identifiers_not_supported}
+
+        0xA2 ->
+          {:error, :wildcard_subscriptions_not_supported}
+      end
+    ] ++ return_codes_to_list(rest)
+  end
 
   # Protocols ----------------------------------------------------------
   defimpl Tortoise.Encodable do
@@ -59,12 +107,21 @@ defmodule Tortoise.Package.Suback do
         Package.Meta.encode(t.__META__),
         Package.variable_length_encode([
           <<identifier::big-integer-size(16)>>,
+          Package.Properties.encode(t.properties),
           Enum.map(t.acks, &encode_ack/1)
         ])
       ]
     end
 
     defp encode_ack({:ok, qos}) when qos in 0x00..0x02, do: qos
-    defp encode_ack({:error, _}), do: 0x80
+    defp encode_ack({:error, :unspecified_error}), do: 0x80
+    defp encode_ack({:error, :implementation_specific_error}), do: 0x83
+    defp encode_ack({:error, :not_authorized}), do: 0x87
+    defp encode_ack({:error, :topic_filter_invalid}), do: 0x8F
+    defp encode_ack({:error, :packet_identifier_in_use}), do: 0x91
+    defp encode_ack({:error, :quota_exceeded}), do: 0x97
+    defp encode_ack({:error, :shared_subscriptions_not_supported}), do: 0x9E
+    defp encode_ack({:error, :subscription_identifiers_not_supported}), do: 0xA1
+    defp encode_ack({:error, :wildcard_subscriptions_not_supported}), do: 0xA2
   end
 end
diff --git a/lib/tortoise/package/subscribe.ex b/lib/tortoise/package/subscribe.ex
index 5fee1025..e0528c45 100644
--- a/lib/tortoise/package/subscribe.ex
+++ b/lib/tortoise/package/subscribe.ex
@@ -8,24 +8,37 @@ defmodule Tortoise.Package.Subscribe do
   alias Tortoise.Package
 
   @type qos :: 0 | 1 | 2
-  @type topic :: {binary(), qos}
+  @type topic :: {binary(), topic_opts}
+  @type topic_opts :: [
+          {:qos, qos},
+          {:no_local, boolean()},
+          {:retain_as_published, boolean()},
+          {:retain_handling, 0 | 1 | 2}
+        ]
   @type topics :: [topic]
 
   @opaque t :: %__MODULE__{
             __META__: Package.Meta.t(),
             identifier: Tortoise.package_identifier(),
-            topics: topics()
+            topics: topics(),
+            properties: []
           }
   defstruct __META__: %Package.Meta{opcode: @opcode, flags: 0b0010},
             identifier: nil,
-            topics: []
+            topics: [],
+            properties: []
 
   @spec decode(binary()) :: t
   def decode(<<@opcode::4, 0b0010::4, length_prefixed_payload::binary>>) do
     payload = drop_length(length_prefixed_payload)
-    <<identifier::big-integer-size(16), topics::binary>> = payload
-    topic_list = decode_topics(topics)
-    %__MODULE__{identifier: identifier, topics: topic_list}
+    <<identifier::big-integer-size(16), rest::binary>> = payload
+    {properties, topics} = Package.parse_variable_length(rest)
+
+    %__MODULE__{
+      identifier: identifier,
+      topics: decode_topics(topics),
+      properties: Package.Properties.decode(properties)
+    }
   end
 
   defp drop_length(payload) do
@@ -40,8 +53,25 @@ defmodule Tortoise.Package.Subscribe do
   defp decode_topics(<<>>), do: []
 
   defp decode_topics(<<length::big-integer-size(16), rest::binary>>) do
-    <<topic::binary-size(length), return_code::integer-size(8), rest::binary>> = rest
-    [{topic, return_code}] ++ decode_topics(rest)
+    <<
+      topic::binary-size(length),
+      # reserved
+      0::2,
+      retain_handling::2,
+      retain_as_published::1,
+      no_local::1,
+      qos::2,
+      rest::binary
+    >> = rest
+
+    opts = [
+      qos: qos,
+      no_local: no_local == 1,
+      retain_as_published: retain_as_published == 1,
+      retain_handling: retain_handling
+    ]
+
+    [{topic, opts}] ++ decode_topics(rest)
   end
 
   # PROTOCOLS ==========================================================
@@ -49,25 +79,37 @@ defmodule Tortoise.Package.Subscribe do
     def encode(
           %Package.Subscribe{
             identifier: identifier,
-            # a valid subscribe package has at least one topic/qos pair
-            topics: [{<<_topic_filter::binary>>, qos} | _]
+            # a valid subscribe package has at least one topic/qos_opts pair
+            topics: [{<<_topic_filter::binary>>, opts} | _]
           } = t
         )
-        when identifier in 0x0001..0xFFFF and qos in 0..2 do
+        when identifier in 0x0001..0xFFFF and is_list(opts) do
       [
         Package.Meta.encode(t.__META__),
         Package.variable_length_encode([
           <<identifier::big-integer-size(16)>>,
+          Package.Properties.encode(t.properties),
           encode_topics(t.topics)
         ])
       ]
     end
 
     defp encode_topics(topics) do
-      Enum.map(topics, fn {topic, qos} ->
-        [Package.length_encode(topic), <<0::6, qos::2>>]
+      Enum.map(topics, fn {topic, opts} ->
+        qos = Keyword.get(opts, :qos, 0)
+        no_local = Keyword.get(opts, :no_local, false)
+        retain_as_published = Keyword.get(opts, :retain_as_published, false)
+        retain_handling = Keyword.get(opts, :retain_handling, 1)
+
+        [
+          Package.length_encode(topic),
+          <<0::2, retain_handling::2, flag(retain_as_published)::1, flag(no_local)::1, qos::2>>
+        ]
       end)
     end
+
+    defp flag(f) when f in [0, nil, false], do: 0
+    defp flag(_), do: 1
   end
 
   defimpl Enumerable do
@@ -93,6 +135,7 @@ defmodule Tortoise.Package.Subscribe do
     def into(%Package.Subscribe{topics: topics} = source) do
       {Enum.into(topics, %{}),
        fn
+         # @todo, switch to opts instead of qos
          acc, {:cont, {<<topic::binary>>, qos}} when qos in 0..2 ->
            # if a topic filter repeat in the input we will pick the
            # biggest one
diff --git a/lib/tortoise/package/unsuback.ex b/lib/tortoise/package/unsuback.ex
index 200386f2..d7bca837 100644
--- a/lib/tortoise/package/unsuback.ex
+++ b/lib/tortoise/package/unsuback.ex
@@ -7,25 +7,97 @@ defmodule Tortoise.Package.Unsuback do
 
   alias Tortoise.Package
 
+  @type refusal ::
+          :no_subscription_existed
+          | :unspecified_error
+          | :implementation_specific_error
+          | :not_authorized
+          | :topic_filter_invalid
+          | :packet_identifier_in_use
+
+  @type result() :: :success | {:error, refusal()}
+
   @opaque t :: %__MODULE__{
             __META__: Package.Meta.t(),
-            identifier: Tortoise.package_identifier()
+            identifier: Tortoise.package_identifier(),
+            results: [],
+            properties: [{:reason_string, any()}, {:user_property, any()}]
           }
   @enforce_keys [:identifier]
   defstruct __META__: %Package.Meta{opcode: @opcode, flags: 0b0000},
-            identifier: nil
+            identifier: nil,
+            results: [],
+            properties: []
+
+  @spec decode(binary()) :: t | {:error, term()}
+  def decode(<<@opcode::4, 0::4, package::binary>>) do
+    with payload <- drop_length(package),
+         <<identifier::big-integer-size(16), rest::binary>> <- payload,
+         {properties, unsubacks} = Package.parse_variable_length(rest) do
+      case return_codes_to_list(unsubacks) do
+        [] ->
+          {:error, {:protocol_violation, :empty_unsubscription_ack}}
+
+        results ->
+          %__MODULE__{
+            identifier: identifier,
+            results: results,
+            properties: Package.Properties.decode(properties)
+          }
+      end
+    end
+  end
 
-  @spec decode(<<_::32>>) :: t
-  def decode(<<@opcode::4, 0::4, 2, identifier::big-integer-size(16)>>)
-      when identifier in 0x0001..0xFFFF do
-    %__MODULE__{identifier: identifier}
+  defp return_codes_to_list(<<>>), do: []
+
+  defp return_codes_to_list(<<reason::8, rest::binary>>) do
+    [
+      case reason do
+        0x00 -> :success
+        0x11 -> {:error, :no_subscription_existed}
+        0x80 -> {:error, :unspecified_error}
+        0x83 -> {:error, :implementation_specific_error}
+        0x87 -> {:error, :not_authorized}
+        0x8F -> {:error, :topic_filter_invalid}
+        0x91 -> {:error, :packet_identifier_in_use}
+      end
+    ] ++ return_codes_to_list(rest)
+  end
+
+  defp drop_length(payload) do
+    case payload do
+      <<0::1, _::7, r::binary>> -> r
+      <<1::1, _::7, 0::1, _::7, r::binary>> -> r
+      <<1::1, _::7, 1::1, _::7, 0::1, _::7, r::binary>> -> r
+      <<1::1, _::7, 1::1, _::7, 1::1, _::7, 0::1, _::7, r::binary>> -> r
+    end
   end
 
   # Protocols ----------------------------------------------------------
   defimpl Tortoise.Encodable do
     def encode(%Package.Unsuback{identifier: identifier} = t)
         when identifier in 0x0001..0xFFFF do
-      [Package.Meta.encode(t.__META__), <<2, identifier::big-integer-size(16)>>]
+      [
+        Package.Meta.encode(t.__META__),
+        Package.variable_length_encode([
+          <<identifier::big-integer-size(16)>>,
+          Package.Properties.encode(t.properties),
+          Enum.map(t.results, &encode_result/1)
+        ])
+      ]
+    end
+
+    defp encode_result(:success), do: 0x00
+
+    defp encode_result({:error, reason}) do
+      case reason do
+        :no_subscription_existed -> 0x11
+        :unspecified_error -> 0x80
+        :implementation_specific_error -> 0x83
+        :not_authorized -> 0x87
+        :topic_filter_invalid -> 0x8F
+        :packet_identifier_in_use -> 0x91
+      end
     end
   end
 end
diff --git a/lib/tortoise/package/unsubscribe.ex b/lib/tortoise/package/unsubscribe.ex
index 57d8b6fc..76f3db83 100644
--- a/lib/tortoise/package/unsubscribe.ex
+++ b/lib/tortoise/package/unsubscribe.ex
@@ -12,18 +12,26 @@ defmodule Tortoise.Package.Unsubscribe do
   @opaque t :: %__MODULE__{
             __META__: Package.Meta.t(),
             identifier: Tortoise.package_identifier(),
-            topics: [topic]
+            topics: [topic],
+            properties: [{:user_property, any()}]
           }
   defstruct __META__: %Package.Meta{opcode: @opcode, flags: 2},
             topics: [],
-            identifier: nil
+            identifier: nil,
+            properties: []
 
   @spec decode(binary()) :: t
   def decode(<<@opcode::4, 0b0010::4, payload::binary>>) do
     with payload <- drop_length(payload),
-         <<identifier::big-integer-size(16), topics::binary>> <- payload,
-         topic_list <- decode_topics(topics),
-         do: %__MODULE__{identifier: identifier, topics: topic_list}
+         <<identifier::big-integer-size(16), rest::binary>> <- payload,
+         {properties, topics} = Package.parse_variable_length(rest),
+         topic_list <- decode_topics(topics) do
+      %__MODULE__{
+        identifier: identifier,
+        topics: topic_list,
+        properties: Package.Properties.decode(properties)
+      }
+    end
   end
 
   defp drop_length(payload) do
@@ -48,14 +56,15 @@ defmodule Tortoise.Package.Unsubscribe do
           %Package.Unsubscribe{
             identifier: identifier,
             # a valid unsubscribe package has at least one topic filter
-            topics: [_topic_filter | _]
+            topics: [topic_filter | _]
           } = t
         )
-        when identifier in 0x0001..0xFFFF do
+        when identifier in 0x0001..0xFFFF and is_binary(topic_filter) do
       [
         Package.Meta.encode(t.__META__),
         Package.variable_length_encode([
           <<identifier::big-integer-size(16)>>,
+          Package.Properties.encode(t.properties),
           Enum.map(t.topics, &Package.length_encode/1)
         ])
       ]
diff --git a/test/tortoise/package/puback_test.exs b/test/tortoise/package/puback_test.exs
index 435123ee..8082f130 100644
--- a/test/tortoise/package/puback_test.exs
+++ b/test/tortoise/package/puback_test.exs
@@ -1,4 +1,20 @@
 defmodule Tortoise.Package.PubackTest do
   use ExUnit.Case
+  use EQC.ExUnit
   doctest Tortoise.Package.Puback
+
+  alias Tortoise.Package
+
+  import Tortoise.TestGenerators, only: [gen_puback: 0]
+
+  property "encoding and decoding puback messages" do
+    forall puback <- gen_puback() do
+      ensure(
+        puback ==
+          puback
+          |> Package.encode()
+          |> Package.decode()
+      )
+    end
+  end
 end
diff --git a/test/tortoise/package/pubcomp_test.exs b/test/tortoise/package/pubcomp_test.exs
index a13052f8..a0df9edb 100644
--- a/test/tortoise/package/pubcomp_test.exs
+++ b/test/tortoise/package/pubcomp_test.exs
@@ -1,4 +1,20 @@
 defmodule Tortoise.Package.PubcompTest do
   use ExUnit.Case
+  use EQC.ExUnit
   doctest Tortoise.Package.Pubcomp
+
+  alias Tortoise.Package
+
+  import Tortoise.TestGenerators, only: [gen_pubcomp: 0]
+
+  property "encoding and decoding pubcomp messages" do
+    forall pubcomp <- gen_pubcomp() do
+      ensure(
+        pubcomp ==
+          pubcomp
+          |> Package.encode()
+          |> Package.decode()
+      )
+    end
+  end
 end
diff --git a/test/tortoise/package/pubrec_test.exs b/test/tortoise/package/pubrec_test.exs
index 023e1833..9d5360eb 100644
--- a/test/tortoise/package/pubrec_test.exs
+++ b/test/tortoise/package/pubrec_test.exs
@@ -1,4 +1,20 @@
 defmodule Tortoise.Package.PubrecTest do
   use ExUnit.Case
+  use EQC.ExUnit
   doctest Tortoise.Package.Pubrec
+
+  alias Tortoise.Package
+
+  import Tortoise.TestGenerators, only: [gen_pubrec: 0]
+
+  property "encoding and decoding pubrec messages" do
+    forall pubrec <- gen_pubrec() do
+      ensure(
+        pubrec ==
+          pubrec
+          |> Package.encode()
+          |> Package.decode()
+      )
+    end
+  end
 end
diff --git a/test/tortoise/package/pubrel_test.exs b/test/tortoise/package/pubrel_test.exs
index 418c3c85..2cba41ed 100644
--- a/test/tortoise/package/pubrel_test.exs
+++ b/test/tortoise/package/pubrel_test.exs
@@ -1,4 +1,20 @@
 defmodule Tortoise.Package.PubrelTest do
   use ExUnit.Case
+  use EQC.ExUnit
   doctest Tortoise.Package.Pubrel
+
+  alias Tortoise.Package
+
+  import Tortoise.TestGenerators, only: [gen_pubrel: 0]
+
+  property "encoding and decoding pubrel messages" do
+    forall pubrel <- gen_pubrel() do
+      ensure(
+        pubrel ==
+          pubrel
+          |> Package.encode()
+          |> Package.decode()
+      )
+    end
+  end
 end
diff --git a/test/tortoise/package/unsuback_test.exs b/test/tortoise/package/unsuback_test.exs
index d3f9b694..cadf49ce 100644
--- a/test/tortoise/package/unsuback_test.exs
+++ b/test/tortoise/package/unsuback_test.exs
@@ -1,4 +1,20 @@
 defmodule Tortoise.Package.UnsubackTest do
   use ExUnit.Case
+  use EQC.ExUnit
   doctest Tortoise.Package.Unsuback
+
+  import Tortoise.TestGenerators, only: [gen_unsuback: 0]
+
+  alias Tortoise.Package
+
+  property "encoding and decoding unsuback messages" do
+    forall unsuback <- gen_unsuback() do
+      ensure(
+        unsuback ==
+          unsuback
+          |> Package.encode()
+          |> Package.decode()
+      )
+    end
+  end
 end

From 36fcd18ffd0bf6349c023291164e20d127d2dd31 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Wed, 3 Oct 2018 22:22:20 +0200
Subject: [PATCH 009/220] Change subscription collectable to last write wins

Also, we accept `[{"foo/bar", qos: 0}]`, `[{"foo/bar", 0}]`, and
`["foo/bar"]` which will all evaluate to: `[{"foo/bar", qos: 0}]`.

There is no validation of the options passed in. It is yet to be
decided where it would belong.
---
 lib/tortoise/package/subscribe.ex        | 12 ++++----
 test/tortoise/package/subscribe_test.exs | 36 ++++++++++++------------
 2 files changed, 25 insertions(+), 23 deletions(-)

diff --git a/lib/tortoise/package/subscribe.ex b/lib/tortoise/package/subscribe.ex
index e0528c45..7560b817 100644
--- a/lib/tortoise/package/subscribe.ex
+++ b/lib/tortoise/package/subscribe.ex
@@ -135,14 +135,16 @@ defmodule Tortoise.Package.Subscribe do
     def into(%Package.Subscribe{topics: topics} = source) do
       {Enum.into(topics, %{}),
        fn
-         # @todo, switch to opts instead of qos
+         acc, {:cont, {<<topic::binary>>, opts}} when is_list(opts) ->
+           Map.put(acc, topic, opts)
+
          acc, {:cont, {<<topic::binary>>, qos}} when qos in 0..2 ->
-           # if a topic filter repeat in the input we will pick the
-           # biggest one
-           Map.update(acc, topic, qos, &max(&1, qos))
+           # if the subscription already exist in the data structure
+           # we will just overwrite the existing one and the options
+           Map.put(acc, topic, qos: qos)
 
          acc, {:cont, <<topic::binary>>} ->
-           Map.put_new(acc, topic, 0)
+           Map.put(acc, topic, qos: 0)
 
          acc, :done ->
            %{source | topics: Map.to_list(acc)}
diff --git a/test/tortoise/package/subscribe_test.exs b/test/tortoise/package/subscribe_test.exs
index d818e612..7cc87343 100644
--- a/test/tortoise/package/subscribe_test.exs
+++ b/test/tortoise/package/subscribe_test.exs
@@ -20,28 +20,28 @@ defmodule Tortoise.Package.SubscribeTest do
   end
 
   describe "Collectable" do
-    test "Pick the largest QoS when topic filters repeat in input" do
-      topic_filters = [{"a", 2}, {"a", 1}, {"a", 0}]
-      assert %Subscribe{topics: [{"a", 2}]} = Enum.into(topic_filters, %Subscribe{})
-
-      topic_filters = [{"a", 0}, {"a", 1}, {"a", 2}]
-      assert %Subscribe{topics: [{"a", 2}]} = Enum.into(topic_filters, %Subscribe{})
-
-      topic_filters = [{"a", 1}, {"a", 0}]
-      assert %Subscribe{topics: [{"a", 1}]} = Enum.into(topic_filters, %Subscribe{})
-
-      topic_filters = [{"a", 0}, {"a", 0}]
-      assert %Subscribe{topics: [{"a", 0}]} = Enum.into(topic_filters, %Subscribe{})
+    test "Accept tuples of {binary(), opts()} as input" do
+      assert %Subscribe{topics: [{"a", [qos: 1, no_local: true]}]} =
+               [{"a", [qos: 1, no_local: true]}]
+               |> Enum.into(%Subscribe{})
+    end
 
-      # if no qos is given it will default to 0, make sure we still
-      # pick the biggest QoS given in the list in that case
-      topic_filters = ["b", {"b", 2}, "b"]
-      assert %Subscribe{topics: [{"b", 2}]} = Enum.into(topic_filters, %Subscribe{})
+    test "Accept tuples of {binary(), qos()} as input" do
+      assert %Subscribe{topics: [{"a", [qos: 0]}]} =
+               [{"a", 0}]
+               |> Enum.into(%Subscribe{})
     end
 
     test "If no QoS is given it should default to zero" do
-      topic_filters = ["a"]
-      assert %Subscribe{topics: [{"a", 0}]} = Enum.into(topic_filters, %Subscribe{})
+      assert %Subscribe{topics: [{"a", [qos: 0]}]} =
+               ["a"]
+               |> Enum.into(%Subscribe{})
+    end
+
+    test "If two topics are the same the last write should win" do
+      assert %Subscribe{topics: [{"a", [qos: 1]}]} =
+               [{"a", qos: 2}, {"a", qos: 0}, {"a", qos: 1}]
+               |> Enum.into(%Subscribe{})
     end
   end
 end

From 975a01cbebf41100e4e16b18ccfc2b55d0a3e22b Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Thu, 4 Oct 2018 21:56:36 +0200
Subject: [PATCH 010/220] Support properties in the options list given to the
 publish command

---
 lib/tortoise.ex | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/tortoise.ex b/lib/tortoise.ex
index 6a928923..e1595429 100644
--- a/lib/tortoise.ex
+++ b/lib/tortoise.ex
@@ -254,13 +254,15 @@ defmodule Tortoise do
                | {:retain, boolean()}
                | {:identifier, package_identifier()}
   def publish(client_id, topic, payload \\ nil, opts \\ []) do
+    {opts, properties} = Keyword.split(opts, [:retain, :qos])
     qos = Keyword.get(opts, :qos, 0)
 
     publish = %Package.Publish{
       topic: topic,
       qos: qos,
       payload: payload,
-      retain: Keyword.get(opts, :retain, false)
+      retain: Keyword.get(opts, :retain, false),
+      properties: properties
     }
 
     with {:ok, {transport, socket}} <- Connection.connection(client_id) do

From 85d5fe6f23ae366fbdd53cc45c9119529c756325 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Thu, 4 Oct 2018 21:57:52 +0200
Subject: [PATCH 011/220] Note that we need to handle error cases in
 unsubscribe now

---
 lib/tortoise/connection/inflight/track.ex | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/tortoise/connection/inflight/track.ex b/lib/tortoise/connection/inflight/track.ex
index 878b7409..7b888e04 100644
--- a/lib/tortoise/connection/inflight/track.ex
+++ b/lib/tortoise/connection/inflight/track.ex
@@ -216,10 +216,11 @@ defmodule Tortoise.Connection.Inflight.Track do
   def result(%State{
         type: Package.Unsubscribe,
         status: [
-          {:received, _},
+          {:received, %Package.Unsuback{results: _results}},
           {:dispatch, %Package.Unsubscribe{topics: topics}} | _other
         ]
       }) do
+    # todo, merge the unsuback results with the topic list
     {:ok, topics}
   end
 

From fa44c8ebd0350b23b06e314aa70c5d6f81984075 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Thu, 4 Oct 2018 21:58:57 +0200
Subject: [PATCH 012/220] Improve the error report when the scripted server
 receive unexpected

---
 test/support/scripted_mqtt_server.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/support/scripted_mqtt_server.exs b/test/support/scripted_mqtt_server.exs
index d13b07d8..b4d748e2 100644
--- a/test/support/scripted_mqtt_server.exs
+++ b/test/support/scripted_mqtt_server.exs
@@ -82,7 +82,7 @@ defmodule Tortoise.Integration.ScriptedMqttServer do
         next_action(%State{state | script: script})
 
       otherwise ->
-        throw({:unexpected_package, otherwise})
+        {:stop, {:unexpected_package, otherwise}, state}
     end
   end
 

From e7b08b4b8b167ddeca520f5e709fb234f2314db3 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Thu, 4 Oct 2018 21:59:40 +0200
Subject: [PATCH 013/220] Make it possible to check membership of a topic in a
 subscription

The format has changed a bit; reflect that in the member? function
---
 lib/tortoise/package/subscribe.ex | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/lib/tortoise/package/subscribe.ex b/lib/tortoise/package/subscribe.ex
index 7560b817..7dd899d9 100644
--- a/lib/tortoise/package/subscribe.ex
+++ b/lib/tortoise/package/subscribe.ex
@@ -117,6 +117,18 @@ defmodule Tortoise.Package.Subscribe do
       Enumerable.List.reduce(topics, acc, fun)
     end
 
+    def member?(%Package.Subscribe{topics: topics}, {<<topic::binary>>, qos})
+        when is_integer(qos) do
+      matcher = fn {current_topic, opts} ->
+        topic == current_topic && opts[:qos] == qos
+      end
+
+      case Enum.find(topics, matcher) do
+        nil -> {:ok, false}
+        _ -> {:ok, true}
+      end
+    end
+
     def member?(%Package.Subscribe{topics: topics}, value) do
       {:ok, Enum.member?(topics, value)}
     end

From 7bd597eb1d0a9cbd6e6a1688dec941049e905d43 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Thu, 4 Oct 2018 22:01:19 +0200
Subject: [PATCH 014/220] Update the finalizing of subscription exchange to
 reflect new world

Subscribe results contain a keyword list now, not just a qos as it did
before. We need to get the qos from the list now.
---
 lib/tortoise/connection/inflight/track.ex | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/lib/tortoise/connection/inflight/track.ex b/lib/tortoise/connection/inflight/track.ex
index 7b888e04..fa7d841f 100644
--- a/lib/tortoise/connection/inflight/track.ex
+++ b/lib/tortoise/connection/inflight/track.ex
@@ -234,14 +234,17 @@ defmodule Tortoise.Connection.Inflight.Track do
     result =
       List.zip([topics, acks])
       |> Enum.reduce(%{error: [], warn: [], ok: []}, fn
-        {{topic, level}, {:ok, level}}, %{ok: oks} = acc ->
-          %{acc | ok: oks ++ [{topic, level}]}
+        {{topic, opts}, {:ok, actual}}, %{ok: oks, warn: warns} = acc ->
+          case Keyword.get(opts, :qos) do
+            ^actual ->
+              %{acc | ok: oks ++ [{topic, actual}]}
 
-        {{topic, requested}, {:ok, actual}}, %{warn: warns} = acc ->
-          %{acc | warn: warns ++ [{topic, [requested: requested, accepted: actual]}]}
+            requested ->
+              %{acc | warn: warns ++ [{topic, [requested: requested, accepted: actual]}]}
+          end
 
-        {{topic, level}, {:error, :access_denied}}, %{error: errors} = acc ->
-          %{acc | error: errors ++ [{:access_denied, {topic, level}}]}
+        {{topic, opts}, {:error, reason}}, %{error: errors} = acc ->
+          %{acc | error: errors ++ [{:reason, {topic, opts}}]}
       end)
 
     {:ok, result}

From 0dcfc4a785bbb292ffa3c22aca0f69e74c34ea3d Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Thu, 4 Oct 2018 22:02:29 +0200
Subject: [PATCH 015/220] The handler sit on the connection_opts, not the opts

---
 lib/tortoise/connection.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index ab23b7e4..32db9f2e 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -76,7 +76,7 @@ defmodule Tortoise.Connection do
 
     # @todo, validate that the handler is valid
     handler =
-      opts
+      connection_opts
       |> Keyword.get(:handler, %Handler{module: Handler.Default, initial_args: []})
       |> Handler.new()
 

From e999d8be8ac26f0206011ec79caa80e0f62674f7 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Thu, 4 Oct 2018 22:03:08 +0200
Subject: [PATCH 016/220] A proper unsuback message contain at least one result

Crash on anything else!
---
 lib/tortoise/connection.ex | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 32db9f2e..8c8429a9 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -620,7 +620,7 @@ defmodule Tortoise.Connection do
 
   def handle_event(
         :internal,
-        {:received, %Package.Unsuback{} = unsuback},
+        {:received, %Package.Unsuback{results: [_ | _]} = unsuback},
         :connected,
         data
       ) do
@@ -628,6 +628,7 @@ defmodule Tortoise.Connection do
     :keep_state_and_data
   end
 
+  # todo; handle the unsuback error cases !
   def handle_event(
         :info,
         {{Tortoise, client_id}, {Package.Unsubscribe, ref}, unsubbed},

From 0ec35f2399ca3b4e9f44a5d656f3be19c9161a16 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Thu, 4 Oct 2018 22:03:46 +0200
Subject: [PATCH 017/220] Update the subscription tests in connection to
 reflect the new world

---
 test/tortoise/connection_test.exs | 51 +++++++++++++++++++++++++------
 1 file changed, 42 insertions(+), 9 deletions(-)

diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index 28d32994..ee4a5cfa 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -250,9 +250,30 @@ defmodule Tortoise.ConnectionTest do
       client_id = context.client_id
 
       connect = %Package.Connect{client_id: client_id, clean_start: true}
-      subscription_foo = Enum.into([{"foo", 0}], %Package.Subscribe{identifier: 1})
-      subscription_bar = Enum.into([{"bar", 1}], %Package.Subscribe{identifier: 2})
-      subscription_baz = Enum.into([{"baz", 2}], %Package.Subscribe{identifier: 3})
+
+      default_subscription_opts = [
+        no_local: false,
+        retain_as_published: false,
+        retain_handling: 1
+      ]
+
+      subscription_foo =
+        Enum.into(
+          [{"foo", [{:qos, 0} | default_subscription_opts]}],
+          %Package.Subscribe{identifier: 1}
+        )
+
+      subscription_bar =
+        Enum.into(
+          [{"bar", [{:qos, 1} | default_subscription_opts]}],
+          %Package.Subscribe{identifier: 2}
+        )
+
+      subscription_baz =
+        Enum.into(
+          [{"baz", [{:qos, 2} | default_subscription_opts]}],
+          %Package.Subscribe{identifier: 3}
+        )
 
       script = [
         {:receive, connect},
@@ -314,19 +335,32 @@ defmodule Tortoise.ConnectionTest do
       script = [
         {:receive, connect},
         {:send, %Package.Connack{reason: :success, session_present: false}},
-        {:receive, %Package.Subscribe{topics: [{"foo", 0}, {"bar", 2}], identifier: 1}},
+        {:receive,
+         %Package.Subscribe{
+           topics: [
+             {"foo", [qos: 0, no_local: false, retain_as_published: false, retain_handling: 1]},
+             {"bar", [qos: 2, no_local: false, retain_as_published: false, retain_handling: 1]}
+           ],
+           identifier: 1
+         }},
         {:send, %Package.Suback{acks: [ok: 0, ok: 2], identifier: 1}},
         # unsubscribe foo
         {:receive, unsubscribe_foo},
-        {:send, %Package.Unsuback{identifier: 2}},
+        {:send, %Package.Unsuback{results: [:success], identifier: 2}},
         # unsubscribe bar
         {:receive, unsubscribe_bar},
-        {:send, %Package.Unsuback{identifier: 3}}
+        {:send, %Package.Unsuback{results: [:success], identifier: 3}}
       ]
 
       {:ok, {ip, port}} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
 
-      subscribe = %Package.Subscribe{topics: [{"foo", 0}, {"bar", 2}], identifier: 1}
+      subscribe = %Package.Subscribe{
+        topics: [
+          {"foo", [qos: 0, no_local: false, retain_as_published: false, retain_handling: 1]},
+          {"bar", [qos: 2, no_local: false, retain_as_published: false, retain_handling: 1]}
+        ],
+        identifier: 1
+      }
 
       opts = [
         client_id: client_id,
@@ -344,7 +378,7 @@ defmodule Tortoise.ConnectionTest do
       :ok = Tortoise.Connection.unsubscribe_sync(client_id, "foo", identifier: 2)
       assert_receive {ScriptedMqttServer, {:received, ^unsubscribe_foo}}
 
-      assert %Package.Subscribe{topics: [{"bar", 2}]} =
+      assert %Package.Subscribe{topics: [{"bar", qos: 2}]} =
                Tortoise.Connection.subscriptions(client_id)
 
       # and unsubscribe from bar
@@ -352,7 +386,6 @@ defmodule Tortoise.ConnectionTest do
       assert_receive {{Tortoise, ^client_id}, ^ref, :ok}
       assert_receive {ScriptedMqttServer, {:received, ^unsubscribe_bar}}
       assert %Package.Subscribe{topics: []} = Tortoise.Connection.subscriptions(client_id)
-
       assert_receive {ScriptedMqttServer, :completed}
     end
   end

From d3a357a8959fc4a6d7c762c455a4cbe676c0d017 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 6 Oct 2018 15:37:50 +0200
Subject: [PATCH 018/220] Make `Tortoise.publish_sync/4` work with the new
 return format

Added support for properties as well.
---
 lib/tortoise.ex                     | 6 ++++--
 lib/tortoise/connection/inflight.ex | 4 ++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/lib/tortoise.ex b/lib/tortoise.ex
index e1595429..b58f2860 100644
--- a/lib/tortoise.ex
+++ b/lib/tortoise.ex
@@ -315,14 +315,16 @@ defmodule Tortoise do
                | {:identifier, package_identifier()}
                | {:timeout, timeout()}
   def publish_sync(client_id, topic, payload \\ nil, opts \\ []) do
-    timeout = Keyword.get(opts, :timeout, :infinity)
+    {opts, properties} = Keyword.split(opts, [:retain, :qos])
     qos = Keyword.get(opts, :qos, 0)
+    timeout = Keyword.get(opts, :timeout, :infinity)
 
     publish = %Package.Publish{
       topic: topic,
       qos: qos,
       payload: payload,
-      retain: Keyword.get(opts, :retain, false)
+      retain: Keyword.get(opts, :retain, false),
+      properties: properties
     }
 
     with {:ok, {transport, socket}} <- Connection.connection(client_id) do
diff --git a/lib/tortoise/connection/inflight.ex b/lib/tortoise/connection/inflight.ex
index 37bab254..6da982c3 100644
--- a/lib/tortoise/connection/inflight.ex
+++ b/lib/tortoise/connection/inflight.ex
@@ -57,11 +57,11 @@ defmodule Tortoise.Connection.Inflight do
   end
 
   @doc false
-  def track_sync(client_id, {:outgoing, _} = command, timeout \\ :infinity) do
+  def track_sync(client_id, {:outgoing, %Package.Publish{}} = command, timeout \\ :infinity) do
     {:ok, ref} = track(client_id, command)
 
     receive do
-      {{Tortoise, ^client_id}, ^ref, result} ->
+      {{Tortoise, ^client_id}, {Package.Publish, ^ref}, result} ->
         result
     after
       timeout -> {:error, :timeout}

From 2097178ba162c521a8142600c40e5e7be86dd171 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 6 Oct 2018 15:47:07 +0200
Subject: [PATCH 019/220] Make the inflight test suite pass

Updates were needed to the format of the subscription packages.
---
 test/tortoise/connection/inflight_test.exs | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/test/tortoise/connection/inflight_test.exs b/test/tortoise/connection/inflight_test.exs
index af53b4c5..63d99c84 100644
--- a/test/tortoise/connection/inflight_test.exs
+++ b/test/tortoise/connection/inflight_test.exs
@@ -25,7 +25,7 @@ defmodule Tortoise.Connection.InflightTest do
   end
 
   def setup_inflight(context) do
-    {:ok, pid} = Inflight.start_link(client_id: context.client_id)
+    {:ok, pid} = Inflight.start_link(client_id: context.client_id, parent: self())
     {:ok, %{inflight_pid: pid}}
   end
 
@@ -33,7 +33,7 @@ defmodule Tortoise.Connection.InflightTest do
     setup [:setup_connection]
 
     test "start/stop", context do
-      assert {:ok, pid} = Inflight.start_link(client_id: context.client_id)
+      assert {:ok, pid} = Inflight.start_link(client_id: context.client_id, parent: self())
       assert Process.alive?(pid)
       assert :ok = Inflight.stop(pid)
       refute Process.alive?(pid)
@@ -133,9 +133,15 @@ defmodule Tortoise.Connection.InflightTest do
     setup [:setup_connection, :setup_inflight]
 
     test "subscription", %{client_id: client_id} = context do
+      opts = [no_local: false, retain_as_published: false, retain_handling: 1]
+
       subscribe = %Package.Subscribe{
         identifier: 1,
-        topics: [{"foo", 0}, {"bar", 1}, {"baz", 2}]
+        topics: [
+          {"foo", [{:qos, 0} | opts]},
+          {"bar", [{:qos, 1} | opts]},
+          {"baz", [{:qos, 2} | opts]}
+        ]
       }
 
       {:ok, ref} = Inflight.track(client_id, {:outgoing, subscribe})

From c14bf37327d81e76eb1ccc859897a35491f45d67 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 6 Oct 2018 16:23:21 +0200
Subject: [PATCH 020/220] The atom :reason was returned instead of the `reason`

This fixes returning an error back to the user defined handler when a
subscription fails. Note that the subscription callback is currently
not called in the connection. This will come in a later commit.
---
 lib/tortoise/connection/inflight/track.ex | 2 +-
 test/tortoise/handler_test.exs            | 6 +++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/lib/tortoise/connection/inflight/track.ex b/lib/tortoise/connection/inflight/track.ex
index fa7d841f..00152aa3 100644
--- a/lib/tortoise/connection/inflight/track.ex
+++ b/lib/tortoise/connection/inflight/track.ex
@@ -244,7 +244,7 @@ defmodule Tortoise.Connection.Inflight.Track do
           end
 
         {{topic, opts}, {:error, reason}}, %{error: errors} = acc ->
-          %{acc | error: errors ++ [{:reason, {topic, opts}}]}
+          %{acc | error: errors ++ [{reason, {topic, opts}}]}
       end)
 
     {:ok, result}
diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index 29b25eba..da30e5ed 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -149,7 +149,11 @@ defmodule Tortoise.HandlerTest do
 
   describe "execute subscribe/2" do
     test "return ok", context do
-      subscribe = %Package.Subscribe{identifier: 1, topics: [{"foo", 0}, {"bar", 1}, {"baz", 0}]}
+      subscribe = %Package.Subscribe{
+        identifier: 1,
+        topics: [{"foo", qos: 0}, {"bar", qos: 1}, {"baz", qos: 0}]
+      }
+
       suback = %Package.Suback{identifier: 1, acks: [ok: 0, ok: 0, error: :access_denied]}
       caller = {self(), make_ref()}
 

From 56f34b5b960ffe92f6dfb1f8491b8fc8107b52e5 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 6 Oct 2018 16:58:05 +0200
Subject: [PATCH 021/220] Change the pipe await and pipe tests to use the new
 format

---
 lib/tortoise/pipe.ex        | 2 +-
 test/tortoise/pipe_test.exs | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/tortoise/pipe.ex b/lib/tortoise/pipe.ex
index 221f2eda..903a24f1 100644
--- a/lib/tortoise/pipe.ex
+++ b/lib/tortoise/pipe.ex
@@ -147,7 +147,7 @@ defmodule Tortoise.Pipe do
 
   def await(%Pipe{client_id: client_id, pending: [ref | rest]} = pipe, timeout) do
     receive do
-      {{Tortoise, ^client_id}, ^ref, :ok} ->
+      {{Tortoise, ^client_id}, {Package.Publish, ^ref}, :ok} ->
         await(%Pipe{pipe | pending: rest})
     after
       timeout ->
diff --git a/test/tortoise/pipe_test.exs b/test/tortoise/pipe_test.exs
index e09272f5..71b75e83 100644
--- a/test/tortoise/pipe_test.exs
+++ b/test/tortoise/pipe_test.exs
@@ -10,7 +10,7 @@ defmodule Tortoise.PipeTest do
   end
 
   def setup_inflight(context) do
-    opts = [client_id: context.client_id]
+    opts = [client_id: context.client_id, parent: self()]
     {:ok, inflight_pid} = Inflight.start_link(opts)
     {:ok, %{inflight_pid: inflight_pid}}
   end

From e22827094b3db0ed15cb108b0f7d3b46ee265d3f Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 6 Oct 2018 17:05:31 +0200
Subject: [PATCH 022/220] Fix the tortoise_test.exs suite

---
 test/tortoise_test.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/tortoise_test.exs b/test/tortoise_test.exs
index aaccb7a6..2df5bc48 100644
--- a/test/tortoise_test.exs
+++ b/test/tortoise_test.exs
@@ -17,7 +17,7 @@ defmodule TortoiseTest do
   end
 
   def setup_inflight(context) do
-    opts = [client_id: context.client_id]
+    opts = [client_id: context.client_id, parent: self()]
     {:ok, pid} = Inflight.start_link(opts)
     {:ok, %{inflight_pid: pid}}
   end

From fe52fb61f2ceb2b029e35a0021ff3388715758e1 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 6 Oct 2018 18:15:07 +0200
Subject: [PATCH 023/220] Move the ping tests from the controller to the
 connection process

Remove the ping related code in the former controller as will
---
 lib/tortoise/connection.ex                   | 17 ++++-
 lib/tortoise/connection/controller.ex        | 73 +-------------------
 test/tortoise/connection/controller_test.exs | 57 +--------------
 test/tortoise/connection_test.exs            | 49 ++++++-------
 4 files changed, 40 insertions(+), 156 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 8c8429a9..4d8c955c 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -329,8 +329,21 @@ defmodule Tortoise.Connection do
   response.
   """
   @spec ping_sync(Tortoise.client_id(), timeout()) :: {:ok, reference()} | {:error, :timeout}
-  defdelegate ping_sync(client_id, timeout \\ :infinity),
-    to: Tortoise.Connection.Controller
+  def ping_sync(client_id, timeout \\ :infinity) do
+    {:ok, ref} = ping(client_id)
+
+    receive do
+      {Tortoise, {:ping_response, ^ref, round_trip_time}} ->
+        {:ok, round_trip_time}
+
+      otherwise ->
+        IO.inspect(otherwise)
+        :ok
+    after
+      timeout ->
+        {:error, :timeout}
+    end
+  end
 
   @doc false
   @spec connection(Tortoise.client_id(), [opts]) ::
diff --git a/lib/tortoise/connection/controller.ex b/lib/tortoise/connection/controller.ex
index 66c1c748..6508e8a3 100644
--- a/lib/tortoise/connection/controller.ex
+++ b/lib/tortoise/connection/controller.ex
@@ -18,16 +18,13 @@ defmodule Tortoise.Connection.Controller do
     Subscribe,
     Suback,
     Unsubscribe,
-    Unsuback,
-    Pingreq,
-    Pingresp
+    Unsuback
   }
 
   use GenServer
 
   @enforce_keys [:client_id, :handler]
   defstruct client_id: nil,
-            ping: :queue.new(),
             status: :down,
             awaiting: %{},
             handler: %Handler{module: Handler.Default, initial_args: []}
@@ -59,47 +56,11 @@ defmodule Tortoise.Connection.Controller do
     GenServer.call(via_name(client_id), :info)
   end
 
-  @spec ping(Tortoise.client_id()) :: {:ok, reference()}
-  def ping(client_id) do
-    ref = make_ref()
-    :ok = GenServer.cast(via_name(client_id), {:ping, {self(), ref}})
-    {:ok, ref}
-  end
-
-  @spec ping_sync(Tortoise.client_id(), timeout()) :: {:ok, reference()} | {:error, :timeout}
-  def ping_sync(client_id, timeout \\ :infinity) do
-    {:ok, ref} = ping(client_id)
-
-    receive do
-      {Tortoise, {:ping_response, ^ref, round_trip_time}} ->
-        {:ok, round_trip_time}
-    after
-      timeout ->
-        {:error, :timeout}
-    end
-  end
-
   @doc false
   def handle_incoming(client_id, package) do
     GenServer.cast(via_name(client_id), {:incoming, package})
   end
 
-  @doc false
-  def handle_result(client_id, {{pid, ref}, Package.Publish, result}) do
-    send(pid, {{Tortoise, client_id}, ref, result})
-    :ok
-  end
-
-  def handle_result(client_id, {{pid, ref}, type, result}) do
-    send(pid, {{Tortoise, client_id}, ref, result})
-    GenServer.cast(via_name(client_id), {:result, {type, result}})
-  end
-
-  @doc false
-  def handle_onward(client_id, %Package.Publish{} = publish) do
-    GenServer.cast(via_name(client_id), {:onward, publish})
-  end
-
   # Server callbacks
   @impl true
   def init(%State{handler: handler} = opts) do
@@ -136,18 +97,6 @@ defmodule Tortoise.Connection.Controller do
     handle_package(package, state)
   end
 
-  def handle_cast({:ping, caller}, state) do
-    with {:ok, {transport, socket}} <- Connection.connection(state.client_id) do
-      time = System.monotonic_time(:microsecond)
-      apply(transport, :send, [socket, Package.encode(%Package.Pingreq{})])
-      ping = :queue.in({caller, time}, state.ping)
-      {:noreply, %State{state | ping: ping}}
-    else
-      {:error, :unknown_connection} ->
-        {:stop, :unknown_connection, state}
-    end
-  end
-
   def handle_cast(
         {:result, {Package.Subscribe, subacks}},
         %State{handler: handler} = state
@@ -312,26 +261,6 @@ defmodule Tortoise.Connection.Controller do
     {:noreply, state}
   end
 
-  # PING MESSAGES ======================================================
-  # command ------------------------------------------------------------
-  defp handle_package(%Pingresp{}, %State{ping: ping} = state)
-       when is_nil(ping) or ping == {[], []} do
-    {:noreply, state}
-  end
-
-  defp handle_package(%Pingresp{}, %State{ping: ping} = state) do
-    {{:value, {{caller, ref}, start_time}}, ping} = :queue.out(ping)
-    round_trip_time = System.monotonic_time(:microsecond) - start_time
-    send(caller, {Tortoise, {:ping_response, ref, round_trip_time}})
-    {:noreply, %State{state | ping: ping}}
-  end
-
-  # response -----------------------------------------------------------
-  defp handle_package(%Pingreq{} = pingreq, state) do
-    # not a server!
-    {:stop, {:protocol_violation, {:unexpected_package_from_remote, pingreq}}, state}
-  end
-
   # CONNECTING =========================================================
   # command ------------------------------------------------------------
   defp handle_package(%Connect{} = connect, state) do
diff --git a/test/tortoise/connection/controller_test.exs b/test/tortoise/connection/controller_test.exs
index 48fe30f1..f803a593 100644
--- a/test/tortoise/connection/controller_test.exs
+++ b/test/tortoise/connection/controller_test.exs
@@ -108,7 +108,7 @@ defmodule Tortoise.Connection.ControllerTest do
   end
 
   def setup_inflight(context) do
-    opts = [client_id: context.client_id]
+    opts = [client_id: context.client_id, parent: self()]
     {:ok, pid} = Inflight.start_link(opts)
     {:ok, %{inflight_pid: pid}}
   end
@@ -185,61 +185,6 @@ defmodule Tortoise.Connection.ControllerTest do
     end
   end
 
-  describe "Ping Control Packets" do
-    setup [:setup_connection, :setup_controller]
-
-    test "send a ping request", context do
-      # send a ping request to the server
-      assert {:ok, ping_ref} = Controller.ping(context.client_id)
-      # assert that the server receives a ping request package
-      {:ok, package} = :gen_tcp.recv(context.server, 0, 200)
-      assert %Package.Pingreq{} = Package.decode(package)
-      # the server will respond with an pingresp (ping response)
-      Controller.handle_incoming(context.client_id, %Package.Pingresp{})
-      assert_receive {Tortoise, {:ping_response, ^ping_ref, _ping_time}}
-    end
-
-    test "send a sync ping request", context do
-      # send a ping request to the server
-      parent = self()
-
-      spawn_link(fn ->
-        {:ok, time} = Controller.ping_sync(context.client_id)
-        send(parent, {:ping_result, time})
-      end)
-
-      # assert that the server receives a ping request package
-      {:ok, package} = :gen_tcp.recv(context.server, 0, 200)
-      assert %Package.Pingreq{} = Package.decode(package)
-      # the server will respond with an pingresp (ping response)
-      Controller.handle_incoming(context.client_id, %Package.Pingresp{})
-      assert_receive {:ping_result, _time}
-    end
-
-    # done
-    # test "receiving a ping request", %{controller_pid: pid} = context do
-    #   Process.flag(:trap_exit, true)
-    #   # receiving a ping request from the server is a protocol violation
-    #   pingreq = %Package.Pingreq{}
-    #   Controller.handle_incoming(context.client_id, pingreq)
-
-    #   assert_receive {:EXIT, ^pid,
-    #                   {:protocol_violation, {:unexpected_package_from_remote, ^pingreq}}}
-    # end
-
-    test "ping request reports are sent in the correct order", context do
-      # send two ping requests to the server
-      assert {:ok, first_ping_ref} = Controller.ping(context.client_id)
-      assert {:ok, second_ping_ref} = Controller.ping(context.client_id)
-
-      # the controller should respond to ping requests in FIFO order
-      Controller.handle_incoming(context.client_id, %Package.Pingresp{})
-      assert_receive {Tortoise, {:ping_response, ^first_ping_ref, _}}
-      Controller.handle_incoming(context.client_id, %Package.Pingresp{})
-      assert_receive {Tortoise, {:ping_response, ^second_ping_ref, _}}
-    end
-  end
-
   describe "publish" do
     setup [:setup_controller]
 
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index ee4a5cfa..9187c40f 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -678,44 +678,41 @@ defmodule Tortoise.ConnectionTest do
   end
 
   describe "ping" do
-    setup [:setup_scripted_mqtt_server]
+    setup [:setup_scripted_mqtt_server, :setup_connection_and_perform_handshake]
 
-    test "send pingreq and receive a pingresp", context do
-      Process.flag(:trap_exit, true)
-      client_id = context.client_id
+    test "send pingreq and receive a pingresp", %{client_id: client_id} = context do
+      {:ok, _} = Tortoise.Events.register(client_id, :status)
+      assert_receive {{Tortoise, ^client_id}, :status, :connected}
 
-      connect = %Package.Connect{client_id: client_id}
-      expected_connack = %Package.Connack{reason: :success, session_present: false}
       ping_request = %Package.Pingreq{}
       expected_pingresp = %Package.Pingresp{}
+      script = [{:receive, ping_request}, {:send, expected_pingresp}]
 
-      script = [
-        {:receive, connect},
-        {:send, expected_connack},
-        {:receive, ping_request},
-        {:send, expected_pingresp}
-      ]
+      {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
 
-      {:ok, {ip, port}} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
+      {:ok, ref} = Connection.ping(context.client_id)
+      assert_receive {ScriptedMqttServer, {:received, ^ping_request}}
+      assert_receive {Tortoise, {:ping_response, ^ref, _}}
+    end
 
-      opts = [
-        client_id: client_id,
-        server: {Tortoise.Transport.Tcp, [host: ip, port: port]},
-        handler: {Tortoise.Handler.Default, []}
-      ]
+    test "ping_sync/2", %{client_id: client_id} = context do
+      {:ok, _} = Tortoise.Events.register(client_id, :status)
+      assert_receive {{Tortoise, ^client_id}, :status, :connected}
 
-      {:ok, _pid} = Tortoise.Events.register(client_id, :status)
+      ping_request = %Package.Pingreq{}
+      expected_pingresp = %Package.Pingresp{}
+      script = [{:receive, ping_request}, {:send, expected_pingresp}]
 
-      assert {:ok, pid} = Connection.start_link(opts)
-      assert_receive {ScriptedMqttServer, {:received, ^connect}}
+      {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
 
-      # assure that we are connected
-      assert_receive {{Tortoise, ^client_id}, :status, :connected}
+      {parent, ref} = {self(), make_ref()}
 
-      {:ok, ref} = Connection.ping(client_id)
-      assert_receive {ScriptedMqttServer, {:received, ^ping_request}}
+      spawn_link(fn ->
+        send(parent, {{:child_result, ref}, Connection.ping_sync(client_id)})
+      end)
 
-      assert_receive {Tortoise, {:ping_response, ^ref, _}}
+      assert_receive {ScriptedMqttServer, {:received, ^ping_request}}
+      assert_receive {{:child_result, ^ref}, {:ok, time}}
       assert_receive {ScriptedMqttServer, :completed}
     end
   end

From 0cf1b55b6a468bc6568b347abf77f11463b2dbd8 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 6 Oct 2018 20:57:31 +0200
Subject: [PATCH 024/220] Make sure we close down all processes when we
 disconnect

---
 lib/tortoise/connection/inflight.ex          |  6 ++++++
 lib/tortoise/connection/receiver.ex          |  6 ++++++
 lib/tortoise/connection/supervisor.ex        |  6 ++++++
 test/tortoise/connection/controller_test.exs | 14 --------------
 test/tortoise/connection_test.exs            | 11 +++++++++++
 5 files changed, 29 insertions(+), 14 deletions(-)

diff --git a/lib/tortoise/connection/inflight.ex b/lib/tortoise/connection/inflight.ex
index 6da982c3..7ef342b4 100644
--- a/lib/tortoise/connection/inflight.ex
+++ b/lib/tortoise/connection/inflight.ex
@@ -23,6 +23,12 @@ defmodule Tortoise.Connection.Inflight do
     Tortoise.Registry.via_name(__MODULE__, client_id)
   end
 
+  def whereis(client_id) do
+    __MODULE__
+    |> Tortoise.Registry.reg_name(client_id)
+    |> Registry.whereis_name()
+  end
+
   def stop(client_id) do
     GenStateMachine.stop(via_name(client_id))
   end
diff --git a/lib/tortoise/connection/receiver.ex b/lib/tortoise/connection/receiver.ex
index 73591703..b6c53567 100644
--- a/lib/tortoise/connection/receiver.ex
+++ b/lib/tortoise/connection/receiver.ex
@@ -24,6 +24,12 @@ defmodule Tortoise.Connection.Receiver do
     Tortoise.Registry.via_name(__MODULE__, client_id)
   end
 
+  def whereis(client_id) do
+    __MODULE__
+    |> Tortoise.Registry.reg_name(client_id)
+    |> Registry.whereis_name()
+  end
+
   def child_spec(opts) do
     %{
       id: __MODULE__,
diff --git a/lib/tortoise/connection/supervisor.ex b/lib/tortoise/connection/supervisor.ex
index 730059ad..8cc90162 100644
--- a/lib/tortoise/connection/supervisor.ex
+++ b/lib/tortoise/connection/supervisor.ex
@@ -14,6 +14,12 @@ defmodule Tortoise.Connection.Supervisor do
     Tortoise.Registry.via_name(__MODULE__, client_id)
   end
 
+  def whereis(client_id) do
+    __MODULE__
+    |> Tortoise.Registry.reg_name(client_id)
+    |> Registry.whereis_name()
+  end
+
   @impl true
   def init(opts) do
     children = [
diff --git a/test/tortoise/connection/controller_test.exs b/test/tortoise/connection/controller_test.exs
index f803a593..7433760a 100644
--- a/test/tortoise/connection/controller_test.exs
+++ b/test/tortoise/connection/controller_test.exs
@@ -114,20 +114,6 @@ defmodule Tortoise.Connection.ControllerTest do
   end
 
   # tests --------------------------------------------------------------
-  test "life cycle", context do
-    handler = %Tortoise.Handler{
-      module: __MODULE__.TestHandler,
-      initial_args: [context.client_id, self()]
-    }
-
-    opts = [client_id: context.client_id, handler: handler]
-    assert {:ok, pid} = Controller.start_link(opts)
-    assert Process.alive?(pid)
-    assert :ok = Controller.stop(context.client_id)
-    refute Process.alive?(pid)
-    assert_receive {:terminating, :normal}
-  end
-
   describe "Connection callback" do
     setup [:setup_controller]
 
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index 9187c40f..639e1165 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -668,12 +668,23 @@ defmodule Tortoise.ConnectionTest do
 
       assert {:ok, pid} = Connection.start_link(opts)
       assert_receive {ScriptedMqttServer, {:received, ^connect}}
+
+      cs_pid = Connection.Supervisor.whereis(client_id)
+      cs_ref = Process.monitor(cs_pid)
+
+      inflight_pid = Connection.Inflight.whereis(client_id)
+      receiver_pid = Connection.Receiver.whereis(client_id)
+
       assert :ok = Tortoise.Connection.disconnect(client_id)
 
       assert_receive {ScriptedMqttServer, {:received, ^disconnect}}
       assert_receive {:EXIT, ^pid, :shutdown}
 
       assert_receive {ScriptedMqttServer, :completed}
+
+      assert_receive {:DOWN, ^cs_ref, :process, ^cs_pid, :shutdown}
+      refute Process.alive?(inflight_pid)
+      refute Process.alive?(receiver_pid)
     end
   end
 

From 329b8688d737d863e2694ecce443da82ac5ca512 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 6 Oct 2018 21:10:15 +0200
Subject: [PATCH 025/220] Remove test from controller_test that is covered in
 connection_test

---
 test/tortoise/connection/controller_test.exs | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/test/tortoise/connection/controller_test.exs b/test/tortoise/connection/controller_test.exs
index 7433760a..6c813a29 100644
--- a/test/tortoise/connection/controller_test.exs
+++ b/test/tortoise/connection/controller_test.exs
@@ -201,16 +201,6 @@ defmodule Tortoise.Connection.ControllerTest do
   describe "Publish Control Packets with Quality of Service level 1" do
     setup [:setup_connection, :setup_controller, :setup_inflight]
 
-    test "incoming publish with qos 1", context do
-      # receive a publish message with a qos of 1
-      publish = %Package.Publish{identifier: 1, topic: "a", qos: 1}
-      Controller.handle_incoming(context.client_id, publish)
-
-      # a puback message should get transmitted
-      {:ok, package} = :gen_tcp.recv(context.server, 0, 200)
-      assert %Package.Puback{identifier: 1} = Package.decode(package)
-    end
-
     test "outgoing publish with qos 1", context do
       client_id = context.client_id
       publish = %Package.Publish{identifier: 1, topic: "a", qos: 1}

From 07255f2ff9590200089b68f0ec30ca78de4b3cc0 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 6 Oct 2018 21:38:33 +0200
Subject: [PATCH 026/220] Added more protocol violation tests

Test that receiving subscription and unsubscribe packages are marked
as protocol violations.
---
 test/tortoise/connection/controller_test.exs | 30 ----------------
 test/tortoise/connection_test.exs            | 36 ++++++++++++++++++++
 2 files changed, 36 insertions(+), 30 deletions(-)

diff --git a/test/tortoise/connection/controller_test.exs b/test/tortoise/connection/controller_test.exs
index 6c813a29..4c93ed93 100644
--- a/test/tortoise/connection/controller_test.exs
+++ b/test/tortoise/connection/controller_test.exs
@@ -409,36 +409,6 @@ defmodule Tortoise.Connection.ControllerTest do
       # the callback module should get the error
       assert_receive {:subscription_error, {"foo", :access_denied}}
     end
-
-    test "Receiving a subscribe package is a protocol violation",
-         %{controller_pid: pid} = context do
-      Process.flag(:trap_exit, true)
-      # receiving a subscribe from the server is a protocol violation
-      subscribe = %Package.Subscribe{
-        identifier: 1,
-        topics: [{"foo/bar", 0}]
-      }
-
-      Controller.handle_incoming(context.client_id, subscribe)
-
-      assert_receive {:EXIT, ^pid,
-                      {:protocol_violation, {:unexpected_package_from_remote, ^subscribe}}}
-    end
-
-    test "Receiving an unsubscribe package is a protocol violation",
-         %{controller_pid: pid} = context do
-      Process.flag(:trap_exit, true)
-      # receiving an unsubscribe from the server is a protocol violation
-      unsubscribe = %Package.Unsubscribe{
-        identifier: 1,
-        topics: ["foo/bar"]
-      }
-
-      Controller.handle_incoming(context.client_id, unsubscribe)
-
-      assert_receive {:EXIT, ^pid,
-                      {:protocol_violation, {:unexpected_package_from_remote, ^unsubscribe}}}
-    end
   end
 
   describe "next actions" do
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index 639e1165..46d4cc68 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -766,6 +766,42 @@ defmodule Tortoise.ConnectionTest do
       assert_receive {:EXIT, ^pid,
                       {:protocol_violation, {:unexpected_package, ^unexpected_pingreq}}}
     end
+
+    test "Receiving a subscribe package from the server is a protocol violation", context do
+      Process.flag(:trap_exit, true)
+
+      unexpected_subscribe = %Package.Subscribe{
+        topics: [
+          {"foo/bar", [qos: 0, no_local: false, retain_as_published: false, retain_handling: 1]}
+        ],
+        identifier: 1
+      }
+
+      script = [{:send, unexpected_subscribe}]
+
+      {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
+      pid = context.connection_pid
+
+      assert_receive {:EXIT, ^pid,
+                      {:protocol_violation, {:unexpected_package, ^unexpected_subscribe}}}
+    end
+
+    test "Receiving an unsubscribe package from the server is a protocol violation", context do
+      Process.flag(:trap_exit, true)
+
+      unexpected_unsubscribe = %Package.Unsubscribe{
+        topics: ["foo/bar"],
+        identifier: 1
+      }
+
+      script = [{:send, unexpected_unsubscribe}]
+
+      {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
+      pid = context.connection_pid
+
+      assert_receive {:EXIT, ^pid,
+                      {:protocol_violation, {:unexpected_package, ^unexpected_unsubscribe}}}
+    end
   end
 
   describe "Publish with QoS=0" do

From df3c11c515fb14dd55a670c8cd98a7dd3e225114 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 6 Oct 2018 22:09:03 +0200
Subject: [PATCH 027/220] Removing more of the old controller code

---
 lib/tortoise/connection/controller.ex        | 199 +------------------
 test/tortoise/connection/controller_test.exs |  25 ---
 2 files changed, 1 insertion(+), 223 deletions(-)

diff --git a/lib/tortoise/connection/controller.ex b/lib/tortoise/connection/controller.ex
index 6508e8a3..4eeb77e2 100644
--- a/lib/tortoise/connection/controller.ex
+++ b/lib/tortoise/connection/controller.ex
@@ -3,23 +3,7 @@ defmodule Tortoise.Connection.Controller do
 
   require Logger
 
-  alias Tortoise.{Package, Connection, Handler}
-  alias Tortoise.Connection.Inflight
-
-  alias Tortoise.Package.{
-    Connect,
-    Connack,
-    Disconnect,
-    Publish,
-    Puback,
-    Pubrec,
-    Pubrel,
-    Pubcomp,
-    Subscribe,
-    Suback,
-    Unsubscribe,
-    Unsuback
-  }
+  alias Tortoise.{Package, Handler}
 
   use GenServer
 
@@ -56,11 +40,6 @@ defmodule Tortoise.Connection.Controller do
     GenServer.call(via_name(client_id), :info)
   end
 
-  @doc false
-  def handle_incoming(client_id, package) do
-    GenServer.cast(via_name(client_id), {:incoming, package})
-  end
-
   # Server callbacks
   @impl true
   def init(%State{handler: handler} = opts) do
@@ -84,42 +63,6 @@ defmodule Tortoise.Connection.Controller do
   end
 
   @impl true
-  def handle_cast({:incoming, <<package::binary>>}, state) do
-    package
-    |> Package.decode()
-    |> handle_package(state)
-  end
-
-  # allow for passing in already decoded packages into the controller,
-  # this allow us to test the controller without having to pass in
-  # binaries
-  def handle_cast({:incoming, %{:__META__ => _} = package}, state) do
-    handle_package(package, state)
-  end
-
-  def handle_cast(
-        {:result, {Package.Subscribe, subacks}},
-        %State{handler: handler} = state
-      ) do
-    case Handler.execute(handler, {:subscribe, subacks}) do
-      {:ok, updated_handler} ->
-        {:noreply, %State{state | handler: updated_handler}}
-    end
-  end
-
-  def handle_cast(
-        {:result, {Package.Unsubscribe, unsubacks}},
-        %State{handler: handler} = state
-      ) do
-    case Handler.execute(handler, {:unsubscribe, unsubacks}) do
-      {:ok, updated_handler} ->
-        {:noreply, %State{state | handler: updated_handler}}
-    end
-  end
-
-  # an incoming publish with QoS=2 will get parked in the inflight
-  # manager process, which will onward it to the controller, making
-  # sure we will only dispatch it once to the publish-handler.
   def handle_cast(
         {:onward, %Package.Publish{qos: 2, dup: false} = publish},
         %State{handler: handler} = state
@@ -131,42 +74,6 @@ defmodule Tortoise.Connection.Controller do
   end
 
   @impl true
-  def handle_info({:next_action, {:subscribe, topic, opts} = action}, state) do
-    {qos, opts} = Keyword.pop_first(opts, :qos, 0)
-
-    case Tortoise.Connection.subscribe(state.client_id, [{topic, qos}], opts) do
-      {:ok, ref} ->
-        updated_awaiting = Map.put_new(state.awaiting, ref, action)
-        {:noreply, %State{state | awaiting: updated_awaiting}}
-    end
-  end
-
-  def handle_info({:next_action, {:unsubscribe, topic} = action}, state) do
-    case Tortoise.Connection.unsubscribe(state.client_id, topic) do
-      {:ok, ref} ->
-        updated_awaiting = Map.put_new(state.awaiting, ref, action)
-        {:noreply, %State{state | awaiting: updated_awaiting}}
-    end
-  end
-
-  # connection changes
-  def handle_info(
-        {{Tortoise, client_id}, :status, same},
-        %State{client_id: client_id, status: same} = state
-      ) do
-    {:noreply, state}
-  end
-
-  def handle_info(
-        {{Tortoise, client_id}, :status, new_status},
-        %State{client_id: client_id, handler: handler} = state
-      ) do
-    case Handler.execute(handler, {:connection, new_status}) do
-      {:ok, updated_handler} ->
-        {:noreply, %State{state | handler: updated_handler, status: new_status}}
-    end
-  end
-
   def handle_info({{Tortoise, client_id}, ref, result}, %{client_id: client_id} = state) do
     case {result, Map.pop(state.awaiting, ref)} do
       {_, {nil, _}} ->
@@ -177,108 +84,4 @@ defmodule Tortoise.Connection.Controller do
         {:noreply, %State{state | awaiting: updated_awaiting}}
     end
   end
-
-  # QoS LEVEL 0 ========================================================
-  # commands -----------------------------------------------------------
-  defp handle_package(
-         %Publish{qos: 0, dup: false} = publish,
-         %State{handler: handler} = state
-       ) do
-    case Handler.execute(handler, {:publish, publish}) do
-      {:ok, updated_handler} ->
-        {:noreply, %State{state | handler: updated_handler}}
-
-        # handle stop
-    end
-  end
-
-  # QoS LEVEL 1 ========================================================
-  # commands -----------------------------------------------------------
-  defp handle_package(
-         %Publish{qos: 1} = publish,
-         %State{handler: handler} = state
-       ) do
-    :ok = Inflight.track(state.client_id, {:incoming, publish})
-
-    case Handler.execute(handler, {:publish, publish}) do
-      {:ok, updated_handler} ->
-        {:noreply, %State{state | handler: updated_handler}}
-    end
-  end
-
-  # response -----------------------------------------------------------
-  defp handle_package(%Puback{} = puback, state) do
-    :ok = Inflight.update(state.client_id, {:received, puback})
-    {:noreply, state}
-  end
-
-  # QoS LEVEL 2 ========================================================
-  # commands -----------------------------------------------------------
-  defp handle_package(%Publish{qos: 2} = publish, %State{} = state) do
-    :ok = Inflight.track(state.client_id, {:incoming, publish})
-    {:noreply, state}
-  end
-
-  defp handle_package(%Pubrel{} = pubrel, state) do
-    :ok = Inflight.update(state.client_id, {:received, pubrel})
-    {:noreply, state}
-  end
-
-  # response -----------------------------------------------------------
-  defp handle_package(%Pubrec{} = pubrec, state) do
-    :ok = Inflight.update(state.client_id, {:received, pubrec})
-    {:noreply, state}
-  end
-
-  defp handle_package(%Pubcomp{} = pubcomp, state) do
-    :ok = Inflight.update(state.client_id, {:received, pubcomp})
-    {:noreply, state}
-  end
-
-  # SUBSCRIBING ========================================================
-  # command ------------------------------------------------------------
-  defp handle_package(%Subscribe{} = subscribe, state) do
-    # not a server! (yet)
-    {:stop, {:protocol_violation, {:unexpected_package_from_remote, subscribe}}, state}
-  end
-
-  # response -----------------------------------------------------------
-  defp handle_package(%Suback{} = suback, state) do
-    :ok = Inflight.update(state.client_id, {:received, suback})
-    {:noreply, state}
-  end
-
-  # UNSUBSCRIBING ======================================================
-  # command ------------------------------------------------------------
-  defp handle_package(%Unsubscribe{} = unsubscribe, state) do
-    # not a server
-    {:stop, {:protocol_violation, {:unexpected_package_from_remote, unsubscribe}}, state}
-  end
-
-  # response -----------------------------------------------------------
-  defp handle_package(%Unsuback{} = unsuback, state) do
-    :ok = Inflight.update(state.client_id, {:received, unsuback})
-    {:noreply, state}
-  end
-
-  # CONNECTING =========================================================
-  # command ------------------------------------------------------------
-  defp handle_package(%Connect{} = connect, state) do
-    # not a server!
-    {:stop, {:protocol_violation, {:unexpected_package_from_remote, connect}}, state}
-  end
-
-  # response -----------------------------------------------------------
-  defp handle_package(%Connack{} = connack, state) do
-    # receiving a connack at this point would be a protocol violation
-    {:stop, {:protocol_violation, {:unexpected_package_from_remote, connack}}, state}
-  end
-
-  # DISCONNECTING ======================================================
-  # command ------------------------------------------------------------
-  defp handle_package(%Disconnect{} = disconnect, state) do
-    # This should be allowed when we implement MQTT 5. Remember there
-    # is a test that assert this as a protocol violation!
-    {:stop, {:protocol_violation, {:unexpected_package_from_remote, disconnect}}, state}
-  end
 end
diff --git a/test/tortoise/connection/controller_test.exs b/test/tortoise/connection/controller_test.exs
index 4c93ed93..e3149f66 100644
--- a/test/tortoise/connection/controller_test.exs
+++ b/test/tortoise/connection/controller_test.exs
@@ -133,31 +133,6 @@ defmodule Tortoise.Connection.ControllerTest do
   describe "Connection Control Packets" do
     setup [:setup_controller]
 
-    # done
-    # test "receiving a connect from the server is a protocol violation",
-    #      %{controller_pid: pid} = context do
-    #   Process.flag(:trap_exit, true)
-    #   # receiving a connect from the server is a protocol violation
-    #   connect = %Package.Connect{client_id: "foo"}
-    #   Controller.handle_incoming(context.client_id, connect)
-
-    #   assert_receive {:EXIT, ^pid,
-    #                   {:protocol_violation, {:unexpected_package_from_remote, ^connect}}}
-    # end
-
-    # done
-    # test "receiving a connack at this point is a protocol violation",
-    #      %{controller_pid: pid} = context do
-    #   Process.flag(:trap_exit, true)
-    #   # receiving a connack from the server *after* the connection has
-    #   # been acknowledged is a protocol violation
-    #   connack = %Package.Connack{reason: :success}
-    #   Controller.handle_incoming(context.client_id, connack)
-
-    #   assert_receive {:EXIT, ^pid,
-    #                   {:protocol_violation, {:unexpected_package_from_remote, ^connack}}}
-    # end
-
     test "receiving a disconnect from the server is a protocol violation",
          %{controller_pid: pid} = context do
       Process.flag(:trap_exit, true)

From d8a278266c006ea70e3237470c4160cce9ee1344 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 6 Oct 2018 22:16:51 +0200
Subject: [PATCH 028/220] The return format of ping requests is now shaped
 other results

The format is now:

`{{Tortoise, ^client_id}, {Package.Pingreq, ^ref}, round_trip_time}`
---
 lib/tortoise/connection.ex        | 6 +++---
 test/tortoise/connection_test.exs | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 4d8c955c..20a34181 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -333,7 +333,7 @@ defmodule Tortoise.Connection do
     {:ok, ref} = ping(client_id)
 
     receive do
-      {Tortoise, {:ping_response, ^ref, round_trip_time}} ->
+      {{Tortoise, ^client_id}, {Package.Pingreq, ^ref}, round_trip_time} ->
         {:ok, round_trip_time}
 
       otherwise ->
@@ -771,11 +771,11 @@ defmodule Tortoise.Connection do
         :internal,
         {:received, %Package.Pingresp{}},
         :connected,
-        %State{ping: ping} = data
+        %State{client_id: client_id, ping: ping} = data
       ) do
     {{:value, {{caller, ref}, start_time}}, ping} = :queue.out(ping)
     round_trip_time = System.monotonic_time(:microsecond) - start_time
-    send(caller, {Tortoise, {:ping_response, ref, round_trip_time}})
+    send(caller, {{Tortoise, client_id}, {Package.Pingreq, ref}, round_trip_time})
     {:keep_state, %State{data | ping: ping}}
   end
 
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index 46d4cc68..e5acc6d3 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -703,7 +703,7 @@ defmodule Tortoise.ConnectionTest do
 
       {:ok, ref} = Connection.ping(context.client_id)
       assert_receive {ScriptedMqttServer, {:received, ^ping_request}}
-      assert_receive {Tortoise, {:ping_response, ^ref, _}}
+      assert_receive {{Tortoise, ^client_id}, {Package.Pingreq, ^ref}, _}
     end
 
     test "ping_sync/2", %{client_id: client_id} = context do

From d9e7fe0308ca8b663ac3554f16213ddd84b25337 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sun, 7 Oct 2018 13:33:58 +0200
Subject: [PATCH 029/220] Test that the connection handler callbacks are called

---
 lib/tortoise/connection.ex        |  6 +---
 test/support/test_handler.exs     | 20 +++++++++++++
 test/tortoise/connection_test.exs | 50 +++++++++++++++++++++++--------
 3 files changed, 59 insertions(+), 17 deletions(-)
 create mode 100644 test/support/test_handler.exs

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 20a34181..39c403d3 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -335,10 +335,6 @@ defmodule Tortoise.Connection do
     receive do
       {{Tortoise, ^client_id}, {Package.Pingreq, ^ref}, round_trip_time} ->
         {:ok, round_trip_time}
-
-      otherwise ->
-        IO.inspect(otherwise)
-        :ok
     after
       timeout ->
         {:error, :timeout}
@@ -698,7 +694,7 @@ defmodule Tortoise.Connection do
         :info,
         {{Tortoise, client_id}, :status, current_state},
         current_state,
-        %State{} = data
+        %State{}
       ) do
     :keep_state_and_data
   end
diff --git a/test/support/test_handler.exs b/test/support/test_handler.exs
new file mode 100644
index 00000000..f099408b
--- /dev/null
+++ b/test/support/test_handler.exs
@@ -0,0 +1,20 @@
+defmodule TestHandler do
+  use Tortoise.Handler
+
+  def init(opts) do
+    state = Enum.into(opts, %{})
+    send(state[:parent], {{__MODULE__, :init}, opts})
+    {:ok, state}
+  end
+
+  def terminate(reason, state) do
+    send(state[:parent], {{__MODULE__, :terminate}, reason})
+    :ok
+  end
+
+  def handle_message(topic, payload, state) do
+    data = %{topic: Enum.join(topic, "/"), payload: payload}
+    send(state[:parent], {{__MODULE__, :handle_message}, data})
+    {:ok, state}
+  end
+end
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index e5acc6d3..18a77a28 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -1,5 +1,6 @@
 Code.require_file("../support/scripted_mqtt_server.exs", __DIR__)
 Code.require_file("../support/scripted_transport.exs", __DIR__)
+Code.require_file("../support/test_handler.exs", __DIR__)
 
 defmodule Tortoise.ConnectionTest do
   use ExUnit.Case, async: true
@@ -60,7 +61,7 @@ defmodule Tortoise.ConnectionTest do
     opts = [
       client_id: client_id,
       server: {Tortoise.Transport.Tcp, [host: ip, port: port]},
-      handler: {Tortoise.Handler.Default, []}
+      handler: {TestHandler, [parent: self()]}
     ]
 
     assert {:ok, connection_pid} = Connection.start_link(opts)
@@ -660,10 +661,12 @@ defmodule Tortoise.ConnectionTest do
 
       {:ok, {ip, port}} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
 
+      handler = {TestHandler, [parent: self()]}
+
       opts = [
         client_id: client_id,
         server: {Tortoise.Transport.Tcp, [host: ip, port: port]},
-        handler: {Tortoise.Handler.Default, []}
+        handler: handler
       ]
 
       assert {:ok, pid} = Connection.start_link(opts)
@@ -685,6 +688,10 @@ defmodule Tortoise.ConnectionTest do
       assert_receive {:DOWN, ^cs_ref, :process, ^cs_pid, :shutdown}
       refute Process.alive?(inflight_pid)
       refute Process.alive?(receiver_pid)
+
+      # The user defined handler should have had its init/1 triggered
+      {handler_mod, handler_init_opts} = handler
+      assert_receive {{^handler_mod, :init}, ^handler_init_opts}
     end
   end
 
@@ -738,9 +745,11 @@ defmodule Tortoise.ConnectionTest do
 
       {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
       pid = context.connection_pid
+      expected_reason = {:protocol_violation, {:unexpected_package, unexpected_connect}}
+      assert_receive {:EXIT, ^pid, ^expected_reason}
 
-      assert_receive {:EXIT, ^pid,
-                      {:protocol_violation, {:unexpected_package, ^unexpected_connect}}}
+      # the terminate/2 callback should get triggered
+      assert_receive {{TestHandler, :terminate}, ^expected_reason}
     end
 
     test "Receiving a connack after the handshake is a protocol violation", context do
@@ -750,9 +759,11 @@ defmodule Tortoise.ConnectionTest do
 
       {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
       pid = context.connection_pid
+      expected_reason = {:protocol_violation, {:unexpected_package, unexpected_connack}}
+      assert_receive {:EXIT, ^pid, ^expected_reason}
 
-      assert_receive {:EXIT, ^pid,
-                      {:protocol_violation, {:unexpected_package, ^unexpected_connack}}}
+      # the terminate/2 callback should get triggered
+      assert_receive {{TestHandler, :terminate}, ^expected_reason}
     end
 
     test "Receiving a ping request from the server is a protocol violation", context do
@@ -762,9 +773,11 @@ defmodule Tortoise.ConnectionTest do
 
       {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
       pid = context.connection_pid
+      expected_reason = {:protocol_violation, {:unexpected_package, unexpected_pingreq}}
+      assert_receive {:EXIT, ^pid, ^expected_reason}
 
-      assert_receive {:EXIT, ^pid,
-                      {:protocol_violation, {:unexpected_package, ^unexpected_pingreq}}}
+      # the terminate/2 callback should get triggered
+      assert_receive {{TestHandler, :terminate}, ^expected_reason}
     end
 
     test "Receiving a subscribe package from the server is a protocol violation", context do
@@ -781,9 +794,11 @@ defmodule Tortoise.ConnectionTest do
 
       {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
       pid = context.connection_pid
+      expected_reason = {:protocol_violation, {:unexpected_package, unexpected_subscribe}}
+      assert_receive {:EXIT, ^pid, ^expected_reason}
 
-      assert_receive {:EXIT, ^pid,
-                      {:protocol_violation, {:unexpected_package, ^unexpected_subscribe}}}
+      # the terminate/2 callback should get triggered
+      assert_receive {{TestHandler, :terminate}, ^expected_reason}
     end
 
     test "Receiving an unsubscribe package from the server is a protocol violation", context do
@@ -798,9 +813,11 @@ defmodule Tortoise.ConnectionTest do
 
       {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
       pid = context.connection_pid
+      expected_reason = {:protocol_violation, {:unexpected_package, unexpected_unsubscribe}}
+      assert_receive {:EXIT, ^pid, ^expected_reason}
 
-      assert_receive {:EXIT, ^pid,
-                      {:protocol_violation, {:unexpected_package, ^unexpected_unsubscribe}}}
+      # the terminate/2 callback should get triggered
+      assert_receive {{TestHandler, :terminate}, ^expected_reason}
     end
   end
 
@@ -818,6 +835,9 @@ defmodule Tortoise.ConnectionTest do
 
       refute_receive {:EXIT, ^pid, {:protocol_violation, {:unexpected_package, ^publish}}}
       assert_receive {ScriptedMqttServer, :completed}
+
+      # the handle message callback should have been called
+      assert_receive {{TestHandler, :handle_message}, %{topic: "foo/bar", payload: nil}}
     end
   end
 
@@ -836,6 +856,9 @@ defmodule Tortoise.ConnectionTest do
 
       {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
       assert_receive {ScriptedMqttServer, :completed}
+
+      # the handle message callback should have been called
+      assert_receive {{TestHandler, :handle_message}, %{topic: "foo/bar", payload: nil}}
     end
 
     test "outgoing publish with QoS=1", context do
@@ -883,6 +906,9 @@ defmodule Tortoise.ConnectionTest do
 
       refute_receive {:EXIT, ^pid, {:protocol_violation, {:unexpected_package, _}}}
       assert_receive {ScriptedMqttServer, :completed}
+
+      # the handle message callback should have been called
+      assert_receive {{TestHandler, :handle_message}, %{topic: "foo/bar", payload: nil}}
     end
 
     test "outgoing publish with QoS=2", context do

From c9ed2be812840d7d0f9fd50c4c01c7125b4bb109 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sun, 7 Oct 2018 13:37:20 +0200
Subject: [PATCH 030/220] Dispatch round-trip time to events when a ping
 response is returned

---
 lib/tortoise/connection.ex | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 39c403d3..18c61bae 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -772,6 +772,7 @@ defmodule Tortoise.Connection do
     {{:value, {{caller, ref}, start_time}}, ping} = :queue.out(ping)
     round_trip_time = System.monotonic_time(:microsecond) - start_time
     send(caller, {{Tortoise, client_id}, {Package.Pingreq, ref}, round_trip_time})
+    :ok = Events.dispatch(client_id, :ping_response, round_trip_time)
     {:keep_state, %State{data | ping: ping}}
   end
 

From 5caf609a96f14345c88f4fc2db64fa79d177e7ad Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sun, 7 Oct 2018 14:15:59 +0200
Subject: [PATCH 031/220] Minor clean up

---
 lib/tortoise/connection.ex | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 18c61bae..808a6cf2 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -665,9 +665,8 @@ defmodule Tortoise.Connection do
         :connecting,
         %State{connect: connect, backoff: backoff} = data
       ) do
-    :ok = Tortoise.Registry.put_meta(via_name(data.client_id), :connecting)
-
-    with :ok = start_connection_supervisor([{:parent, self()} | data.opts]),
+    with :ok = Tortoise.Registry.put_meta(via_name(data.client_id), :connecting),
+         :ok = start_connection_supervisor([{:parent, self()} | data.opts]),
          {:ok, {transport, socket}} <- Tortoise.Connection.Receiver.connect(data.client_id),
          :ok = transport.send(socket, Package.encode(data.connect)) do
       new_data = %State{

From 13417a659e7e485fb58cc2d2f0932a9859063996 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sun, 7 Oct 2018 15:12:33 +0200
Subject: [PATCH 032/220] Make :ignore or {:stop, reason} possible returns on
 handler init

---
 lib/tortoise/connection.ex | 6 +++++-
 lib/tortoise/handler.ex    | 9 ++++++++-
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 808a6cf2..c5e64155 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -409,7 +409,11 @@ defmodule Tortoise.Connection do
         updated_data = %State{data | handler: updated_handler}
         {:ok, :connecting, updated_data, next_events}
 
-        # todo, handle ignore from handler:init/1
+      :ignore ->
+        :ignore
+
+      {:stop, reason} ->
+        {:stop, reason}
     end
   end
 
diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index 627cfe92..4d7b85c7 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -294,7 +294,8 @@ defmodule Tortoise.Handler do
                  ignored: term()
 
   @doc false
-  @spec execute(t, action) :: :ok | {:ok, t} | {:error, {:invalid_next_action, term()}}
+  @spec execute(t, action) ::
+          :ok | {:ok, t} | {:error, {:invalid_next_action, term()}} | :ignore | {:stop, term()}
         when action:
                :init
                | {:subscribe, [term()]}
@@ -306,6 +307,12 @@ defmodule Tortoise.Handler do
     case apply(handler.module, :init, [handler.initial_args]) do
       {:ok, initial_state} ->
         {:ok, %__MODULE__{handler | state: initial_state}}
+
+      :ignore ->
+        :ignore
+
+      {:stop, reason} ->
+        {:stop, reason}
     end
   end
 

From e6992780dc074634b3de9bb413ad9fb193a1cf2f Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sun, 7 Oct 2018 15:14:13 +0200
Subject: [PATCH 033/220] Clean controller_test up and move more into
 connection_test

---
 test/tortoise/connection/controller_test.exs | 69 --------------------
 test/tortoise/connection_test.exs            | 29 ++++++++
 2 files changed, 29 insertions(+), 69 deletions(-)

diff --git a/test/tortoise/connection/controller_test.exs b/test/tortoise/connection/controller_test.exs
index e3149f66..727ccaee 100644
--- a/test/tortoise/connection/controller_test.exs
+++ b/test/tortoise/connection/controller_test.exs
@@ -130,38 +130,9 @@ defmodule Tortoise.Connection.ControllerTest do
     end
   end
 
-  describe "Connection Control Packets" do
-    setup [:setup_controller]
-
-    test "receiving a disconnect from the server is a protocol violation",
-         %{controller_pid: pid} = context do
-      Process.flag(:trap_exit, true)
-      # receiving a disconnect request from the server is a (3.1.1)
-      # protocol violation
-      disconnect = %Package.Disconnect{}
-      Controller.handle_incoming(context.client_id, disconnect)
-
-      assert_receive {:EXIT, ^pid,
-                      {:protocol_violation, {:unexpected_package_from_remote, ^disconnect}}}
-    end
-  end
-
   describe "publish" do
     setup [:setup_controller]
 
-    test "receive a publish", context do
-      publish = %Package.Publish{
-        topic: "foo/bar/baz",
-        payload: "how do you do?",
-        qos: 0
-      }
-
-      assert :ok = Controller.handle_incoming(context.client_id, publish)
-      topic_list = String.split(publish.topic, "/")
-      payload = publish.payload
-      assert_receive(%TestHandler{received: [{^topic_list, ^payload} | _]})
-    end
-
     test "update callback module state between publishes", context do
       publish = %Package.Publish{topic: "a", qos: 0}
       # Our callback module will increment a counter when it receives
@@ -173,46 +144,6 @@ defmodule Tortoise.Connection.ControllerTest do
     end
   end
 
-  describe "Publish Control Packets with Quality of Service level 1" do
-    setup [:setup_connection, :setup_controller, :setup_inflight]
-
-    test "outgoing publish with qos 1", context do
-      client_id = context.client_id
-      publish = %Package.Publish{identifier: 1, topic: "a", qos: 1}
-      # we will get a reference (not the message id).
-      assert {:ok, ref} = Inflight.track(client_id, {:outgoing, publish})
-
-      # assert that the server receives a publish package
-      {:ok, package} = :gen_tcp.recv(context.server, 0, 200)
-      assert ^publish = Package.decode(package)
-      # the server will send back an ack message
-      Controller.handle_incoming(client_id, %Package.Puback{identifier: 1})
-      # the caller should get a message in its mailbox
-      assert_receive {{Tortoise, ^client_id}, {Package.Publish, ^ref}, :ok}
-    end
-
-    test "outgoing publish with qos 1 sync call", context do
-      client_id = context.client_id
-      publish = %Package.Publish{identifier: 1, topic: "a", qos: 1}
-
-      # setup a blocking call
-      {caller, test_ref} = {self(), make_ref()}
-
-      spawn_link(fn ->
-        test_result = Inflight.track_sync(client_id, {:outgoing, publish})
-        send(caller, {:sync_call_result, test_ref, test_result})
-      end)
-
-      # assert that the server receives a publish package
-      {:ok, package} = :gen_tcp.recv(context.server, 0, 200)
-      assert ^publish = Package.decode(package)
-      # the server will send back an ack message
-      Controller.handle_incoming(client_id, %Package.Puback{identifier: 1})
-      # the blocking call should receive :ok when the message is acked
-      assert_receive {:sync_call_result, ^test_ref, :ok}
-    end
-  end
-
   describe "Publish Quality of Service level 2" do
     setup [:setup_connection, :setup_controller, :setup_inflight]
 
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index 18a77a28..5268cde9 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -880,8 +880,37 @@ defmodule Tortoise.ConnectionTest do
       refute_receive {:EXIT, ^pid, {:protocol_violation, {:unexpected_package, _}}}
       assert_receive {ScriptedMqttServer, {:received, ^publish}}
       assert_receive {ScriptedMqttServer, :completed}
+      # the caller should receive an :ok for the ref when it is published
       assert_receive {{Tortoise, ^client_id}, {Package.Publish, ^ref}, :ok}
     end
+
+    test "outgoing publish with QoS=1 (sync call)", %{client_id: client_id} = context do
+      Process.flag(:trap_exit, true)
+      publish = %Package.Publish{identifier: 1, topic: "foo/bar", qos: 1}
+      puback = %Package.Puback{identifier: 1}
+
+      script = [
+        {:receive, publish},
+        {:send, puback}
+      ]
+
+      {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
+
+      # setup a blocking call
+      {parent, test_ref} = {self(), make_ref()}
+
+      spawn_link(fn ->
+        test_result = Inflight.track_sync(client_id, {:outgoing, publish})
+        send(parent, {:sync_call_result, test_ref, test_result})
+      end)
+
+      pid = context.connection_pid
+      refute_receive {:EXIT, ^pid, {:protocol_violation, {:unexpected_package, _}}}
+      assert_receive {ScriptedMqttServer, {:received, ^publish}}
+      assert_receive {ScriptedMqttServer, :completed}
+      # the caller should receive an :ok for the ref when it is published
+      assert_receive {:sync_call_result, ^test_ref, :ok}
+    end
   end
 
   describe "Publish with QoS=2" do

From 2ca95d7d25b43b90df79b1cdf26db4452c642187 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sun, 7 Oct 2018 16:26:57 +0200
Subject: [PATCH 034/220] Call the user defined callbacks on subscription
 changes

---
 lib/tortoise/connection.ex        | 23 +++++++++++++++++++++--
 test/support/test_handler.exs     |  6 ++++++
 test/tortoise/connection_test.exs | 14 ++++++++++++--
 3 files changed, 39 insertions(+), 4 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index c5e64155..504e83d5 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -424,6 +424,21 @@ defmodule Tortoise.Connection do
   end
 
   @impl true
+  def handle_event(
+        :internal,
+        {:execute_handler, cmd},
+        _,
+        %State{handler: handler} = data
+      ) do
+    case Handler.execute(handler, cmd) do
+      {:ok, %Handler{} = updated_handler} ->
+        updated_data = %State{data | handler: updated_handler}
+        {:keep_state, updated_data}
+
+        # handle stop
+    end
+  end
+
   def handle_event(:info, {:incoming, package}, _, _data) when is_binary(package) do
     next_actions = [{:next_event, :internal, {:received, Package.decode(package)}}]
     {:keep_state_and_data, next_actions}
@@ -617,7 +632,9 @@ defmodule Tortoise.Connection do
       {{pid, msg_ref}, updated_pending} when is_pid(pid) and is_reference(msg_ref) ->
         unless pid == self(), do: send(pid, {{Tortoise, client_id}, msg_ref, :ok})
         subscriptions = Enum.into(result[:ok] ++ result[:warn], data.subscriptions)
-        {:keep_state, %State{data | subscriptions: subscriptions, pending_refs: updated_pending}}
+        updated_data = %State{data | subscriptions: subscriptions, pending_refs: updated_pending}
+        next_actions = [{:next_event, :internal, {:execute_handler, {:subscribe, result}}}]
+        {:keep_state, updated_data, next_actions}
     end
   end
 
@@ -653,7 +670,9 @@ defmodule Tortoise.Connection do
         topics = Keyword.drop(data.subscriptions.topics, unsubbed)
         subscriptions = %Package.Subscribe{data.subscriptions | topics: topics}
         unless pid == self(), do: send(pid, {{Tortoise, client_id}, msg_ref, :ok})
-        {:keep_state, %State{data | pending_refs: updated_pending, subscriptions: subscriptions}}
+        updated_data = %State{data | pending_refs: updated_pending, subscriptions: subscriptions}
+        next_actions = [{:next_event, :internal, {:execute_handler, {:unsubscribe, unsubbed}}}]
+        {:keep_state, updated_data, next_actions}
     end
   end
 
diff --git a/test/support/test_handler.exs b/test/support/test_handler.exs
index f099408b..e04aaf01 100644
--- a/test/support/test_handler.exs
+++ b/test/support/test_handler.exs
@@ -17,4 +17,10 @@ defmodule TestHandler do
     send(state[:parent], {{__MODULE__, :handle_message}, data})
     {:ok, state}
   end
+
+  def subscription(status, topic_filter, state) do
+    data = %{status: status, topic_filter: topic_filter}
+    send(state[:parent], {{__MODULE__, :subscription}, data})
+    {:ok, state}
+  end
 end
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index 5268cde9..d2f34457 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -294,7 +294,7 @@ defmodule Tortoise.ConnectionTest do
       opts = [
         client_id: client_id,
         server: {Tortoise.Transport.Tcp, [host: ip, port: port]},
-        handler: {Tortoise.Handler.Default, []}
+        handler: {TestHandler, [parent: self()]}
       ]
 
       # connection
@@ -305,16 +305,19 @@ defmodule Tortoise.ConnectionTest do
       :ok = Tortoise.Connection.subscribe_sync(client_id, {"foo", 0}, identifier: 1)
       assert_receive {ScriptedMqttServer, {:received, ^subscription_foo}}
       assert Enum.member?(Tortoise.Connection.subscriptions(client_id), {"foo", 0})
+      assert_receive {{TestHandler, :subscription}, %{status: :up, topic_filter: "foo"}}
 
       # subscribe to a bar
       assert {:ok, ref} = Tortoise.Connection.subscribe(client_id, {"bar", 1}, identifier: 2)
       assert_receive {{Tortoise, ^client_id}, ^ref, :ok}
       assert_receive {ScriptedMqttServer, {:received, ^subscription_bar}}
+      assert_receive {{TestHandler, :subscription}, %{status: :up, topic_filter: "bar"}}
 
       # subscribe to a baz
       assert {:ok, ref} = Tortoise.Connection.subscribe(client_id, "baz", qos: 2, identifier: 3)
       assert_receive {{Tortoise, ^client_id}, ^ref, :ok}
       assert_receive {ScriptedMqttServer, {:received, ^subscription_baz}}
+      assert_receive {{TestHandler, :subscription}, %{status: :up, topic_filter: "baz"}}
 
       # foo, bar, and baz should now be in the subscription list
       subscriptions = Tortoise.Connection.subscriptions(client_id)
@@ -366,7 +369,7 @@ defmodule Tortoise.ConnectionTest do
       opts = [
         client_id: client_id,
         server: {Tortoise.Transport.Tcp, [host: ip, port: port]},
-        handler: {Tortoise.Handler.Default, []},
+        handler: {TestHandler, [parent: self()]},
         subscriptions: subscribe
       ]
 
@@ -375,9 +378,14 @@ defmodule Tortoise.ConnectionTest do
 
       assert_receive {ScriptedMqttServer, {:received, ^subscribe}}
 
+      assert_receive {{TestHandler, :subscription}, %{status: :up, topic_filter: "foo"}}
+      assert_receive {{TestHandler, :subscription}, %{status: :up, topic_filter: "bar"}}
+
       # now let us try to unsubscribe from foo
       :ok = Tortoise.Connection.unsubscribe_sync(client_id, "foo", identifier: 2)
       assert_receive {ScriptedMqttServer, {:received, ^unsubscribe_foo}}
+      # the callback handler should get a :down message for the foo subscription
+      assert_receive {{TestHandler, :subscription}, %{status: :down, topic_filter: "foo"}}
 
       assert %Package.Subscribe{topics: [{"bar", qos: 2}]} =
                Tortoise.Connection.subscriptions(client_id)
@@ -386,6 +394,8 @@ defmodule Tortoise.ConnectionTest do
       assert {:ok, ref} = Tortoise.Connection.unsubscribe(client_id, "bar", identifier: 3)
       assert_receive {{Tortoise, ^client_id}, ^ref, :ok}
       assert_receive {ScriptedMqttServer, {:received, ^unsubscribe_bar}}
+      # the callback handler should get a :down message for the bar subscription
+      assert_receive {{TestHandler, :subscription}, %{status: :down, topic_filter: "bar"}}
       assert %Package.Subscribe{topics: []} = Tortoise.Connection.subscriptions(client_id)
       assert_receive {ScriptedMqttServer, :completed}
     end

From fd89114b1b69972aaa598dd4c7d3a3b9e701c53e Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sun, 7 Oct 2018 18:05:33 +0200
Subject: [PATCH 035/220] Make sure the connection callback is called on
 connection

---
 lib/tortoise/connection.ex                   | 23 +++++----
 test/support/test_handler.exs                |  5 ++
 test/tortoise/connection/controller_test.exs | 54 --------------------
 test/tortoise/connection_test.exs            |  6 ++-
 4 files changed, 24 insertions(+), 64 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 504e83d5..6b7165ac 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -448,23 +448,30 @@ defmodule Tortoise.Connection do
   def handle_event(
         :internal,
         {:received, %Package.Connack{reason: :success} = connack},
-        # :handshake,
         :connecting,
-        %State{server: %Transport{type: transport}, connection: {transport, socket}} = data
+        %State{
+          client_id: client_id,
+          server: %Transport{type: transport},
+          connection: {transport, _}
+        } = data
       ) do
-    connection = {transport, socket}
-    :ok = Tortoise.Registry.put_meta(via_name(data.client_id), connection)
-    :ok = Events.dispatch(data.client_id, :connection, connection)
-    :ok = Events.dispatch(data.client_id, :status, :connected)
+    :ok = Tortoise.Registry.put_meta(via_name(client_id), data.connection)
+    :ok = Events.dispatch(client_id, :connection, data.connection)
+    :ok = Events.dispatch(client_id, :status, :connected)
 
     case connack do
       %Package.Connack{session_present: true} ->
-        {:next_state, :connected, data}
+        next_actions = [
+          {:next_event, :internal, {:execute_handler, {:connection, :up}}}
+        ]
+
+        {:next_state, :connected, data, next_actions}
 
       %Package.Connack{session_present: false} ->
         caller = {self(), make_ref()}
 
         next_actions = [
+          {:next_event, :internal, {:execute_handler, {:connection, :up}}},
           {:next_event, :cast, {:subscribe, caller, data.subscriptions, []}}
         ]
 
@@ -475,7 +482,6 @@ defmodule Tortoise.Connection do
   def handle_event(
         :internal,
         {:received, %Package.Connack{reason: {:refused, reason}}},
-        # :handshake,
         :connecting,
         %State{} = data
       ) do
@@ -485,7 +491,6 @@ defmodule Tortoise.Connection do
   def handle_event(
         :internal,
         {:received, package},
-        # :handshake,
         :connecting,
         %State{} = data
       ) do
diff --git a/test/support/test_handler.exs b/test/support/test_handler.exs
index e04aaf01..e145862a 100644
--- a/test/support/test_handler.exs
+++ b/test/support/test_handler.exs
@@ -12,6 +12,11 @@ defmodule TestHandler do
     :ok
   end
 
+  def connection(status, state) do
+    send(state[:parent], {{__MODULE__, :connection}, status})
+    {:ok, state}
+  end
+
   def handle_message(topic, payload, state) do
     data = %{topic: Enum.join(topic, "/"), payload: payload}
     send(state[:parent], {{__MODULE__, :handle_message}, data})
diff --git a/test/tortoise/connection/controller_test.exs b/test/tortoise/connection/controller_test.exs
index 727ccaee..1724be63 100644
--- a/test/tortoise/connection/controller_test.exs
+++ b/test/tortoise/connection/controller_test.exs
@@ -114,22 +114,6 @@ defmodule Tortoise.Connection.ControllerTest do
   end
 
   # tests --------------------------------------------------------------
-  describe "Connection callback" do
-    setup [:setup_controller]
-
-    test "Callback is triggered on connection status change", context do
-      # tell the controller that we are up
-      :ok = Tortoise.Events.dispatch(context.client_id, :status, :up)
-      assert_receive(%TestHandler{status: :up})
-      # switch to offline
-      :ok = Tortoise.Events.dispatch(context.client_id, :status, :down)
-      assert_receive(%TestHandler{status: :down})
-      # ... and back up
-      :ok = Tortoise.Events.dispatch(context.client_id, :status, :up)
-      assert_receive(%TestHandler{status: :up})
-    end
-  end
-
   describe "publish" do
     setup [:setup_controller]
 
@@ -220,44 +204,6 @@ defmodule Tortoise.Connection.ControllerTest do
   describe "Subscription" do
     setup [:setup_connection, :setup_controller, :setup_inflight]
 
-    test "Subscribe to multiple topics", context do
-      client_id = context.client_id
-
-      subscribe = %Package.Subscribe{
-        identifier: 1,
-        topics: [{"foo", 0}, {"bar", 1}, {"baz", 2}]
-      }
-
-      suback = %Package.Suback{identifier: 1, acks: [{:ok, 0}, {:ok, 1}, {:ok, 2}]}
-
-      assert {:ok, ref} = Inflight.track(client_id, {:outgoing, subscribe})
-
-      # assert that the server receives a subscribe package
-      {:ok, package} = :gen_tcp.recv(context.server, 0, 200)
-      assert ^subscribe = Package.decode(package)
-      # the server will send back a subscription acknowledgement message
-      :ok = Controller.handle_incoming(client_id, suback)
-
-      assert_receive {{Tortoise, ^client_id}, {Package.Subscribe, ^ref}, _}
-      # the client callback module should get the subscribe notifications in order
-      assert_receive %TestHandler{subscriptions: [{"foo", :ok}]}
-      assert_receive %TestHandler{subscriptions: [{"bar", :ok} | _]}
-      assert_receive %TestHandler{subscriptions: [{"baz", :ok} | _]}
-
-      # unsubscribe from a topic
-      unsubscribe = %Package.Unsubscribe{identifier: 2, topics: ["foo", "baz"]}
-      unsuback = %Package.Unsuback{identifier: 2}
-      assert {:ok, ref} = Inflight.track(client_id, {:outgoing, unsubscribe})
-      {:ok, package} = :gen_tcp.recv(context.server, 0, 200)
-      assert ^unsubscribe = Package.decode(package)
-      :ok = Controller.handle_incoming(client_id, unsuback)
-      assert_receive {{Tortoise, ^client_id}, ^ref, _}
-
-      # the client callback module should remove the subscriptions in order
-      assert_receive %TestHandler{subscriptions: [{"baz", :ok}, {"bar", :ok}]}
-      assert_receive %TestHandler{subscriptions: [{"bar", :ok}]}
-    end
-
     test "Subscribe to a topic that return different QoS than requested", context do
       client_id = context.client_id
 
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index d2f34457..8c2d6bc9 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -699,9 +699,13 @@ defmodule Tortoise.ConnectionTest do
       refute Process.alive?(inflight_pid)
       refute Process.alive?(receiver_pid)
 
-      # The user defined handler should have had its init/1 triggered
+      # The user defined handler should have the following callbacks
+      # triggered during this exchange
       {handler_mod, handler_init_opts} = handler
       assert_receive {{^handler_mod, :init}, ^handler_init_opts}
+      assert_receive {{^handler_mod, :connection}, :up}
+      assert_receive {{^handler_mod, :terminate}, :shutdown}
+      refute_receive {{^handler_mod, _}, _}
     end
   end
 

From 91384bd8517b658adc1b286b1911aef322a53a6f Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sun, 7 Oct 2018 20:32:37 +0200
Subject: [PATCH 036/220] Add modules from test/support to load path when
 running in test

This allow us to use the modules without requiring them.
---
 mix.exs                                                     | 6 +++++-
 .../{scripted_mqtt_server.exs => scripted_mqtt_server.ex}   | 0
 .../{scripted_transport.exs => scripted_transport.ex}       | 0
 test/support/{test_handler.exs => test_handler.ex}          | 0
 test/support/{test_tcp_tunnel.exs => test_tcp_tunnel.ex}    | 0
 test/test_helper.exs                                        | 2 --
 test/tortoise/connection_test.exs                           | 4 ----
 7 files changed, 5 insertions(+), 7 deletions(-)
 rename test/support/{scripted_mqtt_server.exs => scripted_mqtt_server.ex} (100%)
 rename test/support/{scripted_transport.exs => scripted_transport.ex} (100%)
 rename test/support/{test_handler.exs => test_handler.ex} (100%)
 rename test/support/{test_tcp_tunnel.exs => test_tcp_tunnel.ex} (100%)

diff --git a/mix.exs b/mix.exs
index d36b3b54..b235a6f7 100644
--- a/mix.exs
+++ b/mix.exs
@@ -22,7 +22,8 @@ defmodule Tortoise.MixProject do
         "coveralls.json": :test,
         "coveralls.post": :test,
         docs: :docs
-      ]
+      ],
+      elixirc_paths: elixirc_paths(Mix.env())
     ]
   end
 
@@ -52,6 +53,9 @@ defmodule Tortoise.MixProject do
     ]
   end
 
+  defp elixirc_paths(:test), do: ["lib", "test/support"]
+  defp elixirc_paths(_), do: ["lib"]
+
   defp package() do
     [
       maintainers: ["Martin Gausby"],
diff --git a/test/support/scripted_mqtt_server.exs b/test/support/scripted_mqtt_server.ex
similarity index 100%
rename from test/support/scripted_mqtt_server.exs
rename to test/support/scripted_mqtt_server.ex
diff --git a/test/support/scripted_transport.exs b/test/support/scripted_transport.ex
similarity index 100%
rename from test/support/scripted_transport.exs
rename to test/support/scripted_transport.ex
diff --git a/test/support/test_handler.exs b/test/support/test_handler.ex
similarity index 100%
rename from test/support/test_handler.exs
rename to test/support/test_handler.ex
diff --git a/test/support/test_tcp_tunnel.exs b/test/support/test_tcp_tunnel.ex
similarity index 100%
rename from test/support/test_tcp_tunnel.exs
rename to test/support/test_tcp_tunnel.ex
diff --git a/test/test_helper.exs b/test/test_helper.exs
index ec05c4a5..cde61325 100644
--- a/test/test_helper.exs
+++ b/test/test_helper.exs
@@ -1,5 +1,3 @@
-Code.require_file("./support/test_tcp_tunnel.exs", __DIR__)
-
 defmodule Tortoise.TestGenerators do
   @moduledoc """
   EQC generators for generating variables and data structures useful
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index 8c2d6bc9..7eed4c88 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -1,7 +1,3 @@
-Code.require_file("../support/scripted_mqtt_server.exs", __DIR__)
-Code.require_file("../support/scripted_transport.exs", __DIR__)
-Code.require_file("../support/test_handler.exs", __DIR__)
-
 defmodule Tortoise.ConnectionTest do
   use ExUnit.Case, async: true
   doctest Tortoise.Connection

From f7c034b4541b090e3306446853f6755d20cd2997 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sun, 7 Oct 2018 22:36:39 +0200
Subject: [PATCH 037/220] Get rid of the old controller code and out comment
 the tests for it

Some tests are still there; kept as a reminder to what functionality
we need to implement in the new connection controller.
---
 lib/tortoise/connection/controller.ex        |  84 ---
 test/tortoise/connection/controller_test.exs | 612 +++++++++----------
 2 files changed, 306 insertions(+), 390 deletions(-)

diff --git a/lib/tortoise/connection/controller.ex b/lib/tortoise/connection/controller.ex
index 4eeb77e2..b7e663a6 100644
--- a/lib/tortoise/connection/controller.ex
+++ b/lib/tortoise/connection/controller.ex
@@ -1,87 +1,3 @@
 defmodule Tortoise.Connection.Controller do
   @moduledoc false
-
-  require Logger
-
-  alias Tortoise.{Package, Handler}
-
-  use GenServer
-
-  @enforce_keys [:client_id, :handler]
-  defstruct client_id: nil,
-            status: :down,
-            awaiting: %{},
-            handler: %Handler{module: Handler.Default, initial_args: []}
-
-  alias __MODULE__, as: State
-
-  # Client API
-  def start_link(opts) do
-    client_id = Keyword.fetch!(opts, :client_id)
-    handler = Handler.new(Keyword.fetch!(opts, :handler))
-
-    init_state = %State{
-      client_id: client_id,
-      handler: handler
-    }
-
-    GenServer.start_link(__MODULE__, init_state, name: via_name(client_id))
-  end
-
-  defp via_name(client_id) do
-    Tortoise.Registry.via_name(__MODULE__, client_id)
-  end
-
-  def stop(client_id) do
-    GenServer.stop(via_name(client_id))
-  end
-
-  def info(client_id) do
-    GenServer.call(via_name(client_id), :info)
-  end
-
-  # Server callbacks
-  @impl true
-  def init(%State{handler: handler} = opts) do
-    {:ok, _} = Tortoise.Events.register(opts.client_id, :status)
-
-    case Handler.execute(handler, :init) do
-      {:ok, %Handler{} = updated_handler} ->
-        {:ok, %State{opts | handler: updated_handler}}
-    end
-  end
-
-  @impl true
-  def terminate(reason, %State{handler: handler}) do
-    _ignored = Handler.execute(handler, {:terminate, reason})
-    :ok
-  end
-
-  @impl true
-  def handle_call(:info, _from, state) do
-    {:reply, state, state}
-  end
-
-  @impl true
-  def handle_cast(
-        {:onward, %Package.Publish{qos: 2, dup: false} = publish},
-        %State{handler: handler} = state
-      ) do
-    case Handler.execute(handler, {:publish, publish}) do
-      {:ok, updated_handler} ->
-        {:noreply, %State{state | handler: updated_handler}}
-    end
-  end
-
-  @impl true
-  def handle_info({{Tortoise, client_id}, ref, result}, %{client_id: client_id} = state) do
-    case {result, Map.pop(state.awaiting, ref)} do
-      {_, {nil, _}} ->
-        Logger.warn("Unexpected async result")
-        {:noreply, state}
-
-      {:ok, {_action, updated_awaiting}} ->
-        {:noreply, %State{state | awaiting: updated_awaiting}}
-    end
-  end
 end
diff --git a/test/tortoise/connection/controller_test.exs b/test/tortoise/connection/controller_test.exs
index 1724be63..6dd588ca 100644
--- a/test/tortoise/connection/controller_test.exs
+++ b/test/tortoise/connection/controller_test.exs
@@ -1,306 +1,306 @@
-defmodule Tortoise.Connection.ControllerTest do
-  use ExUnit.Case
-  doctest Tortoise.Connection.Controller
-
-  alias Tortoise.Package
-  alias Tortoise.Connection.{Controller, Inflight}
-
-  import ExUnit.CaptureLog
-
-  defmodule TestHandler do
-    use Tortoise.Handler
-
-    defstruct pid: nil,
-              client_id: nil,
-              status: nil,
-              publish_count: 0,
-              received: [],
-              subscriptions: []
-
-    def init([client_id, caller]) when is_pid(caller) do
-      # We pass in the caller `pid` and keep it in the state so we can
-      # send messages back to the test process, which will make it
-      # possible to make assertions on the changes in the handler
-      # callback module
-      {:ok, %__MODULE__{pid: caller, client_id: client_id}}
-    end
-
-    def connection(status, state) do
-      new_state = %__MODULE__{state | status: status}
-      send(state.pid, new_state)
-      {:ok, new_state}
-    end
-
-    def subscription(:up, topic_filter, state) do
-      new_state = %__MODULE__{
-        state
-        | subscriptions: [{topic_filter, :ok} | state.subscriptions]
-      }
-
-      send(state.pid, new_state)
-      {:ok, new_state}
-    end
-
-    def subscription(:down, topic_filter, state) do
-      new_state = %__MODULE__{
-        state
-        | subscriptions:
-            Enum.reject(state.subscriptions, fn {topic, _} -> topic == topic_filter end)
-      }
-
-      send(state.pid, new_state)
-      {:ok, new_state}
-    end
-
-    def subscription({:warn, warning}, topic_filter, state) do
-      new_state = %__MODULE__{
-        state
-        | subscriptions: [{topic_filter, warning} | state.subscriptions]
-      }
-
-      send(state.pid, new_state)
-      {:ok, new_state}
-    end
-
-    def subscription({:error, reason}, topic_filter, state) do
-      send(state.pid, {:subscription_error, {topic_filter, reason}})
-      {:ok, state}
-    end
-
-    def handle_message(topic, message, %__MODULE__{} = state) do
-      new_state = %__MODULE__{
-        state
-        | publish_count: state.publish_count + 1,
-          received: [{topic, message} | state.received]
-      }
-
-      send(state.pid, new_state)
-      {:ok, new_state}
-    end
-
-    def terminate(reason, state) do
-      send(state.pid, {:terminating, reason})
-      :ok
-    end
-  end
-
-  # Setup ==============================================================
-  setup context do
-    {:ok, %{client_id: context.test}}
-  end
-
-  def setup_controller(context) do
-    handler = %Tortoise.Handler{
-      module: __MODULE__.TestHandler,
-      initial_args: [context.client_id, self()]
-    }
-
-    opts = [client_id: context.client_id, handler: handler]
-    {:ok, pid} = Controller.start_link(opts)
-    {:ok, %{controller_pid: pid}}
-  end
-
-  def setup_connection(context) do
-    {:ok, client_socket, server_socket} = Tortoise.Integration.TestTCPTunnel.new()
-    name = Tortoise.Connection.via_name(context.client_id)
-    :ok = Tortoise.Registry.put_meta(name, {Tortoise.Transport.Tcp, client_socket})
-    {:ok, %{client: client_socket, server: server_socket}}
-  end
-
-  def setup_inflight(context) do
-    opts = [client_id: context.client_id, parent: self()]
-    {:ok, pid} = Inflight.start_link(opts)
-    {:ok, %{inflight_pid: pid}}
-  end
-
-  # tests --------------------------------------------------------------
-  describe "publish" do
-    setup [:setup_controller]
-
-    test "update callback module state between publishes", context do
-      publish = %Package.Publish{topic: "a", qos: 0}
-      # Our callback module will increment a counter when it receives
-      # a publish control packet
-      :ok = Controller.handle_incoming(context.client_id, publish)
-      assert_receive %TestHandler{publish_count: 1}
-      :ok = Controller.handle_incoming(context.client_id, publish)
-      assert_receive %TestHandler{publish_count: 2}
-    end
-  end
-
-  describe "Publish Quality of Service level 2" do
-    setup [:setup_connection, :setup_controller, :setup_inflight]
-
-    test "incoming publish with qos 2", context do
-      client_id = context.client_id
-      # send in an publish message with a QoS of 2
-      publish = %Package.Publish{identifier: 1, topic: "a", qos: 2}
-      :ok = Controller.handle_incoming(client_id, publish)
-      # test sending in a duplicate publish
-      :ok = Controller.handle_incoming(client_id, %Package.Publish{publish | dup: true})
-
-      # assert that the sender receives a pubrec package
-      {:ok, pubrec} = :gen_tcp.recv(context.server, 0, 200)
-      assert %Package.Pubrec{identifier: 1} = Package.decode(pubrec)
-
-      # the publish should get onwarded to the handler
-      assert_receive %TestHandler{publish_count: 1, received: [{["a"], nil}]}
-
-      # the MQTT server will then respond with pubrel
-      Controller.handle_incoming(client_id, %Package.Pubrel{identifier: 1})
-      # a pubcomp message should get transmitted
-      {:ok, pubcomp} = :gen_tcp.recv(context.server, 0, 200)
-
-      assert %Package.Pubcomp{identifier: 1} = Package.decode(pubcomp)
-
-      # the publish should only get onwareded once
-      refute_receive %TestHandler{publish_count: 2}
-    end
-
-    test "incoming publish with qos 2 (first message dup)", context do
-      # send in an publish with dup set to true should succeed if the
-      # id is unknown.
-      client_id = context.client_id
-      # send in an publish message with a QoS of 2
-      publish = %Package.Publish{identifier: 1, topic: "a", qos: 2, dup: true}
-      :ok = Controller.handle_incoming(client_id, publish)
-
-      # assert that the sender receives a pubrec package
-      {:ok, pubrec} = :gen_tcp.recv(context.server, 0, 200)
-      assert %Package.Pubrec{identifier: 1} = Package.decode(pubrec)
-
-      # the MQTT server will then respond with pubrel
-      Controller.handle_incoming(client_id, %Package.Pubrel{identifier: 1})
-      # a pubcomp message should get transmitted
-      {:ok, pubcomp} = :gen_tcp.recv(context.server, 0, 200)
-
-      assert %Package.Pubcomp{identifier: 1} = Package.decode(pubcomp)
-
-      # the publish should get onwarded to the handler
-      assert_receive %TestHandler{publish_count: 1, received: [{["a"], nil}]}
-    end
-
-    test "outgoing publish with qos 2", context do
-      client_id = context.client_id
-      publish = %Package.Publish{identifier: 1, topic: "a", qos: 2}
-
-      assert {:ok, ref} = Inflight.track(client_id, {:outgoing, publish})
-
-      # assert that the server receives a publish package
-      {:ok, package} = :gen_tcp.recv(context.server, 0, 200)
-      assert ^publish = Package.decode(package)
-      # the server will send back a publish received message
-      Controller.handle_incoming(client_id, %Package.Pubrec{identifier: 1})
-      # we should send a publish release (pubrel) to the server
-      {:ok, package} = :gen_tcp.recv(context.server, 0, 200)
-      assert %Package.Pubrel{identifier: 1} = Package.decode(package)
-      # receive pubcomp
-      Controller.handle_incoming(client_id, %Package.Pubcomp{identifier: 1})
-      # the caller should get a message in its mailbox
-      assert_receive {{Tortoise, ^client_id}, {Package.Publish, ^ref}, :ok}
-    end
-  end
-
-  describe "Subscription" do
-    setup [:setup_connection, :setup_controller, :setup_inflight]
-
-    test "Subscribe to a topic that return different QoS than requested", context do
-      client_id = context.client_id
-
-      subscribe = %Package.Subscribe{
-        identifier: 1,
-        topics: [{"foo", 2}]
-      }
-
-      suback = %Package.Suback{identifier: 1, acks: [{:ok, 0}]}
-
-      assert {:ok, ref} = Inflight.track(client_id, {:outgoing, subscribe})
-
-      # assert that the server receives a subscribe package
-      {:ok, package} = :gen_tcp.recv(context.server, 0, 200)
-      assert ^subscribe = Package.decode(package)
-      # the server will send back a subscription acknowledgement message
-      :ok = Controller.handle_incoming(client_id, suback)
-
-      assert_receive {{Tortoise, ^client_id}, ^ref, _}
-      # the client callback module should get the subscribe notifications in order
-      assert_receive %TestHandler{subscriptions: [{"foo", [requested: 2, accepted: 0]}]}
-
-      # unsubscribe from a topic
-      unsubscribe = %Package.Unsubscribe{identifier: 2, topics: ["foo"]}
-      unsuback = %Package.Unsuback{identifier: 2}
-      assert {:ok, ref} = Inflight.track(client_id, {:outgoing, unsubscribe})
-      {:ok, package} = :gen_tcp.recv(context.server, 0, 200)
-      assert ^unsubscribe = Package.decode(package)
-      :ok = Controller.handle_incoming(client_id, unsuback)
-      assert_receive {{Tortoise, ^client_id}, ^ref, _}
-
-      # the client callback module should remove the subscription
-      assert_receive %TestHandler{subscriptions: []}
-    end
-
-    test "Subscribe to a topic resulting in an error", context do
-      client_id = context.client_id
-
-      subscribe = %Package.Subscribe{
-        identifier: 1,
-        topics: [{"foo", 1}]
-      }
-
-      suback = %Package.Suback{identifier: 1, acks: [{:error, :access_denied}]}
-
-      assert {:ok, ref} = Inflight.track(client_id, {:outgoing, subscribe})
-
-      # assert that the server receives a subscribe package
-      {:ok, package} = :gen_tcp.recv(context.server, 0, 200)
-      assert ^subscribe = Package.decode(package)
-      # the server will send back a subscription acknowledgement message
-      :ok = Controller.handle_incoming(client_id, suback)
-
-      assert_receive {{Tortoise, ^client_id}, ^ref, _}
-      # the callback module should get the error
-      assert_receive {:subscription_error, {"foo", :access_denied}}
-    end
-  end
-
-  describe "next actions" do
-    setup [:setup_controller]
-
-    test "subscribe action", context do
-      client_id = context.client_id
-      next_action = {:subscribe, "foo/bar", qos: 0}
-      send(context.controller_pid, {:next_action, next_action})
-      %{awaiting: awaiting} = Controller.info(client_id)
-      assert [{ref, ^next_action}] = Map.to_list(awaiting)
-      response = {{Tortoise, client_id}, ref, :ok}
-      send(context.controller_pid, response)
-      %{awaiting: awaiting} = Controller.info(client_id)
-      assert [] = Map.to_list(awaiting)
-    end
-
-    test "unsubscribe action", context do
-      client_id = context.client_id
-      next_action = {:unsubscribe, "foo/bar"}
-      send(context.controller_pid, {:next_action, next_action})
-      %{awaiting: awaiting} = Controller.info(client_id)
-      assert [{ref, ^next_action}] = Map.to_list(awaiting)
-      response = {{Tortoise, client_id}, ref, :ok}
-      send(context.controller_pid, response)
-      %{awaiting: awaiting} = Controller.info(client_id)
-      assert [] = Map.to_list(awaiting)
-    end
-
-    test "receiving unknown async ref", context do
-      client_id = context.client_id
-      ref = make_ref()
-
-      assert capture_log(fn ->
-               send(context.controller_pid, {{Tortoise, client_id}, ref, :ok})
-               :timer.sleep(100)
-             end) =~ "Unexpected"
-
-      %{awaiting: awaiting} = Controller.info(client_id)
-      assert [] = Map.to_list(awaiting)
-    end
-  end
-end
+# defmodule Tortoise.Connection.ControllerTest do
+#   use ExUnit.Case
+#   doctest Tortoise.Connection.Controller
+
+#   alias Tortoise.Package
+#   alias Tortoise.Connection.{Controller, Inflight}
+
+#   import ExUnit.CaptureLog
+
+#   defmodule TestHandler do
+#     use Tortoise.Handler
+
+#     defstruct pid: nil,
+#               client_id: nil,
+#               status: nil,
+#               publish_count: 0,
+#               received: [],
+#               subscriptions: []
+
+#     def init([client_id, caller]) when is_pid(caller) do
+#       # We pass in the caller `pid` and keep it in the state so we can
+#       # send messages back to the test process, which will make it
+#       # possible to make assertions on the changes in the handler
+#       # callback module
+#       {:ok, %__MODULE__{pid: caller, client_id: client_id}}
+#     end
+
+#     def connection(status, state) do
+#       new_state = %__MODULE__{state | status: status}
+#       send(state.pid, new_state)
+#       {:ok, new_state}
+#     end
+
+#     def subscription(:up, topic_filter, state) do
+#       new_state = %__MODULE__{
+#         state
+#         | subscriptions: [{topic_filter, :ok} | state.subscriptions]
+#       }
+
+#       send(state.pid, new_state)
+#       {:ok, new_state}
+#     end
+
+#     def subscription(:down, topic_filter, state) do
+#       new_state = %__MODULE__{
+#         state
+#         | subscriptions:
+#             Enum.reject(state.subscriptions, fn {topic, _} -> topic == topic_filter end)
+#       }
+
+#       send(state.pid, new_state)
+#       {:ok, new_state}
+#     end
+
+#     def subscription({:warn, warning}, topic_filter, state) do
+#       new_state = %__MODULE__{
+#         state
+#         | subscriptions: [{topic_filter, warning} | state.subscriptions]
+#       }
+
+#       send(state.pid, new_state)
+#       {:ok, new_state}
+#     end
+
+#     def subscription({:error, reason}, topic_filter, state) do
+#       send(state.pid, {:subscription_error, {topic_filter, reason}})
+#       {:ok, state}
+#     end
+
+#     def handle_message(topic, message, %__MODULE__{} = state) do
+#       new_state = %__MODULE__{
+#         state
+#         | publish_count: state.publish_count + 1,
+#           received: [{topic, message} | state.received]
+#       }
+
+#       send(state.pid, new_state)
+#       {:ok, new_state}
+#     end
+
+#     def terminate(reason, state) do
+#       send(state.pid, {:terminating, reason})
+#       :ok
+#     end
+#   end
+
+#   # Setup ==============================================================
+#   setup context do
+#     {:ok, %{client_id: context.test}}
+#   end
+
+#   def setup_controller(context) do
+#     handler = %Tortoise.Handler{
+#       module: __MODULE__.TestHandler,
+#       initial_args: [context.client_id, self()]
+#     }
+
+#     opts = [client_id: context.client_id, handler: handler]
+#     {:ok, pid} = Controller.start_link(opts)
+#     {:ok, %{controller_pid: pid}}
+#   end
+
+#   def setup_connection(context) do
+#     {:ok, client_socket, server_socket} = Tortoise.Integration.TestTCPTunnel.new()
+#     name = Tortoise.Connection.via_name(context.client_id)
+#     :ok = Tortoise.Registry.put_meta(name, {Tortoise.Transport.Tcp, client_socket})
+#     {:ok, %{client: client_socket, server: server_socket}}
+#   end
+
+#   def setup_inflight(context) do
+#     opts = [client_id: context.client_id, parent: self()]
+#     {:ok, pid} = Inflight.start_link(opts)
+#     {:ok, %{inflight_pid: pid}}
+#   end
+
+#   # tests --------------------------------------------------------------
+#   describe "publish" do
+#     setup [:setup_controller]
+
+#     test "update callback module state between publishes", context do
+#       publish = %Package.Publish{topic: "a", qos: 0}
+#       # Our callback module will increment a counter when it receives
+#       # a publish control packet
+#       :ok = Controller.handle_incoming(context.client_id, publish)
+#       assert_receive %TestHandler{publish_count: 1}
+#       :ok = Controller.handle_incoming(context.client_id, publish)
+#       assert_receive %TestHandler{publish_count: 2}
+#     end
+#   end
+
+#   describe "Publish Quality of Service level 2" do
+#     setup [:setup_connection, :setup_controller, :setup_inflight]
+
+#     test "incoming publish with qos 2", context do
+#       client_id = context.client_id
+#       # send in an publish message with a QoS of 2
+#       publish = %Package.Publish{identifier: 1, topic: "a", qos: 2}
+#       :ok = Controller.handle_incoming(client_id, publish)
+#       # test sending in a duplicate publish
+#       :ok = Controller.handle_incoming(client_id, %Package.Publish{publish | dup: true})
+
+#       # assert that the sender receives a pubrec package
+#       {:ok, pubrec} = :gen_tcp.recv(context.server, 0, 200)
+#       assert %Package.Pubrec{identifier: 1} = Package.decode(pubrec)
+
+#       # the publish should get onwarded to the handler
+#       assert_receive %TestHandler{publish_count: 1, received: [{["a"], nil}]}
+
+#       # the MQTT server will then respond with pubrel
+#       Controller.handle_incoming(client_id, %Package.Pubrel{identifier: 1})
+#       # a pubcomp message should get transmitted
+#       {:ok, pubcomp} = :gen_tcp.recv(context.server, 0, 200)
+
+#       assert %Package.Pubcomp{identifier: 1} = Package.decode(pubcomp)
+
+#       # the publish should only get onwareded once
+#       refute_receive %TestHandler{publish_count: 2}
+#     end
+
+#     test "incoming publish with qos 2 (first message dup)", context do
+#       # send in an publish with dup set to true should succeed if the
+#       # id is unknown.
+#       client_id = context.client_id
+#       # send in an publish message with a QoS of 2
+#       publish = %Package.Publish{identifier: 1, topic: "a", qos: 2, dup: true}
+#       :ok = Controller.handle_incoming(client_id, publish)
+
+#       # assert that the sender receives a pubrec package
+#       {:ok, pubrec} = :gen_tcp.recv(context.server, 0, 200)
+#       assert %Package.Pubrec{identifier: 1} = Package.decode(pubrec)
+
+#       # the MQTT server will then respond with pubrel
+#       Controller.handle_incoming(client_id, %Package.Pubrel{identifier: 1})
+#       # a pubcomp message should get transmitted
+#       {:ok, pubcomp} = :gen_tcp.recv(context.server, 0, 200)
+
+#       assert %Package.Pubcomp{identifier: 1} = Package.decode(pubcomp)
+
+#       # the publish should get onwarded to the handler
+#       assert_receive %TestHandler{publish_count: 1, received: [{["a"], nil}]}
+#     end
+
+#     test "outgoing publish with qos 2", context do
+#       client_id = context.client_id
+#       publish = %Package.Publish{identifier: 1, topic: "a", qos: 2}
+
+#       assert {:ok, ref} = Inflight.track(client_id, {:outgoing, publish})
+
+#       # assert that the server receives a publish package
+#       {:ok, package} = :gen_tcp.recv(context.server, 0, 200)
+#       assert ^publish = Package.decode(package)
+#       # the server will send back a publish received message
+#       Controller.handle_incoming(client_id, %Package.Pubrec{identifier: 1})
+#       # we should send a publish release (pubrel) to the server
+#       {:ok, package} = :gen_tcp.recv(context.server, 0, 200)
+#       assert %Package.Pubrel{identifier: 1} = Package.decode(package)
+#       # receive pubcomp
+#       Controller.handle_incoming(client_id, %Package.Pubcomp{identifier: 1})
+#       # the caller should get a message in its mailbox
+#       assert_receive {{Tortoise, ^client_id}, {Package.Publish, ^ref}, :ok}
+#     end
+#   end
+
+#   describe "Subscription" do
+#     setup [:setup_connection, :setup_controller, :setup_inflight]
+
+#     test "Subscribe to a topic that return different QoS than requested", context do
+#       client_id = context.client_id
+
+#       subscribe = %Package.Subscribe{
+#         identifier: 1,
+#         topics: [{"foo", 2}]
+#       }
+
+#       suback = %Package.Suback{identifier: 1, acks: [{:ok, 0}]}
+
+#       assert {:ok, ref} = Inflight.track(client_id, {:outgoing, subscribe})
+
+#       # assert that the server receives a subscribe package
+#       {:ok, package} = :gen_tcp.recv(context.server, 0, 200)
+#       assert ^subscribe = Package.decode(package)
+#       # the server will send back a subscription acknowledgement message
+#       :ok = Controller.handle_incoming(client_id, suback)
+
+#       assert_receive {{Tortoise, ^client_id}, ^ref, _}
+#       # the client callback module should get the subscribe notifications in order
+#       assert_receive %TestHandler{subscriptions: [{"foo", [requested: 2, accepted: 0]}]}
+
+#       # unsubscribe from a topic
+#       unsubscribe = %Package.Unsubscribe{identifier: 2, topics: ["foo"]}
+#       unsuback = %Package.Unsuback{identifier: 2}
+#       assert {:ok, ref} = Inflight.track(client_id, {:outgoing, unsubscribe})
+#       {:ok, package} = :gen_tcp.recv(context.server, 0, 200)
+#       assert ^unsubscribe = Package.decode(package)
+#       :ok = Controller.handle_incoming(client_id, unsuback)
+#       assert_receive {{Tortoise, ^client_id}, ^ref, _}
+
+#       # the client callback module should remove the subscription
+#       assert_receive %TestHandler{subscriptions: []}
+#     end
+
+#     test "Subscribe to a topic resulting in an error", context do
+#       client_id = context.client_id
+
+#       subscribe = %Package.Subscribe{
+#         identifier: 1,
+#         topics: [{"foo", 1}]
+#       }
+
+#       suback = %Package.Suback{identifier: 1, acks: [{:error, :access_denied}]}
+
+#       assert {:ok, ref} = Inflight.track(client_id, {:outgoing, subscribe})
+
+#       # assert that the server receives a subscribe package
+#       {:ok, package} = :gen_tcp.recv(context.server, 0, 200)
+#       assert ^subscribe = Package.decode(package)
+#       # the server will send back a subscription acknowledgement message
+#       :ok = Controller.handle_incoming(client_id, suback)
+
+#       assert_receive {{Tortoise, ^client_id}, ^ref, _}
+#       # the callback module should get the error
+#       assert_receive {:subscription_error, {"foo", :access_denied}}
+#     end
+#   end
+
+#   describe "next actions" do
+#     setup [:setup_controller]
+
+#     test "subscribe action", context do
+#       client_id = context.client_id
+#       next_action = {:subscribe, "foo/bar", qos: 0}
+#       send(context.controller_pid, {:next_action, next_action})
+#       %{awaiting: awaiting} = Controller.info(client_id)
+#       assert [{ref, ^next_action}] = Map.to_list(awaiting)
+#       response = {{Tortoise, client_id}, ref, :ok}
+#       send(context.controller_pid, response)
+#       %{awaiting: awaiting} = Controller.info(client_id)
+#       assert [] = Map.to_list(awaiting)
+#     end
+
+#     test "unsubscribe action", context do
+#       client_id = context.client_id
+#       next_action = {:unsubscribe, "foo/bar"}
+#       send(context.controller_pid, {:next_action, next_action})
+#       %{awaiting: awaiting} = Controller.info(client_id)
+#       assert [{ref, ^next_action}] = Map.to_list(awaiting)
+#       response = {{Tortoise, client_id}, ref, :ok}
+#       send(context.controller_pid, response)
+#       %{awaiting: awaiting} = Controller.info(client_id)
+#       assert [] = Map.to_list(awaiting)
+#     end
+
+#     test "receiving unknown async ref", context do
+#       client_id = context.client_id
+#       ref = make_ref()
+
+#       assert capture_log(fn ->
+#                send(context.controller_pid, {{Tortoise, client_id}, ref, :ok})
+#                :timer.sleep(100)
+#              end) =~ "Unexpected"
+
+#       %{awaiting: awaiting} = Controller.info(client_id)
+#       assert [] = Map.to_list(awaiting)
+#     end
+#   end
+# end

From e359fe4ffdc9c52c7c43b0cfaee1c7dbe68a29b1 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Wed, 10 Oct 2018 21:51:57 +0200
Subject: [PATCH 038/220] Make sure the receiver is up before trying to connect

This is put in place as the receiver would reconnect too slowly after
a crash; the connection handler would attempt to reconnect before a
new receiver was ready.

The solution is to await a ready message from the receiver, which will
get dispatched when the receiver is in its init function. This is only
done if we do not have a reference to an existing receiver.

This could perhaps be solved more elegantly if the Elixir Registry had
some kind of "await" function like gproc does. If Registry get an
await function we should replace this with it.
---
 lib/tortoise/connection.ex          | 39 ++++++++++++++++++++++++-----
 lib/tortoise/connection/receiver.ex | 35 ++------------------------
 2 files changed, 35 insertions(+), 39 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 6b7165ac..911aff89 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -20,13 +20,14 @@ defmodule Tortoise.Connection do
     :pending_refs,
     :connection,
     :ping,
-    :handler
+    :handler,
+    :receiver
   ]
 
   alias __MODULE__, as: State
 
   alias Tortoise.{Handler, Transport, Package, Events}
-  alias Tortoise.Connection.{Inflight, Backoff}
+  alias Tortoise.Connection.{Receiver, Inflight, Backoff}
   alias Tortoise.Package.Connect
 
   @doc """
@@ -693,9 +694,11 @@ defmodule Tortoise.Connection do
         :connecting,
         %State{connect: connect, backoff: backoff} = data
       ) do
-    with :ok = Tortoise.Registry.put_meta(via_name(data.client_id), :connecting),
-         :ok = start_connection_supervisor([{:parent, self()} | data.opts]),
-         {:ok, {transport, socket}} <- Tortoise.Connection.Receiver.connect(data.client_id),
+    :ok = Tortoise.Registry.put_meta(via_name(data.client_id), :connecting)
+    :ok = start_connection_supervisor([{:parent, self()} | data.opts])
+    {:ok, data} = await_and_monitor_receiver(data)
+
+    with {:ok, {transport, socket}} <- Receiver.connect(data.client_id),
          :ok = transport.send(socket, Package.encode(data.connect)) do
       new_data = %State{
         data
@@ -703,7 +706,6 @@ defmodule Tortoise.Connection do
           connection: {transport, socket}
       }
 
-      # {:next_state, :handshake, new_state}
       {:keep_state, new_data}
     else
       {:error, {:stop, reason}} ->
@@ -812,6 +814,31 @@ defmodule Tortoise.Connection do
     {:stop, {:protocol_violation, {:unexpected_package, package}}, data}
   end
 
+  def handle_event(
+        :info,
+        {:DOWN, receiver_ref, :process, receiver_pid, _reason},
+        :connected,
+        %State{receiver: {receiver_pid, receiver_ref}} = data
+      ) do
+    next_actions = [{:next_event, :internal, :connect}]
+    updated_data = %State{data | receiver: nil}
+    {:next_state, :connecting, updated_data, next_actions}
+  end
+
+  defp await_and_monitor_receiver(%State{client_id: client_id, receiver: nil} = data) do
+    receive do
+      {{Tortoise, ^client_id}, Receiver, {:ready, pid}} ->
+        {:ok, %State{data | receiver: {pid, Process.monitor(pid)}}}
+    after
+      5000 ->
+        {:error, :receiver_timeout}
+    end
+  end
+
+  defp await_and_monitor_receiver(data) do
+    {:ok, data}
+  end
+
   defp start_connection_supervisor(opts) do
     case Tortoise.Connection.Supervisor.start_link(opts) do
       {:ok, _pid} ->
diff --git a/lib/tortoise/connection/receiver.ex b/lib/tortoise/connection/receiver.ex
index b6c53567..2869b1f7 100644
--- a/lib/tortoise/connection/receiver.ex
+++ b/lib/tortoise/connection/receiver.ex
@@ -44,21 +44,9 @@ defmodule Tortoise.Connection.Receiver do
     GenStateMachine.call(via_name(client_id), :connect)
   end
 
-  def handle_socket(client_id, {transport, socket}) do
-    {:ok, pid} = GenStateMachine.call(via_name(client_id), {:handle_socket, transport, socket})
-
-    case transport.controlling_process(socket, pid) do
-      :ok ->
-        :ok
-
-      {:error, reason} when reason in [:closed, :einval] ->
-        # todo, this is an edge case, figure out what to do here
-        :ok
-    end
-  end
-
   @impl true
   def init(%State{} = data) do
+    send(data.parent, {{Tortoise, data.client_id}, __MODULE__, {:ready, self()}})
     {:ok, :disconnected, data}
   end
 
@@ -79,24 +67,7 @@ defmodule Tortoise.Connection.Receiver do
     {:keep_state, new_data, next_actions}
   end
 
-  # Dropped connection: tell the connection process that it should
-  # attempt to get a new network socket; unfortunately we cannot just
-  # monitor the socket port in the connection process as a transport
-  # method such as the SSL based one will pass an opaque data
-  # structure around instead of a port that can be monitored.
-  def handle_event(:info, {transport, socket}, _state, %{socket: socket} = data)
-      when transport in [:tcp_closed, :ssl_closed] do
-    # should we empty the buffer?
-    # communicate to the world that we have dropped the connection
-    :ok = Events.dispatch(data.client_id, :status, :down)
-    {:next_state, :disconnected, %{data | socket: nil}}
-  end
-
   # activate network socket for incoming traffic
-  def handle_event(:internal, :activate_socket, _state_name, %State{transport: nil}) do
-    {:stop, :no_transport}
-  end
-
   def handle_event(
         :internal,
         :activate_socket,
@@ -114,7 +85,7 @@ defmodule Tortoise.Connection.Receiver do
   end
 
   # consume buffer
-  def handle_event(:internal, :consume_buffer, state_name, %{buffer: <<>>}) do
+  def handle_event(:internal, :consume_buffer, _current_name, %{buffer: <<>>}) do
     :keep_state_and_data
   end
 
@@ -201,8 +172,6 @@ defmodule Tortoise.Connection.Receiver do
           transport: %Transport{type: transport, host: host, port: port, opts: opts}
         } = data
       ) do
-    Events.dispatch(data.client_id, :status, :connecting)
-
     case transport.connect(host, port, opts, 10000) do
       {:ok, socket} ->
         new_state = {:connected, :receiving_fixed_header}

From 458c8ebcfbdeb36e2fa65d150176cf362dcd34a1 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Wed, 10 Oct 2018 22:08:12 +0200
Subject: [PATCH 039/220] Start implementing disconnect messages and
 handle_disconnect

Now I will start to change the user facing callback module a bit. The
old style of handler was perhaps a bit too simple; MQTT 5 require us
to make the actions returned from the callbacks a bit more advanced;
for instance when we handle a disconnect we might want to reconnect,
or stop the client--at the end the user need to make this decision
because there are no one size fits all solution to this.

The execute function in the handler will get split into multiple
functions; in future commits we will see other states such as
handle_auth and handle_connack. They will map into a state in a state
machine, and the possible options will depend on where we are in said
state machine.

Unfortunately this will force the user to know more about the
internals of MQTT, but with some good sane defaults for the states we
should still be able to make an easy to use interface.
---
 lib/tortoise/connection.ex         | 13 ++++++++
 lib/tortoise/handler.ex            | 12 +++++++
 lib/tortoise/package/disconnect.ex |  2 +-
 test/support/test_handler.ex       |  5 +++
 test/tortoise/connection_test.exs  | 50 ++++++++++++++++++++++++++++++
 5 files changed, 81 insertions(+), 1 deletion(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 911aff89..4e264f91 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -805,6 +805,19 @@ defmodule Tortoise.Connection do
     {:keep_state, %State{data | ping: ping}}
   end
 
+  def handle_event(
+        :internal,
+        {:received, %Package.Disconnect{} = disconnect},
+        _current_state,
+        %State{handler: handler} = data
+      ) do
+    case Handler.execute_disconnect(handler, disconnect) do
+      {:stop, reason, updated_handler} ->
+        {:stop, reason, %State{data | handler: updated_handler}}
+    end
+  end
+
+  # unexpected package
   def handle_event(
         :internal,
         {:received, package},
diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index 4d7b85c7..e974d9b3 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -293,6 +293,18 @@ defmodule Tortoise.Handler do
             when reason: :normal | :shutdown | {:shutdown, term()},
                  ignored: term()
 
+  @doc false
+  @spec execute_disconnect(t, %Package.Disconnect{}) :: {:stop, term(), t}
+  def execute_disconnect(handler, %Package.Disconnect{} = disconnect) do
+    handler.module
+    |> apply(:handle_disconnect, [disconnect, handler.state])
+    |> case do
+      {:stop, reason, updated_state} ->
+        {:stop, reason, %__MODULE__{handler | state: updated_state}}
+    end
+  end
+
+  # legacy, should get converted to execute_*type*(handler)
   @doc false
   @spec execute(t, action) ::
           :ok | {:ok, t} | {:error, {:invalid_next_action, term()}} | :ignore | {:stop, term()}
diff --git a/lib/tortoise/package/disconnect.ex b/lib/tortoise/package/disconnect.ex
index 6509669d..df053e63 100644
--- a/lib/tortoise/package/disconnect.ex
+++ b/lib/tortoise/package/disconnect.ex
@@ -42,7 +42,7 @@ defmodule Tortoise.Package.Disconnect do
             __META__: Package.Meta.t(),
             reason: reason(),
             # todo, let this live in the properties module
-            properties: [{atom(), String.t()}]
+            properties: [{atom(), any()}]
           }
   defstruct __META__: %Package.Meta{opcode: @opcode, flags: 0},
             reason: :normal_disconnection,
diff --git a/test/support/test_handler.ex b/test/support/test_handler.ex
index e145862a..405b139f 100644
--- a/test/support/test_handler.ex
+++ b/test/support/test_handler.ex
@@ -17,6 +17,11 @@ defmodule TestHandler do
     {:ok, state}
   end
 
+  def handle_disconnect(disconnect, state) do
+    send(state[:parent], {{__MODULE__, :handle_disconnect}, disconnect})
+    {:stop, :normal, state}
+  end
+
   def handle_message(topic, payload, state) do
     data = %{topic: Enum.join(topic, "/"), payload: payload}
     send(state[:parent], {{__MODULE__, :handle_message}, data})
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index 7eed4c88..04498934 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -977,4 +977,54 @@ defmodule Tortoise.ConnectionTest do
       assert_receive {{Tortoise, ^client_id}, {Package.Publish, ^ref}, :ok}
     end
   end
+
+  describe "Disconnect" do
+    setup [:setup_scripted_mqtt_server, :setup_connection_and_perform_handshake]
+
+    # [x] :normal_disconnection
+    # [ ] :unspecified_error
+    # [ ] :malformed_packet
+    # [ ] :protocol_error
+    # [ ] :implementation_specific_error
+    # [ ] :not_authorized
+    # [ ] :server_busy
+    # [ ] :server_shutting_down
+    # [ ] :keep_alive_timeout
+    # [ ] :session_taken_over
+    # [ ] :topic_filter_invalid
+    # [ ] :topic_name_invalid
+    # [ ] :receive_maximum_exceeded
+    # [ ] :topic_alias_invalid
+    # [ ] :packet_too_large
+    # [ ] :message_rate_too_high
+    # [ ] :quota_exceeded
+    # [ ] :administrative_action
+    # [ ] :payload_format_invalid
+    # [ ] :retain_not_supported
+    # [ ] :qos_not_supported
+    # [ ] :use_another_server (has :server_reference in properties)
+    # [ ] :server_moved (has :server_reference in properties)
+    # [ ] :shared_subscriptions_not_supported
+    # [ ] :connection_rate_exceeded
+    # [ ] :maximum_connect_time
+    # [ ] :subscription_identifiers_not_supported
+    # [ ] :wildcard_subscriptions_not_supported
+
+    test "normal disconnection", context do
+      Process.flag(:trap_exit, true)
+      disconnect = %Package.Disconnect{reason: :normal_disconnection}
+      script = [{:send, disconnect}]
+
+      {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
+      pid = context.connection_pid
+
+      refute_receive {:EXIT, ^pid, {:protocol_violation, {:unexpected_package, ^disconnect}}}
+      assert_receive {ScriptedMqttServer, :completed}
+
+      # the handle disconnect callback should have been called
+      assert_receive {{TestHandler, :handle_disconnect}, ^disconnect}
+      # the callback handler will tell it to stop normally
+      assert_receive {:EXIT, ^pid, :normal}
+    end
+  end
 end

From d736426401e6ce125da9265b9ec316dc32827376 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Thu, 11 Oct 2018 23:40:29 +0200
Subject: [PATCH 040/220] Reintroduce the keep alive ping interval

We cancel it on connection initialization ensuring we wont send a ping
request to the server during a reconnect.

Once connected we will setup a call to the ping function on an
interval based on the keep alive interval defined by the user.
---
 lib/tortoise/connection.ex | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 4e264f91..a7521e63 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -459,6 +459,7 @@ defmodule Tortoise.Connection do
     :ok = Tortoise.Registry.put_meta(via_name(client_id), data.connection)
     :ok = Events.dispatch(client_id, :connection, data.connection)
     :ok = Events.dispatch(client_id, :status, :connected)
+    data = setup_keep_alive(data)
 
     case connack do
       %Package.Connack{session_present: true} ->
@@ -694,6 +695,8 @@ defmodule Tortoise.Connection do
         :connecting,
         %State{connect: connect, backoff: backoff} = data
       ) do
+    # stop the keep alive timer (if running)
+    data = stop_keep_alive(data)
     :ok = Tortoise.Registry.put_meta(via_name(data.client_id), :connecting)
     :ok = start_connection_supervisor([{:parent, self()} | data.opts])
     {:ok, data} = await_and_monitor_receiver(data)
@@ -861,4 +864,25 @@ defmodule Tortoise.Connection do
         :ok
     end
   end
+
+  defp setup_keep_alive(%State{client_id: client_id, keep_alive: nil} = data) do
+    keep_alive = data.connect.keep_alive * 1000
+    {:ok, keep_alive_ref} = :timer.apply_interval(keep_alive, __MODULE__, :ping, [client_id])
+    data = %State{data | keep_alive: keep_alive_ref}
+  end
+
+  defp setup_keep_alive(%State{keep_alive: ref} = data) when is_reference(ref) do
+    data
+    |> stop_keep_alive()
+    |> setup_keep_alive()
+  end
+
+  defp stop_keep_alive(%State{keep_alive: nil} = data) do
+    data
+  end
+
+  defp stop_keep_alive(%State{keep_alive: ref} = data) do
+    {:ok, :cancel} = :timer.cancel(ref)
+    setup_keep_alive(%State{data | keep_alive: nil})
+  end
 end

From 54a8a4490df9e5dde2ebc1ac7cab4072b6070d2d Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 13 Oct 2018 14:34:00 +0200
Subject: [PATCH 041/220] Re-do the ping/keep-alive logic

The old version seemed to leak because the keep alive messages created
references in the ping queue. The new implementation will keep a
timeout for the keep alive message, and dispatch a ping once it is
triggered. If an outside process request a ping we will cancel the
keep alive timeout reference and dispatch a ping. If another outside
process request a ping while the ping has been dispatched to the
server it will latch on to the result and respond to the process with
that value.

This allow us to get rid of the ping-queue; which admittedly was a bit
over-engineered--and as the keep alive is reset after we receive the
ping response we will ping the server less.
---
 lib/tortoise/connection.ex | 140 +++++++++++++++++++++++--------------
 1 file changed, 88 insertions(+), 52 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index a7521e63..58f32ed6 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -9,20 +9,18 @@ defmodule Tortoise.Connection do
 
   require Logger
 
-  defstruct [
-    :client_id,
-    :connect,
-    :server,
-    :backoff,
-    :subscriptions,
-    :keep_alive,
-    :opts,
-    :pending_refs,
-    :connection,
-    :ping,
-    :handler,
-    :receiver
-  ]
+  defstruct client_id: nil,
+            connect: nil,
+            server: nil,
+            backoff: nil,
+            subscriptions: nil,
+            keep_alive: nil,
+            opts: nil,
+            pending_refs: nil,
+            connection: nil,
+            ping: nil,
+            handler: nil,
+            receiver: nil
 
   alias __MODULE__, as: State
 
@@ -398,7 +396,6 @@ defmodule Tortoise.Connection do
       subscriptions: subscriptions,
       opts: opts,
       pending_refs: %{},
-      ping: :queue.new(),
       handler: handler
     }
 
@@ -459,12 +456,12 @@ defmodule Tortoise.Connection do
     :ok = Tortoise.Registry.put_meta(via_name(client_id), data.connection)
     :ok = Events.dispatch(client_id, :connection, data.connection)
     :ok = Events.dispatch(client_id, :status, :connected)
-    data = setup_keep_alive(data)
 
     case connack do
       %Package.Connack{session_present: true} ->
         next_actions = [
-          {:next_event, :internal, {:execute_handler, {:connection, :up}}}
+          {:next_event, :internal, {:execute_handler, {:connection, :up}}},
+          {:next_event, :internal, :setup_keep_alive}
         ]
 
         {:next_state, :connected, data, next_actions}
@@ -474,7 +471,8 @@ defmodule Tortoise.Connection do
 
         next_actions = [
           {:next_event, :internal, {:execute_handler, {:connection, :up}}},
-          {:next_event, :cast, {:subscribe, caller, data.subscriptions, []}}
+          {:next_event, :cast, {:subscribe, caller, data.subscriptions, []}},
+          {:next_event, :internal, :setup_keep_alive}
         ]
 
         {:next_state, :connected, data, next_actions}
@@ -695,8 +693,6 @@ defmodule Tortoise.Connection do
         :connecting,
         %State{connect: connect, backoff: backoff} = data
       ) do
-    # stop the keep alive timer (if running)
-    data = stop_keep_alive(data)
     :ok = Tortoise.Registry.put_meta(via_name(data.client_id), :connecting)
     :ok = start_connection_supervisor([{:parent, self()} | data.opts])
     {:ok, data} = await_and_monitor_receiver(data)
@@ -783,31 +779,92 @@ defmodule Tortoise.Connection do
         :cast,
         {:ping, caller},
         :connected,
-        %State{connection: {transport, socket}} = data
+        %State{connection: {transport, socket}, ping: ping} = data
+      ) do
+    case ping do
+      {{:pinging, start_time}, awaiting} ->
+        ping = {{:pinging, start_time}, [caller | awaiting]}
+        {:keep_state, %State{data | ping: ping}}
+
+      {{:idle, ref}, awaiting} when is_reference(ref) ->
+        ping = {{:idle, ref}, [caller | awaiting]}
+        next_actions = [{:next_event, :info, :keep_alive}]
+        {:keep_state, %State{data | ping: ping}, next_actions}
+    end
+  end
+
+  # not connected yet
+  def handle_event(:cast, {:ping, {caller_pid, ref}}, _, %State{client_id: client_id}) do
+    send(caller_pid, {{Tortoise, client_id}, {Package.Pingreq, ref}, :not_connected})
+    :keep_state_and_data
+  end
+
+  def handle_event(
+        :internal,
+        :setup_keep_alive,
+        :connected,
+        %State{ping: nil} = data
       ) do
-    time = System.monotonic_time(:microsecond)
-    apply(transport, :send, [socket, Package.encode(%Package.Pingreq{})])
-    ping = :queue.in({caller, time}, data.ping)
+    timeout = data.connect.keep_alive * 1000
+    ref = Process.send_after(self(), :keep_alive, timeout)
+    ping = {{:idle, ref}, []}
     {:keep_state, %State{data | ping: ping}}
   end
 
-  # def handle_event(:cast, {:ping, _}, _, %State{}) do
-  #   {:keep_state_and_data, [:postpone]}
-  # end
+  def handle_event(
+        :info,
+        :keep_alive,
+        :connected,
+        %State{connection: {transport, socket}, ping: {{:idle, keep_alive_ref}, awaiting}} = data
+      ) do
+    # the keep alive timeout ref has most likely been triggered, but
+    # if we get here via a user triggered ping we have to cancel the
+    # timer.
+    _ = Process.cancel_timer(keep_alive_ref)
+
+    start_time = System.monotonic_time(:microsecond)
+    ping = {{:pinging, start_time}, awaiting}
+
+    :ok = transport.send(socket, Package.encode(%Package.Pingreq{}))
+
+    {:keep_state, %State{data | ping: ping}}
+  end
+
+  def handle_event(
+        :info,
+        :keep_alive,
+        _,
+        %State{client_id: client_id, ping: {_, awaiting}} = data
+      ) do
+    for {caller, ref} <- awaiting do
+      send(caller, {{Tortoise, client_id}, {Package.Pingreq, ref}, :not_connected})
+    end
+
+    {:keep_state, %State{data | ping: nil}}
+  end
 
   def handle_event(
         :internal,
         {:received, %Package.Pingresp{}},
         :connected,
-        %State{client_id: client_id, ping: ping} = data
+        %State{
+          client_id: client_id,
+          ping: {{:pinging, start_time}, awaiting}
+        } = data
       ) do
-    {{:value, {{caller, ref}, start_time}}, ping} = :queue.out(ping)
     round_trip_time = System.monotonic_time(:microsecond) - start_time
-    send(caller, {{Tortoise, client_id}, {Package.Pingreq, ref}, round_trip_time})
     :ok = Events.dispatch(client_id, :ping_response, round_trip_time)
-    {:keep_state, %State{data | ping: ping}}
+
+    for {caller, ref} <- awaiting do
+      send(caller, {{Tortoise, client_id}, {Package.Pingreq, ref}, round_trip_time})
+    end
+
+    next_actions = [{:next_event, :internal, :setup_keep_alive}]
+
+    {:keep_state, %State{data | ping: nil}, next_actions}
   end
 
+  # disconnect packages
   def handle_event(
         :internal,
         {:received, %Package.Disconnect{} = disconnect},
@@ -864,25 +921,4 @@ defmodule Tortoise.Connection do
         :ok
     end
   end
-
-  defp setup_keep_alive(%State{client_id: client_id, keep_alive: nil} = data) do
-    keep_alive = data.connect.keep_alive * 1000
-    {:ok, keep_alive_ref} = :timer.apply_interval(keep_alive, __MODULE__, :ping, [client_id])
-    data = %State{data | keep_alive: keep_alive_ref}
-  end
-
-  defp setup_keep_alive(%State{keep_alive: ref} = data) when is_reference(ref) do
-    data
-    |> stop_keep_alive()
-    |> setup_keep_alive()
-  end
-
-  defp stop_keep_alive(%State{keep_alive: nil} = data) do
-    data
-  end
-
-  defp stop_keep_alive(%State{keep_alive: ref} = data) do
-    {:ok, :cancel} = :timer.cancel(ref)
-    setup_keep_alive(%State{data | keep_alive: nil})
-  end
 end

From 5be84bc7c8c0c3932468626320646fd548a1469f Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 13 Oct 2018 14:41:56 +0200
Subject: [PATCH 042/220] Clean up unused variable warnings

---
 lib/tortoise/connection.ex | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 58f32ed6..b64be7ef 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -691,7 +691,7 @@ defmodule Tortoise.Connection do
         :internal,
         :connect,
         :connecting,
-        %State{connect: connect, backoff: backoff} = data
+        %State{connect: connect, backoff: _backoff} = data
       ) do
     :ok = Tortoise.Registry.put_meta(via_name(data.client_id), :connecting)
     :ok = start_connection_supervisor([{:parent, self()} | data.opts])
@@ -722,7 +722,7 @@ defmodule Tortoise.Connection do
         :info,
         {{Tortoise, client_id}, :status, current_state},
         current_state,
-        %State{}
+        %State{client_id: client_id}
       ) do
     :keep_state_and_data
   end
@@ -779,7 +779,7 @@ defmodule Tortoise.Connection do
         :cast,
         {:ping, caller},
         :connected,
-        %State{connection: {transport, socket}, ping: ping} = data
+        %State{ping: ping} = data
       ) do
     case ping do
       {{:pinging, start_time}, awaiting} ->

From 066bb5bca346694b1388b8fd5c06dddf68d79418 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 13 Oct 2018 16:31:39 +0200
Subject: [PATCH 043/220] No need for the status event after we started
 monitoring receiver

Previously the connection knew that it should reconnect based on an
event dispatched via Tortoise events. This is no longer needed as we
now monitor the receiver and rely on it to crash when we drop the
connection.
---
 lib/tortoise/connection.ex | 26 --------------------------
 1 file changed, 26 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index b64be7ef..5eb8bb09 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -401,8 +401,6 @@ defmodule Tortoise.Connection do
 
     case Handler.execute(handler, :init) do
       {:ok, %Handler{} = updated_handler} ->
-        {:ok, _pid} = Tortoise.Events.register(data.client_id, :status)
-
         next_events = [{:next_event, :internal, :connect}]
         updated_data = %State{data | handler: updated_handler}
         {:ok, :connecting, updated_data, next_events}
@@ -727,30 +725,6 @@ defmodule Tortoise.Connection do
     :keep_state_and_data
   end
 
-  def handle_event(
-        :info,
-        {{Tortoise, client_id}, :status, status},
-        _current_state,
-        %State{handler: handler, client_id: client_id} = data
-      ) do
-    case status do
-      :down ->
-        next_actions = [{:next_event, :internal, :connect}]
-
-        case Handler.execute(handler, {:connection, status}) do
-          {:ok, updated_handler} ->
-            updated_data = %State{data | handler: updated_handler}
-            {:next_state, :connecting, updated_data, next_actions}
-        end
-
-      _otherwise ->
-        case Handler.execute(handler, {:connection, status}) do
-          {:ok, updated_handler} ->
-            {:keep_state, %State{data | handler: updated_handler}}
-        end
-    end
-  end
-
   # disconnect protocol messages ---------------------------------------
   def handle_event(
         {:call, from},

From 8f69784a985aa3a313589783b069539e617089ca Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 13 Oct 2018 16:39:03 +0200
Subject: [PATCH 044/220] Remove unused subscribe_all/1 from
 Tortoise.Connection

---
 lib/tortoise/connection.ex | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 5eb8bb09..10d0f61c 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -379,12 +379,6 @@ defmodule Tortoise.Connection do
     unless active?, do: Events.unregister(client_id, :connection)
   end
 
-  @doc false
-  @spec subscribe_all(Tortoise.client_id()) :: :ok
-  def subscribe_all(client_id) do
-    GenStateMachine.cast(via_name(client_id), :subscribe_all)
-  end
-
   # Callbacks
   @impl true
   def init({transport, connect, backoff_opts, subscriptions, handler, opts}) do

From ae1a4911116f006dfe2b95e311af41fe245e67f8 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 13 Oct 2018 20:39:51 +0200
Subject: [PATCH 045/220] Reintroduce incremental backoff on reconnects

Had to change the current backoff logic a bit; it will start
suggesting a timeout of zero; then it will backoff as before.
---
 lib/tortoise/connection.ex                | 51 ++++++++++++++---------
 lib/tortoise/connection/backoff.ex        | 12 +++---
 lib/tortoise/connection/receiver.ex       |  4 ++
 lib/tortoise/connection/supervisor.ex     |  2 +-
 test/tortoise/connection/backoff_test.exs |  6 ++-
 5 files changed, 47 insertions(+), 28 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 10d0f61c..8e51dcee 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -14,9 +14,8 @@ defmodule Tortoise.Connection do
             server: nil,
             backoff: nil,
             subscriptions: nil,
-            keep_alive: nil,
             opts: nil,
-            pending_refs: nil,
+            pending_refs: %{},
             connection: nil,
             ping: nil,
             handler: nil,
@@ -389,7 +388,6 @@ defmodule Tortoise.Connection do
       backoff: Backoff.new(backoff_opts),
       subscriptions: subscriptions,
       opts: opts,
-      pending_refs: %{},
       handler: handler
     }
 
@@ -448,6 +446,7 @@ defmodule Tortoise.Connection do
     :ok = Tortoise.Registry.put_meta(via_name(client_id), data.connection)
     :ok = Events.dispatch(client_id, :connection, data.connection)
     :ok = Events.dispatch(client_id, :status, :connected)
+    data = %State{data | backoff: Backoff.reset(data.backoff)}
 
     case connack do
       %Package.Connack{session_present: true} ->
@@ -679,18 +678,26 @@ defmodule Tortoise.Connection do
   end
 
   # connection logic ===================================================
-  def handle_event(
-        :internal,
-        :connect,
-        :connecting,
-        %State{connect: connect, backoff: _backoff} = data
-      ) do
+  def handle_event(:internal, :connect, :connecting, %State{} = data) do
     :ok = Tortoise.Registry.put_meta(via_name(data.client_id), :connecting)
     :ok = start_connection_supervisor([{:parent, self()} | data.opts])
-    {:ok, data} = await_and_monitor_receiver(data)
 
+    case await_and_monitor_receiver(data) do
+      {:ok, data} ->
+        {timeout, updated_data} = Map.get_and_update(data, :backoff, &Backoff.next/1)
+        next_actions = [{:state_timeout, timeout, :attempt_connection}]
+        {:keep_state, updated_data, next_actions}
+    end
+  end
+
+  def handle_event(
+        :state_timeout,
+        :attempt_connection,
+        :connecting,
+        %State{connect: connect} = data
+      ) do
     with {:ok, {transport, socket}} <- Receiver.connect(data.client_id),
-         :ok = transport.send(socket, Package.encode(data.connect)) do
+         :ok = transport.send(socket, Package.encode(connect)) do
       new_data = %State{
         data
         | connect: %Connect{connect | clean_start: false},
@@ -703,7 +710,6 @@ defmodule Tortoise.Connection do
         {:stop, reason, data}
 
       {:error, {:retry, _reason}} ->
-        # {timeout, updated_data} = Map.get_and_update(data, :backoff, &Backoff.next/1)
         next_actions = [{:next_event, :internal, :connect}]
         {:keep_state, data, next_actions}
     end
@@ -771,12 +777,18 @@ defmodule Tortoise.Connection do
         :internal,
         :setup_keep_alive,
         :connected,
-        %State{ping: nil} = data
+        %State{} = data
       ) do
-    timeout = data.connect.keep_alive * 1000
-    ref = Process.send_after(self(), :keep_alive, timeout)
-    ping = {{:idle, ref}, []}
-    {:keep_state, %State{data | ping: ping}}
+    case data.ping do
+      {{:idle, _}, []} ->
+        :keep_state_and_data
+
+      nil ->
+        timeout = data.connect.keep_alive * 1000
+        ref = Process.send_after(self(), :keep_alive, timeout)
+        ping = {{:idle, ref}, []}
+        {:keep_state, %State{data | ping: ping}}
+    end
   end
 
   def handle_event(
@@ -858,9 +870,10 @@ defmodule Tortoise.Connection do
   def handle_event(
         :info,
         {:DOWN, receiver_ref, :process, receiver_pid, _reason},
-        :connected,
+        state,
         %State{receiver: {receiver_pid, receiver_ref}} = data
-      ) do
+      )
+      when state in [:connected, :connecting] do
     next_actions = [{:next_event, :internal, :connect}]
     updated_data = %State{data | receiver: nil}
     {:next_state, :connecting, updated_data, next_actions}
diff --git a/lib/tortoise/connection/backoff.ex b/lib/tortoise/connection/backoff.ex
index a7a4a155..6af2be1f 100644
--- a/lib/tortoise/connection/backoff.ex
+++ b/lib/tortoise/connection/backoff.ex
@@ -16,18 +16,16 @@ defmodule Tortoise.Connection.Backoff do
   end
 
   def next(%State{value: nil} = state) do
-    current = state.min_interval
-    {current, %State{state | value: current}}
+    {0, %State{state | value: state.min_interval}}
   end
 
-  def next(%State{max_interval: same, value: same} = state) do
-    current = state.min_interval
-    {current, %State{state | value: current}}
+  def next(%State{max_interval: value, value: value} = state) do
+    {value, %State{state | value: nil}}
   end
 
   def next(%State{value: value} = state) do
-    current = min(value * 2, state.max_interval)
-    {current, %State{state | value: current}}
+    next = min(value * 2, state.max_interval)
+    {value, %State{state | value: next}}
   end
 
   def reset(%State{} = state) do
diff --git a/lib/tortoise/connection/receiver.ex b/lib/tortoise/connection/receiver.ex
index 2869b1f7..d7a2eee4 100644
--- a/lib/tortoise/connection/receiver.ex
+++ b/lib/tortoise/connection/receiver.ex
@@ -67,6 +67,10 @@ defmodule Tortoise.Connection.Receiver do
     {:keep_state, new_data, next_actions}
   end
 
+  def handle_event(:info, unknown_info, _, data) do
+    {:stop, {:unknown_info, unknown_info}, data}
+  end
+
   # activate network socket for incoming traffic
   def handle_event(
         :internal,
diff --git a/lib/tortoise/connection/supervisor.ex b/lib/tortoise/connection/supervisor.ex
index 8cc90162..9d8bfdbe 100644
--- a/lib/tortoise/connection/supervisor.ex
+++ b/lib/tortoise/connection/supervisor.ex
@@ -27,6 +27,6 @@ defmodule Tortoise.Connection.Supervisor do
       {Receiver, Keyword.take(opts, [:client_id, :transport, :parent])}
     ]
 
-    Supervisor.init(children, strategy: :rest_for_one)
+    Supervisor.init(children, strategy: :rest_for_one, max_seconds: 30, max_restarts: 10)
   end
 end
diff --git a/test/tortoise/connection/backoff_test.exs b/test/tortoise/connection/backoff_test.exs
index b9ed7de0..b89be98d 100644
--- a/test/tortoise/connection/backoff_test.exs
+++ b/test/tortoise/connection/backoff_test.exs
@@ -8,18 +8,22 @@ defmodule Tortoise.Connection.BackoffTest do
     min = 100
     max = 300
     backoff = Backoff.new(min_interval: 100, max_interval: 300)
+    assert {0, backoff} = Backoff.next(backoff)
     assert {^min, backoff} = Backoff.next(backoff)
     {_, backoff} = Backoff.next(backoff)
     assert {^max, backoff} = Backoff.next(backoff)
-    # should roll back to min interval now
+    # should start over now
+    assert {0, backoff} = Backoff.next(backoff)
     assert {^min, _} = Backoff.next(backoff)
   end
 
   test "reset" do
     backoff = Backoff.new(min_interval: 10)
+    assert {0, backoff} = Backoff.next(backoff)
     assert {_, backoff = snapshot} = Backoff.next(backoff)
     assert {_, backoff} = Backoff.next(backoff)
     assert %Backoff{} = backoff = Backoff.reset(backoff)
+    assert {0, backoff} = Backoff.next(backoff)
     assert {_, ^snapshot} = Backoff.next(backoff)
   end
 end

From 6eb8bef28c4169c27b3c06d6fd1b7e57ee341a26 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sun, 14 Oct 2018 17:02:52 +0200
Subject: [PATCH 046/220] Replace the ping/keep-alive logic with a
 stage-timeout approach

This will ensure we don't have to reset timers, because if the state
should change the stage-timeout will get canceled
automatically. gen_statem is quite smart that way.
---
 lib/tortoise/connection.ex | 92 +++++++++++---------------------------
 1 file changed, 27 insertions(+), 65 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 8e51dcee..0b220fb7 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -17,7 +17,7 @@ defmodule Tortoise.Connection do
             opts: nil,
             pending_refs: %{},
             connection: nil,
-            ping: nil,
+            ping: {:idle, []},
             handler: nil,
             receiver: nil
 
@@ -440,7 +440,8 @@ defmodule Tortoise.Connection do
         %State{
           client_id: client_id,
           server: %Transport{type: transport},
-          connection: {transport, _}
+          connection: {transport, _},
+          connect: %Package.Connect{keep_alive: keep_alive}
         } = data
       ) do
     :ok = Tortoise.Registry.put_meta(via_name(client_id), data.connection)
@@ -451,8 +452,8 @@ defmodule Tortoise.Connection do
     case connack do
       %Package.Connack{session_present: true} ->
         next_actions = [
-          {:next_event, :internal, {:execute_handler, {:connection, :up}}},
-          {:next_event, :internal, :setup_keep_alive}
+          {:state_timeout, keep_alive * 1000, :keep_alive},
+          {:next_event, :internal, {:execute_handler, {:connection, :up}}}
         ]
 
         {:next_state, :connected, data, next_actions}
@@ -461,9 +462,9 @@ defmodule Tortoise.Connection do
         caller = {self(), make_ref()}
 
         next_actions = [
+          {:state_timeout, keep_alive * 1000, :keep_alive},
           {:next_event, :internal, {:execute_handler, {:connection, :up}}},
-          {:next_event, :cast, {:subscribe, caller, data.subscriptions, []}},
-          {:next_event, :internal, :setup_keep_alive}
+          {:next_event, :cast, {:subscribe, caller, data.subscriptions, []}}
         ]
 
         {:next_state, :connected, data, next_actions}
@@ -749,21 +750,15 @@ defmodule Tortoise.Connection do
   end
 
   # ping handling ------------------------------------------------------
-  def handle_event(
-        :cast,
-        {:ping, caller},
-        :connected,
-        %State{ping: ping} = data
-      ) do
-    case ping do
-      {{:pinging, start_time}, awaiting} ->
-        ping = {{:pinging, start_time}, [caller | awaiting]}
-        {:keep_state, %State{data | ping: ping}}
+  def handle_event(:cast, {:ping, caller}, :connected, %State{} = data) do
+    case data.ping do
+      {:idle, awaiting} ->
+        # set the keep alive timeout to trigger instantly
+        next_actions = [{:state_timeout, 0, :keep_alive}]
+        {:keep_state, %State{data | ping: {:idle, [caller | awaiting]}}, next_actions}
 
-      {{:idle, ref}, awaiting} when is_reference(ref) ->
-        ping = {{:idle, ref}, [caller | awaiting]}
-        next_actions = [{:next_event, :info, :keep_alive}]
-        {:keep_state, %State{data | ping: ping}, next_actions}
+      {{:pinging, start_time}, awaiting} ->
+        {:keep_state, %State{data | ping: {{:pinging, start_time}, [caller | awaiting]}}}
     end
   end
 
@@ -774,53 +769,16 @@ defmodule Tortoise.Connection do
   end
 
   def handle_event(
-        :internal,
-        :setup_keep_alive,
-        :connected,
-        %State{} = data
-      ) do
-    case data.ping do
-      {{:idle, _}, []} ->
-        :keep_state_and_data
-
-      nil ->
-        timeout = data.connect.keep_alive * 1000
-        ref = Process.send_after(self(), :keep_alive, timeout)
-        ping = {{:idle, ref}, []}
-        {:keep_state, %State{data | ping: ping}}
-    end
-  end
-
-  def handle_event(
-        :info,
+        :state_timeout,
         :keep_alive,
         :connected,
-        %State{connection: {transport, socket}, ping: {{:idle, keep_alive_ref}, awaiting}} = data
+        %State{connection: {transport, socket}, ping: {:idle, awaiting}} = data
       ) do
-    # the keep alive timeout ref has most likely been triggered, but
-    # if we get here via a user triggered ping we have to cancel the
-    # timer.
-    _ = Process.cancel_timer(keep_alive_ref)
-
-    start_time = System.monotonic_time(:microsecond)
-    ping = {{:pinging, start_time}, awaiting}
+    start_time = System.monotonic_time()
 
     :ok = transport.send(socket, Package.encode(%Package.Pingreq{}))
 
-    {:keep_state, %State{data | ping: ping}}
-  end
-
-  def handle_event(
-        :info,
-        :keep_alive,
-        _,
-        %State{client_id: client_id, ping: {_, awaiting}} = data
-      ) do
-    for {caller, ref} <- awaiting do
-      send(caller, {{Tortoise, client_id}, {Package.Pingreq, ref}, :not_connected})
-    end
-
-    {:keep_state, %State{data | ping: nil}}
+    {:keep_state, %State{data | ping: {{:pinging, start_time}, awaiting}}}
   end
 
   def handle_event(
@@ -829,19 +787,23 @@ defmodule Tortoise.Connection do
         :connected,
         %State{
           client_id: client_id,
-          ping: {{:pinging, start_time}, awaiting}
+          ping: {{:pinging, start_time}, awaiting},
+          connect: %Package.Connect{keep_alive: keep_alive}
         } = data
       ) do
-    round_trip_time = System.monotonic_time(:microsecond) - start_time
+    round_trip_time =
+      (System.monotonic_time() - start_time)
+      |> System.convert_time_unit(:native, :microsecond)
+
     :ok = Events.dispatch(client_id, :ping_response, round_trip_time)
 
     for {caller, ref} <- awaiting do
       send(caller, {{Tortoise, client_id}, {Package.Pingreq, ref}, round_trip_time})
     end
 
-    next_actions = [{:next_event, :internal, :setup_keep_alive}]
+    next_actions = [{:state_timeout, keep_alive * 1000, :keep_alive}]
 
-    {:keep_state, %State{data | ping: nil}, next_actions}
+    {:keep_state, %State{data | ping: {:idle, []}}, next_actions}
   end
 
   # disconnect packages

From d1931119f1ef62288f248005a1955f9fd7593741 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sun, 14 Oct 2018 19:24:27 +0200
Subject: [PATCH 047/220] Remove some dead code

We do no longer register for messages posted to :status so this will
never get run.
---
 lib/tortoise/connection.ex | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 0b220fb7..a4b488df 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -716,16 +716,6 @@ defmodule Tortoise.Connection do
     end
   end
 
-  # system state changes; we need to react to connection down messages
-  def handle_event(
-        :info,
-        {{Tortoise, client_id}, :status, current_state},
-        current_state,
-        %State{client_id: client_id}
-      ) do
-    :keep_state_and_data
-  end
-
   # disconnect protocol messages ---------------------------------------
   def handle_event(
         {:call, from},

From c8a686cf64289cb8cdb03b416bd15b92cb86d1d2 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sun, 14 Oct 2018 19:48:09 +0200
Subject: [PATCH 048/220] Extract the handle init callback to its own function

This will allow me to define better specs for the handle_init/1
function.
---
 lib/tortoise/connection.ex     |  2 +-
 lib/tortoise/handler.ex        | 34 +++++++++++++++++++---------------
 test/tortoise/handler_test.exs |  4 ++--
 3 files changed, 22 insertions(+), 18 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index a4b488df..6d8fa9da 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -391,7 +391,7 @@ defmodule Tortoise.Connection do
       handler: handler
     }
 
-    case Handler.execute(handler, :init) do
+    case Handler.execute_init(handler) do
       {:ok, %Handler{} = updated_handler} ->
         next_events = [{:next_event, :internal, :connect}]
         updated_data = %State{data | handler: updated_handler}
diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index e974d9b3..0b9febe9 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -293,6 +293,23 @@ defmodule Tortoise.Handler do
             when reason: :normal | :shutdown | {:shutdown, term()},
                  ignored: term()
 
+  @doc false
+  @spec execute_init(t) :: {:ok, t} | :ignore | {:stop, term()}
+  def execute_init(handler) do
+    handler.module
+    |> apply(:init, [handler.initial_args])
+    |> case do
+      {:ok, initial_state} ->
+        {:ok, %__MODULE__{handler | state: initial_state}}
+
+      :ignore ->
+        :ignore
+
+      {:stop, reason} ->
+        {:stop, reason}
+    end
+  end
+
   @doc false
   @spec execute_disconnect(t, %Package.Disconnect{}) :: {:stop, term(), t}
   def execute_disconnect(handler, %Package.Disconnect{} = disconnect) do
@@ -307,26 +324,13 @@ defmodule Tortoise.Handler do
   # legacy, should get converted to execute_*type*(handler)
   @doc false
   @spec execute(t, action) ::
-          :ok | {:ok, t} | {:error, {:invalid_next_action, term()}} | :ignore | {:stop, term()}
+          :ok | {:ok, t} | {:error, {:invalid_next_action, term()}} | {:stop, term()}
         when action:
-               :init
-               | {:subscribe, [term()]}
+               {:subscribe, [term()]}
                | {:unsubscribe, [term()]}
                | {:publish, Tortoise.Package.Publish.t()}
                | {:connection, :up | :down}
                | {:terminate, reason :: term()}
-  def execute(handler, :init) do
-    case apply(handler.module, :init, [handler.initial_args]) do
-      {:ok, initial_state} ->
-        {:ok, %__MODULE__{handler | state: initial_state}}
-
-      :ignore ->
-        :ignore
-
-      {:stop, reason} ->
-        {:stop, reason}
-    end
-  end
 
   def execute(handler, {:connection, status}) do
     handler.module
diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index da30e5ed..9ebafa5c 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -55,9 +55,9 @@ defmodule Tortoise.HandlerTest do
     %Handler{handler | state: update}
   end
 
-  describe "execute init/1" do
+  describe "execute_init/1" do
     test "return ok-tuple", context do
-      assert {:ok, %Handler{}} = Handler.execute(context.handler, :init)
+      assert {:ok, %Handler{}} = Handler.execute_init(context.handler)
       assert_receive :init
     end
   end

From 644bfe5f186a33b581bff7a4f64a6348522c75bd Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sun, 14 Oct 2018 19:59:37 +0200
Subject: [PATCH 049/220] Extract the execute terminate to execute_terminate/2

This will make it easier to write better type specs
---
 lib/tortoise/connection.ex     |  3 +--
 lib/tortoise/handler.ex        | 13 ++++++++-----
 test/tortoise/handler_test.exs |  2 +-
 3 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 6d8fa9da..71d77e81 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -407,8 +407,7 @@ defmodule Tortoise.Connection do
 
   @impl true
   def terminate(reason, _state, %State{handler: handler}) do
-    _ignored = Handler.execute(handler, {:terminate, reason})
-    :ok
+    _ignored = Handler.execute_terminate(handler, reason)
   end
 
   @impl true
diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index 0b9febe9..db767cba 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -321,6 +321,14 @@ defmodule Tortoise.Handler do
     end
   end
 
+  @doc false
+  @spec execute_terminate(t, reason) :: ignored
+        when reason: term(),
+             ignored: term()
+  def execute_terminate(handler, reason) do
+    _ignored = apply(handler.module, :terminate, [reason, handler.state])
+  end
+
   # legacy, should get converted to execute_*type*(handler)
   @doc false
   @spec execute(t, action) ::
@@ -370,11 +378,6 @@ defmodule Tortoise.Handler do
     end)
   end
 
-  def execute(handler, {:terminate, reason}) do
-    _ignored = apply(handler.module, :terminate, [reason, handler.state])
-    :ok
-  end
-
   # Subacks will come in a map with three keys in the form of tuples
   # where the fist element is one of `:ok`, `:warn`, or `:error`. This
   # is done to make it easy to pattern match in other parts of the
diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index 9ebafa5c..77458c48 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -191,7 +191,7 @@ defmodule Tortoise.HandlerTest do
   describe "execute terminate/2" do
     test "return ok", context do
       handler = set_state(context.handler, pid: self())
-      assert :ok = Handler.execute(handler, {:terminate, :normal})
+      assert :ok = Handler.execute_terminate(handler, :normal)
       assert_receive {:terminate, :normal}
     end
   end

From b1878e9634d6d99981e49f95c638cfaecbb2aece Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sun, 14 Oct 2018 20:15:25 +0200
Subject: [PATCH 050/220] Extract the connection callback handler into
 handle_connection

This is just a stepping stone; I might remove this callback in favor
of a handle_connack function that will receive the %Package.Connack{}
so the user can get access to the user properties and such, and this
will allow us to implement return statements that would fit for this
state in the life-cycle.
---
 lib/tortoise/connection.ex     | 15 +++++++++++++++
 lib/tortoise/handler.ex        | 16 +++++++++-------
 test/tortoise/handler_test.exs |  8 ++++----
 3 files changed, 28 insertions(+), 11 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 71d77e81..261e3123 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -411,6 +411,21 @@ defmodule Tortoise.Connection do
   end
 
   @impl true
+  def handle_event(
+        :internal,
+        {:execute_handler, {:connection, status}},
+        :connected,
+        %State{handler: handler} = data
+      ) do
+    case Handler.execute_connection(handler, status) do
+      {:ok, %Handler{} = updated_handler} ->
+        updated_data = %State{data | handler: updated_handler}
+        {:keep_state, updated_data}
+
+        # handle stop
+    end
+  end
+
   def handle_event(
         :internal,
         {:execute_handler, cmd},
diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index db767cba..6e7c80c2 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -310,6 +310,15 @@ defmodule Tortoise.Handler do
     end
   end
 
+  @doc false
+  @spec execute_connection(t, status) :: {:ok, t}
+        when status: :up | :down
+  def execute_connection(handler, status) do
+    handler.module
+    |> apply(:connection, [status, handler.state])
+    |> handle_result(handler)
+  end
+
   @doc false
   @spec execute_disconnect(t, %Package.Disconnect{}) :: {:stop, term(), t}
   def execute_disconnect(handler, %Package.Disconnect{} = disconnect) do
@@ -337,15 +346,8 @@ defmodule Tortoise.Handler do
                {:subscribe, [term()]}
                | {:unsubscribe, [term()]}
                | {:publish, Tortoise.Package.Publish.t()}
-               | {:connection, :up | :down}
                | {:terminate, reason :: term()}
 
-  def execute(handler, {:connection, status}) do
-    handler.module
-    |> apply(:connection, [status, handler.state])
-    |> handle_result(handler)
-  end
-
   def execute(handler, {:publish, %Package.Publish{} = publish}) do
     topic_list = String.split(publish.topic, "/")
 
diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index 77458c48..7d927428 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -65,10 +65,10 @@ defmodule Tortoise.HandlerTest do
   describe "execute connection/2" do
     test "return ok-tuple", context do
       handler = set_state(context.handler, %{pid: self()})
-      assert {:ok, %Handler{}} = Handler.execute(handler, {:connection, :up})
+      assert {:ok, %Handler{}} = Handler.execute_connection(handler, :up)
       assert_receive {:connection, :up}
 
-      assert {:ok, %Handler{}} = Handler.execute(handler, {:connection, :down})
+      assert {:ok, %Handler{}} = Handler.execute_connection(handler, :down)
       assert_receive {:connection, :down}
     end
 
@@ -79,12 +79,12 @@ defmodule Tortoise.HandlerTest do
         context.handler
         |> set_state(%{pid: self(), next_actions: next_actions})
 
-      assert {:ok, %Handler{}} = Handler.execute(handler, {:connection, :up})
+      assert {:ok, %Handler{}} = Handler.execute_connection(handler, :up)
 
       assert_receive {:connection, :up}
       assert_receive {:next_action, {:subscribe, "foo/bar", qos: 0}}
 
-      assert {:ok, %Handler{}} = Handler.execute(handler, {:connection, :down})
+      assert {:ok, %Handler{}} = Handler.execute_connection(handler, :down)
 
       assert_receive {:connection, :down}
       assert_receive {:next_action, {:subscribe, "foo/bar", qos: 0}}

From e3e3ac9fa7cb17154598a63d50f94e317bc13230 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sun, 14 Oct 2018 22:24:31 +0200
Subject: [PATCH 051/220] Clean up the typespec of Handler.execute after
 refactoring

---
 lib/tortoise/handler.ex | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index 6e7c80c2..3c6496c4 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -341,12 +341,11 @@ defmodule Tortoise.Handler do
   # legacy, should get converted to execute_*type*(handler)
   @doc false
   @spec execute(t, action) ::
-          :ok | {:ok, t} | {:error, {:invalid_next_action, term()}} | {:stop, term()}
+          {:ok, t} | {:error, {:invalid_next_action, term()}} | {:stop, term()}
         when action:
                {:subscribe, [term()]}
                | {:unsubscribe, [term()]}
                | {:publish, Tortoise.Package.Publish.t()}
-               | {:terminate, reason :: term()}
 
   def execute(handler, {:publish, %Package.Publish{} = publish}) do
     topic_list = String.split(publish.topic, "/")

From 8017cb51c6541eb07e5ad421a2b10b4bab799553 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Wed, 17 Oct 2018 15:20:44 +0200
Subject: [PATCH 052/220] Extract unsubscribe handling from Handler.execute

Will make it easier to spec and will make it possible to define
sensible next actions for the user specified return.
---
 lib/tortoise/connection.ex     | 15 +++++++++++++++
 lib/tortoise/handler.ex        | 25 ++++++++++++++-----------
 test/tortoise/handler_test.exs |  2 +-
 3 files changed, 30 insertions(+), 12 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 261e3123..1c5effa7 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -426,6 +426,21 @@ defmodule Tortoise.Connection do
     end
   end
 
+  def handle_event(
+        :internal,
+        {:execute_handler, {:unsubscribe, result}},
+        _current_state,
+        %State{handler: handler} = data
+      ) do
+    case Handler.execute_unsubscribe(handler, result) do
+      {:ok, %Handler{} = updated_handler} ->
+        updated_data = %State{data | handler: updated_handler}
+        {:keep_state, updated_data}
+
+        # handle stop
+    end
+  end
+
   def handle_event(
         :internal,
         {:execute_handler, cmd},
diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index 3c6496c4..8086dda1 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -338,6 +338,20 @@ defmodule Tortoise.Handler do
     _ignored = apply(handler.module, :terminate, [reason, handler.state])
   end
 
+  @doc false
+  @spec execute_unsubscribe(t, result) :: {:ok, t}
+        when result: [Tortoise.topic_filter()]
+  def execute_unsubscribe(handler, results) do
+    Enum.reduce(results, {:ok, handler}, fn topic_filter, {:ok, handler} ->
+      handler.module
+      |> apply(:subscription, [:down, topic_filter, handler.state])
+      |> handle_result(handler)
+
+      # _, {:stop, acc} ->
+      #   {:stop, acc}
+    end)
+  end
+
   # legacy, should get converted to execute_*type*(handler)
   @doc false
   @spec execute(t, action) ::
@@ -355,17 +369,6 @@ defmodule Tortoise.Handler do
     |> handle_result(handler)
   end
 
-  def execute(handler, {:unsubscribe, unsubacks}) do
-    Enum.reduce(unsubacks, {:ok, handler}, fn topic_filter, {:ok, handler} ->
-      handler.module
-      |> apply(:subscription, [:down, topic_filter, handler.state])
-      |> handle_result(handler)
-
-      # _, {:stop, acc} ->
-      #   {:stop, acc}
-    end)
-  end
-
   def execute(handler, {:subscribe, subacks}) do
     subacks
     |> flatten_subacks()
diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index 7d927428..e8f6346f 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -181,7 +181,7 @@ defmodule Tortoise.HandlerTest do
       {:ok, result} = Track.result(track)
 
       handler = set_state(context.handler, pid: self())
-      assert {:ok, %Handler{}} = Handler.execute(handler, {:unsubscribe, result})
+      assert {:ok, %Handler{}} = Handler.execute_unsubscribe(handler, result)
       # we should receive two subscription down messages
       assert_receive {:subscription, :down, "foo/bar"}
       assert_receive {:subscription, :down, "baz/quux"}

From ac84183706ac1fec5785a15f3d566378107e8455 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Wed, 17 Oct 2018 15:27:51 +0200
Subject: [PATCH 053/220] Extract the handle subscribe from the user defined
 callback

---
 lib/tortoise/connection.ex     | 15 ++++++++++++++
 lib/tortoise/handler.ex        | 37 ++++++++++++++++------------------
 test/tortoise/handler_test.exs |  2 +-
 3 files changed, 33 insertions(+), 21 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 1c5effa7..a9b89e41 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -426,6 +426,21 @@ defmodule Tortoise.Connection do
     end
   end
 
+  def handle_event(
+        :internal,
+        {:execute_handler, {:subscribe, results}},
+        _,
+        %State{handler: handler} = data
+      ) do
+    case Handler.execute_subscribe(handler, results) do
+      {:ok, %Handler{} = updated_handler} ->
+        updated_data = %State{data | handler: updated_handler}
+        {:keep_state, updated_data}
+
+        # handle stop
+    end
+  end
+
   def handle_event(
         :internal,
         {:execute_handler, {:unsubscribe, result}},
diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index 8086dda1..daa62313 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -338,6 +338,21 @@ defmodule Tortoise.Handler do
     _ignored = apply(handler.module, :terminate, [reason, handler.state])
   end
 
+  @doc false
+  @spec execute_subscribe(t, [term()]) :: {:ok, t}
+  def execute_subscribe(handler, result) do
+    result
+    |> flatten_subacks()
+    |> Enum.reduce({:ok, handler}, fn {op, topic_filter}, {:ok, handler} ->
+      handler.module
+      |> apply(:subscription, [op, topic_filter, handler.state])
+      |> handle_result(handler)
+
+      # _, {:stop, acc} ->
+      #   {:stop, acc}
+    end)
+  end
+
   @doc false
   @spec execute_unsubscribe(t, result) :: {:ok, t}
         when result: [Tortoise.topic_filter()]
@@ -354,13 +369,8 @@ defmodule Tortoise.Handler do
 
   # legacy, should get converted to execute_*type*(handler)
   @doc false
-  @spec execute(t, action) ::
-          {:ok, t} | {:error, {:invalid_next_action, term()}} | {:stop, term()}
-        when action:
-               {:subscribe, [term()]}
-               | {:unsubscribe, [term()]}
-               | {:publish, Tortoise.Package.Publish.t()}
-
+  @spec execute(t, action) :: {:ok, t} | {:error, {:invalid_next_action, term()}}
+        when action: {:publish, Tortoise.Package.Publish.t()}
   def execute(handler, {:publish, %Package.Publish{} = publish}) do
     topic_list = String.split(publish.topic, "/")
 
@@ -369,19 +379,6 @@ defmodule Tortoise.Handler do
     |> handle_result(handler)
   end
 
-  def execute(handler, {:subscribe, subacks}) do
-    subacks
-    |> flatten_subacks()
-    |> Enum.reduce({:ok, handler}, fn {op, topic_filter}, {:ok, handler} ->
-      handler.module
-      |> apply(:subscription, [op, topic_filter, handler.state])
-      |> handle_result(handler)
-
-      # _, {:stop, acc} ->
-      #   {:stop, acc}
-    end)
-  end
-
   # Subacks will come in a map with three keys in the form of tuples
   # where the fist element is one of `:ok`, `:warn`, or `:error`. This
   # is done to make it easy to pattern match in other parts of the
diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index e8f6346f..a761e6b1 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -162,7 +162,7 @@ defmodule Tortoise.HandlerTest do
       {:ok, result} = Track.result(track)
 
       handler = set_state(context.handler, pid: self())
-      assert {:ok, %Handler{}} = Handler.execute(handler, {:subscribe, result})
+      assert {:ok, %Handler{}} = Handler.execute_subscribe(handler, result)
 
       assert_receive {:subscription, :up, "foo"}
       assert_receive {:subscription, {:error, :access_denied}, "baz"}

From 780c5272accbc46d9c5ec516216ae2ba50b9552f Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Wed, 17 Oct 2018 15:36:14 +0200
Subject: [PATCH 054/220] Extract handle_message from the old Handle.execute
 function

This will make it easier to spec and define sensible next actions the
user can return in the return statement from the callback.
---
 lib/tortoise/connection.ex     | 10 +++++-----
 lib/tortoise/handler.ex        |  7 +++----
 test/tortoise/handler_test.exs |  6 +++---
 3 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index a9b89e41..6aa55b11 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -458,11 +458,11 @@ defmodule Tortoise.Connection do
 
   def handle_event(
         :internal,
-        {:execute_handler, cmd},
+        {:execute_handler, {:publish, %Package.Publish{} = publish}},
         _,
         %State{handler: handler} = data
       ) do
-    case Handler.execute(handler, cmd) do
+    case Handler.execute_handle_message(handler, publish) do
       {:ok, %Handler{} = updated_handler} ->
         updated_data = %State{data | handler: updated_handler}
         {:keep_state, updated_data}
@@ -541,7 +541,7 @@ defmodule Tortoise.Connection do
         _,
         %State{handler: handler} = data
       ) do
-    case Handler.execute(handler, {:publish, publish}) do
+    case Handler.execute_handle_message(handler, publish) do
       {:ok, updated_handler} ->
         {:keep_state, %State{data | handler: updated_handler}}
 
@@ -558,7 +558,7 @@ defmodule Tortoise.Connection do
       ) do
     :ok = Inflight.track(client_id, {:incoming, publish})
 
-    case Handler.execute(handler, {:publish, publish}) do
+    case Handler.execute_handle_message(handler, publish) do
       {:ok, updated_handler} ->
         {:keep_state, %State{data | handler: updated_handler}}
     end
@@ -605,7 +605,7 @@ defmodule Tortoise.Connection do
         _,
         %State{client_id: client_id, handler: handler} = data
       ) do
-    case Handler.execute(handler, {:publish, publish}) do
+    case Handler.execute_handle_message(handler, publish) do
       {:ok, updated_handler} ->
         {:keep_state, %State{data | handler: updated_handler}}
     end
diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index daa62313..756f1170 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -367,11 +367,10 @@ defmodule Tortoise.Handler do
     end)
   end
 
-  # legacy, should get converted to execute_*type*(handler)
   @doc false
-  @spec execute(t, action) :: {:ok, t} | {:error, {:invalid_next_action, term()}}
-        when action: {:publish, Tortoise.Package.Publish.t()}
-  def execute(handler, {:publish, %Package.Publish{} = publish}) do
+  @spec execute_handle_message(t, Package.Publish.t()) ::
+          {:ok, t} | {:error, {:invalid_next_action, term()}}
+  def execute_handle_message(handler, %Package.Publish{} = publish) do
     topic_list = String.split(publish.topic, "/")
 
     handler.module
diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index a761e6b1..489d34cb 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -98,7 +98,7 @@ defmodule Tortoise.HandlerTest do
       topic = "foo/bar"
       publish = %Package.Publish{topic: topic, payload: payload}
 
-      assert {:ok, %Handler{}} = Handler.execute(handler, {:publish, publish})
+      assert {:ok, %Handler{}} = Handler.execute_handle_message(handler, publish)
       # the topic will be in the form of a list making it possible to
       # pattern match on the topic levels
       assert_receive {:publish, topic_list, ^payload}
@@ -114,7 +114,7 @@ defmodule Tortoise.HandlerTest do
       topic = "foo/bar"
       publish = %Package.Publish{topic: topic, payload: payload}
 
-      assert {:ok, %Handler{}} = Handler.execute(handler, {:publish, publish})
+      assert {:ok, %Handler{}} = Handler.execute_handle_message(handler, publish)
 
       assert_receive {:next_action, {:subscribe, "foo/bar", qos: 0}}
 
@@ -134,7 +134,7 @@ defmodule Tortoise.HandlerTest do
       publish = %Package.Publish{topic: topic, payload: payload}
 
       assert {:error, {:invalid_next_action, [{:invalid, "bar"}]}} =
-               Handler.execute(handler, {:publish, publish})
+               Handler.execute_handle_message(handler, publish)
 
       refute_receive {:next_action, {:invalid, "bar"}}
       # we should not receive the otherwise valid next_action

From 0baeaaef6d22ebdd83abe416fd014475e486e873 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Mon, 22 Oct 2018 21:53:05 +0200
Subject: [PATCH 055/220] Have the callbacks for publish handled in one place

---
 lib/tortoise/connection.ex | 26 +++++++++-----------------
 1 file changed, 9 insertions(+), 17 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 6aa55b11..e63b4dcd 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -539,14 +539,10 @@ defmodule Tortoise.Connection do
         :internal,
         {:received, %Package.Publish{qos: 0, dup: false} = publish},
         _,
-        %State{handler: handler} = data
+        %State{}
       ) do
-    case Handler.execute_handle_message(handler, publish) do
-      {:ok, updated_handler} ->
-        {:keep_state, %State{data | handler: updated_handler}}
-
-        # handle stop
-    end
+    next_actions = [{:next_event, :internal, {:execute_handler, {:publish, publish}}}]
+    {:keep_state_and_data, next_actions}
   end
 
   # incoming publish QoS=1 ---------------------------------------------
@@ -554,14 +550,12 @@ defmodule Tortoise.Connection do
         :internal,
         {:received, %Package.Publish{qos: 1} = publish},
         :connected,
-        %State{client_id: client_id, handler: handler} = data
+        %State{client_id: client_id}
       ) do
     :ok = Inflight.track(client_id, {:incoming, publish})
 
-    case Handler.execute_handle_message(handler, publish) do
-      {:ok, updated_handler} ->
-        {:keep_state, %State{data | handler: updated_handler}}
-    end
+    next_actions = [{:next_event, :internal, {:execute_handler, {:publish, publish}}}]
+    {:keep_state_and_data, next_actions}
   end
 
   # outgoing publish QoS=1 ---------------------------------------------
@@ -603,12 +597,10 @@ defmodule Tortoise.Connection do
         :info,
         {{Inflight, client_id}, %Package.Publish{qos: 2} = publish},
         _,
-        %State{client_id: client_id, handler: handler} = data
+        %State{client_id: client_id}
       ) do
-    case Handler.execute_handle_message(handler, publish) do
-      {:ok, updated_handler} ->
-        {:keep_state, %State{data | handler: updated_handler}}
-    end
+    next_actions = [{:next_event, :internal, {:execute_handler, {:publish, publish}}}]
+    {:keep_state_and_data, next_actions}
   end
 
   # outgoing publish QoS=2 ---------------------------------------------

From f10b080670fd0fb5f2fa49f9c1dfc57c56aea3db Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Thu, 25 Oct 2018 20:54:07 +0200
Subject: [PATCH 056/220] Add a handle_puback callback

I am still very confused to why the standard is dead set on making it
possible to send user (or server) defined properties on these
packages; nobody seems to have a good use case, but hey. Now we need
to expose them to the end user.
---
 lib/tortoise/connection.ex     | 16 ++++++++++++----
 lib/tortoise/handler.ex        | 18 ++++++++++++++++++
 lib/tortoise/package/puback.ex | 12 ++++++++++--
 test/tortoise/handler_test.exs | 18 ++++++++++++++++++
 4 files changed, 58 insertions(+), 6 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index e63b4dcd..ea15e302 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -549,7 +549,7 @@ defmodule Tortoise.Connection do
   def handle_event(
         :internal,
         {:received, %Package.Publish{qos: 1} = publish},
-        :connected,
+        _,
         %State{client_id: client_id}
       ) do
     :ok = Inflight.track(client_id, {:incoming, publish})
@@ -563,10 +563,18 @@ defmodule Tortoise.Connection do
         :internal,
         {:received, %Package.Puback{} = puback},
         _,
-        %State{client_id: client_id}
+        %State{client_id: client_id, handler: handler} = data
       ) do
-    :ok = Inflight.update(client_id, {:received, puback})
-    :keep_state_and_data
+    case Handler.execute_handle_puback(handler, puback) do
+      {:ok, %Handler{} = updated_handler} ->
+        :ok = Inflight.update(client_id, {:received, puback})
+        updated_data = %State{data | handler: updated_handler}
+        {:keep_state, updated_data}
+
+      {:error, reason} ->
+        # todo
+        {:stop, reason, data}
+    end
   end
 
   # incoming publish QoS=2 ---------------------------------------------
diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index 756f1170..17cc78a8 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -149,6 +149,11 @@ defmodule Tortoise.Handler do
         {:ok, state}
       end
 
+      @impl true
+      def handle_puback(_puback, state) do
+        {:ok, state}
+      end
+
       defoverridable Tortoise.Handler
     end
   end
@@ -283,6 +288,10 @@ defmodule Tortoise.Handler do
                  topic_levels: [String.t()],
                  payload: Tortoise.payload()
 
+  @callback handle_puback(puback, state :: term()) :: {:ok, new_state}
+            when puback: Package.Puback.t(),
+                 new_state: term()
+
   @doc """
   Invoked when the connection process is about to exit.
 
@@ -378,6 +387,15 @@ defmodule Tortoise.Handler do
     |> handle_result(handler)
   end
 
+  @doc false
+  # @spec execute_handle_puback(t, Package.Puback.t()) ::
+  #         {:ok, t} | {:error, {:invalid_next_action, term()}}
+  def execute_handle_puback(handler, %Package.Puback{} = puback) do
+    handler.module
+    |> apply(:handle_puback, [puback, handler.state])
+    |> handle_result(handler)
+  end
+
   # Subacks will come in a map with three keys in the form of tuples
   # where the fist element is one of `:ok`, `:warn`, or `:error`. This
   # is done to make it easy to pattern match in other parts of the
diff --git a/lib/tortoise/package/puback.ex b/lib/tortoise/package/puback.ex
index e351b68c..36103de2 100644
--- a/lib/tortoise/package/puback.ex
+++ b/lib/tortoise/package/puback.ex
@@ -8,13 +8,21 @@ defmodule Tortoise.Package.Puback do
   alias Tortoise.Package
 
   @type reason :: :success | {:refused, refusal_reasons()}
-  @type refusal_reasons :: :test
+  @type refusal_reasons ::
+          :no_matching_subscribers
+          | :unspecified_error
+          | :implementation_specific_error
+          | :not_authorized
+          | :topic_Name_invalid
+          | :packet_identifier_in_use
+          | :quota_exceeded
+          | :payload_format_invalid
 
   @opaque t :: %__MODULE__{
             __META__: Package.Meta.t(),
             identifier: Tortoise.package_identifier(),
             reason: reason(),
-            properties: [{any(), any()}]
+            properties: [{:reason_string, String.t()}, {:user_property, String.t()}]
           }
   @enforce_keys [:identifier]
   defstruct __META__: %Package.Meta{opcode: @opcode, flags: 0b0000},
diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index 489d34cb..5cd81b8e 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -44,6 +44,11 @@ defmodule Tortoise.HandlerTest do
       send(state[:pid], {:terminate, reason})
       :ok
     end
+
+    def handle_puback(puback, state) do
+      send(state[:pid], {:puback, puback})
+      {:ok, state}
+    end
   end
 
   setup _context do
@@ -195,4 +200,17 @@ defmodule Tortoise.HandlerTest do
       assert_receive {:terminate, :normal}
     end
   end
+
+  describe "execute handle_puback/2" do
+    test "return ok", context do
+      handler = set_state(context.handler, pid: self())
+      puback = %Package.Puback{identifier: 1}
+
+      assert {:ok, %Handler{} = state} =
+               handler
+               |> Handler.execute_handle_puback(puback)
+
+      assert_receive {:puback, ^puback}
+    end
+  end
 end

From 528eec6bb528c127228a8e25f37eba0a8d3a7fc4 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Thu, 25 Oct 2018 21:05:38 +0200
Subject: [PATCH 057/220] MASSIVE BREAKING CHANGE handle_message is now called
 handle_publish

This is done because we need to change the interface to accommodate
the MQTT 5 version of the protocol. Every protocol message basically
need to get handed to the user, because they could contain user
defined properties (including pubrel, pubrec, etc), so we can no
longer make a client that hides that protocol details from the user;
they need to know about pubrel, pubcomp, pubrec, etc, so they will get
a callback each prefixed with `handle_*`

To make the interface consistent I have renamed handle_message to
handle_publish. I am aware that every implementation that uses
tortoise will need to update their callback implementations. Sorry.
---
 lib/tortoise/connection.ex        |  2 +-
 lib/tortoise/handler.ex           | 28 ++++++++++++++--------------
 test/support/test_handler.ex      |  4 ++--
 test/tortoise/connection_test.exs | 12 ++++++------
 test/tortoise/handler_test.exs    | 12 ++++++------
 5 files changed, 29 insertions(+), 29 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index ea15e302..2fb8ac58 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -462,7 +462,7 @@ defmodule Tortoise.Connection do
         _,
         %State{handler: handler} = data
       ) do
-    case Handler.execute_handle_message(handler, publish) do
+    case Handler.execute_handle_publish(handler, publish) do
       {:ok, %Handler{} = updated_handler} ->
         updated_data = %State{data | handler: updated_handler}
         {:keep_state, updated_data}
diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index 17cc78a8..0038c1ba 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -25,7 +25,7 @@ defmodule Tortoise.Handler do
       behavior for when the subscription is accepted, declined, as
       well as unsubscribed.
 
-    - `handle_message/3` is run when the client receive a message on
+    - `handle_publish/3` is run when the client receive a publish on
       one of the subscribed topic filters.
 
   Because the callback-module will run inside the connection
@@ -63,7 +63,7 @@ defmodule Tortoise.Handler do
   require the user to peek into the process mailbox to fetch the
   result of the operation. To allow for changes in the subscriptions
   one can define a set of next actions that should happen as part of
-  the return value to the `handle_message/3`, `subscription/3`, and
+  the return value to the `handle_publish/3`, `subscription/3`, and
   `connection/3` callbacks by returning a `{:ok, state, next_actions}`
   where `next_actions` is a list of commands of:
 
@@ -78,9 +78,9 @@ defmodule Tortoise.Handler do
       from.
 
   If we want to unsubscribe from the current topic when we receive a
-  message on it we could write a `handle_message/3` as follows:
+  publish on it we could write a `handle_publish/3` as follows:
 
-      def handle_message(topic, _payload, state) do
+      def handle_publish(topic, _payload, state) do
         topic = Enum.join(topic, "/")
         next_actions = [{:unsubscribe, topic}]
         {:ok, state, next_actions}
@@ -89,7 +89,7 @@ defmodule Tortoise.Handler do
   Note that the `topic` is received as a list of topic levels, and
   that the next actions has to be a list, even if there is only one
   next action; multiple actions can be given at once. Read more about
-  this in the `handle_message/3` documentation.
+  this in the `handle_publish/3` documentation.
   """
 
   alias Tortoise.Package
@@ -145,7 +145,7 @@ defmodule Tortoise.Handler do
       end
 
       @impl true
-      def handle_message(_topic, _payload, state) do
+      def handle_publish(_topic, _payload, state) do
         {:ok, state}
       end
 
@@ -251,7 +251,7 @@ defmodule Tortoise.Handler do
 
   The `topic` comes in the form of a list of binaries, making it
   possible to pattern match on the topic levels of the retrieved
-  message, store the individual topic levels as variables and use it
+  publish, store the individual topic levels as variables and use it
   in the function body.
 
   `Payload` is a binary. MQTT 3.1.1 does not specify any format of the
@@ -260,15 +260,15 @@ defmodule Tortoise.Handler do
 
   In an example where we are already subscribed to the topic filter
   `room/+/temp` and want to dispatch the received messages to a
-  `Temperature` application we could set up our `handle_message` as
+  `Temperature` application we could set up our `handle_publish` as
   such:
 
-      def handle_message(["room", room, "temp"], payload, state) do
+      def handle_publish(["room", room, "temp"], payload, state) do
         :ok = Temperature.record(room, payload)
         {:ok, state}
       end
 
-  Notice; the `handle_message/3`-callback run inside the connection
+  Notice; the `handle_publish/3`-callback run inside the connection
   controller process, so for handlers that are subscribing to topics
   with heavy traffic should do as little as possible in the callback
   handler and dispatch to other parts of the application using
@@ -281,7 +281,7 @@ defmodule Tortoise.Handler do
   a list of next actions such as `{:unsubscribe, "foo/bar"}` will
   reenter the loop and perform the listed actions.
   """
-  @callback handle_message(topic_levels, payload, state :: term()) ::
+  @callback handle_publish(topic_levels, payload, state :: term()) ::
               {:ok, new_state}
               | {:ok, new_state, [next_action()]}
             when new_state: term(),
@@ -377,13 +377,13 @@ defmodule Tortoise.Handler do
   end
 
   @doc false
-  @spec execute_handle_message(t, Package.Publish.t()) ::
+  @spec execute_handle_publish(t, Package.Publish.t()) ::
           {:ok, t} | {:error, {:invalid_next_action, term()}}
-  def execute_handle_message(handler, %Package.Publish{} = publish) do
+  def execute_handle_publish(handler, %Package.Publish{} = publish) do
     topic_list = String.split(publish.topic, "/")
 
     handler.module
-    |> apply(:handle_message, [topic_list, publish.payload, handler.state])
+    |> apply(:handle_publish, [topic_list, publish.payload, handler.state])
     |> handle_result(handler)
   end
 
diff --git a/test/support/test_handler.ex b/test/support/test_handler.ex
index 405b139f..2385990e 100644
--- a/test/support/test_handler.ex
+++ b/test/support/test_handler.ex
@@ -22,9 +22,9 @@ defmodule TestHandler do
     {:stop, :normal, state}
   end
 
-  def handle_message(topic, payload, state) do
+  def handle_publish(topic, payload, state) do
     data = %{topic: Enum.join(topic, "/"), payload: payload}
-    send(state[:parent], {{__MODULE__, :handle_message}, data})
+    send(state[:parent], {{__MODULE__, :handle_publish}, data})
     {:ok, state}
   end
 
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index 04498934..6c8ffa9a 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -846,8 +846,8 @@ defmodule Tortoise.ConnectionTest do
       refute_receive {:EXIT, ^pid, {:protocol_violation, {:unexpected_package, ^publish}}}
       assert_receive {ScriptedMqttServer, :completed}
 
-      # the handle message callback should have been called
-      assert_receive {{TestHandler, :handle_message}, %{topic: "foo/bar", payload: nil}}
+      # the handle publish callback should have been called
+      assert_receive {{TestHandler, :handle_publish}, %{topic: "foo/bar", payload: nil}}
     end
   end
 
@@ -867,8 +867,8 @@ defmodule Tortoise.ConnectionTest do
       {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
       assert_receive {ScriptedMqttServer, :completed}
 
-      # the handle message callback should have been called
-      assert_receive {{TestHandler, :handle_message}, %{topic: "foo/bar", payload: nil}}
+      # the handle publish callback should have been called
+      assert_receive {{TestHandler, :handle_publish}, %{topic: "foo/bar", payload: nil}}
     end
 
     test "outgoing publish with QoS=1", context do
@@ -946,8 +946,8 @@ defmodule Tortoise.ConnectionTest do
       refute_receive {:EXIT, ^pid, {:protocol_violation, {:unexpected_package, _}}}
       assert_receive {ScriptedMqttServer, :completed}
 
-      # the handle message callback should have been called
-      assert_receive {{TestHandler, :handle_message}, %{topic: "foo/bar", payload: nil}}
+      # the handle publish callback should have been called
+      assert_receive {{TestHandler, :handle_publish}, %{topic: "foo/bar", payload: nil}}
     end
 
     test "outgoing publish with QoS=2", context do
diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index 5cd81b8e..a5ae8ad1 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -30,12 +30,12 @@ defmodule Tortoise.HandlerTest do
     end
 
     # with next actions
-    def handle_message(topic, payload, %{next_actions: next_actions} = state) do
+    def handle_publish(topic, payload, %{next_actions: next_actions} = state) do
       send(state[:pid], {:publish, topic, payload})
       {:ok, state, next_actions}
     end
 
-    def handle_message(topic, payload, state) do
+    def handle_publish(topic, payload, state) do
       send(state[:pid], {:publish, topic, payload})
       {:ok, state}
     end
@@ -96,14 +96,14 @@ defmodule Tortoise.HandlerTest do
     end
   end
 
-  describe "execute handle_message/2" do
+  describe "execute handle_publish/2" do
     test "return ok-2", context do
       handler = set_state(context.handler, %{pid: self()})
       payload = :crypto.strong_rand_bytes(5)
       topic = "foo/bar"
       publish = %Package.Publish{topic: topic, payload: payload}
 
-      assert {:ok, %Handler{}} = Handler.execute_handle_message(handler, publish)
+      assert {:ok, %Handler{}} = Handler.execute_handle_publish(handler, publish)
       # the topic will be in the form of a list making it possible to
       # pattern match on the topic levels
       assert_receive {:publish, topic_list, ^payload}
@@ -119,7 +119,7 @@ defmodule Tortoise.HandlerTest do
       topic = "foo/bar"
       publish = %Package.Publish{topic: topic, payload: payload}
 
-      assert {:ok, %Handler{}} = Handler.execute_handle_message(handler, publish)
+      assert {:ok, %Handler{}} = Handler.execute_handle_publish(handler, publish)
 
       assert_receive {:next_action, {:subscribe, "foo/bar", qos: 0}}
 
@@ -139,7 +139,7 @@ defmodule Tortoise.HandlerTest do
       publish = %Package.Publish{topic: topic, payload: payload}
 
       assert {:error, {:invalid_next_action, [{:invalid, "bar"}]}} =
-               Handler.execute_handle_message(handler, publish)
+               Handler.execute_handle_publish(handler, publish)
 
       refute_receive {:next_action, {:invalid, "bar"}}
       # we should not receive the otherwise valid next_action

From fa7e01856b78254eb58b1d79c5c17dd9e5573399 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Thu, 25 Oct 2018 21:59:56 +0200
Subject: [PATCH 058/220] Add and expose a handle_pubrel callback

I have no idea what this should be used for, but the pubrel package
can contain a user defined property, so better expose it to the
end-user. Also, the end user need to be able to define user defined
properties on the pubcomp; that will come in a later commit.
---
 lib/tortoise/connection.ex        | 16 ++++++++++++----
 lib/tortoise/handler.ex           | 18 ++++++++++++++++++
 test/support/test_handler.ex      |  5 +++++
 test/tortoise/connection_test.exs |  3 ++-
 test/tortoise/handler_test.exs    | 18 ++++++++++++++++++
 5 files changed, 55 insertions(+), 5 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 2fb8ac58..49a94a11 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -591,11 +591,19 @@ defmodule Tortoise.Connection do
   def handle_event(
         :internal,
         {:received, %Package.Pubrel{} = pubrel},
-        :connected,
-        %State{client_id: client_id}
+        _,
+        %State{client_id: client_id, handler: handler} = data
       ) do
-    :ok = Inflight.update(client_id, {:received, pubrel})
-    :keep_state_and_data
+    case Handler.execute_handle_pubrel(handler, pubrel) do
+      {:ok, %Handler{} = updated_handler} ->
+        :ok = Inflight.update(client_id, {:received, pubrel})
+        updated_data = %State{data | handler: updated_handler}
+        {:keep_state, updated_data}
+
+      {:error, reason} ->
+        # todo
+        {:stop, reason, data}
+    end
   end
 
   # an incoming publish with QoS=2 will get parked in the inflight
diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index 0038c1ba..953a8d3f 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -154,6 +154,11 @@ defmodule Tortoise.Handler do
         {:ok, state}
       end
 
+      @impl true
+      def handle_pubrel(_pubrel, state) do
+        {:ok, state}
+      end
+
       defoverridable Tortoise.Handler
     end
   end
@@ -292,6 +297,10 @@ defmodule Tortoise.Handler do
             when puback: Package.Puback.t(),
                  new_state: term()
 
+  @callback handle_pubrel(puback, state :: term()) :: {:ok, new_state}
+            when puback: Package.Pubrel.t(),
+                 new_state: term()
+
   @doc """
   Invoked when the connection process is about to exit.
 
@@ -396,6 +405,15 @@ defmodule Tortoise.Handler do
     |> handle_result(handler)
   end
 
+  @doc false
+  # @spec execute_handle_pubrel(t, Package.Pubrel.t()) ::
+  #         {:ok, t} | {:error, {:invalid_next_action, term()}}
+  def execute_handle_pubrel(handler, %Package.Pubrel{} = pubrel) do
+    handler.module
+    |> apply(:handle_pubrel, [pubrel, handler.state])
+    |> handle_result(handler)
+  end
+
   # Subacks will come in a map with three keys in the form of tuples
   # where the fist element is one of `:ok`, `:warn`, or `:error`. This
   # is done to make it easy to pattern match in other parts of the
diff --git a/test/support/test_handler.ex b/test/support/test_handler.ex
index 2385990e..552a30f5 100644
--- a/test/support/test_handler.ex
+++ b/test/support/test_handler.ex
@@ -28,6 +28,11 @@ defmodule TestHandler do
     {:ok, state}
   end
 
+  def handle_pubrel(pubrel, state) do
+    send(state[:parent], {{__MODULE__, :handle_pubrel}, pubrel})
+    {:ok, state}
+  end
+
   def subscription(status, topic_filter, state) do
     data = %{status: status, topic_filter: topic_filter}
     send(state[:parent], {{__MODULE__, :subscription}, data})
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index 6c8ffa9a..f2583bf0 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -946,7 +946,8 @@ defmodule Tortoise.ConnectionTest do
       refute_receive {:EXIT, ^pid, {:protocol_violation, {:unexpected_package, _}}}
       assert_receive {ScriptedMqttServer, :completed}
 
-      # the handle publish callback should have been called
+      # the handle publish, and handle_pubrel callbacks should have been called
+      assert_receive {{TestHandler, :handle_pubrel}, ^pubrel}
       assert_receive {{TestHandler, :handle_publish}, %{topic: "foo/bar", payload: nil}}
     end
 
diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index a5ae8ad1..05358ab1 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -49,6 +49,11 @@ defmodule Tortoise.HandlerTest do
       send(state[:pid], {:puback, puback})
       {:ok, state}
     end
+
+    def handle_pubrel(pubrel, state) do
+      send(state[:pid], {:pubrel, pubrel})
+      {:ok, state}
+    end
   end
 
   setup _context do
@@ -213,4 +218,17 @@ defmodule Tortoise.HandlerTest do
       assert_receive {:puback, ^puback}
     end
   end
+
+  describe "execute handle_pubrel/2" do
+    test "return ok", context do
+      handler = set_state(context.handler, pid: self())
+      pubrel = %Package.Pubrel{identifier: 1}
+
+      assert {:ok, %Handler{} = state} =
+               handler
+               |> Handler.execute_handle_pubrel(pubrel)
+
+      assert_receive {:pubrel, ^pubrel}
+    end
+  end
 end

From fcb260900ef8fe3e959cfbaea69364ae078e4838 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Thu, 25 Oct 2018 22:14:25 +0200
Subject: [PATCH 059/220] Add and expose a handle_pubrec callback

This will allow the end user to handle user properties on the pubrec
protocol packages.
---
 lib/tortoise/connection.ex        | 16 ++++++++++++----
 lib/tortoise/handler.ex           | 22 ++++++++++++++++++++--
 test/support/test_handler.ex      |  5 +++++
 test/tortoise/connection_test.exs |  3 +++
 test/tortoise/handler_test.exs    | 19 +++++++++++++++++++
 5 files changed, 59 insertions(+), 6 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 49a94a11..bc334a32 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -623,11 +623,19 @@ defmodule Tortoise.Connection do
   def handle_event(
         :internal,
         {:received, %Package.Pubrec{} = pubrec},
-        :connected,
-        %State{client_id: client_id}
+        _,
+        %State{client_id: client_id, handler: handler} = data
       ) do
-    :ok = Inflight.update(client_id, {:received, pubrec})
-    :keep_state_and_data
+    case Handler.execute_handle_pubrec(handler, pubrec) do
+      {:ok, %Handler{} = updated_handler} ->
+        :ok = Inflight.update(client_id, {:received, pubrec})
+        updated_data = %State{data | handler: updated_handler}
+        {:keep_state, updated_data}
+
+      {:error, reason} ->
+        # todo
+        {:stop, reason, data}
+    end
   end
 
   def handle_event(
diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index 953a8d3f..169f2dc4 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -154,6 +154,11 @@ defmodule Tortoise.Handler do
         {:ok, state}
       end
 
+      @impl true
+      def handle_pubrec(_pubrec, state) do
+        {:ok, state}
+      end
+
       @impl true
       def handle_pubrel(_pubrel, state) do
         {:ok, state}
@@ -297,8 +302,12 @@ defmodule Tortoise.Handler do
             when puback: Package.Puback.t(),
                  new_state: term()
 
-  @callback handle_pubrel(puback, state :: term()) :: {:ok, new_state}
-            when puback: Package.Pubrel.t(),
+  @callback handle_pubrec(pubrec, state :: term()) :: {:ok, new_state}
+            when pubrec: Package.Pubrec.t(),
+                 new_state: term()
+
+  @callback handle_pubrel(pubrel, state :: term()) :: {:ok, new_state}
+            when pubrel: Package.Pubrel.t(),
                  new_state: term()
 
   @doc """
@@ -405,6 +414,15 @@ defmodule Tortoise.Handler do
     |> handle_result(handler)
   end
 
+  @doc false
+  # @spec execute_handle_pubrec(t, Package.Pubrec.t()) ::
+  #         {:ok, t} | {:error, {:invalid_next_action, term()}}
+  def execute_handle_pubrec(handler, %Package.Pubrec{} = pubrec) do
+    handler.module
+    |> apply(:handle_pubrec, [pubrec, handler.state])
+    |> handle_result(handler)
+  end
+
   @doc false
   # @spec execute_handle_pubrel(t, Package.Pubrel.t()) ::
   #         {:ok, t} | {:error, {:invalid_next_action, term()}}
diff --git a/test/support/test_handler.ex b/test/support/test_handler.ex
index 552a30f5..435a5b5c 100644
--- a/test/support/test_handler.ex
+++ b/test/support/test_handler.ex
@@ -28,6 +28,11 @@ defmodule TestHandler do
     {:ok, state}
   end
 
+  def handle_pubrec(pubrec, state) do
+    send(state[:parent], {{__MODULE__, :handle_pubrec}, pubrec})
+    {:ok, state}
+  end
+
   def handle_pubrel(pubrel, state) do
     send(state[:parent], {{__MODULE__, :handle_pubrel}, pubrel})
     {:ok, state}
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index f2583bf0..b248f5a2 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -976,6 +976,9 @@ defmodule Tortoise.ConnectionTest do
       assert_receive {ScriptedMqttServer, {:received, ^pubrel}}
       assert_receive {ScriptedMqttServer, :completed}
       assert_receive {{Tortoise, ^client_id}, {Package.Publish, ^ref}, :ok}
+
+      # the handle_pubrec callback should have been called
+      assert_receive {{TestHandler, :handle_pubrec}, ^pubrec}
     end
   end
 
diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index 05358ab1..51467111 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -50,6 +50,11 @@ defmodule Tortoise.HandlerTest do
       {:ok, state}
     end
 
+    def handle_pubrec(pubrec, state) do
+      send(state[:pid], {:pubrec, pubrec})
+      {:ok, state}
+    end
+
     def handle_pubrel(pubrel, state) do
       send(state[:pid], {:pubrel, pubrel})
       {:ok, state}
@@ -219,6 +224,20 @@ defmodule Tortoise.HandlerTest do
     end
   end
 
+  # callbacks for the QoS=2 message exchange
+  describe "execute handle_pubrec/2" do
+    test "return ok", context do
+      handler = set_state(context.handler, pid: self())
+      pubrec = %Package.Pubrec{identifier: 1}
+
+      assert {:ok, %Handler{} = state} =
+               handler
+               |> Handler.execute_handle_pubrec(pubrec)
+
+      assert_receive {:pubrec, ^pubrec}
+    end
+  end
+
   describe "execute handle_pubrel/2" do
     test "return ok", context do
       handler = set_state(context.handler, pid: self())

From 6bddacd9a98501e23998be63ade5e36192af4b59 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Thu, 25 Oct 2018 22:39:03 +0200
Subject: [PATCH 060/220] Introduce and expose a handle_pubcomp callback

---
 lib/tortoise/connection.ex        | 14 +++++++++++---
 lib/tortoise/handler.ex           | 18 ++++++++++++++++++
 test/support/test_handler.ex      |  5 +++++
 test/tortoise/connection_test.exs |  1 +
 test/tortoise/handler_test.exs    | 18 ++++++++++++++++++
 5 files changed, 53 insertions(+), 3 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index bc334a32..fb58ad9d 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -642,10 +642,18 @@ defmodule Tortoise.Connection do
         :internal,
         {:received, %Package.Pubcomp{} = pubcomp},
         :connected,
-        %State{client_id: client_id}
+        %State{client_id: client_id, handler: handler} = data
       ) do
-    :ok = Inflight.update(client_id, {:received, pubcomp})
-    :keep_state_and_data
+    case Handler.execute_handle_pubcomp(handler, pubcomp) do
+      {:ok, %Handler{} = updated_handler} ->
+        :ok = Inflight.update(client_id, {:received, pubcomp})
+        updated_data = %State{data | handler: updated_handler}
+        {:keep_state, updated_data}
+
+      {:error, reason} ->
+        # todo
+        {:stop, reason, data}
+    end
   end
 
   # subscription logic
diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index 169f2dc4..4a3a07d6 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -164,6 +164,11 @@ defmodule Tortoise.Handler do
         {:ok, state}
       end
 
+      @impl true
+      def handle_pubcomp(_pubcomp, state) do
+        {:ok, state}
+      end
+
       defoverridable Tortoise.Handler
     end
   end
@@ -310,6 +315,10 @@ defmodule Tortoise.Handler do
             when pubrel: Package.Pubrel.t(),
                  new_state: term()
 
+  @callback handle_pubcomp(pubcomp, state :: term()) :: {:ok, new_state}
+            when pubcomp: Package.Pubcomp.t(),
+                 new_state: term()
+
   @doc """
   Invoked when the connection process is about to exit.
 
@@ -432,6 +441,15 @@ defmodule Tortoise.Handler do
     |> handle_result(handler)
   end
 
+  @doc false
+  # @spec execute_handle_pubcomp(t, Package.Pubcomp.t()) ::
+  #         {:ok, t} | {:error, {:invalid_next_action, term()}}
+  def execute_handle_pubcomp(handler, %Package.Pubcomp{} = pubcomp) do
+    handler.module
+    |> apply(:handle_pubcomp, [pubcomp, handler.state])
+    |> handle_result(handler)
+  end
+
   # Subacks will come in a map with three keys in the form of tuples
   # where the fist element is one of `:ok`, `:warn`, or `:error`. This
   # is done to make it easy to pattern match in other parts of the
diff --git a/test/support/test_handler.ex b/test/support/test_handler.ex
index 435a5b5c..947d95dc 100644
--- a/test/support/test_handler.ex
+++ b/test/support/test_handler.ex
@@ -38,6 +38,11 @@ defmodule TestHandler do
     {:ok, state}
   end
 
+  def handle_pubcomp(pubcomp, state) do
+    send(state[:parent], {{__MODULE__, :handle_pubcomp}, pubcomp})
+    {:ok, state}
+  end
+
   def subscription(status, topic_filter, state) do
     data = %{status: status, topic_filter: topic_filter}
     send(state[:parent], {{__MODULE__, :subscription}, data})
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index b248f5a2..027f4e21 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -979,6 +979,7 @@ defmodule Tortoise.ConnectionTest do
 
       # the handle_pubrec callback should have been called
       assert_receive {{TestHandler, :handle_pubrec}, ^pubrec}
+      assert_receive {{TestHandler, :handle_pubcomp}, ^pubcomp}
     end
   end
 
diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index 51467111..7f79e6ca 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -59,6 +59,11 @@ defmodule Tortoise.HandlerTest do
       send(state[:pid], {:pubrel, pubrel})
       {:ok, state}
     end
+
+    def handle_pubcomp(pubcomp, state) do
+      send(state[:pid], {:pubcomp, pubcomp})
+      {:ok, state}
+    end
   end
 
   setup _context do
@@ -250,4 +255,17 @@ defmodule Tortoise.HandlerTest do
       assert_receive {:pubrel, ^pubrel}
     end
   end
+
+  describe "execute handle_pubcomp/2" do
+    test "return ok", context do
+      handler = set_state(context.handler, pid: self())
+      pubcomp = %Package.Pubcomp{identifier: 1}
+
+      assert {:ok, %Handler{} = state} =
+               handler
+               |> Handler.execute_handle_pubcomp(pubcomp)
+
+      assert_receive {:pubcomp, ^pubcomp}
+    end
+  end
 end

From f7b200bf0122197d398c14ec74c56a4389bcb261 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sun, 4 Nov 2018 13:29:29 +0100
Subject: [PATCH 061/220] Support user defined properties for protocol packages

The inflight process will now await a package to dispatch before
progressing tracking states. Before it just dispatched the next
message in the plan, but that is not possible anymore as we need to
support user defined properties on packages such as pubrel, pubrec,
pubcomp, etc.

The controller has been updated so it will pass the dispatch update
into the inflight process; the tests has been updated as well.
---
 lib/tortoise/connection.ex                 | 198 +++++++++++----------
 lib/tortoise/connection/inflight.ex        |  63 +++++--
 lib/tortoise/connection/inflight/track.ex  |  12 ++
 lib/tortoise/package/pubcomp.ex            |   9 +-
 lib/tortoise/package/pubrec.ex             |   2 +-
 lib/tortoise/package/pubrel.ex             |   7 +-
 lib/tortoise/package/subscribe.ex          |   5 +-
 test/tortoise/connection/inflight_test.exs |  34 ++--
 test/tortoise_test.exs                     |   5 +-
 9 files changed, 207 insertions(+), 128 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index fb58ad9d..75a506f8 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -411,66 +411,6 @@ defmodule Tortoise.Connection do
   end
 
   @impl true
-  def handle_event(
-        :internal,
-        {:execute_handler, {:connection, status}},
-        :connected,
-        %State{handler: handler} = data
-      ) do
-    case Handler.execute_connection(handler, status) do
-      {:ok, %Handler{} = updated_handler} ->
-        updated_data = %State{data | handler: updated_handler}
-        {:keep_state, updated_data}
-
-        # handle stop
-    end
-  end
-
-  def handle_event(
-        :internal,
-        {:execute_handler, {:subscribe, results}},
-        _,
-        %State{handler: handler} = data
-      ) do
-    case Handler.execute_subscribe(handler, results) do
-      {:ok, %Handler{} = updated_handler} ->
-        updated_data = %State{data | handler: updated_handler}
-        {:keep_state, updated_data}
-
-        # handle stop
-    end
-  end
-
-  def handle_event(
-        :internal,
-        {:execute_handler, {:unsubscribe, result}},
-        _current_state,
-        %State{handler: handler} = data
-      ) do
-    case Handler.execute_unsubscribe(handler, result) do
-      {:ok, %Handler{} = updated_handler} ->
-        updated_data = %State{data | handler: updated_handler}
-        {:keep_state, updated_data}
-
-        # handle stop
-    end
-  end
-
-  def handle_event(
-        :internal,
-        {:execute_handler, {:publish, %Package.Publish{} = publish}},
-        _,
-        %State{handler: handler} = data
-      ) do
-    case Handler.execute_handle_publish(handler, publish) do
-      {:ok, %Handler{} = updated_handler} ->
-        updated_data = %State{data | handler: updated_handler}
-        {:keep_state, updated_data}
-
-        # handle stop
-    end
-  end
-
   def handle_event(:info, {:incoming, package}, _, _data) when is_binary(package) do
     next_actions = [{:next_event, :internal, {:received, Package.decode(package)}}]
     {:keep_state_and_data, next_actions}
@@ -534,28 +474,32 @@ defmodule Tortoise.Connection do
     {:stop, {:protocol_violation, reason}, data}
   end
 
-  # publish packages
+  # publish packages ===================================================
   def handle_event(
         :internal,
         {:received, %Package.Publish{qos: 0, dup: false} = publish},
         _,
-        %State{}
+        %State{handler: handler} = data
       ) do
-    next_actions = [{:next_event, :internal, {:execute_handler, {:publish, publish}}}]
-    {:keep_state_and_data, next_actions}
+    case Handler.execute_handle_publish(handler, publish) do
+      {:ok, %Handler{} = updated_handler} ->
+        updated_data = %State{data | handler: updated_handler}
+        {:keep_state, updated_data}
+
+        # handle stop
+    end
   end
 
-  # incoming publish QoS=1 ---------------------------------------------
+  # incoming publish QoS>0 ---------------------------------------------
   def handle_event(
         :internal,
-        {:received, %Package.Publish{qos: 1} = publish},
+        {:received, %Package.Publish{qos: qos} = publish},
         _,
         %State{client_id: client_id}
-      ) do
+      )
+      when qos in 1..2 do
     :ok = Inflight.track(client_id, {:incoming, publish})
-
-    next_actions = [{:next_event, :internal, {:execute_handler, {:publish, publish}}}]
-    {:keep_state_and_data, next_actions}
+    :keep_state_and_data
   end
 
   # outgoing publish QoS=1 ---------------------------------------------
@@ -565,9 +509,10 @@ defmodule Tortoise.Connection do
         _,
         %State{client_id: client_id, handler: handler} = data
       ) do
+    :ok = Inflight.update(client_id, {:received, puback})
+
     case Handler.execute_handle_puback(handler, puback) do
       {:ok, %Handler{} = updated_handler} ->
-        :ok = Inflight.update(client_id, {:received, puback})
         updated_data = %State{data | handler: updated_handler}
         {:keep_state, updated_data}
 
@@ -580,23 +525,17 @@ defmodule Tortoise.Connection do
   # incoming publish QoS=2 ---------------------------------------------
   def handle_event(
         :internal,
-        {:received, %Package.Publish{qos: 2} = publish},
-        :connected,
-        %State{client_id: client_id}
-      ) do
-    :ok = Inflight.track(client_id, {:incoming, publish})
-    :keep_state_and_data
-  end
-
-  def handle_event(
-        :internal,
-        {:received, %Package.Pubrel{} = pubrel},
+        {:received, %Package.Pubrel{identifier: id} = pubrel},
         _,
         %State{client_id: client_id, handler: handler} = data
       ) do
+    :ok = Inflight.update(client_id, {:received, pubrel})
+
     case Handler.execute_handle_pubrel(handler, pubrel) do
       {:ok, %Handler{} = updated_handler} ->
-        :ok = Inflight.update(client_id, {:received, pubrel})
+        # dispatch a pubcomp
+        pubcomp = %Package.Pubcomp{identifier: id}
+        :ok = Inflight.update(client_id, {:dispatch, pubcomp})
         updated_data = %State{data | handler: updated_handler}
         {:keep_state, updated_data}
 
@@ -606,29 +545,59 @@ defmodule Tortoise.Connection do
     end
   end
 
-  # an incoming publish with QoS=2 will get parked in the inflight
+  # an incoming publish with QoS>0 will get parked in the inflight
   # manager process, which will onward it to the controller, making
   # sure we will only dispatch it once to the publish-handler.
   def handle_event(
         :info,
-        {{Inflight, client_id}, %Package.Publish{qos: 2} = publish},
+        {{Inflight, client_id}, %Package.Publish{identifier: id, qos: 1} = publish},
         _,
-        %State{client_id: client_id}
+        %State{client_id: client_id, handler: handler} = data
       ) do
-    next_actions = [{:next_event, :internal, {:execute_handler, {:publish, publish}}}]
-    {:keep_state_and_data, next_actions}
+    case Handler.execute_handle_publish(handler, publish) do
+      {:ok, %Handler{} = updated_handler} ->
+        # respond with a puback
+        puback = %Package.Puback{identifier: id}
+        :ok = Inflight.update(client_id, {:dispatch, puback})
+        # - - -
+        updated_data = %State{data | handler: updated_handler}
+        {:keep_state, updated_data}
+
+        # handle stop
+    end
+  end
+
+  def handle_event(
+        :info,
+        {{Inflight, client_id}, %Package.Publish{identifier: id, qos: 2} = publish},
+        _,
+        %State{client_id: client_id, handler: handler} = data
+      ) do
+    case Handler.execute_handle_publish(handler, publish) do
+      {:ok, %Handler{} = updated_handler} ->
+        # respond with pubrec
+        pubrec = %Package.Pubrec{identifier: id}
+        :ok = Inflight.update(client_id, {:dispatch, pubrec})
+        # - - -
+        updated_data = %State{data | handler: updated_handler}
+        {:keep_state, updated_data}
+    end
   end
 
   # outgoing publish QoS=2 ---------------------------------------------
   def handle_event(
         :internal,
-        {:received, %Package.Pubrec{} = pubrec},
+        {:received, %Package.Pubrec{identifier: id} = pubrec},
         _,
         %State{client_id: client_id, handler: handler} = data
       ) do
+    :ok = Inflight.update(client_id, {:received, pubrec})
+
     case Handler.execute_handle_pubrec(handler, pubrec) do
       {:ok, %Handler{} = updated_handler} ->
-        :ok = Inflight.update(client_id, {:received, pubrec})
+        pubrel = %Package.Pubrel{identifier: id}
+        :ok = Inflight.update(client_id, {:dispatch, pubrel})
+
         updated_data = %State{data | handler: updated_handler}
         {:keep_state, updated_data}
 
@@ -644,9 +613,10 @@ defmodule Tortoise.Connection do
         :connected,
         %State{client_id: client_id, handler: handler} = data
       ) do
+    :ok = Inflight.update(client_id, {:received, pubcomp})
+
     case Handler.execute_handle_pubcomp(handler, pubcomp) do
       {:ok, %Handler{} = updated_handler} ->
-        :ok = Inflight.update(client_id, {:received, pubcomp})
         updated_data = %State{data | handler: updated_handler}
         {:keep_state, updated_data}
 
@@ -746,6 +716,52 @@ defmodule Tortoise.Connection do
     {:keep_state_and_data, next_actions}
   end
 
+  # dispatch to user defined handler
+  def handle_event(
+        :internal,
+        {:execute_handler, {:connection, status}},
+        :connected,
+        %State{handler: handler} = data
+      ) do
+    case Handler.execute_connection(handler, status) do
+      {:ok, %Handler{} = updated_handler} ->
+        updated_data = %State{data | handler: updated_handler}
+        {:keep_state, updated_data}
+
+        # handle stop
+    end
+  end
+
+  def handle_event(
+        :internal,
+        {:execute_handler, {:subscribe, results}},
+        _,
+        %State{handler: handler} = data
+      ) do
+    case Handler.execute_subscribe(handler, results) do
+      {:ok, %Handler{} = updated_handler} ->
+        updated_data = %State{data | handler: updated_handler}
+        {:keep_state, updated_data}
+
+        # handle stop
+    end
+  end
+
+  def handle_event(
+        :internal,
+        {:execute_handler, {:unsubscribe, result}},
+        _current_state,
+        %State{handler: handler} = data
+      ) do
+    case Handler.execute_unsubscribe(handler, result) do
+      {:ok, %Handler{} = updated_handler} ->
+        updated_data = %State{data | handler: updated_handler}
+        {:keep_state, updated_data}
+
+        # handle stop
+    end
+  end
+
   # connection logic ===================================================
   def handle_event(:internal, :connect, :connecting, %State{} = data) do
     :ok = Tortoise.Registry.put_meta(via_name(data.client_id), :connecting)
diff --git a/lib/tortoise/connection/inflight.ex b/lib/tortoise/connection/inflight.ex
index 7ef342b4..39c58439 100644
--- a/lib/tortoise/connection/inflight.ex
+++ b/lib/tortoise/connection/inflight.ex
@@ -171,8 +171,7 @@ defmodule Tortoise.Connection.Inflight do
     }
 
     next_actions = [
-      {:next_event, :internal, {:onward_publish, package}},
-      {:next_event, :internal, {:execute, track}}
+      {:next_event, :internal, {:onward_publish, package}}
     ]
 
     {:keep_state, data, next_actions}
@@ -234,23 +233,32 @@ defmodule Tortoise.Connection.Inflight do
 
   def handle_event(
         :cast,
-        {:update, {_, %{identifier: identifier}} = update},
+        {:update, {:received, %{identifier: identifier}} = update},
         _state,
         %State{pending: pending} = data
       ) do
     with {:ok, track} <- Map.fetch(pending, identifier),
          {:ok, track} <- Track.resolve(track, update) do
-      next_actions = [
-        {:next_event, :internal, {:execute, track}}
-      ]
-
       data = %State{
         data
         | pending: Map.put(pending, identifier, track),
           order: [identifier | data.order -- [identifier]]
       }
 
-      {:keep_state, data, next_actions}
+      case track do
+        %Track{pending: [[{:respond, _}, _] | _]} ->
+          next_actions = [
+            {:next_event, :internal, {:execute, track}}
+          ]
+
+          {:keep_state, data, next_actions}
+
+        # to support user defined properties we need to await a
+        # dispatch command from the controller before we can
+        # progress.
+        %Track{pending: [[{:dispatch, _}, _] | _]} ->
+          {:keep_state, data}
+      end
     else
       :error ->
         {:stop, {:protocol_violation, :unknown_identifier}, data}
@@ -260,6 +268,28 @@ defmodule Tortoise.Connection.Inflight do
     end
   end
 
+  def handle_event(
+        :cast,
+        {:update, {:dispatch, %{identifier: identifier}} = update},
+        _state,
+        %State{pending: pending} = data
+      ) do
+    with {:ok, track} <- Map.fetch(pending, identifier),
+         {:ok, track} <- Track.resolve(track, update) do
+      data = %State{
+        data
+        | pending: Map.put(pending, identifier, track),
+          order: [identifier | data.order -- [identifier]]
+      }
+
+      next_actions = [
+        {:next_event, :internal, {:execute, track}}
+      ]
+
+      {:keep_state, data, next_actions}
+    end
+  end
+
   def handle_event(:cast, :reset, _, %State{pending: pending} = data) do
     # cancel all currently outgoing messages
     for {_, %Track{polarity: :negative, caller: {pid, ref}}} <- pending do
@@ -269,25 +299,20 @@ defmodule Tortoise.Connection.Inflight do
     {:keep_state, %State{data | pending: %{}, order: []}}
   end
 
-  # We trap the incoming QoS 2 packages in the inflight manager so we
-  # can make sure we will not onward them to the connection handler
-  # more than once.
+  # We trap the incoming QoS>0 packages in the inflight manager so we
+  # can make sure we will not onward the same publish to the user
+  # defined callback more than once.
   def handle_event(
         :internal,
-        {:onward_publish, %Package.Publish{qos: 2} = publish},
+        {:onward_publish, %Package.Publish{qos: qos} = publish},
         _,
         %State{client_id: client_id, parent: parent_pid} = data
-      ) do
+      )
+      when qos in 1..2 do
     send(parent_pid, {{__MODULE__, client_id}, publish})
     :keep_state_and_data
   end
 
-  # The other package types should not get onwarded to the controller
-  # handler
-  def handle_event(:internal, {:onward_publish, _}, _, %State{}) do
-    :keep_state_and_data
-  end
-
   def handle_event(:internal, {:execute, _}, :draining, _) do
     :keep_state_and_data
   end
diff --git a/lib/tortoise/connection/inflight/track.ex b/lib/tortoise/connection/inflight/track.ex
index 00152aa3..a63f9920 100644
--- a/lib/tortoise/connection/inflight/track.ex
+++ b/lib/tortoise/connection/inflight/track.ex
@@ -64,6 +64,18 @@ defmodule Tortoise.Connection.Inflight.Track do
     {:ok, %State{state | pending: rest, status: [expected, action | state.status]}}
   end
 
+  # When we are awaiting a package to dispatch, replace the package
+  # with the message given by the user; this allow us to support user
+  # defined properties on packages such as pubrec, pubrel, pubcomp,
+  # etc.
+  def resolve(
+        %State{pending: [[{:dispatch, %{__struct__: t, identifier: id}}, resolution] | rest]} =
+          state,
+        {:dispatch, %{__struct__: t, identifier: id}} = dispatch
+      ) do
+    {:ok, %State{state | pending: [[dispatch, resolution] | rest]}}
+  end
+
   # the value has previously been received; here we should stay where
   # we are at and retry the transmission
   def resolve(
diff --git a/lib/tortoise/package/pubcomp.ex b/lib/tortoise/package/pubcomp.ex
index 94273471..45a497ea 100644
--- a/lib/tortoise/package/pubcomp.ex
+++ b/lib/tortoise/package/pubcomp.ex
@@ -3,15 +3,18 @@ defmodule Tortoise.Package.Pubcomp do
 
   @opcode 7
 
-  @allowed_properties [:reason_string, :user_property]
+  # @allowed_properties [:reason_string, :user_property]
 
   alias Tortoise.Package
 
+  @type reason :: :success | {:refused, refusal_reasons()}
+  @type refusal_reasons :: :packet_identifier_not_found
+
   @opaque t :: %__MODULE__{
             __META__: Package.Meta.t(),
             identifier: Tortoise.package_identifier(),
-            reason: :success,
-            properties: []
+            reason: reason(),
+            properties: [{:reason_string, String.t()}, {:user_property, String.t()}]
           }
   defstruct __META__: %Package.Meta{opcode: @opcode, flags: 0},
             identifier: nil,
diff --git a/lib/tortoise/package/pubrec.ex b/lib/tortoise/package/pubrec.ex
index 29fc23c0..b01329e9 100644
--- a/lib/tortoise/package/pubrec.ex
+++ b/lib/tortoise/package/pubrec.ex
@@ -22,7 +22,7 @@ defmodule Tortoise.Package.Pubrec do
             __META__: Package.Meta.t(),
             identifier: Tortoise.package_identifier(),
             reason: reason(),
-            properties: [{any(), any()}]
+            properties: [{:reason_string, String.t()}, {:user_property, String.t()}]
           }
   defstruct __META__: %Package.Meta{opcode: @opcode, flags: 0b0000},
             identifier: nil,
diff --git a/lib/tortoise/package/pubrel.ex b/lib/tortoise/package/pubrel.ex
index fdd1123b..6a5a639d 100644
--- a/lib/tortoise/package/pubrel.ex
+++ b/lib/tortoise/package/pubrel.ex
@@ -7,11 +7,14 @@ defmodule Tortoise.Package.Pubrel do
 
   alias Tortoise.Package
 
+  @type reason :: :success | {:refused, refusal_reasons()}
+  @type refusal_reasons :: :packet_identifier_not_found
+
   @opaque t :: %__MODULE__{
             __META__: Package.Meta.t(),
             identifier: Tortoise.package_identifier(),
-            reason: :success | {:refused, :packet_identifier_not_found},
-            properties: []
+            reason: reason(),
+            properties: [{:reason_string, String.t()}, {:user_property, String.t()}]
           }
   @enforce_keys [:identifier]
   defstruct __META__: %Package.Meta{opcode: @opcode, flags: 0b0010},
diff --git a/lib/tortoise/package/subscribe.ex b/lib/tortoise/package/subscribe.ex
index 7dd899d9..427f82f0 100644
--- a/lib/tortoise/package/subscribe.ex
+++ b/lib/tortoise/package/subscribe.ex
@@ -21,7 +21,10 @@ defmodule Tortoise.Package.Subscribe do
             __META__: Package.Meta.t(),
             identifier: Tortoise.package_identifier(),
             topics: topics(),
-            properties: []
+            properties: [
+              {:subscription_identifier, 0x1..0xFFFFFFF},
+              {:user_property, binary()}
+            ]
           }
   defstruct __META__: %Package.Meta{opcode: @opcode, flags: 0b0010},
             identifier: nil,
diff --git a/test/tortoise/connection/inflight_test.exs b/test/tortoise/connection/inflight_test.exs
index 63d99c84..6de6cea3 100644
--- a/test/tortoise/connection/inflight_test.exs
+++ b/test/tortoise/connection/inflight_test.exs
@@ -46,8 +46,11 @@ defmodule Tortoise.Connection.InflightTest do
     test "incoming publish QoS=1", %{client_id: client_id} = context do
       publish = %Package.Publish{identifier: 1, topic: "foo", qos: 1}
       :ok = Inflight.track(client_id, {:incoming, publish})
-      assert {:ok, puback} = :gen_tcp.recv(context.server, 0, 500)
-      assert %Package.Puback{identifier: 1} = Package.decode(puback)
+
+      puback = %Package.Puback{identifier: 1}
+      :ok = Inflight.update(client_id, {:dispatch, puback})
+      assert {:ok, data} = :gen_tcp.recv(context.server, 0, 500)
+      assert ^puback = Package.decode(data)
     end
 
     test "outgoing publish QoS=1", %{client_id: client_id} = context do
@@ -79,21 +82,29 @@ defmodule Tortoise.Connection.InflightTest do
     test "incoming publish QoS=2", %{client_id: client_id} = context do
       publish = %Package.Publish{identifier: 1, topic: "foo", qos: 2}
       :ok = Inflight.track(client_id, {:incoming, publish})
-      assert {:ok, pubrec} = :gen_tcp.recv(context.server, 0, 500)
-      assert %Package.Pubrec{identifier: 1} = Package.decode(pubrec)
+
+      pubrec = %Package.Pubrec{identifier: 1}
+      :ok = Inflight.update(client_id, {:dispatch, pubrec})
+
+      assert {:ok, data} = :gen_tcp.recv(context.server, 0, 500)
+      assert ^pubrec = Package.decode(data)
 
       # drop and reestablish the connection
       {:ok, context} = drop_connection(context)
       {:ok, context} = setup_connection(context)
 
       # now we should receive the same pubrec message
-      assert {:ok, ^pubrec} = :gen_tcp.recv(context.server, 0, 500)
+      assert {:ok, ^data} = :gen_tcp.recv(context.server, 0, 500)
 
       # simulate that we receive a pubrel from the server
       Inflight.update(client_id, {:received, %Package.Pubrel{identifier: 1}})
+      # dispatch a pubcomp; we need to feed this because of user
+      # properties that can be set on the pubcomp package by the user
+      pubcomp = %Package.Pubcomp{identifier: 1}
+      Inflight.update(client_id, {:dispatch, pubcomp})
 
-      assert {:ok, pubcomp} = :gen_tcp.recv(context.server, 0, 500)
-      assert %Package.Pubcomp{identifier: 1} = Package.decode(pubcomp)
+      assert {:ok, encoded_pubcomp} = :gen_tcp.recv(context.server, 0, 500)
+      assert ^pubcomp = Package.decode(encoded_pubcomp)
     end
 
     test "outgoing publish QoS=2", %{client_id: client_id} = context do
@@ -115,13 +126,16 @@ defmodule Tortoise.Connection.InflightTest do
       Inflight.update(client_id, {:received, %Package.Pubrec{identifier: 1}})
 
       # we should send the pubrel package
-      assert {:ok, pubrel} = :gen_tcp.recv(context.server, 0, 500)
-      assert %Package.Pubrel{identifier: 1} = Package.decode(pubrel)
+      pubrel = %Package.Pubrel{identifier: 1}
+      :ok = Inflight.update(client_id, {:dispatch, pubrel})
+
+      assert {:ok, pubrel_encoded} = :gen_tcp.recv(context.server, 0, 500)
+      assert ^pubrel = Package.decode(pubrel_encoded)
       # drop and reestablish the connection
       {:ok, context} = drop_connection(context)
       {:ok, context} = setup_connection(context)
       # re-transmit the pubrel
-      assert {:ok, ^pubrel} = :gen_tcp.recv(context.server, 0, 500)
+      assert {:ok, ^pubrel_encoded} = :gen_tcp.recv(context.server, 0, 500)
 
       # When we receive the pubcomp message we should respond the caller
       Inflight.update(client_id, {:received, %Package.Pubcomp{identifier: 1}})
diff --git a/test/tortoise_test.exs b/test/tortoise_test.exs
index 2df5bc48..5a965288 100644
--- a/test/tortoise_test.exs
+++ b/test/tortoise_test.exs
@@ -86,8 +86,11 @@ defmodule TortoiseTest do
                Package.decode(data)
 
       :ok = Inflight.update(client_id, {:received, %Package.Pubrec{identifier: id}})
+      # respond with a pubrel
+      pubrel = %Package.Pubrel{identifier: id}
+      :ok = Inflight.update(client_id, {:dispatch, pubrel})
       assert {:ok, data} = :gen_tcp.recv(context.server, 0, 500)
-      assert %Package.Pubrel{identifier: ^id} = Package.decode(data)
+      assert ^pubrel = Package.decode(data)
 
       :ok = Inflight.update(client_id, {:received, %Package.Pubcomp{identifier: id}})
       assert_receive :done

From 997c416a371b38b854a77f46ec0384dad0ffd00f Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sun, 4 Nov 2018 16:32:08 +0100
Subject: [PATCH 062/220] Get rid of unused variables

---
 lib/tortoise/connection/inflight.ex | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/tortoise/connection/inflight.ex b/lib/tortoise/connection/inflight.ex
index 39c58439..20b582b9 100644
--- a/lib/tortoise/connection/inflight.ex
+++ b/lib/tortoise/connection/inflight.ex
@@ -2,7 +2,6 @@ defmodule Tortoise.Connection.Inflight do
   @moduledoc false
 
   alias Tortoise.{Package, Connection}
-  alias Tortoise.Connection.Controller
   alias Tortoise.Connection.Inflight.Track
 
   use GenStateMachine
@@ -306,7 +305,7 @@ defmodule Tortoise.Connection.Inflight do
         :internal,
         {:onward_publish, %Package.Publish{qos: qos} = publish},
         _,
-        %State{client_id: client_id, parent: parent_pid} = data
+        %State{client_id: client_id, parent: parent_pid}
       )
       when qos in 1..2 do
     send(parent_pid, {{__MODULE__, client_id}, publish})

From e3acdf2f7d1f154da382fb47bef865e0d411f7d1 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 10 Nov 2018 12:08:43 +0100
Subject: [PATCH 063/220] Started working on adding suback to the user defined
 callbacks

---
 lib/tortoise/connection.ex     | 20 ++++++++++++++++++--
 lib/tortoise/handler.ex        | 18 ++++++++++++++++++
 lib/tortoise/package/suback.ex |  2 +-
 test/support/test_handler.ex   |  5 +++++
 test/tortoise/handler_test.exs | 18 ++++++++++++++++++
 5 files changed, 60 insertions(+), 3 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 75a506f8..0a0bc6ef 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -654,6 +654,7 @@ defmodule Tortoise.Connection do
         data
       ) do
     :ok = Inflight.update(data.client_id, {:received, suback})
+
     :keep_state_and_data
   end
 
@@ -665,14 +666,29 @@ defmodule Tortoise.Connection do
       ) do
     case Map.pop(pending, ref) do
       {{pid, msg_ref}, updated_pending} when is_pid(pid) and is_reference(msg_ref) ->
-        unless pid == self(), do: send(pid, {{Tortoise, client_id}, msg_ref, :ok})
         subscriptions = Enum.into(result[:ok] ++ result[:warn], data.subscriptions)
         updated_data = %State{data | subscriptions: subscriptions, pending_refs: updated_pending}
-        next_actions = [{:next_event, :internal, {:execute_handler, {:subscribe, result}}}]
+
+        next_actions = [
+          {:next_event, :internal, {:reply, {pid, msg_ref}, :ok}},
+          {:next_event, :internal, {:execute_handler, {:subscribe, result}}}
+        ]
+
         {:keep_state, updated_data, next_actions}
     end
   end
 
+  def handle_event(:internal, {:reply, from, result}, _current_state, %State{client_id: client_id}) do
+    case from do
+      {pid, _} when pid == self() ->
+        :keep_state_and_data
+
+      {pid, msg_ref} ->
+        send(pid, {{Tortoise, client_id}, msg_ref, result})
+        :keep_state_and_data
+    end
+  end
+
   def handle_event(:cast, {:unsubscribe, caller, unsubscribe, opts}, :connected, data) do
     client_id = data.client_id
     _timeout = Keyword.get(opts, :timeout, 5000)
diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index 4a3a07d6..020219bf 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -169,6 +169,11 @@ defmodule Tortoise.Handler do
         {:ok, state}
       end
 
+      @impl true
+      def handle_suback(_suback, state) do
+        {:ok, state}
+      end
+
       defoverridable Tortoise.Handler
     end
   end
@@ -261,6 +266,10 @@ defmodule Tortoise.Handler do
                  topic_filter: Tortoise.topic_filter(),
                  new_state: term
 
+  @callback handle_suback(suback, state :: term) :: {:ok, new_state}
+            when suback: Package.Suback.t(),
+                 new_state: term()
+
   @doc """
   Invoked when messages are published to subscribed topics.
 
@@ -389,6 +398,15 @@ defmodule Tortoise.Handler do
     end)
   end
 
+  @doc false
+  # @spec execute_handle_suback(t, Package.Suback.t()) ::
+  #         {:ok, t} | {:error, {:invalid_next_action, term()}}
+  def execute_handle_suback(handler, %Package.Suback{} = suback) do
+    handler.module
+    |> apply(:handle_suback, [suback, handler.state])
+    |> handle_result(handler)
+  end
+
   @doc false
   @spec execute_unsubscribe(t, result) :: {:ok, t}
         when result: [Tortoise.topic_filter()]
diff --git a/lib/tortoise/package/suback.ex b/lib/tortoise/package/suback.ex
index 82cffbc3..d60c59fb 100644
--- a/lib/tortoise/package/suback.ex
+++ b/lib/tortoise/package/suback.ex
@@ -25,7 +25,7 @@ defmodule Tortoise.Package.Suback do
             __META__: Package.Meta.t(),
             identifier: Tortoise.package_identifier(),
             acks: [ack_result],
-            properties: [{any(), any()}]
+            properties: [{:reason_string, String.t()}, {:user_property, String.t()}]
           }
   @enforce_keys [:identifier]
   defstruct __META__: %Package.Meta{opcode: @opcode, flags: 0},
diff --git a/test/support/test_handler.ex b/test/support/test_handler.ex
index 947d95dc..dffb9cf1 100644
--- a/test/support/test_handler.ex
+++ b/test/support/test_handler.ex
@@ -48,4 +48,9 @@ defmodule TestHandler do
     send(state[:parent], {{__MODULE__, :subscription}, data})
     {:ok, state}
   end
+
+  def handle_suback(suback, state) do
+    send(state[:parent], {{__MODULE__, :handle_suback}, suback})
+    {:ok, state}
+  end
 end
diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index 7f79e6ca..cf0bbfe0 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -29,6 +29,11 @@ defmodule Tortoise.HandlerTest do
       {:ok, state}
     end
 
+    def handle_suback(suback, state) do
+      send(state[:pid], {:suback, suback})
+      {:ok, state}
+    end
+
     # with next actions
     def handle_publish(topic, payload, %{next_actions: next_actions} = state) do
       send(state[:pid], {:publish, topic, payload})
@@ -190,6 +195,19 @@ defmodule Tortoise.HandlerTest do
     end
   end
 
+  describe "execute handle_suback/2" do
+    test "return ok", context do
+      handler = set_state(context.handler, pid: self())
+      suback = %Package.Suback{identifier: 1}
+
+      assert {:ok, %Handler{} = state} =
+               handler
+               |> Handler.execute_handle_suback(suback)
+
+      assert_receive {:suback, ^suback}
+    end
+  end
+
   describe "execute unsubscribe/2" do
     test "return ok", context do
       unsubscribe = %Package.Unsubscribe{identifier: 1, topics: ["foo/bar", "baz/quux"]}

From dd1a7a5fae175f5559c6346d8bc85f81fbbe20da Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 17 Nov 2018 15:37:42 +0100
Subject: [PATCH 064/220] Refactor the reply to pid handler code

---
 lib/tortoise/connection.ex | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 0a0bc6ef..7b0d4a21 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -678,15 +678,22 @@ defmodule Tortoise.Connection do
     end
   end
 
-  def handle_event(:internal, {:reply, from, result}, _current_state, %State{client_id: client_id}) do
-    case from do
-      {pid, _} when pid == self() ->
-        :keep_state_and_data
-
-      {pid, msg_ref} ->
-        send(pid, {{Tortoise, client_id}, msg_ref, result})
-        :keep_state_and_data
-    end
+  # Pass on the result of an operation if we have a calling pid. This
+  # can happen if a process order the connection to subscribe to a
+  # topic, or unsubscribe, etc.
+  def handle_event(
+        :internal,
+        {:reply, {pid, msg_ref}, result},
+        _current_state,
+        %State{client_id: client_id}
+      )
+      when pid != self() do
+    send(pid, {{Tortoise, client_id}, msg_ref, result})
+    :keep_state_and_data
+  end
+
+  def handle_event(:internal, {:reply, _from, _}, _current_state, %State{}) do
+    :keep_state_and_data
   end
 
   def handle_event(:cast, {:unsubscribe, caller, unsubscribe, opts}, :connected, data) do

From a52df8e833a3a89bd92954c33b105beca036c5af Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 17 Nov 2018 15:43:09 +0100
Subject: [PATCH 065/220] Use the reply-event to handle reply from unsubscribe
 operations

---
 lib/tortoise/connection.ex | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 7b0d4a21..5058e7e9 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -726,10 +726,15 @@ defmodule Tortoise.Connection do
     case Map.pop(pending, ref) do
       {{pid, msg_ref}, updated_pending} when is_pid(pid) and is_reference(msg_ref) ->
         topics = Keyword.drop(data.subscriptions.topics, unsubbed)
+
+        next_actions = [
+          {:next_event, :internal, {:reply, {pid, msg_ref}, :ok}},
+          {:next_event, :internal, {:execute_handler, {:unsubscribe, unsubbed}}}
+        ]
+
         subscriptions = %Package.Subscribe{data.subscriptions | topics: topics}
-        unless pid == self(), do: send(pid, {{Tortoise, client_id}, msg_ref, :ok})
         updated_data = %State{data | pending_refs: updated_pending, subscriptions: subscriptions}
-        next_actions = [{:next_event, :internal, {:execute_handler, {:unsubscribe, unsubbed}}}]
+
         {:keep_state, updated_data, next_actions}
     end
   end

From 783a9159324a2dfd27adc84545c10f14f9cec748 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Wed, 9 Jan 2019 20:24:05 +0000
Subject: [PATCH 066/220] Revamped the subscription/unsubscribe logic

This is code I wrote a good long while ago. It seems good, so better
commit it and fix it if anything is wrong with it. Let's get back on
track with the MQTT 5 support!
---
 lib/tortoise/connection.ex                | 143 ++++++++++--------
 lib/tortoise/connection/inflight/track.ex |  29 +---
 lib/tortoise/handler.ex                   | 172 ++++++++++------------
 lib/tortoise/package/subscribe.ex         |  14 +-
 test/support/test_handler.ex              |   9 +-
 test/tortoise/connection_test.exs         |  93 +++++-------
 test/tortoise/handler_test.exs            |  61 +++-----
 7 files changed, 244 insertions(+), 277 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 5058e7e9..d0faa993 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -13,7 +13,10 @@ defmodule Tortoise.Connection do
             connect: nil,
             server: nil,
             backoff: nil,
+            # todo, replace subscriptions with a list of topics stored in a
+            # Tortoise.Config
             subscriptions: nil,
+            active_subscriptions: %{},
             opts: nil,
             pending_refs: %{},
             connection: nil,
@@ -161,7 +164,7 @@ defmodule Tortoise.Connection do
                | {:identifier, Tortoise.package_identifier()}
   def subscribe(client_id, topics, opts \\ [])
 
-  def subscribe(client_id, [{_, n} | _] = topics, opts) when is_number(n) do
+  def subscribe(client_id, [{_, topic_opts} | _] = topics, opts) when is_list(topic_opts) do
     caller = {_, ref} = {self(), make_ref()}
     {identifier, opts} = Keyword.pop_first(opts, :identifier, nil)
     subscribe = Enum.into(topics, %Package.Subscribe{identifier: identifier})
@@ -169,7 +172,7 @@ defmodule Tortoise.Connection do
     {:ok, ref}
   end
 
-  def subscribe(client_id, {_, n} = topic, opts) when is_number(n) do
+  def subscribe(client_id, {_, topic_opts} = topic, opts) when is_list(topic_opts) do
     subscribe(client_id, [topic], opts)
   end
 
@@ -179,7 +182,7 @@ defmodule Tortoise.Connection do
         throw("Please specify a quality of service for the subscription")
 
       {qos, opts} when qos in 0..2 ->
-        subscribe(client_id, [{topic, qos}], opts)
+        subscribe(client_id, [{topic, [qos: qos]}], opts)
     end
   end
 
@@ -204,7 +207,7 @@ defmodule Tortoise.Connection do
                | {:identifier, Tortoise.package_identifier()}
   def subscribe_sync(client_id, topics, opts \\ [])
 
-  def subscribe_sync(client_id, [{_, n} | _] = topics, opts) when is_number(n) do
+  def subscribe_sync(client_id, [{_, topic_opts} | _] = topics, opts) when is_list(topic_opts) do
     timeout = Keyword.get(opts, :timeout, 5000)
     {:ok, ref} = subscribe(client_id, topics, opts)
 
@@ -216,7 +219,7 @@ defmodule Tortoise.Connection do
     end
   end
 
-  def subscribe_sync(client_id, {_, n} = topic, opts) when is_number(n) do
+  def subscribe_sync(client_id, {_, topic_opts} = topic, opts) when is_list(topic_opts) do
     subscribe_sync(client_id, [topic], opts)
   end
 
@@ -226,7 +229,7 @@ defmodule Tortoise.Connection do
         throw("Please specify a quality of service for the subscription")
 
       {qos, opts} ->
-        subscribe_sync(client_id, [{topic, qos}], opts)
+        subscribe_sync(client_id, [{topic, qos: qos}], opts)
     end
   end
 
@@ -443,12 +446,12 @@ defmodule Tortoise.Connection do
         {:next_state, :connected, data, next_actions}
 
       %Package.Connack{session_present: false} ->
-        caller = {self(), make_ref()}
+        # caller = {self(), make_ref()}
 
         next_actions = [
           {:state_timeout, keep_alive * 1000, :keep_alive},
-          {:next_event, :internal, {:execute_handler, {:connection, :up}}},
-          {:next_event, :cast, {:subscribe, caller, data.subscriptions, []}}
+          {:next_event, :internal, {:execute_handler, {:connection, :up}}}
+          # {:next_event, :cast, {:subscribe, caller, data.subscriptions, []}}
         ]
 
         {:next_state, :connected, data, next_actions}
@@ -660,21 +663,44 @@ defmodule Tortoise.Connection do
 
   def handle_event(
         :info,
-        {{Tortoise, client_id}, {Package.Subscribe, ref}, result},
+        {{Tortoise, client_id}, {Package.Subscribe, ref}, {subscribe, suback}},
         _current_state,
-        %State{client_id: client_id, pending_refs: %{} = pending} = data
+        %State{client_id: client_id, handler: handler, pending_refs: %{} = pending} = data
       ) do
-    case Map.pop(pending, ref) do
-      {{pid, msg_ref}, updated_pending} when is_pid(pid) and is_reference(msg_ref) ->
-        subscriptions = Enum.into(result[:ok] ++ result[:warn], data.subscriptions)
-        updated_data = %State{data | subscriptions: subscriptions, pending_refs: updated_pending}
+    {{pid, msg_ref}, updated_pending} = Map.pop(pending, ref)
+    data = %State{data | pending_refs: updated_pending}
+
+    updated_active_subscriptions =
+      subscribe.topics
+      |> Enum.zip(suback.acks)
+      |> Enum.reduce(data.active_subscriptions, fn
+        {{topic, opts}, {:ok, accepted_qos}}, acc ->
+          Map.put(acc, topic, Keyword.replace!(opts, :qos, accepted_qos))
+
+        {_, {:error, _}}, acc ->
+          acc
+      end)
+
+    case Handler.execute_handle_suback(handler, subscribe, suback) do
+      {:ok, %Handler{} = updated_handler, next_actions} ->
+        data = %State{
+          data
+          | handler: updated_handler,
+            active_subscriptions: updated_active_subscriptions
+        }
 
         next_actions = [
-          {:next_event, :internal, {:reply, {pid, msg_ref}, :ok}},
-          {:next_event, :internal, {:execute_handler, {:subscribe, result}}}
+          {:next_event, :internal, {:reply, {pid, msg_ref}, :ok}}
+          | for action <- next_actions do
+              {:next_event, :internal, action}
+            end
         ]
 
-        {:keep_state, updated_data, next_actions}
+        {:keep_state, data, next_actions}
+
+      {:error, reason} ->
+        # todo
+        {:stop, reason, data}
     end
   end
 
@@ -719,27 +745,43 @@ defmodule Tortoise.Connection do
   # todo; handle the unsuback error cases !
   def handle_event(
         :info,
-        {{Tortoise, client_id}, {Package.Unsubscribe, ref}, unsubbed},
+        {{Tortoise, client_id}, {Package.Unsubscribe, ref}, {unsubscribe, unsuback}},
         _current_state,
-        %State{client_id: client_id, pending_refs: %{} = pending} = data
+        %State{client_id: client_id, handler: handler, pending_refs: %{} = pending} = data
       ) do
-    case Map.pop(pending, ref) do
-      {{pid, msg_ref}, updated_pending} when is_pid(pid) and is_reference(msg_ref) ->
-        topics = Keyword.drop(data.subscriptions.topics, unsubbed)
+    # todo, call a handle_unsuback callback!
+    {{pid, msg_ref}, updated_pending} = Map.pop(pending, ref)
+    data = %State{data | pending_refs: updated_pending}
+
+    # todo, if the results in unsuback contain an error, such as
+    # `{:error, :no_subscription_existed}` then we would be out of
+    # sync! What should we do here?
+    active_subscriptions = Map.drop(data.active_subscriptions, unsubscribe.topics)
+
+    case Handler.execute_handle_unsuback(handler, unsubscribe, unsuback) do
+      {:ok, %Handler{} = updated_handler, next_actions} ->
+        data = %State{
+          data
+          | handler: updated_handler,
+            active_subscriptions: active_subscriptions
+        }
 
         next_actions = [
-          {:next_event, :internal, {:reply, {pid, msg_ref}, :ok}},
-          {:next_event, :internal, {:execute_handler, {:unsubscribe, unsubbed}}}
+          {:next_event, :internal, {:reply, {pid, msg_ref}, :ok}}
+          | for action <- next_actions do
+              {:next_event, :internal, action}
+            end
         ]
 
-        subscriptions = %Package.Subscribe{data.subscriptions | topics: topics}
-        updated_data = %State{data | pending_refs: updated_pending, subscriptions: subscriptions}
+        {:keep_state, data, next_actions}
 
-        {:keep_state, updated_data, next_actions}
+      {:error, reason} ->
+        # todo
+        {:stop, reason, data}
     end
   end
 
-  def handle_event({:call, from}, :subscriptions, _, %State{subscriptions: subscriptions}) do
+  def handle_event({:call, from}, :subscriptions, _, %State{active_subscriptions: subscriptions}) do
     next_actions = [{:reply, from, subscriptions}]
     {:keep_state_and_data, next_actions}
   end
@@ -760,35 +802,20 @@ defmodule Tortoise.Connection do
     end
   end
 
-  def handle_event(
-        :internal,
-        {:execute_handler, {:subscribe, results}},
-        _,
-        %State{handler: handler} = data
-      ) do
-    case Handler.execute_subscribe(handler, results) do
-      {:ok, %Handler{} = updated_handler} ->
-        updated_data = %State{data | handler: updated_handler}
-        {:keep_state, updated_data}
-
-        # handle stop
-    end
-  end
-
-  def handle_event(
-        :internal,
-        {:execute_handler, {:unsubscribe, result}},
-        _current_state,
-        %State{handler: handler} = data
-      ) do
-    case Handler.execute_unsubscribe(handler, result) do
-      {:ok, %Handler{} = updated_handler} ->
-        updated_data = %State{data | handler: updated_handler}
-        {:keep_state, updated_data}
-
-        # handle stop
-    end
-  end
+  # def handle_event(
+  #       :internal,
+  #       {:execute_handler, {:unsubscribe, result}},
+  #       _current_state,
+  #       %State{handler: handler} = data
+  #     ) do
+  #   case Handler.execute_unsubscribe(handler, result) do
+  #     {:ok, %Handler{} = updated_handler} ->
+  #       updated_data = %State{data | handler: updated_handler}
+  #       {:keep_state, updated_data}
+
+  #       # handle stop
+  #   end
+  # end
 
   # connection logic ===================================================
   def handle_event(:internal, :connect, :connecting, %State{} = data) do
diff --git a/lib/tortoise/connection/inflight/track.ex b/lib/tortoise/connection/inflight/track.ex
index a63f9920..cae64c79 100644
--- a/lib/tortoise/connection/inflight/track.ex
+++ b/lib/tortoise/connection/inflight/track.ex
@@ -228,37 +228,20 @@ defmodule Tortoise.Connection.Inflight.Track do
   def result(%State{
         type: Package.Unsubscribe,
         status: [
-          {:received, %Package.Unsuback{results: _results}},
-          {:dispatch, %Package.Unsubscribe{topics: topics}} | _other
+          {:received, %Package.Unsuback{} = unsuback},
+          {:dispatch, %Package.Unsubscribe{} = unsubscribe} | _other
         ]
       }) do
-    # todo, merge the unsuback results with the topic list
-    {:ok, topics}
+    {:ok, {unsubscribe, unsuback}}
   end
 
   def result(%State{
         type: Package.Subscribe,
         status: [
-          {:received, %Package.Suback{acks: acks}},
-          {:dispatch, %Package.Subscribe{topics: topics}} | _other
+          {:received, %Package.Suback{} = suback},
+          {:dispatch, %Package.Subscribe{} = subscribe} | _other
         ]
       }) do
-    result =
-      List.zip([topics, acks])
-      |> Enum.reduce(%{error: [], warn: [], ok: []}, fn
-        {{topic, opts}, {:ok, actual}}, %{ok: oks, warn: warns} = acc ->
-          case Keyword.get(opts, :qos) do
-            ^actual ->
-              %{acc | ok: oks ++ [{topic, actual}]}
-
-            requested ->
-              %{acc | warn: warns ++ [{topic, [requested: requested, accepted: actual]}]}
-          end
-
-        {{topic, opts}, {:error, reason}}, %{error: errors} = acc ->
-          %{acc | error: errors ++ [{reason, {topic, opts}}]}
-      end)
-
-    {:ok, result}
+    {:ok, {subscribe, suback}}
   end
 end
diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index 020219bf..7f14a357 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -139,10 +139,10 @@ defmodule Tortoise.Handler do
         {:ok, state}
       end
 
-      @impl true
-      def subscription(_status, _topic_filter, state) do
-        {:ok, state}
-      end
+      # @impl true
+      # def subscription(_status, _topic_filter, state) do
+      #   {:ok, state}
+      # end
 
       @impl true
       def handle_publish(_topic, _payload, state) do
@@ -170,7 +170,7 @@ defmodule Tortoise.Handler do
       end
 
       @impl true
-      def handle_suback(_suback, state) do
+      def handle_suback(_subscribe, _suback, state) do
         {:ok, state}
       end
 
@@ -225,49 +225,55 @@ defmodule Tortoise.Handler do
             when status: :up | :down,
                  new_state: term()
 
-  @doc """
-  Invoked when the subscription of a topic filter changes status.
-
-  The `status` of a subscription can be one of:
-
-    - `:up`, triggered when the subscription has been accepted by the
-      MQTT broker with the requested quality of service
-
-    - `{:warn, [requested: req_qos, accepted: qos]}`, triggered when
-       the subscription is accepted by the MQTT broker, but with a
-       different quality of service `qos` than the one requested
-       `req_qos`
-
-    - `{:error, reason}`, triggered when the subscription is rejected
-      with the reason `reason` such as `:access_denied`
-
-    - `:down`, triggered when the subscription of the given topic
-      filter has been successfully acknowledged as unsubscribed by the
-      MQTT broker
-
-  The `topic_filter` is the topic filter in question, and the `state`
-  is the internal state being passed through transitions.
-
-  Returning `{:ok, new_state}` will set the state for later
-  invocations.
+  # @doc """
+  # Invoked when the subscription of a topic filter changes status.
+
+  # The `status` of a subscription can be one of:
+
+  #   - `:up`, triggered when the subscription has been accepted by the
+  #     MQTT broker with the requested quality of service
+
+  #   - `{:warn, [requested: req_qos, accepted: qos]}`, triggered when
+  #      the subscription is accepted by the MQTT broker, but with a
+  #      different quality of service `qos` than the one requested
+  #      `req_qos`
+
+  #   - `{:error, reason}`, triggered when the subscription is rejected
+  #     with the reason `reason` such as `:access_denied`
+
+  #   - `:down`, triggered when the subscription of the given topic
+  #     filter has been successfully acknowledged as unsubscribed by the
+  #     MQTT broker
+
+  # The `topic_filter` is the topic filter in question, and the `state`
+  # is the internal state being passed through transitions.
+
+  # Returning `{:ok, new_state}` will set the state for later
+  # invocations.
+
+  # Returning `{:ok, new_state, next_actions}`, where `next_actions` is
+  # a list of next actions such as `{:unsubscribe, "foo/bar"}` will
+  # result in the state being returned and the next actions performed.
+  # """
+  # @callback subscription(status, topic_filter, state :: term) ::
+  #             {:ok, new_state}
+  #             | {:ok, new_state, [next_action()]}
+  #           when status:
+  #                  :up
+  #                  | :down
+  #                  | {:warn, [requested: Tortoise.qos(), accepted: Tortoise.qos()]}
+  #                  | {:error, term()},
+  #                topic_filter: Tortoise.topic_filter(),
+  #                new_state: term
+
+  @callback handle_suback(subscribe, suback, state :: term) :: {:ok, new_state}
+            when subscribe: Package.Subscribe.t(),
+                 suback: Package.Suback.t(),
+                 new_state: term()
 
-  Returning `{:ok, new_state, next_actions}`, where `next_actions` is
-  a list of next actions such as `{:unsubscribe, "foo/bar"}` will
-  result in the state being returned and the next actions performed.
-  """
-  @callback subscription(status, topic_filter, state :: term) ::
-              {:ok, new_state}
-              | {:ok, new_state, [next_action()]}
-            when status:
-                   :up
-                   | :down
-                   | {:warn, [requested: Tortoise.qos(), accepted: Tortoise.qos()]}
-                   | {:error, term()},
-                 topic_filter: Tortoise.topic_filter(),
-                 new_state: term
-
-  @callback handle_suback(suback, state :: term) :: {:ok, new_state}
-            when suback: Package.Suback.t(),
+  @callback handle_unsuback(unsubscribe, unsuback, state :: term) :: {:ok, new_state}
+            when unsubscribe: Package.Unsubscribe.t(),
+                 unsuback: Package.Unsuback.t(),
                  new_state: term()
 
   @doc """
@@ -384,27 +390,37 @@ defmodule Tortoise.Handler do
   end
 
   @doc false
-  @spec execute_subscribe(t, [term()]) :: {:ok, t}
-  def execute_subscribe(handler, result) do
-    result
-    |> flatten_subacks()
-    |> Enum.reduce({:ok, handler}, fn {op, topic_filter}, {:ok, handler} ->
-      handler.module
-      |> apply(:subscription, [op, topic_filter, handler.state])
-      |> handle_result(handler)
+  @spec execute_handle_suback(t, Package.Subscribe.t(), Package.Suback.t()) :: {:ok, t}
+  def execute_handle_suback(handler, subscribe, suback) do
+    handler.module
+    |> apply(:handle_suback, [subscribe, suback, handler.state])
+    |> handle_suback_result(handler)
+  end
 
-      # _, {:stop, acc} ->
-      #   {:stop, acc}
-    end)
+  defp handle_suback_result({:ok, updated_state}, handler) do
+    {:ok, %__MODULE__{handler | state: updated_state}, []}
+  end
+
+  defp handle_suback_result({:ok, updated_state, next_actions}, handler)
+       when is_list(next_actions) do
+    {:ok, %__MODULE__{handler | state: updated_state}, next_actions}
   end
 
   @doc false
-  # @spec execute_handle_suback(t, Package.Suback.t()) ::
-  #         {:ok, t} | {:error, {:invalid_next_action, term()}}
-  def execute_handle_suback(handler, %Package.Suback{} = suback) do
+  @spec execute_handle_unsuback(t, Package.Unsubscribe.t(), Package.Unsuback.t()) :: {:ok, t}
+  def execute_handle_unsuback(handler, unsubscribe, unsuback) do
     handler.module
-    |> apply(:handle_suback, [suback, handler.state])
-    |> handle_result(handler)
+    |> apply(:handle_unsuback, [unsubscribe, unsuback, handler.state])
+    |> handle_unsuback_result(handler)
+  end
+
+  defp handle_unsuback_result({:ok, updated_state}, handler) do
+    {:ok, %__MODULE__{handler | state: updated_state}, []}
+  end
+
+  defp handle_unsuback_result({:ok, updated_state, next_actions}, handler)
+       when is_list(next_actions) do
+    {:ok, %__MODULE__{handler | state: updated_state}, next_actions}
   end
 
   @doc false
@@ -468,36 +484,6 @@ defmodule Tortoise.Handler do
     |> handle_result(handler)
   end
 
-  # Subacks will come in a map with three keys in the form of tuples
-  # where the fist element is one of `:ok`, `:warn`, or `:error`. This
-  # is done to make it easy to pattern match in other parts of the
-  # system, and error out early if the result set contain errors. In
-  # this part of the system it is more convenient to transform the
-  # data to a flat list containing tuples of `{operation, data}` so we
-  # can reduce the handler state to collect the possible next actions,
-  # and pass through if there is an :error or :disconnect return.
-  defp flatten_subacks(subacks) do
-    Enum.reduce(subacks, [], fn
-      {_, []}, acc ->
-        acc
-
-      {:ok, entries}, acc ->
-        for {topic_filter, _qos} <- entries do
-          {:up, topic_filter}
-        end ++ acc
-
-      {:warn, entries}, acc ->
-        for {topic_filter, warning} <- entries do
-          {{:warn, warning}, topic_filter}
-        end ++ acc
-
-      {:error, entries}, acc ->
-        for {reason, {topic_filter, _qos}} <- entries do
-          {{:error, reason}, topic_filter}
-        end ++ acc
-    end)
-  end
-
   # handle the user defined return from the callback
   defp handle_result({:ok, updated_state}, handler) do
     {:ok, %__MODULE__{handler | state: updated_state}}
diff --git a/lib/tortoise/package/subscribe.ex b/lib/tortoise/package/subscribe.ex
index 427f82f0..11bc7fd8 100644
--- a/lib/tortoise/package/subscribe.ex
+++ b/lib/tortoise/package/subscribe.ex
@@ -147,22 +147,20 @@ defmodule Tortoise.Package.Subscribe do
   end
 
   defimpl Collectable do
-    def into(%Package.Subscribe{topics: topics} = source) do
-      {Enum.into(topics, %{}),
+    def into(%Package.Subscribe{topics: current_topics} = source) do
+      {current_topics,
        fn
          acc, {:cont, {<<topic::binary>>, opts}} when is_list(opts) ->
-           Map.put(acc, topic, opts)
+           List.keystore(acc, topic, 0, {topic, opts})
 
          acc, {:cont, {<<topic::binary>>, qos}} when qos in 0..2 ->
-           # if the subscription already exist in the data structure
-           # we will just overwrite the existing one and the options
-           Map.put(acc, topic, qos: qos)
+           List.keystore(acc, topic, 0, {topic, qos: qos})
 
          acc, {:cont, <<topic::binary>>} ->
-           Map.put(acc, topic, qos: 0)
+           List.keystore(acc, topic, 0, {topic, qos: 0})
 
          acc, :done ->
-           %{source | topics: Map.to_list(acc)}
+           %{source | topics: acc}
 
          _, :halt ->
            :ok
diff --git a/test/support/test_handler.ex b/test/support/test_handler.ex
index dffb9cf1..c39efa1d 100644
--- a/test/support/test_handler.ex
+++ b/test/support/test_handler.ex
@@ -49,8 +49,13 @@ defmodule TestHandler do
     {:ok, state}
   end
 
-  def handle_suback(suback, state) do
-    send(state[:parent], {{__MODULE__, :handle_suback}, suback})
+  def handle_suback(subscribe, suback, state) do
+    send(state[:parent], {{__MODULE__, :handle_suback}, {subscribe, suback}})
+    {:ok, state}
+  end
+
+  def handle_unsuback(unsubscribe, unsuback, state) do
+    send(state[:parent], {{__MODULE__, :handle_unsuback}, {unsubscribe, unsuback}})
     {:ok, state}
   end
 end
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index 027f4e21..2aa6f11f 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -241,13 +241,11 @@ defmodule Tortoise.ConnectionTest do
   end
 
   describe "subscriptions" do
-    setup [:setup_scripted_mqtt_server]
+    setup [:setup_scripted_mqtt_server, :setup_connection_and_perform_handshake]
 
     test "successful subscription", context do
       client_id = context.client_id
 
-      connect = %Package.Connect{client_id: client_id, clean_start: true}
-
       default_subscription_opts = [
         no_local: false,
         retain_as_published: false,
@@ -260,66 +258,61 @@ defmodule Tortoise.ConnectionTest do
           %Package.Subscribe{identifier: 1}
         )
 
+      suback_foo = %Package.Suback{identifier: 1, acks: [{:ok, 0}]}
+
       subscription_bar =
         Enum.into(
           [{"bar", [{:qos, 1} | default_subscription_opts]}],
           %Package.Subscribe{identifier: 2}
         )
 
+      suback_bar = %Package.Suback{identifier: 2, acks: [{:ok, 1}]}
+
       subscription_baz =
         Enum.into(
           [{"baz", [{:qos, 2} | default_subscription_opts]}],
           %Package.Subscribe{identifier: 3}
         )
 
+      suback_baz = %Package.Suback{identifier: 3, acks: [{:ok, 2}]}
+
       script = [
-        {:receive, connect},
-        {:send, %Package.Connack{reason: :success, session_present: false}},
         # subscribe to foo with qos 0
         {:receive, subscription_foo},
-        {:send, %Package.Suback{identifier: 1, acks: [{:ok, 0}]}},
-        # subscribe to bar with qos 0
+        {:send, suback_foo},
+        # subscribe to bar with qos 1
         {:receive, subscription_bar},
-        {:send, %Package.Suback{identifier: 2, acks: [{:ok, 1}]}},
+        {:send, suback_bar},
+        # subscribe to baz with qos 2
         {:receive, subscription_baz},
-        {:send, %Package.Suback{identifier: 3, acks: [{:ok, 2}]}}
-      ]
-
-      {:ok, {ip, port}} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
-
-      opts = [
-        client_id: client_id,
-        server: {Tortoise.Transport.Tcp, [host: ip, port: port]},
-        handler: {TestHandler, [parent: self()]}
+        {:send, suback_baz}
       ]
 
-      # connection
-      assert {:ok, _pid} = Connection.start_link(opts)
-      assert_receive {ScriptedMqttServer, {:received, ^connect}}
+      {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
 
       # subscribe to a foo
-      :ok = Tortoise.Connection.subscribe_sync(client_id, {"foo", 0}, identifier: 1)
+      :ok = Tortoise.Connection.subscribe_sync(client_id, {"foo", qos: 0}, identifier: 1)
       assert_receive {ScriptedMqttServer, {:received, ^subscription_foo}}
-      assert Enum.member?(Tortoise.Connection.subscriptions(client_id), {"foo", 0})
-      assert_receive {{TestHandler, :subscription}, %{status: :up, topic_filter: "foo"}}
+      assert Map.has_key?(Tortoise.Connection.subscriptions(client_id), "foo")
+      assert_receive {{TestHandler, :handle_suback}, {%Package.Subscribe{}, ^suback_foo}}
 
       # subscribe to a bar
-      assert {:ok, ref} = Tortoise.Connection.subscribe(client_id, {"bar", 1}, identifier: 2)
+      assert {:ok, ref} = Tortoise.Connection.subscribe(client_id, {"bar", qos: 1}, identifier: 2)
       assert_receive {{Tortoise, ^client_id}, ^ref, :ok}
       assert_receive {ScriptedMqttServer, {:received, ^subscription_bar}}
-      assert_receive {{TestHandler, :subscription}, %{status: :up, topic_filter: "bar"}}
+      assert_receive {{TestHandler, :handle_suback}, {%Package.Subscribe{}, ^suback_bar}}
 
       # subscribe to a baz
       assert {:ok, ref} = Tortoise.Connection.subscribe(client_id, "baz", qos: 2, identifier: 3)
       assert_receive {{Tortoise, ^client_id}, ^ref, :ok}
       assert_receive {ScriptedMqttServer, {:received, ^subscription_baz}}
-      assert_receive {{TestHandler, :subscription}, %{status: :up, topic_filter: "baz"}}
+      assert_receive {{TestHandler, :handle_suback}, {%Package.Subscribe{}, ^suback_baz}}
 
       # foo, bar, and baz should now be in the subscription list
       subscriptions = Tortoise.Connection.subscriptions(client_id)
-      assert Enum.member?(subscriptions, {"foo", 0})
-      assert Enum.member?(subscriptions, {"bar", 1})
-      assert Enum.member?(subscriptions, {"baz", 2})
+      assert Map.has_key?(subscriptions, "foo")
+      assert Map.has_key?(subscriptions, "bar")
+      assert Map.has_key?(subscriptions, "baz")
 
       # done
       assert_receive {ScriptedMqttServer, :completed}
@@ -328,13 +321,12 @@ defmodule Tortoise.ConnectionTest do
     test "successful unsubscribe", context do
       client_id = context.client_id
 
-      connect = %Package.Connect{client_id: client_id, clean_start: true}
       unsubscribe_foo = %Package.Unsubscribe{identifier: 2, topics: ["foo"]}
+      unsuback_foo = %Package.Unsuback{results: [:success], identifier: 2}
       unsubscribe_bar = %Package.Unsubscribe{identifier: 3, topics: ["bar"]}
+      unsuback_bar = %Package.Unsuback{results: [:success], identifier: 3}
 
       script = [
-        {:receive, connect},
-        {:send, %Package.Connack{reason: :success, session_present: false}},
         {:receive,
          %Package.Subscribe{
            topics: [
@@ -346,13 +338,13 @@ defmodule Tortoise.ConnectionTest do
         {:send, %Package.Suback{acks: [ok: 0, ok: 2], identifier: 1}},
         # unsubscribe foo
         {:receive, unsubscribe_foo},
-        {:send, %Package.Unsuback{results: [:success], identifier: 2}},
+        {:send, unsuback_foo},
         # unsubscribe bar
         {:receive, unsubscribe_bar},
-        {:send, %Package.Unsuback{results: [:success], identifier: 3}}
+        {:send, unsuback_bar}
       ]
 
-      {:ok, {ip, port}} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
+      {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
 
       subscribe = %Package.Subscribe{
         topics: [
@@ -362,37 +354,32 @@ defmodule Tortoise.ConnectionTest do
         identifier: 1
       }
 
-      opts = [
-        client_id: client_id,
-        server: {Tortoise.Transport.Tcp, [host: ip, port: port]},
-        handler: {TestHandler, [parent: self()]},
-        subscriptions: subscribe
-      ]
-
-      assert {:ok, _pid} = Connection.start_link(opts)
-      assert_receive {ScriptedMqttServer, {:received, ^connect}}
+      Tortoise.Connection.subscribe(client_id, subscribe.topics, identifier: 1)
 
       assert_receive {ScriptedMqttServer, {:received, ^subscribe}}
 
-      assert_receive {{TestHandler, :subscription}, %{status: :up, topic_filter: "foo"}}
-      assert_receive {{TestHandler, :subscription}, %{status: :up, topic_filter: "bar"}}
+      assert_receive {{TestHandler, :handle_suback}, {_, %Package.Suback{identifier: 1}}}
 
       # now let us try to unsubscribe from foo
       :ok = Tortoise.Connection.unsubscribe_sync(client_id, "foo", identifier: 2)
       assert_receive {ScriptedMqttServer, {:received, ^unsubscribe_foo}}
-      # the callback handler should get a :down message for the foo subscription
-      assert_receive {{TestHandler, :subscription}, %{status: :down, topic_filter: "foo"}}
+      # handle_unsuback should get called on the callback handler
+      assert_receive {{TestHandler, :handle_unsuback}, {^unsubscribe_foo, ^unsuback_foo}}
 
-      assert %Package.Subscribe{topics: [{"bar", qos: 2}]} =
-               Tortoise.Connection.subscriptions(client_id)
+      refute Map.has_key?(Tortoise.Connection.subscriptions(client_id), "foo")
+      # should still have bar in active subscriptions
+      assert Map.has_key?(Tortoise.Connection.subscriptions(client_id), "bar")
 
       # and unsubscribe from bar
       assert {:ok, ref} = Tortoise.Connection.unsubscribe(client_id, "bar", identifier: 3)
       assert_receive {{Tortoise, ^client_id}, ^ref, :ok}
       assert_receive {ScriptedMqttServer, {:received, ^unsubscribe_bar}}
-      # the callback handler should get a :down message for the bar subscription
-      assert_receive {{TestHandler, :subscription}, %{status: :down, topic_filter: "bar"}}
-      assert %Package.Subscribe{topics: []} = Tortoise.Connection.subscriptions(client_id)
+      # handle_unsuback should get called on the callback handler
+      assert_receive {{TestHandler, :handle_unsuback}, {^unsubscribe_bar, ^unsuback_bar}}
+
+      refute Map.has_key?(Tortoise.Connection.subscriptions(client_id), "bar")
+      # there should be no subscriptions now
+      assert map_size(Tortoise.Connection.subscriptions(client_id)) == 0
       assert_receive {ScriptedMqttServer, :completed}
     end
   end
diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index cf0bbfe0..0e5e1ecd 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -3,7 +3,6 @@ defmodule Tortoise.HandlerTest do
   doctest Tortoise.Handler
 
   alias Tortoise.Handler
-  alias Tortoise.Connection.Inflight.Track
   alias Tortoise.Package
 
   defmodule TestHandler do
@@ -24,13 +23,13 @@ defmodule Tortoise.HandlerTest do
       {:ok, state}
     end
 
-    def subscription(status, topic, state) do
-      send(state[:pid], {:subscription, status, topic})
+    def handle_suback(subscribe, suback, state) do
+      send(state[:pid], {:suback, {subscribe, suback}})
       {:ok, state}
     end
 
-    def handle_suback(suback, state) do
-      send(state[:pid], {:suback, suback})
+    def handle_unsuback(unsubscribe, unsuback, state) do
+      send(state[:pid], {:unsuback, {unsubscribe, unsuback}})
       {:ok, state}
     end
 
@@ -172,57 +171,39 @@ defmodule Tortoise.HandlerTest do
     end
   end
 
-  describe "execute subscribe/2" do
+  describe "execute handle_suback/3" do
     test "return ok", context do
+      handler = set_state(context.handler, pid: self())
+
       subscribe = %Package.Subscribe{
         identifier: 1,
-        topics: [{"foo", qos: 0}, {"bar", qos: 1}, {"baz", qos: 0}]
+        topics: [{"foo", qos: 0}]
       }
 
-      suback = %Package.Suback{identifier: 1, acks: [ok: 0, ok: 0, error: :access_denied]}
-      caller = {self(), make_ref()}
+      suback = %Package.Suback{identifier: 1, acks: [ok: 0]}
 
-      track = Track.create({:negative, caller}, subscribe)
-      {:ok, track} = Track.resolve(track, {:received, suback})
-      {:ok, result} = Track.result(track)
-
-      handler = set_state(context.handler, pid: self())
-      assert {:ok, %Handler{}} = Handler.execute_subscribe(handler, result)
+      assert {:ok, %Handler{} = state, []} =
+               Handler.execute_handle_suback(handler, subscribe, suback)
 
-      assert_receive {:subscription, :up, "foo"}
-      assert_receive {:subscription, {:error, :access_denied}, "baz"}
-      assert_receive {:subscription, {:warn, requested: 1, accepted: 0}, "bar"}
+      assert_receive {:suback, {^subscribe, ^suback}}
     end
   end
 
-  describe "execute handle_suback/2" do
+  describe "execute handle_unsuback/3" do
     test "return ok", context do
       handler = set_state(context.handler, pid: self())
-      suback = %Package.Suback{identifier: 1}
-
-      assert {:ok, %Handler{} = state} =
-               handler
-               |> Handler.execute_handle_suback(suback)
 
-      assert_receive {:suback, ^suback}
-    end
-  end
+      unsubscribe = %Package.Unsubscribe{
+        identifier: 1,
+        topics: ["foo"]
+      }
 
-  describe "execute unsubscribe/2" do
-    test "return ok", context do
-      unsubscribe = %Package.Unsubscribe{identifier: 1, topics: ["foo/bar", "baz/quux"]}
-      unsuback = %Package.Unsuback{identifier: 1}
-      caller = {self(), make_ref()}
+      unsuback = %Package.Unsuback{identifier: 1, results: [:success]}
 
-      track = Track.create({:negative, caller}, unsubscribe)
-      {:ok, track} = Track.resolve(track, {:received, unsuback})
-      {:ok, result} = Track.result(track)
+      assert {:ok, %Handler{} = state, []} =
+               Handler.execute_handle_unsuback(handler, unsubscribe, unsuback)
 
-      handler = set_state(context.handler, pid: self())
-      assert {:ok, %Handler{}} = Handler.execute_unsubscribe(handler, result)
-      # we should receive two subscription down messages
-      assert_receive {:subscription, :down, "foo/bar"}
-      assert_receive {:subscription, :down, "baz/quux"}
+      assert_receive {:unsuback, {^unsubscribe, ^unsuback}}
     end
   end
 

From 831cccfa3f2307274d49512990fba465699ee7ab Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Wed, 9 Jan 2019 21:07:32 +0000
Subject: [PATCH 067/220] Added some QoS=2 related tests

---
 test/tortoise/connection/controller_test.exs | 73 --------------------
 test/tortoise/connection_test.exs            | 55 +++++++++++++++
 2 files changed, 55 insertions(+), 73 deletions(-)

diff --git a/test/tortoise/connection/controller_test.exs b/test/tortoise/connection/controller_test.exs
index 6dd588ca..885b619c 100644
--- a/test/tortoise/connection/controller_test.exs
+++ b/test/tortoise/connection/controller_test.exs
@@ -128,79 +128,6 @@
 #     end
 #   end
 
-#   describe "Publish Quality of Service level 2" do
-#     setup [:setup_connection, :setup_controller, :setup_inflight]
-
-#     test "incoming publish with qos 2", context do
-#       client_id = context.client_id
-#       # send in an publish message with a QoS of 2
-#       publish = %Package.Publish{identifier: 1, topic: "a", qos: 2}
-#       :ok = Controller.handle_incoming(client_id, publish)
-#       # test sending in a duplicate publish
-#       :ok = Controller.handle_incoming(client_id, %Package.Publish{publish | dup: true})
-
-#       # assert that the sender receives a pubrec package
-#       {:ok, pubrec} = :gen_tcp.recv(context.server, 0, 200)
-#       assert %Package.Pubrec{identifier: 1} = Package.decode(pubrec)
-
-#       # the publish should get onwarded to the handler
-#       assert_receive %TestHandler{publish_count: 1, received: [{["a"], nil}]}
-
-#       # the MQTT server will then respond with pubrel
-#       Controller.handle_incoming(client_id, %Package.Pubrel{identifier: 1})
-#       # a pubcomp message should get transmitted
-#       {:ok, pubcomp} = :gen_tcp.recv(context.server, 0, 200)
-
-#       assert %Package.Pubcomp{identifier: 1} = Package.decode(pubcomp)
-
-#       # the publish should only get onwareded once
-#       refute_receive %TestHandler{publish_count: 2}
-#     end
-
-#     test "incoming publish with qos 2 (first message dup)", context do
-#       # send in an publish with dup set to true should succeed if the
-#       # id is unknown.
-#       client_id = context.client_id
-#       # send in an publish message with a QoS of 2
-#       publish = %Package.Publish{identifier: 1, topic: "a", qos: 2, dup: true}
-#       :ok = Controller.handle_incoming(client_id, publish)
-
-#       # assert that the sender receives a pubrec package
-#       {:ok, pubrec} = :gen_tcp.recv(context.server, 0, 200)
-#       assert %Package.Pubrec{identifier: 1} = Package.decode(pubrec)
-
-#       # the MQTT server will then respond with pubrel
-#       Controller.handle_incoming(client_id, %Package.Pubrel{identifier: 1})
-#       # a pubcomp message should get transmitted
-#       {:ok, pubcomp} = :gen_tcp.recv(context.server, 0, 200)
-
-#       assert %Package.Pubcomp{identifier: 1} = Package.decode(pubcomp)
-
-#       # the publish should get onwarded to the handler
-#       assert_receive %TestHandler{publish_count: 1, received: [{["a"], nil}]}
-#     end
-
-#     test "outgoing publish with qos 2", context do
-#       client_id = context.client_id
-#       publish = %Package.Publish{identifier: 1, topic: "a", qos: 2}
-
-#       assert {:ok, ref} = Inflight.track(client_id, {:outgoing, publish})
-
-#       # assert that the server receives a publish package
-#       {:ok, package} = :gen_tcp.recv(context.server, 0, 200)
-#       assert ^publish = Package.decode(package)
-#       # the server will send back a publish received message
-#       Controller.handle_incoming(client_id, %Package.Pubrec{identifier: 1})
-#       # we should send a publish release (pubrel) to the server
-#       {:ok, package} = :gen_tcp.recv(context.server, 0, 200)
-#       assert %Package.Pubrel{identifier: 1} = Package.decode(package)
-#       # receive pubcomp
-#       Controller.handle_incoming(client_id, %Package.Pubcomp{identifier: 1})
-#       # the caller should get a message in its mailbox
-#       assert_receive {{Tortoise, ^client_id}, {Package.Publish, ^ref}, :ok}
-#     end
-#   end
-
 #   describe "Subscription" do
 #     setup [:setup_connection, :setup_controller, :setup_inflight]
 
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index 2aa6f11f..6250ca7e 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -938,6 +938,61 @@ defmodule Tortoise.ConnectionTest do
       assert_receive {{TestHandler, :handle_publish}, %{topic: "foo/bar", payload: nil}}
     end
 
+    test "incoming publish with QoS=2 with duplicate", context do
+      Process.flag(:trap_exit, true)
+      publish = %Package.Publish{identifier: 1, topic: "foo/bar", qos: 2}
+      expected_pubrec = %Package.Pubrec{identifier: 1}
+      pubrel = %Package.Pubrel{identifier: 1}
+      expected_pubcomp = %Package.Pubcomp{identifier: 1}
+
+      script = [
+        {:send, publish},
+        {:send, %Package.Publish{publish | dup: true}},
+        {:receive, expected_pubrec},
+        {:send, pubrel},
+        {:receive, expected_pubcomp}
+      ]
+
+      {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
+      pid = context.connection_pid
+
+      refute_receive {:EXIT, ^pid, {:protocol_violation, {:unexpected_package, _}}}
+      assert_receive {ScriptedMqttServer, :completed}
+
+      # the handle publish, and handle_pubrel callbacks should have been called
+      assert_receive {{TestHandler, :handle_pubrel}, ^pubrel}
+      assert_receive {{TestHandler, :handle_publish}, %{topic: "foo/bar", payload: nil}}
+      # the handle publish should only get called once, so if the
+      # duplicated publish result in a handle_publish message it would
+      # be a failure.
+      refute_receive {{TestHandler, :handle_publish}, %{topic: "foo/bar", payload: nil}}
+    end
+
+    test "incoming publish with QoS=2 with first message marked as duplicate", context do
+      Process.flag(:trap_exit, true)
+      publish = %Package.Publish{identifier: 1, topic: "foo/bar", qos: 2, dup: true}
+      expected_pubrec = %Package.Pubrec{identifier: 1}
+      pubrel = %Package.Pubrel{identifier: 1}
+      expected_pubcomp = %Package.Pubcomp{identifier: 1}
+
+      script = [
+        {:send, publish},
+        {:receive, expected_pubrec},
+        {:send, pubrel},
+        {:receive, expected_pubcomp}
+      ]
+
+      {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
+      pid = context.connection_pid
+
+      refute_receive {:EXIT, ^pid, {:protocol_violation, {:unexpected_package, _}}}
+      assert_receive {ScriptedMqttServer, :completed}
+
+      # the handle publish, and handle_pubrel callbacks should have been called
+      assert_receive {{TestHandler, :handle_pubrel}, ^pubrel}
+      assert_receive {{TestHandler, :handle_publish}, %{topic: "foo/bar", payload: nil}}
+    end
+
     test "outgoing publish with QoS=2", context do
       Process.flag(:trap_exit, true)
       publish = %Package.Publish{identifier: 1, topic: "foo/bar", qos: 2}

From 0695c353728e7419d15234e26a945be978fc0dc9 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Wed, 9 Jan 2019 21:21:51 +0000
Subject: [PATCH 068/220] Add todo notes for missing tests

---
 test/tortoise/connection/controller_test.exs | 233 -------------------
 test/tortoise/connection_test.exs            |   5 +
 2 files changed, 5 insertions(+), 233 deletions(-)

diff --git a/test/tortoise/connection/controller_test.exs b/test/tortoise/connection/controller_test.exs
index 885b619c..e69de29b 100644
--- a/test/tortoise/connection/controller_test.exs
+++ b/test/tortoise/connection/controller_test.exs
@@ -1,233 +0,0 @@
-# defmodule Tortoise.Connection.ControllerTest do
-#   use ExUnit.Case
-#   doctest Tortoise.Connection.Controller
-
-#   alias Tortoise.Package
-#   alias Tortoise.Connection.{Controller, Inflight}
-
-#   import ExUnit.CaptureLog
-
-#   defmodule TestHandler do
-#     use Tortoise.Handler
-
-#     defstruct pid: nil,
-#               client_id: nil,
-#               status: nil,
-#               publish_count: 0,
-#               received: [],
-#               subscriptions: []
-
-#     def init([client_id, caller]) when is_pid(caller) do
-#       # We pass in the caller `pid` and keep it in the state so we can
-#       # send messages back to the test process, which will make it
-#       # possible to make assertions on the changes in the handler
-#       # callback module
-#       {:ok, %__MODULE__{pid: caller, client_id: client_id}}
-#     end
-
-#     def connection(status, state) do
-#       new_state = %__MODULE__{state | status: status}
-#       send(state.pid, new_state)
-#       {:ok, new_state}
-#     end
-
-#     def subscription(:up, topic_filter, state) do
-#       new_state = %__MODULE__{
-#         state
-#         | subscriptions: [{topic_filter, :ok} | state.subscriptions]
-#       }
-
-#       send(state.pid, new_state)
-#       {:ok, new_state}
-#     end
-
-#     def subscription(:down, topic_filter, state) do
-#       new_state = %__MODULE__{
-#         state
-#         | subscriptions:
-#             Enum.reject(state.subscriptions, fn {topic, _} -> topic == topic_filter end)
-#       }
-
-#       send(state.pid, new_state)
-#       {:ok, new_state}
-#     end
-
-#     def subscription({:warn, warning}, topic_filter, state) do
-#       new_state = %__MODULE__{
-#         state
-#         | subscriptions: [{topic_filter, warning} | state.subscriptions]
-#       }
-
-#       send(state.pid, new_state)
-#       {:ok, new_state}
-#     end
-
-#     def subscription({:error, reason}, topic_filter, state) do
-#       send(state.pid, {:subscription_error, {topic_filter, reason}})
-#       {:ok, state}
-#     end
-
-#     def handle_message(topic, message, %__MODULE__{} = state) do
-#       new_state = %__MODULE__{
-#         state
-#         | publish_count: state.publish_count + 1,
-#           received: [{topic, message} | state.received]
-#       }
-
-#       send(state.pid, new_state)
-#       {:ok, new_state}
-#     end
-
-#     def terminate(reason, state) do
-#       send(state.pid, {:terminating, reason})
-#       :ok
-#     end
-#   end
-
-#   # Setup ==============================================================
-#   setup context do
-#     {:ok, %{client_id: context.test}}
-#   end
-
-#   def setup_controller(context) do
-#     handler = %Tortoise.Handler{
-#       module: __MODULE__.TestHandler,
-#       initial_args: [context.client_id, self()]
-#     }
-
-#     opts = [client_id: context.client_id, handler: handler]
-#     {:ok, pid} = Controller.start_link(opts)
-#     {:ok, %{controller_pid: pid}}
-#   end
-
-#   def setup_connection(context) do
-#     {:ok, client_socket, server_socket} = Tortoise.Integration.TestTCPTunnel.new()
-#     name = Tortoise.Connection.via_name(context.client_id)
-#     :ok = Tortoise.Registry.put_meta(name, {Tortoise.Transport.Tcp, client_socket})
-#     {:ok, %{client: client_socket, server: server_socket}}
-#   end
-
-#   def setup_inflight(context) do
-#     opts = [client_id: context.client_id, parent: self()]
-#     {:ok, pid} = Inflight.start_link(opts)
-#     {:ok, %{inflight_pid: pid}}
-#   end
-
-#   # tests --------------------------------------------------------------
-#   describe "publish" do
-#     setup [:setup_controller]
-
-#     test "update callback module state between publishes", context do
-#       publish = %Package.Publish{topic: "a", qos: 0}
-#       # Our callback module will increment a counter when it receives
-#       # a publish control packet
-#       :ok = Controller.handle_incoming(context.client_id, publish)
-#       assert_receive %TestHandler{publish_count: 1}
-#       :ok = Controller.handle_incoming(context.client_id, publish)
-#       assert_receive %TestHandler{publish_count: 2}
-#     end
-#   end
-
-#   describe "Subscription" do
-#     setup [:setup_connection, :setup_controller, :setup_inflight]
-
-#     test "Subscribe to a topic that return different QoS than requested", context do
-#       client_id = context.client_id
-
-#       subscribe = %Package.Subscribe{
-#         identifier: 1,
-#         topics: [{"foo", 2}]
-#       }
-
-#       suback = %Package.Suback{identifier: 1, acks: [{:ok, 0}]}
-
-#       assert {:ok, ref} = Inflight.track(client_id, {:outgoing, subscribe})
-
-#       # assert that the server receives a subscribe package
-#       {:ok, package} = :gen_tcp.recv(context.server, 0, 200)
-#       assert ^subscribe = Package.decode(package)
-#       # the server will send back a subscription acknowledgement message
-#       :ok = Controller.handle_incoming(client_id, suback)
-
-#       assert_receive {{Tortoise, ^client_id}, ^ref, _}
-#       # the client callback module should get the subscribe notifications in order
-#       assert_receive %TestHandler{subscriptions: [{"foo", [requested: 2, accepted: 0]}]}
-
-#       # unsubscribe from a topic
-#       unsubscribe = %Package.Unsubscribe{identifier: 2, topics: ["foo"]}
-#       unsuback = %Package.Unsuback{identifier: 2}
-#       assert {:ok, ref} = Inflight.track(client_id, {:outgoing, unsubscribe})
-#       {:ok, package} = :gen_tcp.recv(context.server, 0, 200)
-#       assert ^unsubscribe = Package.decode(package)
-#       :ok = Controller.handle_incoming(client_id, unsuback)
-#       assert_receive {{Tortoise, ^client_id}, ^ref, _}
-
-#       # the client callback module should remove the subscription
-#       assert_receive %TestHandler{subscriptions: []}
-#     end
-
-#     test "Subscribe to a topic resulting in an error", context do
-#       client_id = context.client_id
-
-#       subscribe = %Package.Subscribe{
-#         identifier: 1,
-#         topics: [{"foo", 1}]
-#       }
-
-#       suback = %Package.Suback{identifier: 1, acks: [{:error, :access_denied}]}
-
-#       assert {:ok, ref} = Inflight.track(client_id, {:outgoing, subscribe})
-
-#       # assert that the server receives a subscribe package
-#       {:ok, package} = :gen_tcp.recv(context.server, 0, 200)
-#       assert ^subscribe = Package.decode(package)
-#       # the server will send back a subscription acknowledgement message
-#       :ok = Controller.handle_incoming(client_id, suback)
-
-#       assert_receive {{Tortoise, ^client_id}, ^ref, _}
-#       # the callback module should get the error
-#       assert_receive {:subscription_error, {"foo", :access_denied}}
-#     end
-#   end
-
-#   describe "next actions" do
-#     setup [:setup_controller]
-
-#     test "subscribe action", context do
-#       client_id = context.client_id
-#       next_action = {:subscribe, "foo/bar", qos: 0}
-#       send(context.controller_pid, {:next_action, next_action})
-#       %{awaiting: awaiting} = Controller.info(client_id)
-#       assert [{ref, ^next_action}] = Map.to_list(awaiting)
-#       response = {{Tortoise, client_id}, ref, :ok}
-#       send(context.controller_pid, response)
-#       %{awaiting: awaiting} = Controller.info(client_id)
-#       assert [] = Map.to_list(awaiting)
-#     end
-
-#     test "unsubscribe action", context do
-#       client_id = context.client_id
-#       next_action = {:unsubscribe, "foo/bar"}
-#       send(context.controller_pid, {:next_action, next_action})
-#       %{awaiting: awaiting} = Controller.info(client_id)
-#       assert [{ref, ^next_action}] = Map.to_list(awaiting)
-#       response = {{Tortoise, client_id}, ref, :ok}
-#       send(context.controller_pid, response)
-#       %{awaiting: awaiting} = Controller.info(client_id)
-#       assert [] = Map.to_list(awaiting)
-#     end
-
-#     test "receiving unknown async ref", context do
-#       client_id = context.client_id
-#       ref = make_ref()
-
-#       assert capture_log(fn ->
-#                send(context.controller_pid, {{Tortoise, client_id}, ref, :ok})
-#                :timer.sleep(100)
-#              end) =~ "Unexpected"
-
-#       %{awaiting: awaiting} = Controller.info(client_id)
-#       assert [] = Map.to_list(awaiting)
-#     end
-#   end
-# end
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index 6250ca7e..92a43f4b 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -318,6 +318,9 @@ defmodule Tortoise.ConnectionTest do
       assert_receive {ScriptedMqttServer, :completed}
     end
 
+    # @todo subscribe with a qos but have it accepted with a lower qos
+    # @todo unsuccessful subscribe
+
     test "successful unsubscribe", context do
       client_id = context.client_id
 
@@ -382,6 +385,8 @@ defmodule Tortoise.ConnectionTest do
       assert map_size(Tortoise.Connection.subscriptions(client_id)) == 0
       assert_receive {ScriptedMqttServer, :completed}
     end
+
+    # @todo unsuccessful unsubscribe
   end
 
   describe "encrypted connection" do

From c299dcc09fd0fe84276487f7b468190d6794493a Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Wed, 9 Jan 2019 21:24:09 +0000
Subject: [PATCH 069/220] Remove the old connection controller module

This has been moved into the connection instead.
---
 lib/tortoise/connection/controller.ex        | 3 ---
 test/tortoise/connection/controller_test.exs | 0
 2 files changed, 3 deletions(-)
 delete mode 100644 lib/tortoise/connection/controller.ex
 delete mode 100644 test/tortoise/connection/controller_test.exs

diff --git a/lib/tortoise/connection/controller.ex b/lib/tortoise/connection/controller.ex
deleted file mode 100644
index b7e663a6..00000000
--- a/lib/tortoise/connection/controller.ex
+++ /dev/null
@@ -1,3 +0,0 @@
-defmodule Tortoise.Connection.Controller do
-  @moduledoc false
-end
diff --git a/test/tortoise/connection/controller_test.exs b/test/tortoise/connection/controller_test.exs
deleted file mode 100644
index e69de29b..00000000

From e92f5b822c590cdcfa163a022e360e97520c1dad Mon Sep 17 00:00:00 2001
From: Martin Gausby <martingausby@agner-krarup.local>
Date: Thu, 10 Jan 2019 17:11:12 +0000
Subject: [PATCH 070/220] Add a missing implementation for handle_unsuback/3

---
 lib/tortoise/handler.ex | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index 7f14a357..4b242939 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -174,6 +174,11 @@ defmodule Tortoise.Handler do
         {:ok, state}
       end
 
+      @impl true
+      def handle_unsuback(_unsubscribe, _unsuback, state) do
+        {:ok, state}
+      end
+
       defoverridable Tortoise.Handler
     end
   end

From 1e4fd5116dd2ddffccd2765ea3656a81d34937eb Mon Sep 17 00:00:00 2001
From: Martin Gausby <martingausby@agner-krarup.local>
Date: Thu, 10 Jan 2019 17:11:58 +0000
Subject: [PATCH 071/220] Add notes about what to implement as callbacks in the
 handler

---
 lib/tortoise/handler.ex | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index 4b242939..f0c45b95 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -139,11 +139,6 @@ defmodule Tortoise.Handler do
         {:ok, state}
       end
 
-      # @impl true
-      # def subscription(_status, _topic_filter, state) do
-      #   {:ok, state}
-      # end
-
       @impl true
       def handle_publish(_topic, _payload, state) do
         {:ok, state}
@@ -230,6 +225,12 @@ defmodule Tortoise.Handler do
             when status: :up | :down,
                  new_state: term()
 
+  # todo, connection/2 should perhaps be handle_connack
+  # todo, handle_connack
+
+  # todo, handle_auth
+
+
   # @doc """
   # Invoked when the subscription of a topic filter changes status.
 
@@ -339,6 +340,12 @@ defmodule Tortoise.Handler do
             when pubcomp: Package.Pubcomp.t(),
                  new_state: term()
 
+  @callback handle_disconnect(disconnect, state :: term()) :: {:ok, new_state}
+            when disconnect: Package.Disconnect.t(),
+                 new_state: term()
+
+  # todo, should we do handle_pingresp as well ?
+
   @doc """
   Invoked when the connection process is about to exit.
 

From f69e5dfb72157a96d95f6e7fbd4ed2a27c4815b9 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martingausby@agner-krarup.local>
Date: Thu, 10 Jan 2019 17:18:00 +0000
Subject: [PATCH 072/220] Ignore my personal todo notes

---
 .gitignore | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitignore b/.gitignore
index c18811b2..f282b4ed 100644
--- a/.gitignore
+++ b/.gitignore
@@ -25,3 +25,5 @@ tortoise-*.tar
 # Files generated by Erlang/Elixir QuickCheck
 /current_counterexample.eqc
 /.eqc-info
+
+/todo.org
\ No newline at end of file

From 797d7ceb9a515bd717d280f9bb54de69c57a4dc7 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martingausby@agner-krarup.local>
Date: Thu, 10 Jan 2019 17:18:31 +0000
Subject: [PATCH 073/220] mix format

---
 lib/tortoise/handler.ex | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index f0c45b95..de3315a7 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -230,7 +230,6 @@ defmodule Tortoise.Handler do
 
   # todo, handle_auth
 
-
   # @doc """
   # Invoked when the subscription of a topic filter changes status.
 

From 30840b9018d459df0d003ae0ea7c2f07a68ed226 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 12 Jan 2019 15:02:02 +0000
Subject: [PATCH 074/220] Rename execute_disconnect ->
 handle_execute_disconnect

Add a test for the handler
---
 lib/tortoise/connection.ex     |  5 ++++-
 lib/tortoise/handler.ex        | 18 +++++++++++++-----
 lib/tortoise/handler/logger.ex |  6 ++++++
 test/tortoise/handler_test.exs | 18 ++++++++++++++++++
 4 files changed, 41 insertions(+), 6 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index d0faa993..6a27cacd 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -942,7 +942,10 @@ defmodule Tortoise.Connection do
         _current_state,
         %State{handler: handler} = data
       ) do
-    case Handler.execute_disconnect(handler, disconnect) do
+    case Handler.execute_handle_disconnect(handler, disconnect) do
+      {:ok, updated_handler, []} ->
+        {:keep_state, %State{data | handler: updated_handler}}
+
       {:stop, reason, updated_handler} ->
         {:stop, reason, %State{data | handler: updated_handler}}
     end
diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index de3315a7..067446ef 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -174,6 +174,11 @@ defmodule Tortoise.Handler do
         {:ok, state}
       end
 
+      @impl true
+      def handle_disconnect(_disconnect, state) do
+        {:ok, state}
+      end
+
       defoverridable Tortoise.Handler
     end
   end
@@ -382,14 +387,17 @@ defmodule Tortoise.Handler do
   end
 
   @doc false
-  @spec execute_disconnect(t, %Package.Disconnect{}) :: {:stop, term(), t}
-  def execute_disconnect(handler, %Package.Disconnect{} = disconnect) do
+  @spec execute_handle_disconnect(t, %Package.Disconnect{}) :: {:stop, term(), t}
+  def execute_handle_disconnect(handler, %Package.Disconnect{} = disconnect) do
     handler.module
     |> apply(:handle_disconnect, [disconnect, handler.state])
     |> case do
-      {:stop, reason, updated_state} ->
-        {:stop, reason, %__MODULE__{handler | state: updated_state}}
-    end
+         {:ok, updated_state} ->
+           {:ok, %__MODULE__{handler | state: updated_state}, []}
+
+         {:stop, reason, updated_state} ->
+           {:stop, reason, %__MODULE__{handler | state: updated_state}}
+       end
   end
 
   @doc false
diff --git a/lib/tortoise/handler/logger.ex b/lib/tortoise/handler/logger.ex
index 1b89698c..273e6880 100644
--- a/lib/tortoise/handler/logger.ex
+++ b/lib/tortoise/handler/logger.ex
@@ -53,6 +53,12 @@ defmodule Tortoise.Handler.Logger do
     {:ok, state}
   end
 
+  @impl true
+  def handle_disconnect(disconnect, state) do
+    Logger.info("Received disconnect from server #{inspect disconnect}")
+    {:ok, state}
+  end
+
   def terminate(reason, _state) do
     Logger.warn("Client has been terminated with reason: #{inspect(reason)}")
     :ok
diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index 0e5e1ecd..da8c5146 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -68,6 +68,11 @@ defmodule Tortoise.HandlerTest do
       send(state[:pid], {:pubcomp, pubcomp})
       {:ok, state}
     end
+
+    def handle_disconnect(disconnect, state) do
+      send(state[:pid], {:disconnect, disconnect})
+      {:ok, state}
+    end
   end
 
   setup _context do
@@ -267,4 +272,17 @@ defmodule Tortoise.HandlerTest do
       assert_receive {:pubcomp, ^pubcomp}
     end
   end
+
+  describe "execute handle_disconnect/2" do
+    test "return ok", context do
+      handler = set_state(context.handler, pid: self())
+      disconnect = %Package.Disconnect{}
+
+      assert {:ok, %Handler{} = state, []} =
+               handler
+               |> Handler.execute_handle_disconnect(disconnect)
+
+      assert_receive {:disconnect, ^disconnect}
+    end
+  end
 end

From d4aedb06f11fcd25a056ba61d2b336994308053b Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 12 Jan 2019 21:52:51 +0000
Subject: [PATCH 075/220] Introduce a handle_connack

The user will be able to check the session state and subscribe to
topic filters in the yet to be implemented next actions. The
subscriptions option for the connection will be removed in a later
commit.
---
 lib/tortoise/connection.ex     | 33 +++++++++++---------------------
 lib/tortoise/handler.ex        | 35 +++++++++++++++++++++++++++-------
 lib/tortoise/handler/logger.ex |  2 +-
 test/tortoise/handler_test.exs | 20 +++++++++++++++++++
 4 files changed, 60 insertions(+), 30 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 6a27cacd..2f5de5d7 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -426,32 +426,20 @@ defmodule Tortoise.Connection do
         :connecting,
         %State{
           client_id: client_id,
-          server: %Transport{type: transport},
-          connection: {transport, _},
-          connect: %Package.Connect{keep_alive: keep_alive}
+          connect: %Package.Connect{keep_alive: keep_alive},
+          handler: handler
         } = data
       ) do
-    :ok = Tortoise.Registry.put_meta(via_name(client_id), data.connection)
-    :ok = Events.dispatch(client_id, :connection, data.connection)
-    :ok = Events.dispatch(client_id, :status, :connected)
-    data = %State{data | backoff: Backoff.reset(data.backoff)}
-
-    case connack do
-      %Package.Connack{session_present: true} ->
-        next_actions = [
-          {:state_timeout, keep_alive * 1000, :keep_alive},
-          {:next_event, :internal, {:execute_handler, {:connection, :up}}}
-        ]
-
-        {:next_state, :connected, data, next_actions}
-
-      %Package.Connack{session_present: false} ->
-        # caller = {self(), make_ref()}
+    case Handler.execute_handle_connack(handler, connack) do
+      {:ok, %Handler{} = updated_handler, []} ->
+        :ok = Tortoise.Registry.put_meta(via_name(client_id), data.connection)
+        :ok = Events.dispatch(client_id, :connection, data.connection)
+        :ok = Events.dispatch(client_id, :status, :connected)
+        data = %State{data | backoff: Backoff.reset(data.backoff), handler: updated_handler}
 
         next_actions = [
           {:state_timeout, keep_alive * 1000, :keep_alive},
           {:next_event, :internal, {:execute_handler, {:connection, :up}}}
-          # {:next_event, :cast, {:subscribe, caller, data.subscriptions, []}}
         ]
 
         {:next_state, :connected, data, next_actions}
@@ -460,10 +448,11 @@ defmodule Tortoise.Connection do
 
   def handle_event(
         :internal,
-        {:received, %Package.Connack{reason: {:refused, reason}}},
+        {:received, %Package.Connack{reason: {:refused, reason}} = _connack},
         :connecting,
-        %State{} = data
+        %State{handler: _handler} = data
       ) do
+    # todo, pass this through to the user defined callback handler
     {:stop, {:connection_failed, reason}, data}
   end
 
diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index 067446ef..7e02800b 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -139,6 +139,11 @@ defmodule Tortoise.Handler do
         {:ok, state}
       end
 
+      @impl true
+      def handle_connack(_connack, state) do
+        {:ok, state}
+      end
+
       @impl true
       def handle_publish(_topic, _payload, state) do
         {:ok, state}
@@ -230,8 +235,13 @@ defmodule Tortoise.Handler do
             when status: :up | :down,
                  new_state: term()
 
-  # todo, connection/2 should perhaps be handle_connack
-  # todo, handle_connack
+  # todo, connection/2 should perhaps be handle_connack ?
+
+  @callback handle_connack(connack, state :: term()) ::
+              {:ok, new_state}
+              | {:ok, new_state, [next_action()]}
+            when connack: Package.Connack.t(),
+                 new_state: term()
 
   # todo, handle_auth
 
@@ -386,18 +396,29 @@ defmodule Tortoise.Handler do
     |> handle_result(handler)
   end
 
+  @spec execute_handle_connack(t, Package.Connack.t()) ::
+          {:ok, t} | {:error, {:invalid_next_action, term()}}
+  def execute_handle_connack(handler, %Package.Connack{} = connack) do
+    handler.module
+    |> apply(:handle_connack, [connack, handler.state])
+    |> case do
+      {:ok, updated_state} ->
+        {:ok, %__MODULE__{handler | state: updated_state}, []}
+    end
+  end
+
   @doc false
   @spec execute_handle_disconnect(t, %Package.Disconnect{}) :: {:stop, term(), t}
   def execute_handle_disconnect(handler, %Package.Disconnect{} = disconnect) do
     handler.module
     |> apply(:handle_disconnect, [disconnect, handler.state])
     |> case do
-         {:ok, updated_state} ->
-           {:ok, %__MODULE__{handler | state: updated_state}, []}
+      {:ok, updated_state} ->
+        {:ok, %__MODULE__{handler | state: updated_state}, []}
 
-         {:stop, reason, updated_state} ->
-           {:stop, reason, %__MODULE__{handler | state: updated_state}}
-       end
+      {:stop, reason, updated_state} ->
+        {:stop, reason, %__MODULE__{handler | state: updated_state}}
+    end
   end
 
   @doc false
diff --git a/lib/tortoise/handler/logger.ex b/lib/tortoise/handler/logger.ex
index 273e6880..4c28c2f9 100644
--- a/lib/tortoise/handler/logger.ex
+++ b/lib/tortoise/handler/logger.ex
@@ -55,7 +55,7 @@ defmodule Tortoise.Handler.Logger do
 
   @impl true
   def handle_disconnect(disconnect, state) do
-    Logger.info("Received disconnect from server #{inspect disconnect}")
+    Logger.info("Received disconnect from server #{inspect(disconnect)}")
     {:ok, state}
   end
 
diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index da8c5146..ce3a6cf0 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -23,6 +23,11 @@ defmodule Tortoise.HandlerTest do
       {:ok, state}
     end
 
+    def handle_connack(connack, state) do
+      send(state[:pid], {:connack, connack})
+      {:ok, state}
+    end
+
     def handle_suback(subscribe, suback, state) do
       send(state[:pid], {:suback, {subscribe, suback}})
       {:ok, state}
@@ -120,6 +125,21 @@ defmodule Tortoise.HandlerTest do
     end
   end
 
+  describe "execute handle_connack/2" do
+    test "return ok-tuple", context do
+      handler = set_state(context.handler, pid: self())
+
+      connack = %Package.Connack{
+        reason: :success,
+        session_present: false
+      }
+
+      assert {:ok, %Handler{} = state, []} = Handler.execute_handle_connack(handler, connack)
+
+      assert_receive {:connack, ^connack}
+    end
+  end
+
   describe "execute handle_publish/2" do
     test "return ok-2", context do
       handler = set_state(context.handler, %{pid: self()})

From 864f942ee62ac344cb89afd3540b08a968af1617 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 12 Jan 2019 22:52:36 +0000
Subject: [PATCH 076/220] The handle_publish callback will now receive the
 publish as 2nd arg

Previously the handle_publish received three arguments:

  - the topic as a list, split on '/'
  - the payload
  - the handler state

The second argument has been changed to contain the entire publish
package; the payload can be accessed easily via `publish.payload`.
---
 lib/tortoise/handler.ex           | 18 ++++++++----------
 test/support/test_handler.ex      |  6 +++---
 test/tortoise/connection_test.exs | 20 ++++++++++++--------
 test/tortoise/handler_test.exs    | 14 +++++++-------
 4 files changed, 30 insertions(+), 28 deletions(-)

diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index 7e02800b..5cdf5b14 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -80,16 +80,14 @@ defmodule Tortoise.Handler do
   If we want to unsubscribe from the current topic when we receive a
   publish on it we could write a `handle_publish/3` as follows:
 
-      def handle_publish(topic, _payload, state) do
-        topic = Enum.join(topic, "/")
+      def handle_publish(_, %{topic: topic}, state) do
         next_actions = [{:unsubscribe, topic}]
         {:ok, state, next_actions}
       end
 
-  Note that the `topic` is received as a list of topic levels, and
-  that the next actions has to be a list, even if there is only one
-  next action; multiple actions can be given at once. Read more about
-  this in the `handle_publish/3` documentation.
+  The next actions has to be a list, even if there is only one next
+  action; multiple actions can be given at once. Read more about this
+  in the `handle_publish/3` documentation.
   """
 
   alias Tortoise.Package
@@ -145,7 +143,7 @@ defmodule Tortoise.Handler do
       end
 
       @impl true
-      def handle_publish(_topic, _payload, state) do
+      def handle_publish(_topic_list, _publish, state) do
         {:ok, state}
       end
 
@@ -313,8 +311,8 @@ defmodule Tortoise.Handler do
   `Temperature` application we could set up our `handle_publish` as
   such:
 
-      def handle_publish(["room", room, "temp"], payload, state) do
-        :ok = Temperature.record(room, payload)
+      def handle_publish(["room", room, "temp"], publish, state) do
+        :ok = Temperature.record(room, publish.payload)
         {:ok, state}
       end
 
@@ -484,7 +482,7 @@ defmodule Tortoise.Handler do
     topic_list = String.split(publish.topic, "/")
 
     handler.module
-    |> apply(:handle_publish, [topic_list, publish.payload, handler.state])
+    |> apply(:handle_publish, [topic_list, publish, handler.state])
     |> handle_result(handler)
   end
 
diff --git a/test/support/test_handler.ex b/test/support/test_handler.ex
index c39efa1d..ee63a52e 100644
--- a/test/support/test_handler.ex
+++ b/test/support/test_handler.ex
@@ -22,9 +22,9 @@ defmodule TestHandler do
     {:stop, :normal, state}
   end
 
-  def handle_publish(topic, payload, state) do
-    data = %{topic: Enum.join(topic, "/"), payload: payload}
-    send(state[:parent], {{__MODULE__, :handle_publish}, data})
+  def handle_publish(_topic, publish, state) do
+    # data = %{topic: Enum.join(topic, "/"), payload: payload}
+    send(state[:parent], {{__MODULE__, :handle_publish}, publish})
     {:ok, state}
   end
 
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index 92a43f4b..d19e9471 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -839,7 +839,7 @@ defmodule Tortoise.ConnectionTest do
       assert_receive {ScriptedMqttServer, :completed}
 
       # the handle publish callback should have been called
-      assert_receive {{TestHandler, :handle_publish}, %{topic: "foo/bar", payload: nil}}
+      assert_receive {{TestHandler, :handle_publish}, ^publish}
     end
   end
 
@@ -860,7 +860,7 @@ defmodule Tortoise.ConnectionTest do
       assert_receive {ScriptedMqttServer, :completed}
 
       # the handle publish callback should have been called
-      assert_receive {{TestHandler, :handle_publish}, %{topic: "foo/bar", payload: nil}}
+      assert_receive {{TestHandler, :handle_publish}, ^publish}
     end
 
     test "outgoing publish with QoS=1", context do
@@ -940,19 +940,20 @@ defmodule Tortoise.ConnectionTest do
 
       # the handle publish, and handle_pubrel callbacks should have been called
       assert_receive {{TestHandler, :handle_pubrel}, ^pubrel}
-      assert_receive {{TestHandler, :handle_publish}, %{topic: "foo/bar", payload: nil}}
+      assert_receive {{TestHandler, :handle_publish}, ^publish}
     end
 
     test "incoming publish with QoS=2 with duplicate", context do
       Process.flag(:trap_exit, true)
       publish = %Package.Publish{identifier: 1, topic: "foo/bar", qos: 2}
+      dup_publish = %Package.Publish{publish | dup: true}
       expected_pubrec = %Package.Pubrec{identifier: 1}
       pubrel = %Package.Pubrel{identifier: 1}
       expected_pubcomp = %Package.Pubcomp{identifier: 1}
 
       script = [
         {:send, publish},
-        {:send, %Package.Publish{publish | dup: true}},
+        {:send, dup_publish},
         {:receive, expected_pubrec},
         {:send, pubrel},
         {:receive, expected_pubcomp}
@@ -966,11 +967,11 @@ defmodule Tortoise.ConnectionTest do
 
       # the handle publish, and handle_pubrel callbacks should have been called
       assert_receive {{TestHandler, :handle_pubrel}, ^pubrel}
-      assert_receive {{TestHandler, :handle_publish}, %{topic: "foo/bar", payload: nil}}
+      assert_receive {{TestHandler, :handle_publish}, ^publish}
       # the handle publish should only get called once, so if the
       # duplicated publish result in a handle_publish message it would
       # be a failure.
-      refute_receive {{TestHandler, :handle_publish}, %{topic: "foo/bar", payload: nil}}
+      refute_receive {{TestHandler, :handle_publish}, ^dup_publish}
     end
 
     test "incoming publish with QoS=2 with first message marked as duplicate", context do
@@ -993,9 +994,12 @@ defmodule Tortoise.ConnectionTest do
       refute_receive {:EXIT, ^pid, {:protocol_violation, {:unexpected_package, _}}}
       assert_receive {ScriptedMqttServer, :completed}
 
-      # the handle publish, and handle_pubrel callbacks should have been called
+      # the handle publish, and handle_pubrel callbacks should have
+      # been called; we convert the dup:true package to dup:false if
+      # it is the first message we see with that id
       assert_receive {{TestHandler, :handle_pubrel}, ^pubrel}
-      assert_receive {{TestHandler, :handle_publish}, %{topic: "foo/bar", payload: nil}}
+      non_dup_publish = %Package.Publish{publish | dup: false}
+      assert_receive {{TestHandler, :handle_publish}, ^non_dup_publish}
     end
 
     test "outgoing publish with QoS=2", context do
diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index ce3a6cf0..a8108229 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -39,13 +39,13 @@ defmodule Tortoise.HandlerTest do
     end
 
     # with next actions
-    def handle_publish(topic, payload, %{next_actions: next_actions} = state) do
-      send(state[:pid], {:publish, topic, payload})
+    def handle_publish(topic, publish, %{next_actions: next_actions} = state) do
+      send(state[:pid], {:publish, topic, publish})
       {:ok, state, next_actions}
     end
 
-    def handle_publish(topic, payload, state) do
-      send(state[:pid], {:publish, topic, payload})
+    def handle_publish(topic, publish, state) do
+      send(state[:pid], {:publish, topic, publish})
       {:ok, state}
     end
 
@@ -150,7 +150,7 @@ defmodule Tortoise.HandlerTest do
       assert {:ok, %Handler{}} = Handler.execute_handle_publish(handler, publish)
       # the topic will be in the form of a list making it possible to
       # pattern match on the topic levels
-      assert_receive {:publish, topic_list, ^payload}
+      assert_receive {:publish, topic_list, ^publish}
       assert is_list(topic_list)
       assert topic == Enum.join(topic_list, "/")
     end
@@ -169,7 +169,7 @@ defmodule Tortoise.HandlerTest do
 
       # the topic will be in the form of a list making it possible to
       # pattern match on the topic levels
-      assert_receive {:publish, topic_list, ^payload}
+      assert_receive {:publish, topic_list, ^publish}
       assert is_list(topic_list)
       assert topic == Enum.join(topic_list, "/")
     end
@@ -190,7 +190,7 @@ defmodule Tortoise.HandlerTest do
       refute_receive {:next_action, {:unsubscribe, "foo/bar"}}
 
       # the callback is still run so lets check the received data
-      assert_receive {:publish, topic_list, ^payload}
+      assert_receive {:publish, topic_list, ^publish}
       assert is_list(topic_list)
       assert topic == Enum.join(topic_list, "/")
     end

From b4f130a60905d5e64987dad8ec93511836d50d79 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sun, 27 Jan 2019 19:59:52 +0000
Subject: [PATCH 077/220] Removed the subscriptions field from the internal
 state

The :subscriptions field on the internal connection state was used to
keep a list of user defined subscriptions for a connection. My thought
was to move it to a configuration object, so it would be outside of
the state and thus be able to survive a crash, and have the connection
connect with the same subscription for a given client id, but this is
impossible because of user defined properties!

We cannot just pack all subscriptions in one subscribe call, as the
user might want to set a user defined property for one of the
subscriptions, and we cannot just set the user defined property in the
configs along with the subscription as the user might want to send new
data every time they subscribe.

User defined properties sure are a blessing and a curse.
---
 lib/tortoise/connection.ex | 22 ++--------------------
 1 file changed, 2 insertions(+), 20 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 2f5de5d7..3b76187a 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -13,9 +13,6 @@ defmodule Tortoise.Connection do
             connect: nil,
             server: nil,
             backoff: nil,
-            # todo, replace subscriptions with a list of topics stored in a
-            # Tortoise.Config
-            subscriptions: nil,
             active_subscriptions: %{},
             opts: nil,
             pending_refs: %{},
@@ -43,8 +40,6 @@ defmodule Tortoise.Connection do
                | {:password, String.t()}
                | {:keep_alive, non_neg_integer()}
                | {:will, Tortoise.Package.Publish.t()}
-               | {:subscriptions,
-                  [{Tortoise.topic_filter(), Tortoise.qos()}] | Tortoise.Package.Subscribe.t()}
                | {:handler, {atom(), term()}},
              options: [option]
   def start_link(connection_opts, opts \\ []) do
@@ -63,18 +58,6 @@ defmodule Tortoise.Connection do
 
     backoff = Keyword.get(connection_opts, :backoff, [])
 
-    # This allow us to either pass in a list of topics, or a
-    # subscription struct. Passing in a subscription struct is helpful
-    # in tests.
-    subscriptions =
-      case Keyword.get(connection_opts, :subscriptions, []) do
-        topics when is_list(topics) ->
-          Enum.into(topics, %Package.Subscribe{})
-
-        %Package.Subscribe{} = subscribe ->
-          subscribe
-      end
-
     # @todo, validate that the handler is valid
     handler =
       connection_opts
@@ -85,7 +68,7 @@ defmodule Tortoise.Connection do
       {:transport, server} | Keyword.take(connection_opts, [:client_id])
     ]
 
-    initial = {server, connect, backoff, subscriptions, handler, connection_opts}
+    initial = {server, connect, backoff, handler, connection_opts}
     opts = Keyword.merge(opts, name: via_name(client_id))
     GenStateMachine.start_link(__MODULE__, initial, opts)
   end
@@ -383,13 +366,12 @@ defmodule Tortoise.Connection do
 
   # Callbacks
   @impl true
-  def init({transport, connect, backoff_opts, subscriptions, handler, opts}) do
+  def init({transport, connect, backoff_opts, handler, opts}) do
     data = %State{
       client_id: connect.client_id,
       server: transport,
       connect: connect,
       backoff: Backoff.new(backoff_opts),
-      subscriptions: subscriptions,
       opts: opts,
       handler: handler
     }

From 9f83fd30e1b77a214038f2b19372647a040b746a Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sun, 27 Jan 2019 20:20:20 +0000
Subject: [PATCH 078/220] Rename the :active_subscriptions to :subscriptions in
 connection

Previously a configuration listing the subscriptions that should be
established when a connection had been was stored under the name
subscriptions. That was removed in a previous commit because the new
user defined properties make that approach impossible.

The field containing the active subscriptions was called
:active_subscriptions; for brevity we can not use the now free
:subscriptions to store that information.
---
 lib/tortoise/connection.ex | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 3b76187a..a49ed837 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -13,7 +13,7 @@ defmodule Tortoise.Connection do
             connect: nil,
             server: nil,
             backoff: nil,
-            active_subscriptions: %{},
+            subscriptions: %{},
             opts: nil,
             pending_refs: %{},
             connection: nil,
@@ -641,10 +641,10 @@ defmodule Tortoise.Connection do
     {{pid, msg_ref}, updated_pending} = Map.pop(pending, ref)
     data = %State{data | pending_refs: updated_pending}
 
-    updated_active_subscriptions =
+    updated_subscriptions =
       subscribe.topics
       |> Enum.zip(suback.acks)
-      |> Enum.reduce(data.active_subscriptions, fn
+      |> Enum.reduce(data.subscriptions, fn
         {{topic, opts}, {:ok, accepted_qos}}, acc ->
           Map.put(acc, topic, Keyword.replace!(opts, :qos, accepted_qos))
 
@@ -657,7 +657,7 @@ defmodule Tortoise.Connection do
         data = %State{
           data
           | handler: updated_handler,
-            active_subscriptions: updated_active_subscriptions
+            subscriptions: updated_subscriptions
         }
 
         next_actions = [
@@ -727,14 +727,14 @@ defmodule Tortoise.Connection do
     # todo, if the results in unsuback contain an error, such as
     # `{:error, :no_subscription_existed}` then we would be out of
     # sync! What should we do here?
-    active_subscriptions = Map.drop(data.active_subscriptions, unsubscribe.topics)
+    subscriptions = Map.drop(data.subscriptions, unsubscribe.topics)
 
     case Handler.execute_handle_unsuback(handler, unsubscribe, unsuback) do
       {:ok, %Handler{} = updated_handler, next_actions} ->
         data = %State{
           data
           | handler: updated_handler,
-            active_subscriptions: active_subscriptions
+            subscriptions: subscriptions
         }
 
         next_actions = [
@@ -752,7 +752,7 @@ defmodule Tortoise.Connection do
     end
   end
 
-  def handle_event({:call, from}, :subscriptions, _, %State{active_subscriptions: subscriptions}) do
+  def handle_event({:call, from}, :subscriptions, _, %State{subscriptions: subscriptions}) do
     next_actions = [{:reply, from, subscriptions}]
     {:keep_state_and_data, next_actions}
   end

From 1c12d90cb2e2204ef4f75d7ee19cba418e7f8ce2 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sun, 27 Jan 2019 20:33:35 +0000
Subject: [PATCH 079/220] Remove a section that was previously commented out

---
 lib/tortoise/connection.ex | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index a49ed837..8e3d4a32 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -773,21 +773,6 @@ defmodule Tortoise.Connection do
     end
   end
 
-  # def handle_event(
-  #       :internal,
-  #       {:execute_handler, {:unsubscribe, result}},
-  #       _current_state,
-  #       %State{handler: handler} = data
-  #     ) do
-  #   case Handler.execute_unsubscribe(handler, result) do
-  #     {:ok, %Handler{} = updated_handler} ->
-  #       updated_data = %State{data | handler: updated_handler}
-  #       {:keep_state, updated_data}
-
-  #       # handle stop
-  #   end
-  # end
-
   # connection logic ===================================================
   def handle_event(:internal, :connect, :connecting, %State{} = data) do
     :ok = Tortoise.Registry.put_meta(via_name(data.client_id), :connecting)

From 6a0eb27c60ff0eda0daa3335cde118c63c35682e Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 2 Feb 2019 22:30:31 +0000
Subject: [PATCH 080/220] Remove a resolved todo item

---
 lib/tortoise/connection.ex | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 8e3d4a32..8a642a35 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -720,7 +720,6 @@ defmodule Tortoise.Connection do
         _current_state,
         %State{client_id: client_id, handler: handler, pending_refs: %{} = pending} = data
       ) do
-    # todo, call a handle_unsuback callback!
     {{pid, msg_ref}, updated_pending} = Map.pop(pending, ref)
     data = %State{data | pending_refs: updated_pending}
 

From dbf125af3597ed3385dc816bd92cbbd4b98bedb5 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 2 Feb 2019 22:30:52 +0000
Subject: [PATCH 081/220] Make sure async unsubscribes result in a msg passed
 to the caller

---
 test/tortoise/connection_test.exs | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index d19e9471..d55bf68c 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -357,7 +357,8 @@ defmodule Tortoise.ConnectionTest do
         identifier: 1
       }
 
-      Tortoise.Connection.subscribe(client_id, subscribe.topics, identifier: 1)
+      {:ok, unsub_ref} =
+        Tortoise.Connection.subscribe(client_id, subscribe.topics, identifier: 1)
 
       assert_receive {ScriptedMqttServer, {:received, ^subscribe}}
 
@@ -384,6 +385,10 @@ defmodule Tortoise.ConnectionTest do
       # there should be no subscriptions now
       assert map_size(Tortoise.Connection.subscriptions(client_id)) == 0
       assert_receive {ScriptedMqttServer, :completed}
+
+      # the process calling the async unsubscribe should receive the
+      # result of the unsubscribe as a message
+      assert_receive {{Tortoise, ^client_id}, ^unsub_ref, :ok}, 0
     end
 
     # @todo unsuccessful unsubscribe

From 050f144cd1d8c4f93871f642d1fa66a0a72b0505 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Thu, 7 Feb 2019 21:01:37 +0000
Subject: [PATCH 082/220] Make it possible to transform packages in user
 initialized publishes

The publishes now take a list of transform functions, operating on the
package types send and received in the publish exchange. This makes it
possible to intercept the pubrel and add user defined properties on
them, specified for the individual publish. It has a transaction state
as well, so state can be carried from publish, pubrec, pubrel, and
pubcomp, etc.
---
 lib/tortoise.ex                               | 10 +-
 lib/tortoise/connection.ex                    |  2 +-
 lib/tortoise/connection/inflight.ex           | 97 +++++++++++++++++--
 lib/tortoise/connection/inflight/track.ex     | 22 ++++-
 lib/tortoise/handler.ex                       | 10 +-
 .../connection/inflight/track_test.exs        |  4 +-
 test/tortoise/connection_test.exs             |  3 +-
 test/tortoise/handler_test.exs                | 27 +++++-
 test/tortoise_test.exs                        | 92 ++++++++++++++++++
 9 files changed, 238 insertions(+), 29 deletions(-)

diff --git a/lib/tortoise.ex b/lib/tortoise.ex
index b58f2860..e4c76e84 100644
--- a/lib/tortoise.ex
+++ b/lib/tortoise.ex
@@ -254,7 +254,7 @@ defmodule Tortoise do
                | {:retain, boolean()}
                | {:identifier, package_identifier()}
   def publish(client_id, topic, payload \\ nil, opts \\ []) do
-    {opts, properties} = Keyword.split(opts, [:retain, :qos])
+    {opts, properties} = Keyword.split(opts, [:retain, :qos, :transforms])
     qos = Keyword.get(opts, :qos, 0)
 
     publish = %Package.Publish{
@@ -272,7 +272,8 @@ defmodule Tortoise do
           apply(transport, :send, [socket, encoded_publish])
 
         %Package.Publish{qos: qos} when qos in [1, 2] ->
-          Inflight.track(client_id, {:outgoing, publish})
+          transforms = Keyword.get(opts, :transforms, {[], nil})
+          Inflight.track(client_id, {:outgoing, publish, transforms})
       end
     else
       {:error, :unknown_connection} ->
@@ -315,7 +316,7 @@ defmodule Tortoise do
                | {:identifier, package_identifier()}
                | {:timeout, timeout()}
   def publish_sync(client_id, topic, payload \\ nil, opts \\ []) do
-    {opts, properties} = Keyword.split(opts, [:retain, :qos])
+    {opts, properties} = Keyword.split(opts, [:retain, :qos, :transforms])
     qos = Keyword.get(opts, :qos, 0)
     timeout = Keyword.get(opts, :timeout, :infinity)
 
@@ -334,7 +335,8 @@ defmodule Tortoise do
           apply(transport, :send, [socket, encoded_publish])
 
         %Package.Publish{qos: qos} when qos in [1, 2] ->
-          Inflight.track_sync(client_id, {:outgoing, publish}, timeout)
+          transforms = Keyword.get(opts, :transforms, {[], nil})
+          Inflight.track_sync(client_id, {:outgoing, publish, transforms}, timeout)
       end
     else
       {:error, :unknown_connection} ->
diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 8a642a35..708b47f9 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -506,7 +506,7 @@ defmodule Tortoise.Connection do
     :ok = Inflight.update(client_id, {:received, pubrel})
 
     case Handler.execute_handle_pubrel(handler, pubrel) do
-      {:ok, %Handler{} = updated_handler} ->
+      {:ok, %Handler{} = updated_handler, []} ->
         # dispatch a pubcomp
         pubcomp = %Package.Pubcomp{identifier: id}
         :ok = Inflight.update(client_id, {:dispatch, pubcomp})
diff --git a/lib/tortoise/connection/inflight.ex b/lib/tortoise/connection/inflight.ex
index 20b582b9..001b4777 100644
--- a/lib/tortoise/connection/inflight.ex
+++ b/lib/tortoise/connection/inflight.ex
@@ -44,11 +44,16 @@ defmodule Tortoise.Connection.Inflight do
   end
 
   def track(client_id, {:outgoing, package}) do
+    # no transforms and nil track session state
+    track(client_id, {:outgoing, package, {[], nil}})
+  end
+
+  def track(client_id, {:outgoing, package, opts}) do
     caller = {_, ref} = {self(), make_ref()}
 
     case package do
-      %Package.Publish{qos: qos} when qos in 1..2 ->
-        :ok = GenStateMachine.cast(via_name(client_id), {:outgoing, caller, package})
+      %Package.Publish{qos: qos} when qos in [1, 2] ->
+        :ok = GenStateMachine.cast(via_name(client_id), {:outgoing, caller, {package, opts}})
         {:ok, ref}
 
       %Package.Subscribe{} ->
@@ -62,7 +67,14 @@ defmodule Tortoise.Connection.Inflight do
   end
 
   @doc false
-  def track_sync(client_id, {:outgoing, %Package.Publish{}} = command, timeout \\ :infinity) do
+  def track_sync(client_id, command, timeout \\ :infinity)
+
+  def track_sync(client_id, {:outgoing, %Package.Publish{}} = command, timeout) do
+    # add no transforms and a nil track state
+    track_sync(client_id, Tuple.append(command, {[], nil}), timeout)
+  end
+
+  def track_sync(client_id, {:outgoing, %Package.Publish{}, _transforms} = command, timeout) do
     {:ok, ref} = track(client_id, command)
 
     receive do
@@ -208,6 +220,23 @@ defmodule Tortoise.Connection.Inflight do
     :keep_state_and_data
   end
 
+  def handle_event(:cast, {:outgoing, caller, {package, opts}}, _state, data) do
+    {:ok, package} = assign_identifier(package, data.pending)
+    track = Track.create({:negative, caller}, {package, opts})
+
+    next_actions = [
+      {:next_event, :internal, {:execute, track}}
+    ]
+
+    data = %State{
+      data
+      | pending: Map.put_new(data.pending, track.identifier, track),
+        order: [track.identifier | data.order]
+    }
+
+    {:keep_state, data, next_actions}
+  end
+
   def handle_event(:cast, {:outgoing, caller, package}, _state, data) do
     {:ok, package} = assign_identifier(package, data.pending)
     track = Track.create({:negative, caller}, package)
@@ -232,11 +261,12 @@ defmodule Tortoise.Connection.Inflight do
 
   def handle_event(
         :cast,
-        {:update, {:received, %{identifier: identifier}} = update},
+        {:update, {:received, %{identifier: identifier} = package} = update},
         _state,
         %State{pending: pending} = data
       ) do
     with {:ok, track} <- Map.fetch(pending, identifier),
+         {_package, track} = apply_transform(package, track),
          {:ok, track} <- Track.resolve(track, update) do
       data = %State{
         data
@@ -322,9 +352,13 @@ defmodule Tortoise.Connection.Inflight do
         {:connected, {transport, socket}},
         %State{} = data
       ) do
-    case apply(transport, :send, [socket, Package.encode(package)]) do
-      :ok ->
-        {:keep_state, handle_next(track, data)}
+    with {package, track} <- apply_transform(package, track),
+         :ok = apply(transport, :send, [socket, Package.encode(package)]) do
+      {:keep_state, handle_next(track, data)}
+    else
+      res ->
+        IO.inspect(res, label: __MODULE__)
+        {:stop, res, data}
     end
   end
 
@@ -369,6 +403,51 @@ defmodule Tortoise.Connection.Inflight do
   end
 
   # helpers ------------------------------------------------------------
+  @type_to_cb %{
+    Package.Publish => :publish,
+    Package.Pubrec => :pubrec,
+    Package.Pubrel => :pubrel,
+    Package.Pubcomp => :pubcomp,
+    Package.Puback => :puback
+  }
+
+  defp apply_transform(package, %Track{transforms: []} = track) do
+    # noop, no transforms defined for any package type, so just pass
+    # through
+    {package, track}
+  end
+
+  defp apply_transform(%type{identifier: identifier} = package, %Track{} = track) do
+    # see if the callback for the given type is defined; if so, apply
+    # it and update the track state
+    case track.transforms[Map.get(@type_to_cb, type)] do
+      nil ->
+        {package, track}
+
+      fun when is_function(fun) ->
+        case apply(fun, [package, track.state]) do
+          {:ok, updated_state} ->
+            # just update the track session state
+            updated_track_state = %Track{track | state: updated_state}
+            {package, updated_track_state}
+
+          {:ok, properties, updated_state} when is_list(properties) ->
+            # Update the user defined properties on the package with a
+            # list of `[{string, string}]`
+            updated_package = %{package | properties: properties}
+            updated_track_state = %Track{track | state: updated_state}
+            {updated_package, updated_track_state}
+
+          {:ok, %^type{identifier: ^identifier} = updated_package, updated_state} ->
+            # overwrite the package with a package of the same type
+            # and identifier; allows us to alter properties besides
+            # the user defined properties
+            updated_track_state = %Track{track | state: updated_state}
+            {updated_package, updated_track_state}
+        end
+    end
+  end
+
   defp handle_next(
          %Track{pending: [[_, :cleanup]], identifier: identifier},
          %State{pending: pending} = state
@@ -377,8 +456,8 @@ defmodule Tortoise.Connection.Inflight do
     %State{state | pending: Map.delete(pending, identifier), order: order}
   end
 
-  defp handle_next(_track, %State{} = state) do
-    state
+  defp handle_next(%Track{identifier: identifier} = track, %State{pending: pending} = state) do
+    %State{state | pending: Map.replace!(pending, identifier, track)}
   end
 
   # Assign a random identifier to the tracked package; this will make
diff --git a/lib/tortoise/connection/inflight/track.ex b/lib/tortoise/connection/inflight/track.ex
index cae64c79..47724f93 100644
--- a/lib/tortoise/connection/inflight/track.ex
+++ b/lib/tortoise/connection/inflight/track.ex
@@ -36,7 +36,9 @@ defmodule Tortoise.Connection.Inflight.Track do
             caller: {pid(), reference()} | nil,
             identifier: Tortoise.package_identifier(),
             status: [status_update()],
-            pending: [next_action()]
+            pending: [next_action()],
+            transforms: [function()],
+            state: any()
           }
   @enforce_keys [:type, :identifier, :polarity, :pending]
   defstruct type: nil,
@@ -44,7 +46,9 @@ defmodule Tortoise.Connection.Inflight.Track do
             caller: nil,
             identifier: nil,
             status: [],
-            pending: []
+            pending: [],
+            transforms: [],
+            state: nil
 
   alias __MODULE__, as: State
   alias Tortoise.Package
@@ -116,13 +120,18 @@ defmodule Tortoise.Connection.Inflight.Track do
     }
   end
 
-  def create({:negative, {pid, ref}}, %Package.Publish{qos: 1, identifier: id} = publish)
+  def create(
+        {:negative, {pid, ref}},
+        {%Package.Publish{qos: 1, identifier: id} = publish, {transforms, initial_state}}
+      )
       when is_pid(pid) and is_reference(ref) do
     %State{
       type: Package.Publish,
       polarity: :negative,
       caller: {pid, ref},
       identifier: id,
+      transforms: transforms,
+      state: initial_state,
       pending: [
         [
           {:dispatch, publish},
@@ -155,13 +164,18 @@ defmodule Tortoise.Connection.Inflight.Track do
     }
   end
 
-  def create({:negative, {pid, ref}}, %Package.Publish{identifier: id, qos: 2} = publish)
+  def create(
+        {:negative, {pid, ref}},
+        {%Package.Publish{identifier: id, qos: 2} = publish, {transforms, initial_state}}
+      )
       when is_pid(pid) and is_reference(ref) do
     %State{
       type: Package.Publish,
       polarity: :negative,
       caller: {pid, ref},
       identifier: id,
+      transforms: transforms,
+      state: initial_state,
       pending: [
         [
           {:dispatch, publish},
diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index 5cdf5b14..7cc2652c 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -508,9 +508,13 @@ defmodule Tortoise.Handler do
   # @spec execute_handle_pubrel(t, Package.Pubrel.t()) ::
   #         {:ok, t} | {:error, {:invalid_next_action, term()}}
   def execute_handle_pubrel(handler, %Package.Pubrel{} = pubrel) do
-    handler.module
-    |> apply(:handle_pubrel, [pubrel, handler.state])
-    |> handle_result(handler)
+    case apply(handler.module, :handle_pubrel, [pubrel, handler.state]) do
+      {:ok, updated_state} ->
+        {:ok, %__MODULE__{handler | state: updated_state}, []}
+
+      {:ok, updated_state, next_actions} when is_list(next_actions) ->
+        {:ok, %__MODULE__{handler | state: updated_state}, next_actions}
+    end
   end
 
   @doc false
diff --git a/test/tortoise/connection/inflight/track_test.exs b/test/tortoise/connection/inflight/track_test.exs
index 4fee2888..025ab3a8 100644
--- a/test/tortoise/connection/inflight/track_test.exs
+++ b/test/tortoise/connection/inflight/track_test.exs
@@ -57,7 +57,7 @@ defmodule Tortoise.Connection.Inflight.TrackTest do
       publish = %Package.Publish{qos: 1, identifier: id}
       caller = {self(), make_ref()}
 
-      state = Track.create({:negative, caller}, publish)
+      state = Track.create({:negative, caller}, {publish, {[], nil}})
       assert %Track{pending: [[{:dispatch, ^publish}, _] | _]} = state
 
       {next_action, resolution} = Track.next(state)
@@ -84,7 +84,7 @@ defmodule Tortoise.Connection.Inflight.TrackTest do
       publish = %Package.Publish{qos: 2, identifier: id}
       caller = {self(), make_ref()}
 
-      state = Track.create({:negative, caller}, publish)
+      state = Track.create({:negative, caller}, {publish, {[], nil}})
       assert %Track{pending: [[{:dispatch, ^publish}, _] | _]} = state
 
       {next_action, resolution} = Track.next(state)
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index d55bf68c..896b8736 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -357,8 +357,7 @@ defmodule Tortoise.ConnectionTest do
         identifier: 1
       }
 
-      {:ok, unsub_ref} =
-        Tortoise.Connection.subscribe(client_id, subscribe.topics, identifier: 1)
+      {:ok, unsub_ref} = Tortoise.Connection.subscribe(client_id, subscribe.topics, identifier: 1)
 
       assert_receive {ScriptedMqttServer, {:received, ^subscribe}}
 
diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index a8108229..5de422de 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -66,7 +66,14 @@ defmodule Tortoise.HandlerTest do
 
     def handle_pubrel(pubrel, state) do
       send(state[:pid], {:pubrel, pubrel})
-      {:ok, state}
+
+      case Keyword.get(state, :pubrel) do
+        nil ->
+          {:ok, state}
+
+        fun when is_function(fun) ->
+          apply(fun, [pubrel, state])
+      end
     end
 
     def handle_pubcomp(pubcomp, state) do
@@ -272,9 +279,21 @@ defmodule Tortoise.HandlerTest do
       handler = set_state(context.handler, pid: self())
       pubrel = %Package.Pubrel{identifier: 1}
 
-      assert {:ok, %Handler{} = state} =
-               handler
-               |> Handler.execute_handle_pubrel(pubrel)
+      assert {:ok, %Handler{} = state, []} = Handler.execute_handle_pubrel(handler, pubrel)
+
+      assert_receive {:pubrel, ^pubrel}
+    end
+
+    test "return ok with next_actions", context do
+      pubrel_fn = fn %Package.Pubrel{identifier: 1}, state ->
+        {:ok, state, [{:complete, %Package.Pubcomp{identifier: 1}}]}
+      end
+
+      handler = set_state(context.handler, pid: self(), pubrel: pubrel_fn)
+      pubrel = %Package.Pubrel{identifier: 1}
+
+      assert {:ok, %Handler{} = state, [{:complete, %Package.Pubcomp{identifier: 1}}]} =
+               Handler.execute_handle_pubrel(handler, pubrel)
 
       assert_receive {:pubrel, ^pubrel}
     end
diff --git a/test/tortoise_test.exs b/test/tortoise_test.exs
index 5a965288..ec270057 100644
--- a/test/tortoise_test.exs
+++ b/test/tortoise_test.exs
@@ -37,11 +37,103 @@ defmodule TortoiseTest do
       assert %Package.Publish{topic: "foo/bar", qos: 1, payload: nil} = Package.decode(data)
     end
 
+    test "publish qos=1 with user defined callbacks", %{client_id: client_id} = context do
+      parent = self()
+
+      transforms = [
+        publish: fn %type{properties: properties}, [:init] = state ->
+          send(parent, {:callback, {type, properties}, state})
+          {:ok, properties, [type | state]}
+        end,
+        puback: fn %type{properties: properties}, state ->
+          send(parent, {:callback, {type, properties}, state})
+          {:ok, [type | state]}
+        end
+      ]
+
+      assert {:ok, publish_ref} =
+               Tortoise.publish(context.client_id, "foo/bar", nil,
+                 qos: 1,
+                 transforms: {transforms, [:init]}
+               )
+
+      assert {:ok, data} = :gen_tcp.recv(context.server, 0, 500)
+
+      assert %Package.Publish{identifier: id, topic: "foo/bar", qos: 1, payload: nil} =
+               Package.decode(data)
+
+      :ok = Inflight.update(client_id, {:received, %Package.Puback{identifier: id}})
+      # check the internal transform state
+      assert_receive {:callback, {Package.Publish, []}, [:init]}
+      assert_receive {:callback, {Package.Puback, []}, [Package.Publish, :init]}
+    end
+
     test "publish qos=2", context do
       assert {:ok, _ref} = Tortoise.publish(context.client_id, "foo/bar", nil, qos: 2)
       assert {:ok, data} = :gen_tcp.recv(context.server, 0, 500)
       assert %Package.Publish{topic: "foo/bar", qos: 2, payload: nil} = Package.decode(data)
     end
+
+    test "publish qos=2 with custom callbacks", %{client_id: client_id} = context do
+      parent = self()
+
+      transforms = [
+        publish: fn %type{properties: properties}, [:init] = state ->
+          send(parent, {:callback, {type, properties}, state})
+          {:ok, [{"foo", "bar"} | properties], [type | state]}
+        end,
+        pubrec: fn %type{properties: properties}, state ->
+          send(parent, {:callback, {type, properties}, state})
+          {:ok, [type | state]}
+        end,
+        pubrel: fn %type{properties: properties}, state ->
+          send(parent, {:callback, {type, properties}, state})
+          properties = [{"hello", "world"} | properties]
+          {:ok, properties, [type | state]}
+        end,
+        pubcomp: fn %type{properties: properties}, state ->
+          send(parent, {:callback, {type, properties}, state})
+          {:ok, [type | state]}
+        end
+      ]
+
+      assert {:ok, publish_ref} =
+               Tortoise.publish(context.client_id, "foo/bar", nil,
+                 qos: 2,
+                 transforms: {transforms, [:init]}
+               )
+
+      assert {:ok, data} = :gen_tcp.recv(context.server, 0, 500)
+
+      assert %Package.Publish{
+               identifier: id,
+               topic: "foo/bar",
+               qos: 2,
+               payload: nil,
+               properties: [{"foo", "bar"}]
+             } = Package.decode(data)
+
+      :ok = Inflight.update(context.client_id, {:received, %Package.Pubrec{identifier: id}})
+
+      pubrel = %Package.Pubrel{identifier: id}
+      :ok = Inflight.update(client_id, {:dispatch, pubrel})
+
+      assert {:ok, data} = :gen_tcp.recv(context.server, 0, 500)
+      expected_pubrel = %Package.Pubrel{pubrel | properties: [{"hello", "world"}]}
+      assert expected_pubrel == Package.decode(data)
+
+      :ok = Inflight.update(client_id, {:received, %Package.Pubcomp{identifier: id}})
+
+      assert_receive {{Tortoise, ^client_id}, {Package.Publish, ^publish_ref}, :ok}
+      # check the internal state of the transform; in the test we add
+      # the type of the package to the state, which we have defied as
+      # a list:
+      assert_receive {:callback, {Package.Publish, []}, [:init]}
+      assert_receive {:callback, {Package.Pubrec, []}, [Package.Publish | _]}
+      assert_receive {:callback, {Package.Pubrel, []}, [Package.Pubrec | _]}
+      expected_state = [Package.Pubrel, Package.Pubrec, Package.Publish, :init]
+      assert_receive {:callback, {Package.Pubcomp, []}, ^expected_state}
+    end
   end
 
   describe "publish_sync/4" do

From 8720081e553242fe73a5c27a2bb1a7fc8afd2cc6 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Thu, 7 Feb 2019 22:32:05 +0000
Subject: [PATCH 083/220] Make it possible to define pubrel and pubcomp for
 incoming flows

The callbacks now support definition of custom pubrel and pubcomps for
incoming publishes. The callback handler need to return :cont, or
{:cont, package}.
---
 lib/tortoise/connection.ex     | 12 ++++-----
 lib/tortoise/handler.ex        | 48 +++++++++++++++++++++++++---------
 test/support/test_handler.ex   |  4 +--
 test/tortoise/handler_test.exs | 20 +++++++++-----
 4 files changed, 56 insertions(+), 28 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 708b47f9..ba7be4e7 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -506,9 +506,8 @@ defmodule Tortoise.Connection do
     :ok = Inflight.update(client_id, {:received, pubrel})
 
     case Handler.execute_handle_pubrel(handler, pubrel) do
-      {:ok, %Handler{} = updated_handler, []} ->
-        # dispatch a pubcomp
-        pubcomp = %Package.Pubcomp{identifier: id}
+      {:ok, %Package.Pubcomp{identifier: ^id} = pubcomp, %Handler{} = updated_handler, []} ->
+        # dispatch the pubcomp
         :ok = Inflight.update(client_id, {:dispatch, pubcomp})
         updated_data = %State{data | handler: updated_handler}
         {:keep_state, updated_data}
@@ -568,11 +567,10 @@ defmodule Tortoise.Connection do
     :ok = Inflight.update(client_id, {:received, pubrec})
 
     case Handler.execute_handle_pubrec(handler, pubrec) do
-      {:ok, %Handler{} = updated_handler} ->
-        pubrel = %Package.Pubrel{identifier: id}
-        :ok = Inflight.update(client_id, {:dispatch, pubrel})
-
+      {:ok, %Package.Pubrel{identifier: ^id} = pubrel, %Handler{} = updated_handler,
+       _next_actions} ->
         updated_data = %State{data | handler: updated_handler}
+        :ok = Inflight.update(client_id, {:dispatch, pubrel})
         {:keep_state, updated_data}
 
       {:error, reason} ->
diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index 7cc2652c..e27e8e2a 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -154,12 +154,12 @@ defmodule Tortoise.Handler do
 
       @impl true
       def handle_pubrec(_pubrec, state) do
-        {:ok, state}
+        {:cont, state, []}
       end
 
       @impl true
       def handle_pubrel(_pubrel, state) do
-        {:ok, state}
+        {:cont, state, []}
       end
 
       @impl true
@@ -498,25 +498,47 @@ defmodule Tortoise.Handler do
   @doc false
   # @spec execute_handle_pubrec(t, Package.Pubrec.t()) ::
   #         {:ok, t} | {:error, {:invalid_next_action, term()}}
-  def execute_handle_pubrec(handler, %Package.Pubrec{} = pubrec) do
-    handler.module
-    |> apply(:handle_pubrec, [pubrec, handler.state])
-    |> handle_result(handler)
+  def execute_handle_pubrec(handler, %Package.Pubrec{identifier: id} = pubrec) do
+    apply(handler.module, :handle_pubrec, [pubrec, handler.state])
+    |> transform_result()
+    |> case do
+      {:cont, updated_state, next_actions} ->
+        pubrel = %Package.Pubrel{identifier: id}
+        updated_handler = %__MODULE__{handler | state: updated_state}
+        {:ok, pubrel, updated_handler, next_actions}
+
+      {{:cont, %Package.Pubrel{identifier: ^id} = pubrel}, updated_state, next_actions} ->
+        updated_handler = %__MODULE__{handler | state: updated_state}
+        {:ok, pubrel, updated_handler, next_actions}
+    end
   end
 
   @doc false
   # @spec execute_handle_pubrel(t, Package.Pubrel.t()) ::
   #         {:ok, t} | {:error, {:invalid_next_action, term()}}
-  def execute_handle_pubrel(handler, %Package.Pubrel{} = pubrel) do
-    case apply(handler.module, :handle_pubrel, [pubrel, handler.state]) do
-      {:ok, updated_state} ->
-        {:ok, %__MODULE__{handler | state: updated_state}, []}
-
-      {:ok, updated_state, next_actions} when is_list(next_actions) ->
-        {:ok, %__MODULE__{handler | state: updated_state}, next_actions}
+  def execute_handle_pubrel(handler, %Package.Pubrel{identifier: id} = pubrel) do
+    apply(handler.module, :handle_pubrel, [pubrel, handler.state])
+    |> transform_result()
+    |> case do
+      {:cont, updated_state, next_actions} ->
+        pubcomp = %Package.Pubcomp{identifier: id}
+        updated_handler = %__MODULE__{handler | state: updated_state}
+        {:ok, pubcomp, updated_handler, next_actions}
+
+      {{:cont, %Package.Pubcomp{identifier: ^id} = pubcomp}, updated_state, next_actions} ->
+        updated_handler = %__MODULE__{handler | state: updated_state}
+        {:ok, pubcomp, updated_handler, next_actions}
     end
   end
 
+  defp transform_result({cont, updated_state, next_actions}) when is_list(next_actions) do
+    {cont, updated_state, next_actions}
+  end
+
+  defp transform_result({cont, updated_state}) do
+    transform_result({cont, updated_state, []})
+  end
+
   @doc false
   # @spec execute_handle_pubcomp(t, Package.Pubcomp.t()) ::
   #         {:ok, t} | {:error, {:invalid_next_action, term()}}
diff --git a/test/support/test_handler.ex b/test/support/test_handler.ex
index ee63a52e..1569c67d 100644
--- a/test/support/test_handler.ex
+++ b/test/support/test_handler.ex
@@ -30,12 +30,12 @@ defmodule TestHandler do
 
   def handle_pubrec(pubrec, state) do
     send(state[:parent], {{__MODULE__, :handle_pubrec}, pubrec})
-    {:ok, state}
+    {:cont, state}
   end
 
   def handle_pubrel(pubrel, state) do
     send(state[:parent], {{__MODULE__, :handle_pubrel}, pubrel})
-    {:ok, state}
+    {:cont, state}
   end
 
   def handle_pubcomp(pubcomp, state) do
diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index 5de422de..b524bd76 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -61,7 +61,14 @@ defmodule Tortoise.HandlerTest do
 
     def handle_pubrec(pubrec, state) do
       send(state[:pid], {:pubrec, pubrec})
-      {:ok, state}
+
+      case Keyword.get(state, :pubrec) do
+        nil ->
+          {:cont, state}
+
+        fun when is_function(fun) ->
+          apply(fun, [pubrec, state])
+      end
     end
 
     def handle_pubrel(pubrel, state) do
@@ -69,7 +76,7 @@ defmodule Tortoise.HandlerTest do
 
       case Keyword.get(state, :pubrel) do
         nil ->
-          {:ok, state}
+          {:cont, state}
 
         fun when is_function(fun) ->
           apply(fun, [pubrel, state])
@@ -266,7 +273,7 @@ defmodule Tortoise.HandlerTest do
       handler = set_state(context.handler, pid: self())
       pubrec = %Package.Pubrec{identifier: 1}
 
-      assert {:ok, %Handler{} = state} =
+      assert {:ok, %Package.Pubrel{identifier: 1}, %Handler{}, []} =
                handler
                |> Handler.execute_handle_pubrec(pubrec)
 
@@ -279,20 +286,21 @@ defmodule Tortoise.HandlerTest do
       handler = set_state(context.handler, pid: self())
       pubrel = %Package.Pubrel{identifier: 1}
 
-      assert {:ok, %Handler{} = state, []} = Handler.execute_handle_pubrel(handler, pubrel)
+      assert {:ok, %Package.Pubcomp{identifier: 1}, %Handler{} = state, []} =
+               Handler.execute_handle_pubrel(handler, pubrel)
 
       assert_receive {:pubrel, ^pubrel}
     end
 
     test "return ok with next_actions", context do
       pubrel_fn = fn %Package.Pubrel{identifier: 1}, state ->
-        {:ok, state, [{:complete, %Package.Pubcomp{identifier: 1}}]}
+        {{:cont, %Package.Pubcomp{identifier: 1, properties: [{"foo", "bar"}]}}, state}
       end
 
       handler = set_state(context.handler, pid: self(), pubrel: pubrel_fn)
       pubrel = %Package.Pubrel{identifier: 1}
 
-      assert {:ok, %Handler{} = state, [{:complete, %Package.Pubcomp{identifier: 1}}]} =
+      assert {:ok, %Package.Pubcomp{identifier: 1}, %Handler{} = state, []} =
                Handler.execute_handle_pubrel(handler, pubrel)
 
       assert_receive {:pubrel, ^pubrel}

From 524c6e6c16d0a60a9cc5237a4a30535749cb05fb Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Thu, 7 Feb 2019 22:46:00 +0000
Subject: [PATCH 084/220] Allow a list of properties as continuation value in
 user handler

The list will be turned into a user defined property list for the next
protocol package.
---
 lib/tortoise/handler.ex | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index e27e8e2a..a6b4ee1e 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -507,6 +507,11 @@ defmodule Tortoise.Handler do
         updated_handler = %__MODULE__{handler | state: updated_state}
         {:ok, pubrel, updated_handler, next_actions}
 
+      {{:cont, properties}, updated_state, next_actions} when is_list(properties) ->
+        pubrel = %Package.Pubrel{identifier: id, properties: properties}
+        updated_handler = %__MODULE__{handler | state: updated_state}
+        {:ok, pubrel, updated_handler, next_actions}
+
       {{:cont, %Package.Pubrel{identifier: ^id} = pubrel}, updated_state, next_actions} ->
         updated_handler = %__MODULE__{handler | state: updated_state}
         {:ok, pubrel, updated_handler, next_actions}
@@ -525,6 +530,11 @@ defmodule Tortoise.Handler do
         updated_handler = %__MODULE__{handler | state: updated_state}
         {:ok, pubcomp, updated_handler, next_actions}
 
+      {{:cont, properties}, updated_state, next_actions} when is_list(properties) ->
+        pubcomp = %Package.Pubcomp{identifier: id, properties: properties}
+        updated_handler = %__MODULE__{handler | state: updated_state}
+        {:ok, pubcomp, updated_handler, next_actions}
+
       {{:cont, %Package.Pubcomp{identifier: ^id} = pubcomp}, updated_state, next_actions} ->
         updated_handler = %__MODULE__{handler | state: updated_state}
         {:ok, pubcomp, updated_handler, next_actions}

From 847cdf99581ac9828ae7f7f2a84b3f110d855304 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Thu, 7 Feb 2019 22:57:15 +0000
Subject: [PATCH 085/220] Changed the pubcomp handler to use the :cont
 interface as well

Also, first step to supporting next actions for pubcomp callback
---
 lib/tortoise/connection.ex     |  3 ++-
 lib/tortoise/handler.ex        | 23 ++++++++++++++---------
 test/support/test_handler.ex   |  2 +-
 test/tortoise/handler_test.exs | 11 +++++++++--
 4 files changed, 26 insertions(+), 13 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index ba7be4e7..36a5cec5 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -588,7 +588,8 @@ defmodule Tortoise.Connection do
     :ok = Inflight.update(client_id, {:received, pubcomp})
 
     case Handler.execute_handle_pubcomp(handler, pubcomp) do
-      {:ok, %Handler{} = updated_handler} ->
+      {:ok, %Handler{} = updated_handler, _next_actions} ->
+        # todo, handle next actions
         updated_data = %State{data | handler: updated_handler}
         {:keep_state, updated_data}
 
diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index a6b4ee1e..34d6a2df 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -164,7 +164,7 @@ defmodule Tortoise.Handler do
 
       @impl true
       def handle_pubcomp(_pubcomp, state) do
-        {:ok, state}
+        {:cont, state, []}
       end
 
       @impl true
@@ -541,6 +541,19 @@ defmodule Tortoise.Handler do
     end
   end
 
+  @doc false
+  # @spec execute_handle_pubcomp(t, Package.Pubcomp.t()) ::
+  #         {:ok, t} | {:error, {:invalid_next_action, term()}}
+  def execute_handle_pubcomp(handler, %Package.Pubcomp{} = pubcomp) do
+    apply(handler.module, :handle_pubcomp, [pubcomp, handler.state])
+    |> transform_result()
+    |> case do
+         {:cont, updated_state, next_actions} ->
+           updated_handler = %__MODULE__{handler | state: updated_state}
+           {:ok, updated_handler, next_actions}
+       end
+  end
+
   defp transform_result({cont, updated_state, next_actions}) when is_list(next_actions) do
     {cont, updated_state, next_actions}
   end
@@ -549,14 +562,6 @@ defmodule Tortoise.Handler do
     transform_result({cont, updated_state, []})
   end
 
-  @doc false
-  # @spec execute_handle_pubcomp(t, Package.Pubcomp.t()) ::
-  #         {:ok, t} | {:error, {:invalid_next_action, term()}}
-  def execute_handle_pubcomp(handler, %Package.Pubcomp{} = pubcomp) do
-    handler.module
-    |> apply(:handle_pubcomp, [pubcomp, handler.state])
-    |> handle_result(handler)
-  end
 
   # handle the user defined return from the callback
   defp handle_result({:ok, updated_state}, handler) do
diff --git a/test/support/test_handler.ex b/test/support/test_handler.ex
index 1569c67d..18307cc1 100644
--- a/test/support/test_handler.ex
+++ b/test/support/test_handler.ex
@@ -40,7 +40,7 @@ defmodule TestHandler do
 
   def handle_pubcomp(pubcomp, state) do
     send(state[:parent], {{__MODULE__, :handle_pubcomp}, pubcomp})
-    {:ok, state}
+    {:cont, state}
   end
 
   def subscription(status, topic_filter, state) do
diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index b524bd76..9012474b 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -85,7 +85,14 @@ defmodule Tortoise.HandlerTest do
 
     def handle_pubcomp(pubcomp, state) do
       send(state[:pid], {:pubcomp, pubcomp})
-      {:ok, state}
+
+      case Keyword.get(state, :pubcomp) do
+        nil ->
+          {:cont, state}
+
+        fun when is_function(fun) ->
+          apply(fun, [pubcomp, state])
+      end
     end
 
     def handle_disconnect(disconnect, state) do
@@ -312,7 +319,7 @@ defmodule Tortoise.HandlerTest do
       handler = set_state(context.handler, pid: self())
       pubcomp = %Package.Pubcomp{identifier: 1}
 
-      assert {:ok, %Handler{} = state} =
+      assert {:ok, %Handler{} = state, []} =
                handler
                |> Handler.execute_handle_pubcomp(pubcomp)
 

From 06a5fdae9e8a2f7a51695eec0c0580cdbc7e6ae1 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Thu, 7 Feb 2019 23:38:33 +0000
Subject: [PATCH 086/220] Validate next actions and pass them on to publishes

---
 lib/tortoise/connection.ex     | 12 ++---
 lib/tortoise/handler.ex        | 80 +++++++++++++++++++++++++++-------
 test/support/test_handler.ex   | 16 ++++++-
 test/tortoise/handler_test.exs | 14 ++----
 4 files changed, 91 insertions(+), 31 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 36a5cec5..fe4ea3a2 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -456,7 +456,8 @@ defmodule Tortoise.Connection do
         %State{handler: handler} = data
       ) do
     case Handler.execute_handle_publish(handler, publish) do
-      {:ok, %Handler{} = updated_handler} ->
+      {:ok, %Handler{} = updated_handler, _next_actions} ->
+        # todo, handle next_actions
         updated_data = %State{data | handler: updated_handler}
         {:keep_state, updated_data}
 
@@ -528,9 +529,10 @@ defmodule Tortoise.Connection do
         %State{client_id: client_id, handler: handler} = data
       ) do
     case Handler.execute_handle_publish(handler, publish) do
-      {:ok, %Handler{} = updated_handler} ->
+      {:ok, %Package.Puback{identifier: ^id} = puback, %Handler{} = updated_handler,
+       _next_actions} ->
+        # todo handle next actions
         # respond with a puback
-        puback = %Package.Puback{identifier: id}
         :ok = Inflight.update(client_id, {:dispatch, puback})
         # - - -
         updated_data = %State{data | handler: updated_handler}
@@ -547,9 +549,9 @@ defmodule Tortoise.Connection do
         %State{client_id: client_id, handler: handler} = data
       ) do
     case Handler.execute_handle_publish(handler, publish) do
-      {:ok, %Handler{} = updated_handler} ->
+      {:ok, %Package.Pubrec{identifier: ^id} = pubrec, %Handler{} = updated_handler,
+       _next_actions} ->
         # respond with pubrec
-        pubrec = %Package.Pubrec{identifier: id}
         :ok = Inflight.update(client_id, {:dispatch, pubrec})
         # - - -
         updated_data = %State{data | handler: updated_handler}
diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index 34d6a2df..19de7561 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -82,7 +82,7 @@ defmodule Tortoise.Handler do
 
       def handle_publish(_, %{topic: topic}, state) do
         next_actions = [{:unsubscribe, topic}]
-        {:ok, state, next_actions}
+        {:cont, state, next_actions}
       end
 
   The next actions has to be a list, even if there is only one next
@@ -144,7 +144,7 @@ defmodule Tortoise.Handler do
 
       @impl true
       def handle_publish(_topic_list, _publish, state) do
-        {:ok, state}
+        {:cont, state, []}
       end
 
       @impl true
@@ -313,7 +313,7 @@ defmodule Tortoise.Handler do
 
       def handle_publish(["room", room, "temp"], publish, state) do
         :ok = Temperature.record(room, publish.payload)
-        {:ok, state}
+        {:cont, state}
       end
 
   Notice; the `handle_publish/3`-callback run inside the connection
@@ -330,8 +330,8 @@ defmodule Tortoise.Handler do
   reenter the loop and perform the listed actions.
   """
   @callback handle_publish(topic_levels, payload, state :: term()) ::
-              {:ok, new_state}
-              | {:ok, new_state, [next_action()]}
+              {:cont, new_state}
+              | {:cont, new_state, [next_action()]}
             when new_state: term(),
                  topic_levels: [String.t()],
                  payload: Tortoise.payload()
@@ -478,12 +478,51 @@ defmodule Tortoise.Handler do
   @doc false
   @spec execute_handle_publish(t, Package.Publish.t()) ::
           {:ok, t} | {:error, {:invalid_next_action, term()}}
-  def execute_handle_publish(handler, %Package.Publish{} = publish) do
+  def execute_handle_publish(handler, %Package.Publish{qos: 0} = publish) do
     topic_list = String.split(publish.topic, "/")
 
-    handler.module
-    |> apply(:handle_publish, [topic_list, publish, handler.state])
-    |> handle_result(handler)
+    apply(handler.module, :handle_publish, [topic_list, publish, handler.state])
+    |> transform_result()
+    |> case do
+      {:cont, updated_state, next_actions} ->
+        updated_handler = %__MODULE__{handler | state: updated_state}
+        {:ok, updated_handler, next_actions}
+
+      {:error, reason} ->
+        {:error, reason}
+    end
+  end
+
+  def execute_handle_publish(handler, %Package.Publish{identifier: id, qos: 1} = publish) do
+    topic_list = String.split(publish.topic, "/")
+
+    apply(handler.module, :handle_publish, [topic_list, publish, handler.state])
+    |> transform_result()
+    |> case do
+      {:cont, updated_state, next_actions} ->
+        puback = %Package.Puback{identifier: id}
+        updated_handler = %__MODULE__{handler | state: updated_state}
+        {:ok, puback, updated_handler, next_actions}
+
+      {:error, reason} ->
+        {:error, reason}
+    end
+  end
+
+  def execute_handle_publish(handler, %Package.Publish{identifier: id, qos: 2} = publish) do
+    topic_list = String.split(publish.topic, "/")
+
+    apply(handler.module, :handle_publish, [topic_list, publish, handler.state])
+    |> transform_result()
+    |> case do
+      {:cont, updated_state, next_actions} ->
+        pubrec = %Package.Pubrec{identifier: id}
+        updated_handler = %__MODULE__{handler | state: updated_state}
+        {:ok, pubrec, updated_handler, next_actions}
+
+      {:error, reason} ->
+        {:error, reason}
+    end
   end
 
   @doc false
@@ -515,6 +554,9 @@ defmodule Tortoise.Handler do
       {{:cont, %Package.Pubrel{identifier: ^id} = pubrel}, updated_state, next_actions} ->
         updated_handler = %__MODULE__{handler | state: updated_state}
         {:ok, pubrel, updated_handler, next_actions}
+
+      {:error, reason} ->
+        {:error, reason}
     end
   end
 
@@ -538,6 +580,9 @@ defmodule Tortoise.Handler do
       {{:cont, %Package.Pubcomp{identifier: ^id} = pubcomp}, updated_state, next_actions} ->
         updated_handler = %__MODULE__{handler | state: updated_state}
         {:ok, pubcomp, updated_handler, next_actions}
+
+      {:error, reason} ->
+        {:error, reason}
     end
   end
 
@@ -548,21 +593,26 @@ defmodule Tortoise.Handler do
     apply(handler.module, :handle_pubcomp, [pubcomp, handler.state])
     |> transform_result()
     |> case do
-         {:cont, updated_state, next_actions} ->
-           updated_handler = %__MODULE__{handler | state: updated_state}
-           {:ok, updated_handler, next_actions}
-       end
+      {:cont, updated_state, next_actions} ->
+        updated_handler = %__MODULE__{handler | state: updated_state}
+        {:ok, updated_handler, next_actions}
+    end
   end
 
   defp transform_result({cont, updated_state, next_actions}) when is_list(next_actions) do
-    {cont, updated_state, next_actions}
+    case Enum.split_with(next_actions, &valid_next_action?/1) do
+      {_, []} ->
+        {cont, updated_state, next_actions}
+
+      {_, invalid_next_actions} ->
+        {:error, {:invalid_next_action, invalid_next_actions}}
+    end
   end
 
   defp transform_result({cont, updated_state}) do
     transform_result({cont, updated_state, []})
   end
 
-
   # handle the user defined return from the callback
   defp handle_result({:ok, updated_state}, handler) do
     {:ok, %__MODULE__{handler | state: updated_state}}
diff --git a/test/support/test_handler.ex b/test/support/test_handler.ex
index 18307cc1..e7d49338 100644
--- a/test/support/test_handler.ex
+++ b/test/support/test_handler.ex
@@ -1,6 +1,8 @@
 defmodule TestHandler do
   use Tortoise.Handler
 
+  alias Tortoise.Package
+
   def init(opts) do
     state = Enum.into(opts, %{})
     send(state[:parent], {{__MODULE__, :init}, opts})
@@ -22,10 +24,22 @@ defmodule TestHandler do
     {:stop, :normal, state}
   end
 
+  def handle_publish(_topic, %Package.Publish{qos: 1} = publish, state) do
+    # data = %{topic: Enum.join(topic, "/"), payload: payload}
+    send(state[:parent], {{__MODULE__, :handle_publish}, publish})
+    {:cont, state}
+  end
+
+  def handle_publish(_topic, %Package.Publish{qos: 2} = publish, state) do
+    # data = %{topic: Enum.join(topic, "/"), payload: payload}
+    send(state[:parent], {{__MODULE__, :handle_publish}, publish})
+    {:cont, state}
+  end
+
   def handle_publish(_topic, publish, state) do
     # data = %{topic: Enum.join(topic, "/"), payload: payload}
     send(state[:parent], {{__MODULE__, :handle_publish}, publish})
-    {:ok, state}
+    {:cont, state}
   end
 
   def handle_pubrec(pubrec, state) do
diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index 9012474b..217277e7 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -41,12 +41,12 @@ defmodule Tortoise.HandlerTest do
     # with next actions
     def handle_publish(topic, publish, %{next_actions: next_actions} = state) do
       send(state[:pid], {:publish, topic, publish})
-      {:ok, state, next_actions}
+      {:cont, state, next_actions}
     end
 
     def handle_publish(topic, publish, state) do
       send(state[:pid], {:publish, topic, publish})
-      {:ok, state}
+      {:cont, state}
     end
 
     def terminate(reason, state) do
@@ -168,7 +168,7 @@ defmodule Tortoise.HandlerTest do
       topic = "foo/bar"
       publish = %Package.Publish{topic: topic, payload: payload}
 
-      assert {:ok, %Handler{}} = Handler.execute_handle_publish(handler, publish)
+      assert {:ok, %Handler{}, []} = Handler.execute_handle_publish(handler, publish)
       # the topic will be in the form of a list making it possible to
       # pattern match on the topic levels
       assert_receive {:publish, topic_list, ^publish}
@@ -184,9 +184,7 @@ defmodule Tortoise.HandlerTest do
       topic = "foo/bar"
       publish = %Package.Publish{topic: topic, payload: payload}
 
-      assert {:ok, %Handler{}} = Handler.execute_handle_publish(handler, publish)
-
-      assert_receive {:next_action, {:subscribe, "foo/bar", qos: 0}}
+      assert {:ok, %Handler{}, ^next_actions} = Handler.execute_handle_publish(handler, publish)
 
       # the topic will be in the form of a list making it possible to
       # pattern match on the topic levels
@@ -206,10 +204,6 @@ defmodule Tortoise.HandlerTest do
       assert {:error, {:invalid_next_action, [{:invalid, "bar"}]}} =
                Handler.execute_handle_publish(handler, publish)
 
-      refute_receive {:next_action, {:invalid, "bar"}}
-      # we should not receive the otherwise valid next_action
-      refute_receive {:next_action, {:unsubscribe, "foo/bar"}}
-
       # the callback is still run so lets check the received data
       assert_receive {:publish, topic_list, ^publish}
       assert is_list(topic_list)

From 042eef428a12a1516deb0e628363214b19b9ad58 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Fri, 8 Feb 2019 20:28:47 +0000
Subject: [PATCH 087/220] Use a returned a list of values for publishes as user
 defined props

Returning `{:cont, [{"foo", "bar"}]}` will create a puback or pubrec
(depending on the qos) with foo:bar as a user defined property.
---
 lib/tortoise/handler.ex | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index 19de7561..2bf30547 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -504,6 +504,11 @@ defmodule Tortoise.Handler do
         updated_handler = %__MODULE__{handler | state: updated_state}
         {:ok, puback, updated_handler, next_actions}
 
+      {{:cont, properties}, updated_state, next_actions} when is_list(properties) ->
+        puback = %Package.Puback{identifier: id, properties: properties}
+        updated_handler = %__MODULE__{handler | state: updated_state}
+        {:ok, puback, updated_handler, next_actions}
+
       {:error, reason} ->
         {:error, reason}
     end
@@ -520,6 +525,11 @@ defmodule Tortoise.Handler do
         updated_handler = %__MODULE__{handler | state: updated_state}
         {:ok, pubrec, updated_handler, next_actions}
 
+      {{:cont, properties}, updated_state, next_actions} when is_list(properties) ->
+        pubrec = %Package.Pubrec{identifier: id, properties: properties}
+        updated_handler = %__MODULE__{handler | state: updated_state}
+        {:ok, pubrec, updated_handler, next_actions}
+
       {:error, reason} ->
         {:error, reason}
     end

From 3d28350e29e9eef2f4351eb059f338faffca1814 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Fri, 8 Feb 2019 20:45:37 +0000
Subject: [PATCH 088/220] Update the callback for handle_unsuback

It now support the new callback form, supporting next actions for
handle unsuback.
---
 lib/tortoise/handler.ex        | 37 ++++++++++------------------------
 test/support/test_handler.ex   |  2 +-
 test/tortoise/handler_test.exs |  2 +-
 3 files changed, 13 insertions(+), 28 deletions(-)

diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index 2bf30547..48433736 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -445,34 +445,19 @@ defmodule Tortoise.Handler do
   end
 
   @doc false
-  @spec execute_handle_unsuback(t, Package.Unsubscribe.t(), Package.Unsuback.t()) :: {:ok, t}
+  @spec execute_handle_unsuback(t, Package.Unsubscribe.t(), Package.Unsuback.t()) ::
+          {:ok, t, [any()]}
   def execute_handle_unsuback(handler, unsubscribe, unsuback) do
-    handler.module
-    |> apply(:handle_unsuback, [unsubscribe, unsuback, handler.state])
-    |> handle_unsuback_result(handler)
-  end
-
-  defp handle_unsuback_result({:ok, updated_state}, handler) do
-    {:ok, %__MODULE__{handler | state: updated_state}, []}
-  end
-
-  defp handle_unsuback_result({:ok, updated_state, next_actions}, handler)
-       when is_list(next_actions) do
-    {:ok, %__MODULE__{handler | state: updated_state}, next_actions}
-  end
+    apply(handler.module, :handle_unsuback, [unsubscribe, unsuback, handler.state])
+    |> transform_result()
+    |> case do
+      {:cont, updated_state, next_actions} ->
+        updated_handler = %__MODULE__{handler | state: updated_state}
+        {:ok, updated_handler, next_actions}
 
-  @doc false
-  @spec execute_unsubscribe(t, result) :: {:ok, t}
-        when result: [Tortoise.topic_filter()]
-  def execute_unsubscribe(handler, results) do
-    Enum.reduce(results, {:ok, handler}, fn topic_filter, {:ok, handler} ->
-      handler.module
-      |> apply(:subscription, [:down, topic_filter, handler.state])
-      |> handle_result(handler)
-
-      # _, {:stop, acc} ->
-      #   {:stop, acc}
-    end)
+      {:error, reason} ->
+        {:error, reason}
+    end
   end
 
   @doc false
diff --git a/test/support/test_handler.ex b/test/support/test_handler.ex
index e7d49338..9f90073a 100644
--- a/test/support/test_handler.ex
+++ b/test/support/test_handler.ex
@@ -70,6 +70,6 @@ defmodule TestHandler do
 
   def handle_unsuback(unsubscribe, unsuback, state) do
     send(state[:parent], {{__MODULE__, :handle_unsuback}, {unsubscribe, unsuback}})
-    {:ok, state}
+    {:cont, state}
   end
 end
diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index 217277e7..38ad31ab 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -35,7 +35,7 @@ defmodule Tortoise.HandlerTest do
 
     def handle_unsuback(unsubscribe, unsuback, state) do
       send(state[:pid], {:unsuback, {unsubscribe, unsuback}})
-      {:ok, state}
+      {:cont, state}
     end
 
     # with next actions

From 405ddfb1d0376520845e938ef3a6e9927dc0d375 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Fri, 8 Feb 2019 20:52:49 +0000
Subject: [PATCH 089/220] Support the new callback style for handle_suback

It now takes a `{:cont, updated_state, next_actions}` form as the
return expression.
---
 lib/tortoise/handler.ex        | 20 +++++++++-----------
 test/support/test_handler.ex   |  8 +-------
 test/tortoise/handler_test.exs |  2 +-
 3 files changed, 11 insertions(+), 19 deletions(-)

diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index 48433736..e137f7e9 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -430,18 +430,16 @@ defmodule Tortoise.Handler do
   @doc false
   @spec execute_handle_suback(t, Package.Subscribe.t(), Package.Suback.t()) :: {:ok, t}
   def execute_handle_suback(handler, subscribe, suback) do
-    handler.module
-    |> apply(:handle_suback, [subscribe, suback, handler.state])
-    |> handle_suback_result(handler)
-  end
-
-  defp handle_suback_result({:ok, updated_state}, handler) do
-    {:ok, %__MODULE__{handler | state: updated_state}, []}
-  end
+    apply(handler.module, :handle_suback, [subscribe, suback, handler.state])
+    |> transform_result()
+    |> case do
+      {:cont, updated_state, next_actions} ->
+        updated_handler = %__MODULE__{handler | state: updated_state}
+        {:ok, updated_handler, next_actions}
 
-  defp handle_suback_result({:ok, updated_state, next_actions}, handler)
-       when is_list(next_actions) do
-    {:ok, %__MODULE__{handler | state: updated_state}, next_actions}
+      {:error, reason} ->
+        {:error, reason}
+    end
   end
 
   @doc false
diff --git a/test/support/test_handler.ex b/test/support/test_handler.ex
index 9f90073a..c41866e4 100644
--- a/test/support/test_handler.ex
+++ b/test/support/test_handler.ex
@@ -57,15 +57,9 @@ defmodule TestHandler do
     {:cont, state}
   end
 
-  def subscription(status, topic_filter, state) do
-    data = %{status: status, topic_filter: topic_filter}
-    send(state[:parent], {{__MODULE__, :subscription}, data})
-    {:ok, state}
-  end
-
   def handle_suback(subscribe, suback, state) do
     send(state[:parent], {{__MODULE__, :handle_suback}, {subscribe, suback}})
-    {:ok, state}
+    {:cont, state}
   end
 
   def handle_unsuback(unsubscribe, unsuback, state) do
diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index 38ad31ab..5e257427 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -30,7 +30,7 @@ defmodule Tortoise.HandlerTest do
 
     def handle_suback(subscribe, suback, state) do
       send(state[:pid], {:suback, {subscribe, suback}})
-      {:ok, state}
+      {:cont, state}
     end
 
     def handle_unsuback(unsubscribe, unsuback, state) do

From 3a22285a0fd0f123305fba8635c30f8462149d20 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Fri, 8 Feb 2019 21:00:17 +0000
Subject: [PATCH 090/220] Update handle_puback to the new callback return
 expression

---
 lib/tortoise/connection.ex     |  3 ++-
 lib/tortoise/handler.ex        | 14 +++++++++++---
 test/tortoise/handler_test.exs | 11 +++++++++--
 3 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index fe4ea3a2..feb7b4cd 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -487,7 +487,8 @@ defmodule Tortoise.Connection do
     :ok = Inflight.update(client_id, {:received, puback})
 
     case Handler.execute_handle_puback(handler, puback) do
-      {:ok, %Handler{} = updated_handler} ->
+      {:ok, %Handler{} = updated_handler, _next_actions} ->
+        # todo, handle next_actions
         updated_data = %State{data | handler: updated_handler}
         {:keep_state, updated_data}
 
diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index e137f7e9..60b18c4c 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -522,9 +522,17 @@ defmodule Tortoise.Handler do
   # @spec execute_handle_puback(t, Package.Puback.t()) ::
   #         {:ok, t} | {:error, {:invalid_next_action, term()}}
   def execute_handle_puback(handler, %Package.Puback{} = puback) do
-    handler.module
-    |> apply(:handle_puback, [puback, handler.state])
-    |> handle_result(handler)
+
+    apply(handler.module, :handle_puback, [puback, handler.state])
+    |> transform_result()
+    |> case do
+      {:cont, updated_state, next_actions} ->
+        updated_handler = %__MODULE__{handler | state: updated_state}
+        {:ok, updated_handler, next_actions}
+
+      {:error, reason} ->
+        {:error, reason}
+    end
   end
 
   @doc false
diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index 5e257427..6f2bb662 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -56,7 +56,14 @@ defmodule Tortoise.HandlerTest do
 
     def handle_puback(puback, state) do
       send(state[:pid], {:puback, puback})
-      {:ok, state}
+
+      case Keyword.get(state, :puback) do
+        nil ->
+          {:cont, state}
+
+        fun when is_function(fun) ->
+          apply(fun, [puback, state])
+      end
     end
 
     def handle_pubrec(pubrec, state) do
@@ -260,7 +267,7 @@ defmodule Tortoise.HandlerTest do
       handler = set_state(context.handler, pid: self())
       puback = %Package.Puback{identifier: 1}
 
-      assert {:ok, %Handler{} = state} =
+      assert {:ok, %Handler{} = state, []} =
                handler
                |> Handler.execute_handle_puback(puback)
 

From 1927a381586e716a3cc15702a0b75e3bae0e4383 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Fri, 8 Feb 2019 21:07:14 +0000
Subject: [PATCH 091/220] Support more than just empty lists as next actions
 for pubcomp

Currently it does nothing with the next actions, but the
first (second) step is taken.
---
 lib/tortoise/connection.ex | 4 +++-
 lib/tortoise/handler.ex    | 1 -
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index feb7b4cd..27718d6d 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -508,7 +508,9 @@ defmodule Tortoise.Connection do
     :ok = Inflight.update(client_id, {:received, pubrel})
 
     case Handler.execute_handle_pubrel(handler, pubrel) do
-      {:ok, %Package.Pubcomp{identifier: ^id} = pubcomp, %Handler{} = updated_handler, []} ->
+      {:ok, %Package.Pubcomp{identifier: ^id} = pubcomp, %Handler{} = updated_handler,
+       _next_actions} ->
+        # todo, handle next actions
         # dispatch the pubcomp
         :ok = Inflight.update(client_id, {:dispatch, pubcomp})
         updated_data = %State{data | handler: updated_handler}
diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index 60b18c4c..c7de42a5 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -522,7 +522,6 @@ defmodule Tortoise.Handler do
   # @spec execute_handle_puback(t, Package.Puback.t()) ::
   #         {:ok, t} | {:error, {:invalid_next_action, term()}}
   def execute_handle_puback(handler, %Package.Puback{} = puback) do
-
     apply(handler.module, :handle_puback, [puback, handler.state])
     |> transform_result()
     |> case do

From bee3d68ea4522122ff94d288bea0b0c3128b8451 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Fri, 8 Feb 2019 21:19:55 +0000
Subject: [PATCH 092/220] Updated handle_connack return expression to the new
 format

---
 lib/tortoise/handler.ex        | 14 +++++++++-----
 test/tortoise/handler_test.exs |  2 +-
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index c7de42a5..a61978a0 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -139,7 +139,7 @@ defmodule Tortoise.Handler do
 
       @impl true
       def handle_connack(_connack, state) do
-        {:ok, state}
+        {:cont, state}
       end
 
       @impl true
@@ -397,11 +397,15 @@ defmodule Tortoise.Handler do
   @spec execute_handle_connack(t, Package.Connack.t()) ::
           {:ok, t} | {:error, {:invalid_next_action, term()}}
   def execute_handle_connack(handler, %Package.Connack{} = connack) do
-    handler.module
-    |> apply(:handle_connack, [connack, handler.state])
+    apply(handler.module, :handle_connack, [connack, handler.state])
+    |> transform_result()
     |> case do
-      {:ok, updated_state} ->
-        {:ok, %__MODULE__{handler | state: updated_state}, []}
+      {:cont, updated_state, next_actions} ->
+        updated_handler = %__MODULE__{handler | state: updated_state}
+        {:ok, updated_handler, next_actions}
+
+      {:error, reason} ->
+        {:error, reason}
     end
   end
 
diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index 6f2bb662..add3c2c7 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -25,7 +25,7 @@ defmodule Tortoise.HandlerTest do
 
     def handle_connack(connack, state) do
       send(state[:pid], {:connack, connack})
-      {:ok, state}
+      {:cont, state}
     end
 
     def handle_suback(subscribe, suback, state) do

From c2cb8fef72660453edd18a5944e722ff31e3cf8f Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Fri, 8 Feb 2019 22:02:43 +0000
Subject: [PATCH 093/220] Update the connection callback to follow the new
 format

---
 lib/tortoise/connection.ex     |  6 ++++--
 lib/tortoise/handler.ex        | 15 +++++++++++----
 lib/tortoise/handler/logger.ex |  6 +++---
 test/support/test_handler.ex   |  2 +-
 test/tortoise/handler_test.exs | 14 ++++++--------
 5 files changed, 25 insertions(+), 18 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 27718d6d..76060f94 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -413,7 +413,8 @@ defmodule Tortoise.Connection do
         } = data
       ) do
     case Handler.execute_handle_connack(handler, connack) do
-      {:ok, %Handler{} = updated_handler, []} ->
+      {:ok, %Handler{} = updated_handler, _next_actions} ->
+        # todo, add support for next actions
         :ok = Tortoise.Registry.put_meta(via_name(client_id), data.connection)
         :ok = Events.dispatch(client_id, :connection, data.connection)
         :ok = Events.dispatch(client_id, :status, :connected)
@@ -768,7 +769,8 @@ defmodule Tortoise.Connection do
         %State{handler: handler} = data
       ) do
     case Handler.execute_connection(handler, status) do
-      {:ok, %Handler{} = updated_handler} ->
+      {:ok, %Handler{} = updated_handler, _next_actions} ->
+        # todo handle next_actions
         updated_data = %State{data | handler: updated_handler}
         {:keep_state, updated_data}
 
diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index a61978a0..351efed2 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -134,7 +134,7 @@ defmodule Tortoise.Handler do
 
       @impl true
       def connection(_status, state) do
-        {:ok, state}
+        {:cont, state}
       end
 
       @impl true
@@ -389,9 +389,16 @@ defmodule Tortoise.Handler do
   @spec execute_connection(t, status) :: {:ok, t}
         when status: :up | :down
   def execute_connection(handler, status) do
-    handler.module
-    |> apply(:connection, [status, handler.state])
-    |> handle_result(handler)
+    apply(handler.module, :connection, [status, handler.state])
+    |> transform_result()
+    |> case do
+      {:cont, updated_state, next_actions} ->
+        updated_handler = %__MODULE__{handler | state: updated_state}
+        {:ok, updated_handler, next_actions}
+
+      {:error, reason} ->
+        {:error, reason}
+    end
   end
 
   @spec execute_handle_connack(t, Package.Connack.t()) ::
diff --git a/lib/tortoise/handler/logger.ex b/lib/tortoise/handler/logger.ex
index 4c28c2f9..bb3f9765 100644
--- a/lib/tortoise/handler/logger.ex
+++ b/lib/tortoise/handler/logger.ex
@@ -15,17 +15,17 @@ defmodule Tortoise.Handler.Logger do
 
   def connection(:up, state) do
     Logger.info("Connection has been established")
-    {:ok, state}
+    {:cont, state}
   end
 
   def connection(:down, state) do
     Logger.warn("Connection has been dropped")
-    {:ok, state}
+    {:cont, state}
   end
 
   def connection(:terminating, state) do
     Logger.warn("Connection is terminating")
-    {:ok, state}
+    {:cont, state}
   end
 
   def subscription(:up, topic, state) do
diff --git a/test/support/test_handler.ex b/test/support/test_handler.ex
index c41866e4..c9fcaddb 100644
--- a/test/support/test_handler.ex
+++ b/test/support/test_handler.ex
@@ -16,7 +16,7 @@ defmodule TestHandler do
 
   def connection(status, state) do
     send(state[:parent], {{__MODULE__, :connection}, status})
-    {:ok, state}
+    {:cont, state}
   end
 
   def handle_disconnect(disconnect, state) do
diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index add3c2c7..d1ad2a13 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -15,12 +15,12 @@ defmodule Tortoise.HandlerTest do
 
     def connection(status, %{next_actions: next_actions} = state) do
       send(state[:pid], {:connection, status})
-      {:ok, state, next_actions}
+      {:cont, state, next_actions}
     end
 
     def connection(status, state) do
       send(state[:pid], {:connection, status})
-      {:ok, state}
+      {:cont, state}
     end
 
     def handle_connack(connack, state) do
@@ -127,10 +127,10 @@ defmodule Tortoise.HandlerTest do
   describe "execute connection/2" do
     test "return ok-tuple", context do
       handler = set_state(context.handler, %{pid: self()})
-      assert {:ok, %Handler{}} = Handler.execute_connection(handler, :up)
+      assert {:ok, %Handler{}, []} = Handler.execute_connection(handler, :up)
       assert_receive {:connection, :up}
 
-      assert {:ok, %Handler{}} = Handler.execute_connection(handler, :down)
+      assert {:ok, %Handler{}, []} = Handler.execute_connection(handler, :down)
       assert_receive {:connection, :down}
     end
 
@@ -141,15 +141,13 @@ defmodule Tortoise.HandlerTest do
         context.handler
         |> set_state(%{pid: self(), next_actions: next_actions})
 
-      assert {:ok, %Handler{}} = Handler.execute_connection(handler, :up)
+      assert {:ok, %Handler{}, ^next_actions} = Handler.execute_connection(handler, :up)
 
       assert_receive {:connection, :up}
-      assert_receive {:next_action, {:subscribe, "foo/bar", qos: 0}}
 
-      assert {:ok, %Handler{}} = Handler.execute_connection(handler, :down)
+      assert {:ok, %Handler{}, ^next_actions} = Handler.execute_connection(handler, :down)
 
       assert_receive {:connection, :down}
-      assert_receive {:next_action, {:subscribe, "foo/bar", qos: 0}}
     end
   end
 

From 9d96907c09cba30f32c4159e9cdb5024151428d8 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Fri, 8 Feb 2019 22:10:06 +0000
Subject: [PATCH 094/220] Get rid of the now unused handle_result helper in
 handler.ex

---
 lib/tortoise/handler.ex | 19 -------------------
 1 file changed, 19 deletions(-)

diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index 351efed2..9a38df80 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -624,25 +624,6 @@ defmodule Tortoise.Handler do
     transform_result({cont, updated_state, []})
   end
 
-  # handle the user defined return from the callback
-  defp handle_result({:ok, updated_state}, handler) do
-    {:ok, %__MODULE__{handler | state: updated_state}}
-  end
-
-  defp handle_result({:ok, updated_state, next_actions}, handler)
-       when is_list(next_actions) do
-    case Enum.split_with(next_actions, &valid_next_action?/1) do
-      {next_actions, []} ->
-        # send the next actions to the process mailbox. Notice that
-        # this code is run in the context of the connection controller
-        for action <- next_actions, do: send(self(), {:next_action, action})
-        {:ok, %__MODULE__{handler | state: updated_state}}
-
-      {_, errors} ->
-        {:error, {:invalid_next_action, errors}}
-    end
-  end
-
   defp valid_next_action?({:subscribe, topic, opts}) do
     is_binary(topic) and is_list(opts)
   end

From a1f82109b2656df5942e23ed9c9a20639d06237d Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Fri, 8 Feb 2019 22:50:06 +0000
Subject: [PATCH 095/220] Update the disconnect callback to follow the new
 style

Make the transform result helper accept `{:stop, reason, state}`
---
 lib/tortoise/connection.ex     |  3 ++-
 lib/tortoise/handler.ex        | 13 +++++++++----
 test/tortoise/handler_test.exs |  2 +-
 3 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 76060f94..c7d591de 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -904,7 +904,8 @@ defmodule Tortoise.Connection do
         %State{handler: handler} = data
       ) do
     case Handler.execute_handle_disconnect(handler, disconnect) do
-      {:ok, updated_handler, []} ->
+      {:cont, updated_handler, _next_actions} ->
+        # todo, handle next_actions
         {:keep_state, %State{data | handler: updated_handler}}
 
       {:stop, reason, updated_handler} ->
diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index 9a38df80..bc12fa78 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -419,11 +419,12 @@ defmodule Tortoise.Handler do
   @doc false
   @spec execute_handle_disconnect(t, %Package.Disconnect{}) :: {:stop, term(), t}
   def execute_handle_disconnect(handler, %Package.Disconnect{} = disconnect) do
-    handler.module
-    |> apply(:handle_disconnect, [disconnect, handler.state])
+    apply(handler.module, :handle_disconnect, [disconnect, handler.state])
+    |> transform_result()
     |> case do
-      {:ok, updated_state} ->
-        {:ok, %__MODULE__{handler | state: updated_state}, []}
+      {:cont, updated_state, next_actions} ->
+        updated_handler = %__MODULE__{handler | state: updated_state}
+        {:ok, updated_handler, next_actions}
 
       {:stop, reason, updated_state} ->
         {:stop, reason, %__MODULE__{handler | state: updated_state}}
@@ -610,6 +611,10 @@ defmodule Tortoise.Handler do
     end
   end
 
+  defp transform_result({:stop, reason, updated_state}) do
+    {:stop, reason, updated_state}
+  end
+
   defp transform_result({cont, updated_state, next_actions}) when is_list(next_actions) do
     case Enum.split_with(next_actions, &valid_next_action?/1) do
       {_, []} ->
diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index d1ad2a13..155690d7 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -104,7 +104,7 @@ defmodule Tortoise.HandlerTest do
 
     def handle_disconnect(disconnect, state) do
       send(state[:pid], {:disconnect, disconnect})
-      {:ok, state}
+      {:cont, state}
     end
   end
 

From c2956836a1874ff5fff614f2e4046c8158d723d5 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 9 Feb 2019 10:00:26 +0000
Subject: [PATCH 096/220] Make the handler tests a bit more flexible for the
 publish tests

The caller can now pass in a function that should get executed in the
callback handler, making it possible to return all the responses one
would desire.
---
 test/tortoise/handler_test.exs | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index 155690d7..e2b997b0 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -38,15 +38,16 @@ defmodule Tortoise.HandlerTest do
       {:cont, state}
     end
 
-    # with next actions
-    def handle_publish(topic, publish, %{next_actions: next_actions} = state) do
-      send(state[:pid], {:publish, topic, publish})
-      {:cont, state, next_actions}
-    end
-
     def handle_publish(topic, publish, state) do
       send(state[:pid], {:publish, topic, publish})
-      {:cont, state}
+
+      case Keyword.get(state, :publish) do
+        nil ->
+          {:cont, state}
+
+        fun when is_function(fun) ->
+          apply(fun, [publish, state])
+      end
     end
 
     def terminate(reason, state) do
@@ -168,7 +169,7 @@ defmodule Tortoise.HandlerTest do
 
   describe "execute handle_publish/2" do
     test "return ok-2", context do
-      handler = set_state(context.handler, %{pid: self()})
+      handler = set_state(context.handler, [pid: self()])
       payload = :crypto.strong_rand_bytes(5)
       topic = "foo/bar"
       publish = %Package.Publish{topic: topic, payload: payload}
@@ -183,8 +184,8 @@ defmodule Tortoise.HandlerTest do
 
     test "return ok-3", context do
       next_actions = [{:subscribe, "foo/bar", [qos: 0]}]
-      opts = %{pid: self(), next_actions: next_actions}
-      handler = set_state(context.handler, opts)
+      publish_fn = fn (%Package.Publish{}, state) -> {:cont, state, next_actions} end
+      handler = set_state(context.handler, [pid: self(), publish: publish_fn])
       payload = :crypto.strong_rand_bytes(5)
       topic = "foo/bar"
       publish = %Package.Publish{topic: topic, payload: payload}
@@ -200,8 +201,8 @@ defmodule Tortoise.HandlerTest do
 
     test "return ok-3 with invalid next action", context do
       next_actions = [{:unsubscribe, "foo/bar"}, {:invalid, "bar"}]
-      opts = %{pid: self(), next_actions: next_actions}
-      handler = set_state(context.handler, opts)
+      publish_fn = fn %Package.Publish{}, state -> {:cont, state, next_actions} end
+      handler = set_state(context.handler, [pid: self(), publish: publish_fn])
       payload = :crypto.strong_rand_bytes(5)
       topic = "foo/bar"
       publish = %Package.Publish{topic: topic, payload: payload}

From 3e1eb8a86037c92416cdca0ac360b5193a424764 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 9 Feb 2019 10:05:06 +0000
Subject: [PATCH 097/220] Make sure we only accept arity 2 functions in the
 test handler

---
 test/tortoise/handler_test.exs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index e2b997b0..749399e6 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -45,7 +45,7 @@ defmodule Tortoise.HandlerTest do
         nil ->
           {:cont, state}
 
-        fun when is_function(fun) ->
+        fun when is_function(fun, 2) ->
           apply(fun, [publish, state])
       end
     end
@@ -62,7 +62,7 @@ defmodule Tortoise.HandlerTest do
         nil ->
           {:cont, state}
 
-        fun when is_function(fun) ->
+        fun when is_function(fun, 2) ->
           apply(fun, [puback, state])
       end
     end
@@ -74,7 +74,7 @@ defmodule Tortoise.HandlerTest do
         nil ->
           {:cont, state}
 
-        fun when is_function(fun) ->
+        fun when is_function(fun, 2) ->
           apply(fun, [pubrec, state])
       end
     end
@@ -86,7 +86,7 @@ defmodule Tortoise.HandlerTest do
         nil ->
           {:cont, state}
 
-        fun when is_function(fun) ->
+        fun when is_function(fun, 2) ->
           apply(fun, [pubrel, state])
       end
     end
@@ -98,7 +98,7 @@ defmodule Tortoise.HandlerTest do
         nil ->
           {:cont, state}
 
-        fun when is_function(fun) ->
+        fun when is_function(fun, 2) ->
           apply(fun, [pubcomp, state])
       end
     end

From add5c82029f13f62c10edfd91ae80c859a10a118 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 9 Feb 2019 11:23:25 +0000
Subject: [PATCH 098/220] Abstracted the response in the test handler callback
 module

---
 test/tortoise/handler_test.exs | 89 ++++++++++++++++++----------------
 1 file changed, 47 insertions(+), 42 deletions(-)

diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index 749399e6..1cce46b2 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -25,7 +25,7 @@ defmodule Tortoise.HandlerTest do
 
     def handle_connack(connack, state) do
       send(state[:pid], {:connack, connack})
-      {:cont, state}
+      make_return(connack, state)
     end
 
     def handle_suback(subscribe, suback, state) do
@@ -40,14 +40,7 @@ defmodule Tortoise.HandlerTest do
 
     def handle_publish(topic, publish, state) do
       send(state[:pid], {:publish, topic, publish})
-
-      case Keyword.get(state, :publish) do
-        nil ->
-          {:cont, state}
-
-        fun when is_function(fun, 2) ->
-          apply(fun, [publish, state])
-      end
+      make_return(publish, state)
     end
 
     def terminate(reason, state) do
@@ -57,55 +50,67 @@ defmodule Tortoise.HandlerTest do
 
     def handle_puback(puback, state) do
       send(state[:pid], {:puback, puback})
-
-      case Keyword.get(state, :puback) do
-        nil ->
-          {:cont, state}
-
-        fun when is_function(fun, 2) ->
-          apply(fun, [puback, state])
-      end
+      make_return(puback, state)
     end
 
     def handle_pubrec(pubrec, state) do
       send(state[:pid], {:pubrec, pubrec})
-
-      case Keyword.get(state, :pubrec) do
-        nil ->
-          {:cont, state}
-
-        fun when is_function(fun, 2) ->
-          apply(fun, [pubrec, state])
-      end
+      make_return(pubrec, state)
     end
 
     def handle_pubrel(pubrel, state) do
       send(state[:pid], {:pubrel, pubrel})
-
-      case Keyword.get(state, :pubrel) do
-        nil ->
-          {:cont, state}
-
-        fun when is_function(fun, 2) ->
-          apply(fun, [pubrel, state])
-      end
+      make_return(pubrel, state)
     end
 
     def handle_pubcomp(pubcomp, state) do
       send(state[:pid], {:pubcomp, pubcomp})
+      make_return(pubcomp, state)
+    end
+
+    def handle_disconnect(disconnect, state) do
+      send(state[:pid], {:disconnect, disconnect})
+      make_return(disconnect, state)
+    end
 
-      case Keyword.get(state, :pubcomp) do
+    # `make return` will search the test handler state for a function
+    # with an arity of two that relate to the given package, and if
+    # found it will execute that function with the input package as
+    # the first argument and the handler state as the second. This
+    # allow us to specify the return value in the test itself, and
+    # thereby testing everything the user would return in the
+    # callbacks. If no callback function is defined we will default to
+    # returning `{:cont, state}`.
+    @package_to_type %{
+      Package.Connack => :connack,
+      Package.Publish => :publish,
+      Package.Puback => :puback,
+      Package.Pubrec => :pubrec,
+      Package.Pubrel => :pubrel,
+      Package.Pubcomp => :pubcomp,
+      Package.Disconnect => :disconnect
+    }
+
+    @allowed_package_types Map.keys(@package_to_type)
+
+    defp make_return(%type{} = package, state) when type in @allowed_package_types do
+      type = @package_to_type[type]
+
+      case Keyword.get(state, type) do
         nil ->
           {:cont, state}
 
         fun when is_function(fun, 2) ->
-          apply(fun, [pubcomp, state])
+          apply(fun, [package, state])
+
+        fun when is_function(fun) ->
+          msg = "Callback function for #{type} in #{__MODULE__} should be of arity-two"
+          raise ArgumentError, message: msg
       end
     end
 
-    def handle_disconnect(disconnect, state) do
-      send(state[:pid], {:disconnect, disconnect})
-      {:cont, state}
+    defp make_return(%type{}, _) do
+      raise ArgumentError, message: "Unknown type for #{__MODULE__}: #{type}"
     end
   end
 
@@ -169,7 +174,7 @@ defmodule Tortoise.HandlerTest do
 
   describe "execute handle_publish/2" do
     test "return ok-2", context do
-      handler = set_state(context.handler, [pid: self()])
+      handler = set_state(context.handler, pid: self())
       payload = :crypto.strong_rand_bytes(5)
       topic = "foo/bar"
       publish = %Package.Publish{topic: topic, payload: payload}
@@ -184,8 +189,8 @@ defmodule Tortoise.HandlerTest do
 
     test "return ok-3", context do
       next_actions = [{:subscribe, "foo/bar", [qos: 0]}]
-      publish_fn = fn (%Package.Publish{}, state) -> {:cont, state, next_actions} end
-      handler = set_state(context.handler, [pid: self(), publish: publish_fn])
+      publish_fn = fn %Package.Publish{}, state -> {:cont, state, next_actions} end
+      handler = set_state(context.handler, pid: self(), publish: publish_fn)
       payload = :crypto.strong_rand_bytes(5)
       topic = "foo/bar"
       publish = %Package.Publish{topic: topic, payload: payload}
@@ -202,7 +207,7 @@ defmodule Tortoise.HandlerTest do
     test "return ok-3 with invalid next action", context do
       next_actions = [{:unsubscribe, "foo/bar"}, {:invalid, "bar"}]
       publish_fn = fn %Package.Publish{}, state -> {:cont, state, next_actions} end
-      handler = set_state(context.handler, [pid: self(), publish: publish_fn])
+      handler = set_state(context.handler, pid: self(), publish: publish_fn)
       payload = :crypto.strong_rand_bytes(5)
       topic = "foo/bar"
       publish = %Package.Publish{topic: topic, payload: payload}

From ccc0bb43f995be4bc53d5f31250d2bddf62d2ef8 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 9 Feb 2019 13:32:44 +0000
Subject: [PATCH 099/220] Make the connection callback tests more extendable

---
 test/tortoise/handler_test.exs | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index 1cce46b2..fd6af786 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -13,14 +13,16 @@ defmodule Tortoise.HandlerTest do
       {:ok, opts}
     end
 
-    def connection(status, %{next_actions: next_actions} = state) do
-      send(state[:pid], {:connection, status})
-      {:cont, state, next_actions}
-    end
-
     def connection(status, state) do
       send(state[:pid], {:connection, status})
-      {:cont, state}
+
+      case state[:connection] do
+        nil ->
+          {:cont, state}
+
+        fun when is_function(fun, 2) ->
+          apply(fun, [status, state])
+      end
     end
 
     def handle_connack(connack, state) do
@@ -132,7 +134,7 @@ defmodule Tortoise.HandlerTest do
 
   describe "execute connection/2" do
     test "return ok-tuple", context do
-      handler = set_state(context.handler, %{pid: self()})
+      handler = set_state(context.handler, pid: self())
       assert {:ok, %Handler{}, []} = Handler.execute_connection(handler, :up)
       assert_receive {:connection, :up}
 
@@ -142,10 +144,8 @@ defmodule Tortoise.HandlerTest do
 
     test "return ok-3-tuple", context do
       next_actions = [{:subscribe, "foo/bar", qos: 0}]
-
-      handler =
-        context.handler
-        |> set_state(%{pid: self(), next_actions: next_actions})
+      connection_fn = fn _status, state -> {:cont, state, next_actions} end
+      handler = set_state(context.handler, pid: self(), connection: connection_fn)
 
       assert {:ok, %Handler{}, ^next_actions} = Handler.execute_connection(handler, :up)
 

From 1025a8674df2e7b60a9dcb9c7899929417e27107 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 9 Feb 2019 13:41:46 +0000
Subject: [PATCH 100/220] Test more aspects of pubrel callback handling

Make sure a list of {string, string} becomes a user defined property
list on the resulting pubcomp package, and make sure we crash if the
user define a custom pubcomp with a wrong package identifier.
---
 test/tortoise/handler_test.exs | 38 +++++++++++++++++++++++++++++++---
 1 file changed, 35 insertions(+), 3 deletions(-)

diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index fd6af786..6773cf95 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -304,15 +304,47 @@ defmodule Tortoise.HandlerTest do
       assert_receive {:pubrel, ^pubrel}
     end
 
-    test "return ok with next_actions", context do
+    test "return ok with custom pubrel", context do
+      properties = [{"foo", "bar"}]
       pubrel_fn = fn %Package.Pubrel{identifier: 1}, state ->
-        {{:cont, %Package.Pubcomp{identifier: 1, properties: [{"foo", "bar"}]}}, state}
+        {{:cont, %Package.Pubcomp{identifier: 1, properties: properties}}, state}
       end
 
       handler = set_state(context.handler, pid: self(), pubrel: pubrel_fn)
       pubrel = %Package.Pubrel{identifier: 1}
 
-      assert {:ok, %Package.Pubcomp{identifier: 1}, %Handler{} = state, []} =
+      assert {:ok, %Package.Pubcomp{identifier: 1, properties: ^properties}, %Handler{} = state, []} =
+               Handler.execute_handle_pubrel(handler, pubrel)
+
+      assert_receive {:pubrel, ^pubrel}
+    end
+
+    test "should not allow custom Pubrel with a different id", context do
+      pubrel_fn = fn %Package.Pubrel{identifier: id}, state ->
+        {{:cont, %Package.Pubcomp{identifier: id + 1}}, state}
+      end
+
+      handler = set_state(context.handler, pid: self(), pubrel: pubrel_fn)
+      pubrel = %Package.Pubrel{identifier: 1}
+
+      # todo, consider making an IdentifierMismatchError type
+      assert_raise CaseClauseError, fn ->
+        Handler.execute_handle_pubrel(handler, pubrel)
+      end
+
+      assert_receive {:pubrel, ^pubrel}
+    end
+
+    test "returning {:cont, [{string(), string()}]} become user defined properties", context do
+      properties = [{"foo", "bar"}, {"bar", "baz"}]
+      pubrel_fn = fn %Package.Pubrel{identifier: 1}, state ->
+        {{:cont, properties}, state}
+      end
+
+      handler = set_state(context.handler, pid: self(), pubrel: pubrel_fn)
+      pubrel = %Package.Pubrel{identifier: 1}
+
+      assert {:ok, %Package.Pubcomp{identifier: 1, properties: ^properties}, %Handler{} = state, []} =
                Handler.execute_handle_pubrel(handler, pubrel)
 
       assert_receive {:pubrel, ^pubrel}

From a996e61f0da70d650b34989d21899a6767cd2353 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 9 Feb 2019 14:09:31 +0000
Subject: [PATCH 101/220] Add more tests for the pubcomp callback handler

---
 test/tortoise/handler_test.exs | 33 ++++++++++++++++++++++++++++-----
 1 file changed, 28 insertions(+), 5 deletions(-)

diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index 6773cf95..5d0963b4 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -306,6 +306,7 @@ defmodule Tortoise.HandlerTest do
 
     test "return ok with custom pubrel", context do
       properties = [{"foo", "bar"}]
+
       pubrel_fn = fn %Package.Pubrel{identifier: 1}, state ->
         {{:cont, %Package.Pubcomp{identifier: 1, properties: properties}}, state}
       end
@@ -313,8 +314,8 @@ defmodule Tortoise.HandlerTest do
       handler = set_state(context.handler, pid: self(), pubrel: pubrel_fn)
       pubrel = %Package.Pubrel{identifier: 1}
 
-      assert {:ok, %Package.Pubcomp{identifier: 1, properties: ^properties}, %Handler{} = state, []} =
-               Handler.execute_handle_pubrel(handler, pubrel)
+      assert {:ok, %Package.Pubcomp{identifier: 1, properties: ^properties}, %Handler{} = state,
+              []} = Handler.execute_handle_pubrel(handler, pubrel)
 
       assert_receive {:pubrel, ^pubrel}
     end
@@ -337,6 +338,7 @@ defmodule Tortoise.HandlerTest do
 
     test "returning {:cont, [{string(), string()}]} become user defined properties", context do
       properties = [{"foo", "bar"}, {"bar", "baz"}]
+
       pubrel_fn = fn %Package.Pubrel{identifier: 1}, state ->
         {{:cont, properties}, state}
       end
@@ -344,8 +346,8 @@ defmodule Tortoise.HandlerTest do
       handler = set_state(context.handler, pid: self(), pubrel: pubrel_fn)
       pubrel = %Package.Pubrel{identifier: 1}
 
-      assert {:ok, %Package.Pubcomp{identifier: 1, properties: ^properties}, %Handler{} = state, []} =
-               Handler.execute_handle_pubrel(handler, pubrel)
+      assert {:ok, %Package.Pubcomp{identifier: 1, properties: ^properties}, %Handler{} = state,
+              []} = Handler.execute_handle_pubrel(handler, pubrel)
 
       assert_receive {:pubrel, ^pubrel}
     end
@@ -353,15 +355,36 @@ defmodule Tortoise.HandlerTest do
 
   describe "execute handle_pubcomp/2" do
     test "return ok", context do
-      handler = set_state(context.handler, pid: self())
       pubcomp = %Package.Pubcomp{identifier: 1}
 
+      pubcomp_fn = fn ^pubcomp, state ->
+        {:cont, state}
+      end
+
+      handler = set_state(context.handler, pid: self(), pubcomp: pubcomp_fn)
+
       assert {:ok, %Handler{} = state, []} =
                handler
                |> Handler.execute_handle_pubcomp(pubcomp)
 
       assert_receive {:pubcomp, ^pubcomp}
     end
+
+    test "return ok with next actions", context do
+      pubcomp = %Package.Pubcomp{identifier: 1}
+      next_actions = [{:subscribe, "foo/bar", qos: 0}]
+
+      pubcomp_fn = fn ^pubcomp, state ->
+        {:cont, state, next_actions}
+      end
+
+      handler = set_state(context.handler, pid: self(), pubcomp: pubcomp_fn)
+
+      assert {:ok, %Handler{} = state, ^next_actions} =
+               Handler.execute_handle_pubcomp(handler, pubcomp)
+
+      assert_receive {:pubcomp, ^pubcomp}
+    end
   end
 
   describe "execute handle_disconnect/2" do

From 996ed26f1d849fd0d59acd1dba0a3541f015718c Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 9 Feb 2019 14:28:28 +0000
Subject: [PATCH 102/220] Add more tests to the pubrel callback handler

---
 test/tortoise/handler_test.exs | 45 ++++++++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)

diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index 5d0963b4..faecee08 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -291,6 +291,51 @@ defmodule Tortoise.HandlerTest do
 
       assert_receive {:pubrec, ^pubrec}
     end
+
+    test "return ok with custom pubrel", context do
+      pubrec = %Package.Pubrec{identifier: 1}
+      properties = [{"foo", "bar"}]
+
+      pubrec_fn = fn ^pubrec, state ->
+        {{:cont, %Package.Pubrel{identifier: 1, properties: properties}}, state}
+      end
+
+      handler = set_state(context.handler, pid: self(), pubrec: pubrec_fn)
+
+      assert {:ok, %Package.Pubrel{identifier: 1, properties: ^properties}, %Handler{}, []} =
+               Handler.execute_handle_pubrec(handler, pubrec)
+
+      assert_receive {:pubrec, ^pubrec}
+    end
+
+    test "raise an error if a custom pubrel with the wrong id is returned", context do
+      pubrec = %Package.Pubrec{identifier: 1}
+
+      pubrec_fn = fn %Package.Pubrec{identifier: id}, state ->
+        {{:cont, %Package.Pubrel{identifier: id + 1}}, state}
+      end
+
+      handler = set_state(context.handler, pid: self(), pubrec: pubrec_fn)
+
+      assert_raise CaseClauseError, fn ->
+        Handler.execute_handle_pubrec(handler, pubrec)
+      end
+
+      assert_receive {:pubrec, ^pubrec}
+    end
+
+    test "returning {:cont, [{string(), string()}]} should result in a pubrel with properties",
+         context do
+      pubrec = %Package.Pubrec{identifier: 1}
+      properties = [{"foo", "bar"}]
+      pubrec_fn = fn ^pubrec, state -> {{:cont, properties}, state} end
+      handler = set_state(context.handler, pid: self(), pubrec: pubrec_fn)
+
+      assert {:ok, %Package.Pubrel{identifier: 1, properties: ^properties}, %Handler{}, []} =
+               Handler.execute_handle_pubrec(handler, pubrec)
+
+      assert_receive {:pubrec, ^pubrec}
+    end
   end
 
   describe "execute handle_pubrel/2" do

From 56d47bc0bada576308564992b805a56f11b55d99 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 9 Feb 2019 20:02:38 +0000
Subject: [PATCH 103/220] Correct the names of some tests

---
 test/tortoise/handler_test.exs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index faecee08..38aebc32 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -133,7 +133,7 @@ defmodule Tortoise.HandlerTest do
   end
 
   describe "execute connection/2" do
-    test "return ok-tuple", context do
+    test "return continues", context do
       handler = set_state(context.handler, pid: self())
       assert {:ok, %Handler{}, []} = Handler.execute_connection(handler, :up)
       assert_receive {:connection, :up}
@@ -142,7 +142,7 @@ defmodule Tortoise.HandlerTest do
       assert_receive {:connection, :down}
     end
 
-    test "return ok-3-tuple", context do
+    test "return continue with next actions", context do
       next_actions = [{:subscribe, "foo/bar", qos: 0}]
       connection_fn = fn _status, state -> {:cont, state, next_actions} end
       handler = set_state(context.handler, pid: self(), connection: connection_fn)

From c536b62198849cea805faf2b0aaaf28d6823b68f Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 9 Feb 2019 20:15:38 +0000
Subject: [PATCH 104/220] Test that handle_connack return next_actions

---
 test/tortoise/handler_test.exs | 23 ++++++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index 38aebc32..c744ab78 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -158,18 +158,35 @@ defmodule Tortoise.HandlerTest do
   end
 
   describe "execute handle_connack/2" do
-    test "return ok-tuple", context do
-      handler = set_state(context.handler, pid: self())
-
+    test "return continue", context do
       connack = %Package.Connack{
         reason: :success,
         session_present: false
       }
 
+      connack_fn = fn ^connack, state -> {:cont, state} end
+      handler = set_state(context.handler, pid: self(), connack: connack_fn)
+
       assert {:ok, %Handler{} = state, []} = Handler.execute_handle_connack(handler, connack)
 
       assert_receive {:connack, ^connack}
     end
+
+    test "return continue with next actions", context do
+      connack = %Package.Connack{
+        reason: :success,
+        session_present: false
+      }
+
+      next_actions = [{:subscribe, "foo/bar", [qos: 0]}]
+      connack_fn = fn ^connack, state -> {:cont, state, next_actions} end
+      handler = set_state(context.handler, pid: self(), connack: connack_fn)
+
+      assert {:ok, %Handler{} = state, ^next_actions} =
+               Handler.execute_handle_connack(handler, connack)
+
+      assert_receive {:connack, ^connack}
+    end
   end
 
   describe "execute handle_publish/2" do

From 2e6fcfb9310690a3bdf78c27bda22d11f9219655 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 9 Feb 2019 20:24:33 +0000
Subject: [PATCH 105/220] Update handle_publish callback handler tests

---
 test/tortoise/handler_test.exs | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index c744ab78..fe847665 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -190,11 +190,12 @@ defmodule Tortoise.HandlerTest do
   end
 
   describe "execute handle_publish/2" do
-    test "return ok-2", context do
-      handler = set_state(context.handler, pid: self())
+    test "return continue", context do
       payload = :crypto.strong_rand_bytes(5)
       topic = "foo/bar"
       publish = %Package.Publish{topic: topic, payload: payload}
+      publish_fn = fn ^publish, state -> {:cont, state} end
+      handler = set_state(context.handler, pid: self(), publish: publish_fn)
 
       assert {:ok, %Handler{}, []} = Handler.execute_handle_publish(handler, publish)
       # the topic will be in the form of a list making it possible to
@@ -204,14 +205,13 @@ defmodule Tortoise.HandlerTest do
       assert topic == Enum.join(topic_list, "/")
     end
 
-    test "return ok-3", context do
-      next_actions = [{:subscribe, "foo/bar", [qos: 0]}]
-      publish_fn = fn %Package.Publish{}, state -> {:cont, state, next_actions} end
-      handler = set_state(context.handler, pid: self(), publish: publish_fn)
-      payload = :crypto.strong_rand_bytes(5)
+    test "return continue with next actions", context do
       topic = "foo/bar"
+      payload = :crypto.strong_rand_bytes(5)
+      next_actions = [{:subscribe, "foo/bar", [qos: 0]}]
       publish = %Package.Publish{topic: topic, payload: payload}
-
+      publish_fn = fn ^publish, state -> {:cont, state, next_actions} end
+      handler = set_state(context.handler, pid: self(), publish: publish_fn)
       assert {:ok, %Handler{}, ^next_actions} = Handler.execute_handle_publish(handler, publish)
 
       # the topic will be in the form of a list making it possible to
@@ -221,13 +221,13 @@ defmodule Tortoise.HandlerTest do
       assert topic == Enum.join(topic_list, "/")
     end
 
-    test "return ok-3 with invalid next action", context do
-      next_actions = [{:unsubscribe, "foo/bar"}, {:invalid, "bar"}]
-      publish_fn = fn %Package.Publish{}, state -> {:cont, state, next_actions} end
-      handler = set_state(context.handler, pid: self(), publish: publish_fn)
-      payload = :crypto.strong_rand_bytes(5)
+    test "return continue with invalid next action", context do
       topic = "foo/bar"
+      payload = :crypto.strong_rand_bytes(5)
       publish = %Package.Publish{topic: topic, payload: payload}
+      next_actions = [{:unsubscribe, "foo/bar"}, {:invalid, "bar"}]
+      publish_fn = fn ^publish, state -> {:cont, state, next_actions} end
+      handler = set_state(context.handler, pid: self(), publish: publish_fn)
 
       assert {:error, {:invalid_next_action, [{:invalid, "bar"}]}} =
                Handler.execute_handle_publish(handler, publish)

From c4a5f2da700e28cf017327cd0795cba0cbea469e Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 9 Feb 2019 21:15:17 +0000
Subject: [PATCH 106/220] Update the tests for the handle_suback callback
 handler

---
 test/tortoise/handler_test.exs | 46 +++++++++++++++++++++++++++++++---
 1 file changed, 42 insertions(+), 4 deletions(-)

diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index fe847665..1e88f372 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -32,7 +32,7 @@ defmodule Tortoise.HandlerTest do
 
     def handle_suback(subscribe, suback, state) do
       send(state[:pid], {:suback, {subscribe, suback}})
-      {:cont, state}
+      make_return({subscribe, suback}, state)
     end
 
     def handle_unsuback(unsubscribe, unsuback, state) do
@@ -90,6 +90,8 @@ defmodule Tortoise.HandlerTest do
       Package.Pubrec => :pubrec,
       Package.Pubrel => :pubrel,
       Package.Pubcomp => :pubcomp,
+      Package.Suback => :suback,
+      Package.Unsuback => :unsuback,
       Package.Disconnect => :disconnect
     }
 
@@ -111,6 +113,22 @@ defmodule Tortoise.HandlerTest do
       end
     end
 
+    defp make_return({package, %type{} = ack}, state) when type in @allowed_package_types do
+      type = @package_to_type[type]
+
+      case Keyword.get(state, type) do
+        nil ->
+          {:cont, state}
+
+        fun when is_function(fun, 3) ->
+          apply(fun, [package, ack, state])
+
+        fun when is_function(fun) ->
+          msg = "Callback function for #{type} in #{__MODULE__} should be of arity-three"
+          raise ArgumentError, message: msg
+      end
+    end
+
     defp make_return(%type{}, _) do
       raise ArgumentError, message: "Unknown type for #{__MODULE__}: #{type}"
     end
@@ -240,9 +258,7 @@ defmodule Tortoise.HandlerTest do
   end
 
   describe "execute handle_suback/3" do
-    test "return ok", context do
-      handler = set_state(context.handler, pid: self())
-
+    test "return continue", context do
       subscribe = %Package.Subscribe{
         identifier: 1,
         topics: [{"foo", qos: 0}]
@@ -250,11 +266,33 @@ defmodule Tortoise.HandlerTest do
 
       suback = %Package.Suback{identifier: 1, acks: [ok: 0]}
 
+      suback_fn = fn (^subscribe, ^suback, state) -> {:cont, state} end
+      handler = set_state(context.handler, pid: self(), suback: suback_fn)
+
       assert {:ok, %Handler{} = state, []} =
                Handler.execute_handle_suback(handler, subscribe, suback)
 
       assert_receive {:suback, {^subscribe, ^suback}}
     end
+
+    test "return continue with next actions", context do
+      subscribe = %Package.Subscribe{
+        identifier: 1,
+        topics: [{"foo", qos: 0}]
+      }
+
+      suback = %Package.Suback{identifier: 1, acks: [ok: 0]}
+
+      next_actions = [{:unsubscribe, "foo/bar"}]
+
+      suback_fn = fn (^subscribe, ^suback, state) -> {:cont, state, next_actions} end
+      handler = set_state(context.handler, pid: self(), suback: suback_fn)
+
+      assert {:ok, %Handler{} = state, ^next_actions} =
+               Handler.execute_handle_suback(handler, subscribe, suback)
+
+      assert_receive {:suback, {^subscribe, ^suback}}
+    end
   end
 
   describe "execute handle_unsuback/3" do

From 4b1025171f3c40cd06dcc51f60dbfc8adcb7d4a1 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 9 Feb 2019 21:27:17 +0000
Subject: [PATCH 107/220] More tests for the unsuback callback handler

---
 test/tortoise/handler_test.exs | 29 +++++++++++++++++++----------
 1 file changed, 19 insertions(+), 10 deletions(-)

diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index 1e88f372..65a8db9e 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -37,7 +37,7 @@ defmodule Tortoise.HandlerTest do
 
     def handle_unsuback(unsubscribe, unsuback, state) do
       send(state[:pid], {:unsuback, {unsubscribe, unsuback}})
-      {:cont, state}
+      make_return({unsubscribe, unsuback}, state)
     end
 
     def handle_publish(topic, publish, state) do
@@ -266,7 +266,7 @@ defmodule Tortoise.HandlerTest do
 
       suback = %Package.Suback{identifier: 1, acks: [ok: 0]}
 
-      suback_fn = fn (^subscribe, ^suback, state) -> {:cont, state} end
+      suback_fn = fn ^subscribe, ^suback, state -> {:cont, state} end
       handler = set_state(context.handler, pid: self(), suback: suback_fn)
 
       assert {:ok, %Handler{} = state, []} =
@@ -285,7 +285,7 @@ defmodule Tortoise.HandlerTest do
 
       next_actions = [{:unsubscribe, "foo/bar"}]
 
-      suback_fn = fn (^subscribe, ^suback, state) -> {:cont, state, next_actions} end
+      suback_fn = fn ^subscribe, ^suback, state -> {:cont, state, next_actions} end
       handler = set_state(context.handler, pid: self(), suback: suback_fn)
 
       assert {:ok, %Handler{} = state, ^next_actions} =
@@ -296,17 +296,26 @@ defmodule Tortoise.HandlerTest do
   end
 
   describe "execute handle_unsuback/3" do
-    test "return ok", context do
-      handler = set_state(context.handler, pid: self())
+    test "return continue", context do
+      unsubscribe = %Package.Unsubscribe{identifier: 1, topics: ["foo"]}
+      unsuback = %Package.Unsuback{identifier: 1, results: [:success]}
+      unsuback_fn = fn ^unsubscribe, ^unsuback, state -> {:cont, state} end
+      handler = set_state(context.handler, pid: self(), unsuback: unsuback_fn)
 
-      unsubscribe = %Package.Unsubscribe{
-        identifier: 1,
-        topics: ["foo"]
-      }
+      assert {:ok, %Handler{} = state, []} =
+               Handler.execute_handle_unsuback(handler, unsubscribe, unsuback)
 
+      assert_receive {:unsuback, {^unsubscribe, ^unsuback}}
+    end
+
+    test "return continue with next actions", context do
+      unsubscribe = %Package.Unsubscribe{identifier: 1, topics: ["foo"]}
       unsuback = %Package.Unsuback{identifier: 1, results: [:success]}
+      next_actions = [{:unsubscribe, "foo/bar"}]
+      unsuback_fn = fn ^unsubscribe, ^unsuback, state -> {:cont, state, next_actions} end
+      handler = set_state(context.handler, pid: self(), unsuback: unsuback_fn)
 
-      assert {:ok, %Handler{} = state, []} =
+      assert {:ok, %Handler{} = state, ^next_actions} =
                Handler.execute_handle_unsuback(handler, unsubscribe, unsuback)
 
       assert_receive {:unsuback, {^unsubscribe, ^unsuback}}

From 8084fd401f43609b7f5137076ecad3e51ea25a8d Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 9 Feb 2019 21:28:23 +0000
Subject: [PATCH 108/220] Move terminate to the bottom of the file

---
 test/tortoise/handler_test.exs | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index 65a8db9e..559d81b4 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -322,14 +322,6 @@ defmodule Tortoise.HandlerTest do
     end
   end
 
-  describe "execute terminate/2" do
-    test "return ok", context do
-      handler = set_state(context.handler, pid: self())
-      assert :ok = Handler.execute_terminate(handler, :normal)
-      assert_receive {:terminate, :normal}
-    end
-  end
-
   describe "execute handle_puback/2" do
     test "return ok", context do
       handler = set_state(context.handler, pid: self())
@@ -508,4 +500,12 @@ defmodule Tortoise.HandlerTest do
       assert_receive {:disconnect, ^disconnect}
     end
   end
+
+  describe "execute terminate/2" do
+    test "return ok", context do
+      handler = set_state(context.handler, pid: self())
+      assert :ok = Handler.execute_terminate(handler, :normal)
+      assert_receive {:terminate, :normal}
+    end
+  end
 end

From 4c17cfc7df5ccb092eb8db841fb8654ee65d2972 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 9 Feb 2019 21:37:09 +0000
Subject: [PATCH 109/220] Corrected the names of two tests

---
 test/tortoise/handler_test.exs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index 559d81b4..08518ae1 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -405,7 +405,7 @@ defmodule Tortoise.HandlerTest do
       assert_receive {:pubrel, ^pubrel}
     end
 
-    test "return ok with custom pubrel", context do
+    test "return continue with custom pubcomp", context do
       properties = [{"foo", "bar"}]
 
       pubrel_fn = fn %Package.Pubrel{identifier: 1}, state ->
@@ -421,7 +421,7 @@ defmodule Tortoise.HandlerTest do
       assert_receive {:pubrel, ^pubrel}
     end
 
-    test "should not allow custom Pubrel with a different id", context do
+    test "should not allow custom pubcomp with a different id", context do
       pubrel_fn = fn %Package.Pubrel{identifier: id}, state ->
         {{:cont, %Package.Pubcomp{identifier: id + 1}}, state}
       end

From c1fe1fab897dcd1df88599a0529062c2f8ea88d8 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 9 Feb 2019 21:44:07 +0000
Subject: [PATCH 110/220] Update tests for the handle_puback callback

---
 test/tortoise/handler_test.exs | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index 08518ae1..fdc77c52 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -322,14 +322,26 @@ defmodule Tortoise.HandlerTest do
     end
   end
 
+  # callbacks for the QoS=1 message exchange
   describe "execute handle_puback/2" do
-    test "return ok", context do
-      handler = set_state(context.handler, pid: self())
+    test "return continue", context do
       puback = %Package.Puback{identifier: 1}
+      puback_fn = fn ^puback, state -> {:cont, state} end
+      handler = set_state(context.handler, pid: self(), puback: puback_fn)
 
-      assert {:ok, %Handler{} = state, []} =
-               handler
-               |> Handler.execute_handle_puback(puback)
+      assert {:ok, %Handler{} = state, []} = Handler.execute_handle_puback(handler, puback)
+
+      assert_receive {:puback, ^puback}
+    end
+
+    test "return continue with next actions", context do
+      puback = %Package.Puback{identifier: 1}
+      next_actions = [{:subscribe, "foo/bar", qos: 0}]
+      puback_fn = fn ^puback, state -> {:cont, state, next_actions} end
+      handler = set_state(context.handler, pid: self(), puback: puback_fn)
+
+      assert {:ok, %Handler{} = state, ^next_actions} =
+               Handler.execute_handle_puback(handler, puback)
 
       assert_receive {:puback, ^puback}
     end

From 301a73b13ce2d1e6cabf60cd52a64d2c731af6d8 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 9 Feb 2019 21:51:07 +0000
Subject: [PATCH 111/220] Update handle_pubrec callback tests

---
 test/tortoise/handler_test.exs | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index fdc77c52..237b9ad6 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -349,18 +349,18 @@ defmodule Tortoise.HandlerTest do
 
   # callbacks for the QoS=2 message exchange
   describe "execute handle_pubrec/2" do
-    test "return ok", context do
-      handler = set_state(context.handler, pid: self())
+    test "return continue", context do
       pubrec = %Package.Pubrec{identifier: 1}
+      pubrec_fn = fn ^pubrec, state -> {:cont, state} end
+      handler = set_state(context.handler, pid: self(), pubrec: pubrec_fn)
 
       assert {:ok, %Package.Pubrel{identifier: 1}, %Handler{}, []} =
-               handler
-               |> Handler.execute_handle_pubrec(pubrec)
+               Handler.execute_handle_pubrec(handler, pubrec)
 
       assert_receive {:pubrec, ^pubrec}
     end
 
-    test "return ok with custom pubrel", context do
+    test "return continue with custom pubrel", context do
       pubrec = %Package.Pubrec{identifier: 1}
       properties = [{"foo", "bar"}]
 
@@ -392,8 +392,7 @@ defmodule Tortoise.HandlerTest do
       assert_receive {:pubrec, ^pubrec}
     end
 
-    test "returning {:cont, [{string(), string()}]} should result in a pubrel with properties",
-         context do
+    test "returning continue with a list should result in a pubrel with user props", context do
       pubrec = %Package.Pubrec{identifier: 1}
       properties = [{"foo", "bar"}]
       pubrec_fn = fn ^pubrec, state -> {{:cont, properties}, state} end

From ff9845eae38627eef2549188892c9d372eb943d2 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 9 Feb 2019 21:57:40 +0000
Subject: [PATCH 112/220] Update the handle_pubrel callback tests

---
 test/tortoise/handler_test.exs | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index 237b9ad6..6b7842ac 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -406,9 +406,10 @@ defmodule Tortoise.HandlerTest do
   end
 
   describe "execute handle_pubrel/2" do
-    test "return ok", context do
-      handler = set_state(context.handler, pid: self())
+    test "return continue", context do
       pubrel = %Package.Pubrel{identifier: 1}
+      pubrel_fn = fn ^pubrel, state -> {:cont, state} end
+      handler = set_state(context.handler, pid: self(), pubrel: pubrel_fn)
 
       assert {:ok, %Package.Pubcomp{identifier: 1}, %Handler{} = state, []} =
                Handler.execute_handle_pubrel(handler, pubrel)
@@ -417,14 +418,12 @@ defmodule Tortoise.HandlerTest do
     end
 
     test "return continue with custom pubcomp", context do
+      pubrel = %Package.Pubrel{identifier: 1}
       properties = [{"foo", "bar"}]
-
       pubrel_fn = fn %Package.Pubrel{identifier: 1}, state ->
         {{:cont, %Package.Pubcomp{identifier: 1, properties: properties}}, state}
       end
-
       handler = set_state(context.handler, pid: self(), pubrel: pubrel_fn)
-      pubrel = %Package.Pubrel{identifier: 1}
 
       assert {:ok, %Package.Pubcomp{identifier: 1, properties: ^properties}, %Handler{} = state,
               []} = Handler.execute_handle_pubrel(handler, pubrel)
@@ -433,12 +432,11 @@ defmodule Tortoise.HandlerTest do
     end
 
     test "should not allow custom pubcomp with a different id", context do
+      pubrel = %Package.Pubrel{identifier: 1}
       pubrel_fn = fn %Package.Pubrel{identifier: id}, state ->
         {{:cont, %Package.Pubcomp{identifier: id + 1}}, state}
       end
-
       handler = set_state(context.handler, pid: self(), pubrel: pubrel_fn)
-      pubrel = %Package.Pubrel{identifier: 1}
 
       # todo, consider making an IdentifierMismatchError type
       assert_raise CaseClauseError, fn ->
@@ -450,13 +448,11 @@ defmodule Tortoise.HandlerTest do
 
     test "returning {:cont, [{string(), string()}]} become user defined properties", context do
       properties = [{"foo", "bar"}, {"bar", "baz"}]
-
-      pubrel_fn = fn %Package.Pubrel{identifier: 1}, state ->
+      pubrel = %Package.Pubrel{identifier: 1}
+      pubrel_fn = fn ^pubrel, state ->
         {{:cont, properties}, state}
       end
-
       handler = set_state(context.handler, pid: self(), pubrel: pubrel_fn)
-      pubrel = %Package.Pubrel{identifier: 1}
 
       assert {:ok, %Package.Pubcomp{identifier: 1, properties: ^properties}, %Handler{} = state,
               []} = Handler.execute_handle_pubrel(handler, pubrel)

From a907f7054e05f6c0012671b2ed14c12fe6480707 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 9 Feb 2019 21:59:59 +0000
Subject: [PATCH 113/220] Update the pubcomp tests, ran code formatter

---
 test/tortoise/handler_test.exs | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index 6b7842ac..dac44d57 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -420,9 +420,11 @@ defmodule Tortoise.HandlerTest do
     test "return continue with custom pubcomp", context do
       pubrel = %Package.Pubrel{identifier: 1}
       properties = [{"foo", "bar"}]
+
       pubrel_fn = fn %Package.Pubrel{identifier: 1}, state ->
         {{:cont, %Package.Pubcomp{identifier: 1, properties: properties}}, state}
       end
+
       handler = set_state(context.handler, pid: self(), pubrel: pubrel_fn)
 
       assert {:ok, %Package.Pubcomp{identifier: 1, properties: ^properties}, %Handler{} = state,
@@ -433,9 +435,11 @@ defmodule Tortoise.HandlerTest do
 
     test "should not allow custom pubcomp with a different id", context do
       pubrel = %Package.Pubrel{identifier: 1}
+
       pubrel_fn = fn %Package.Pubrel{identifier: id}, state ->
         {{:cont, %Package.Pubcomp{identifier: id + 1}}, state}
       end
+
       handler = set_state(context.handler, pid: self(), pubrel: pubrel_fn)
 
       # todo, consider making an IdentifierMismatchError type
@@ -449,9 +453,11 @@ defmodule Tortoise.HandlerTest do
     test "returning {:cont, [{string(), string()}]} become user defined properties", context do
       properties = [{"foo", "bar"}, {"bar", "baz"}]
       pubrel = %Package.Pubrel{identifier: 1}
+
       pubrel_fn = fn ^pubrel, state ->
         {{:cont, properties}, state}
       end
+
       handler = set_state(context.handler, pid: self(), pubrel: pubrel_fn)
 
       assert {:ok, %Package.Pubcomp{identifier: 1, properties: ^properties}, %Handler{} = state,
@@ -462,13 +468,9 @@ defmodule Tortoise.HandlerTest do
   end
 
   describe "execute handle_pubcomp/2" do
-    test "return ok", context do
+    test "return continue", context do
       pubcomp = %Package.Pubcomp{identifier: 1}
-
-      pubcomp_fn = fn ^pubcomp, state ->
-        {:cont, state}
-      end
-
+      pubcomp_fn = fn ^pubcomp, state -> {:cont, state} end
       handler = set_state(context.handler, pid: self(), pubcomp: pubcomp_fn)
 
       assert {:ok, %Handler{} = state, []} =
@@ -478,13 +480,10 @@ defmodule Tortoise.HandlerTest do
       assert_receive {:pubcomp, ^pubcomp}
     end
 
-    test "return ok with next actions", context do
+    test "return continue with next actions", context do
       pubcomp = %Package.Pubcomp{identifier: 1}
       next_actions = [{:subscribe, "foo/bar", qos: 0}]
-
-      pubcomp_fn = fn ^pubcomp, state ->
-        {:cont, state, next_actions}
-      end
+      pubcomp_fn = fn ^pubcomp, state -> {:cont, state, next_actions} end
 
       handler = set_state(context.handler, pid: self(), pubcomp: pubcomp_fn)
 

From 7b515bc7bab4e8ccc74d10b72cb473450167cc28 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 9 Feb 2019 22:03:51 +0000
Subject: [PATCH 114/220] Update the handle_disconnect callback tests

---
 test/tortoise/handler_test.exs | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index dac44d57..1ca6aa48 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -495,13 +495,24 @@ defmodule Tortoise.HandlerTest do
   end
 
   describe "execute handle_disconnect/2" do
-    test "return ok", context do
-      handler = set_state(context.handler, pid: self())
+    test "return continue", context do
       disconnect = %Package.Disconnect{}
+      disconnect_fn = fn (^disconnect, state) -> {:cont, state} end
+      handler = set_state(context.handler, pid: self(), disconnect: disconnect_fn)
 
       assert {:ok, %Handler{} = state, []} =
-               handler
-               |> Handler.execute_handle_disconnect(disconnect)
+               Handler.execute_handle_disconnect(handler, disconnect)
+
+      assert_receive {:disconnect, ^disconnect}
+    end
+
+    test "return stop with normal reason", context do
+      disconnect = %Package.Disconnect{}
+      disconnect_fn = fn (^disconnect, state) -> {:stop, :normal, state} end
+      handler = set_state(context.handler, pid: self(), disconnect: disconnect_fn)
+
+      assert {:stop, :normal, %Handler{} = state} =
+               Handler.execute_handle_disconnect(handler, disconnect)
 
       assert_receive {:disconnect, ^disconnect}
     end

From 5a212ac103806d45e41c71c61cb26b91eaaf708c Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sun, 10 Feb 2019 09:20:30 +0000
Subject: [PATCH 115/220] Make sure we can return next actions when receiving a
 disconnect

---
 test/tortoise/handler_test.exs | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index 1ca6aa48..39586cdb 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -497,7 +497,7 @@ defmodule Tortoise.HandlerTest do
   describe "execute handle_disconnect/2" do
     test "return continue", context do
       disconnect = %Package.Disconnect{}
-      disconnect_fn = fn (^disconnect, state) -> {:cont, state} end
+      disconnect_fn = fn ^disconnect, state -> {:cont, state} end
       handler = set_state(context.handler, pid: self(), disconnect: disconnect_fn)
 
       assert {:ok, %Handler{} = state, []} =
@@ -506,9 +506,22 @@ defmodule Tortoise.HandlerTest do
       assert_receive {:disconnect, ^disconnect}
     end
 
+    test "return continue with next actions", context do
+      disconnect = %Package.Disconnect{}
+      next_actions = [{:subscribe, "foo/bar", qos: 2}]
+      disconnect_fn = fn ^disconnect, state -> {:cont, state, next_actions} end
+
+      handler = set_state(context.handler, pid: self(), disconnect: disconnect_fn)
+
+      assert {:ok, %Handler{} = state, ^next_actions} =
+               Handler.execute_handle_disconnect(handler, disconnect)
+
+      assert_receive {:disconnect, ^disconnect}
+    end
+
     test "return stop with normal reason", context do
       disconnect = %Package.Disconnect{}
-      disconnect_fn = fn (^disconnect, state) -> {:stop, :normal, state} end
+      disconnect_fn = fn ^disconnect, state -> {:stop, :normal, state} end
       handler = set_state(context.handler, pid: self(), disconnect: disconnect_fn)
 
       assert {:stop, :normal, %Handler{} = state} =

From 7ad34b5e25c613382ccf77b65ac720afb49f4c52 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sun, 10 Feb 2019 11:15:28 +0000
Subject: [PATCH 116/220] Remove the need for a magic pid variable in the
 callback TestHander

I might have replaced it with some other magic, but for most of the
tests we really did not need to send back the input, as it for the
most part passed in untouched. It is possible to send in a function
that send values back to the calling process, so I have changed it to
just doing that. Also, I have made it possible to send in a function
for the execute_init, and I have added more tests for the init
callback.
---
 test/tortoise/handler_test.exs | 237 +++++++++++++++++----------------
 1 file changed, 124 insertions(+), 113 deletions(-)

diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index 39586cdb..13ffc38a 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -6,16 +6,46 @@ defmodule Tortoise.HandlerTest do
   alias Tortoise.Package
 
   defmodule TestHandler do
+    @moduledoc """
+    A Tortoise callback handler for testing the input given and the
+    returned output.
+
+    This callback handler rely on having a keyword list as the state;
+    when a callback is called it will look for an entry relating to
+    that callback, and if an anonymous function is found it will be
+    used as the return value. For instance, if the `handle_pubrel/2`
+    callback is called, if will look for the `pubrel` in the state
+    keyword list; if the value is nil an `{:cont, state}` will get
+    returned, if an anonymous function of arity two is found it will
+    get called with `apply(fun, [pubrel, state])`. This makes it
+    possible to pass in a known state and set expectations in our
+    tests.
+    """
+
     @behaviour Handler
 
-    def init(opts) do
-      send(opts[:pid], :init)
-      {:ok, opts}
+    # For these tests, if the initial_arg is a two-tuple of a function
+    # and a term we will execute the function with the term as an
+    # argument and use the return value as the return; otherwise we
+    # will just return `{:ok, initial_arg}`
+    def init({fun, opts}) when is_function(fun, 1), do: apply(fun, [opts])
+    def init(opts), do: {:ok, opts}
+
+    def terminate(reason, state) do
+      case Keyword.get(state, :terminate) do
+        nil ->
+          :ok
+
+        fun when is_function(fun, 2) ->
+          apply(fun, [reason, state])
+
+        fun when is_function(fun) ->
+          msg = "Callback function for terminate in #{__MODULE__} should be of arity-two"
+          raise ArgumentError, message: msg
+      end
     end
 
     def connection(status, state) do
-      send(state[:pid], {:connection, status})
-
       case state[:connection] do
         nil ->
           {:cont, state}
@@ -25,53 +55,49 @@ defmodule Tortoise.HandlerTest do
       end
     end
 
+    def handle_publish(topic_list, publish, state) do
+      case Keyword.get(state, :publish) do
+        nil ->
+          {:cont, state}
+
+        fun when is_function(fun, 3) ->
+          apply(fun, [topic_list, publish, state])
+
+        fun when is_function(fun) ->
+          msg = "Callback function for Publish in #{__MODULE__} should be of arity-three"
+          raise ArgumentError, message: msg
+      end
+    end
+
     def handle_connack(connack, state) do
-      send(state[:pid], {:connack, connack})
       make_return(connack, state)
     end
 
     def handle_suback(subscribe, suback, state) do
-      send(state[:pid], {:suback, {subscribe, suback}})
       make_return({subscribe, suback}, state)
     end
 
     def handle_unsuback(unsubscribe, unsuback, state) do
-      send(state[:pid], {:unsuback, {unsubscribe, unsuback}})
       make_return({unsubscribe, unsuback}, state)
     end
 
-    def handle_publish(topic, publish, state) do
-      send(state[:pid], {:publish, topic, publish})
-      make_return(publish, state)
-    end
-
-    def terminate(reason, state) do
-      send(state[:pid], {:terminate, reason})
-      :ok
-    end
-
     def handle_puback(puback, state) do
-      send(state[:pid], {:puback, puback})
       make_return(puback, state)
     end
 
     def handle_pubrec(pubrec, state) do
-      send(state[:pid], {:pubrec, pubrec})
       make_return(pubrec, state)
     end
 
     def handle_pubrel(pubrel, state) do
-      send(state[:pid], {:pubrel, pubrel})
       make_return(pubrel, state)
     end
 
     def handle_pubcomp(pubcomp, state) do
-      send(state[:pid], {:pubcomp, pubcomp})
       make_return(pubcomp, state)
     end
 
     def handle_disconnect(disconnect, state) do
-      send(state[:pid], {:disconnect, disconnect})
       make_return(disconnect, state)
     end
 
@@ -85,7 +111,6 @@ defmodule Tortoise.HandlerTest do
     # returning `{:cont, state}`.
     @package_to_type %{
       Package.Connack => :connack,
-      Package.Publish => :publish,
       Package.Puback => :puback,
       Package.Pubrec => :pubrec,
       Package.Pubrel => :pubrel,
@@ -135,7 +160,7 @@ defmodule Tortoise.HandlerTest do
   end
 
   setup _context do
-    handler = %Tortoise.Handler{module: TestHandler, initial_args: [pid: self()]}
+    handler = %Tortoise.Handler{module: TestHandler, initial_args: nil}
     {:ok, %{handler: handler}}
   end
 
@@ -144,15 +169,35 @@ defmodule Tortoise.HandlerTest do
   end
 
   describe "execute_init/1" do
-    test "return ok-tuple", context do
-      assert {:ok, %Handler{}} = Handler.execute_init(context.handler)
-      assert_receive :init
+    test "return ok-tuple should set the handler state" do
+      handler = %Handler{module: TestHandler, state: nil, initial_args: make_ref()}
+      assert {:ok, %Handler{state: state, initial_args: state}} = Handler.execute_init(handler)
+    end
+
+    test "returning ignore" do
+      init_fn = fn nil -> :ignore end
+      handler = %Handler{module: TestHandler, state: nil, initial_args: {init_fn, nil}}
+      assert :ignore = Handler.execute_init(handler)
+    end
+
+    test "returning stop with a reason" do
+      reason = make_ref()
+      init_fn = fn nil -> {:stop, reason} end
+      handler = %Handler{module: TestHandler, state: nil, initial_args: {init_fn, nil}}
+      assert {:stop, ^reason} = Handler.execute_init(handler)
     end
   end
 
   describe "execute connection/2" do
     test "return continues", context do
-      handler = set_state(context.handler, pid: self())
+      parent = self()
+
+      connection_fn = fn status, state ->
+        send(parent, {:connection, status})
+        {:cont, state}
+      end
+
+      handler = set_state(context.handler, connection: connection_fn)
       assert {:ok, %Handler{}, []} = Handler.execute_connection(handler, :up)
       assert_receive {:connection, :up}
 
@@ -162,8 +207,14 @@ defmodule Tortoise.HandlerTest do
 
     test "return continue with next actions", context do
       next_actions = [{:subscribe, "foo/bar", qos: 0}]
-      connection_fn = fn _status, state -> {:cont, state, next_actions} end
-      handler = set_state(context.handler, pid: self(), connection: connection_fn)
+      parent = self()
+
+      connection_fn = fn status, state ->
+        send(parent, {:connection, status})
+        {:cont, state, next_actions}
+      end
+
+      handler = set_state(context.handler, connection: connection_fn)
 
       assert {:ok, %Handler{}, ^next_actions} = Handler.execute_connection(handler, :up)
 
@@ -183,11 +234,9 @@ defmodule Tortoise.HandlerTest do
       }
 
       connack_fn = fn ^connack, state -> {:cont, state} end
-      handler = set_state(context.handler, pid: self(), connack: connack_fn)
+      handler = set_state(context.handler, connack: connack_fn)
 
       assert {:ok, %Handler{} = state, []} = Handler.execute_handle_connack(handler, connack)
-
-      assert_receive {:connack, ^connack}
     end
 
     test "return continue with next actions", context do
@@ -198,12 +247,10 @@ defmodule Tortoise.HandlerTest do
 
       next_actions = [{:subscribe, "foo/bar", [qos: 0]}]
       connack_fn = fn ^connack, state -> {:cont, state, next_actions} end
-      handler = set_state(context.handler, pid: self(), connack: connack_fn)
+      handler = set_state(context.handler, connack: connack_fn)
 
       assert {:ok, %Handler{} = state, ^next_actions} =
                Handler.execute_handle_connack(handler, connack)
-
-      assert_receive {:connack, ^connack}
     end
   end
 
@@ -212,13 +259,19 @@ defmodule Tortoise.HandlerTest do
       payload = :crypto.strong_rand_bytes(5)
       topic = "foo/bar"
       publish = %Package.Publish{topic: topic, payload: payload}
-      publish_fn = fn ^publish, state -> {:cont, state} end
-      handler = set_state(context.handler, pid: self(), publish: publish_fn)
+      parent = self()
+
+      publish_fn = fn topic_list, ^publish, state ->
+        send(parent, {:received_topic_list, topic_list})
+        {:cont, state}
+      end
+
+      handler = set_state(context.handler, publish: publish_fn)
 
       assert {:ok, %Handler{}, []} = Handler.execute_handle_publish(handler, publish)
       # the topic will be in the form of a list making it possible to
       # pattern match on the topic levels
-      assert_receive {:publish, topic_list, ^publish}
+      assert_receive {:received_topic_list, topic_list}
       assert is_list(topic_list)
       assert topic == Enum.join(topic_list, "/")
     end
@@ -228,15 +281,9 @@ defmodule Tortoise.HandlerTest do
       payload = :crypto.strong_rand_bytes(5)
       next_actions = [{:subscribe, "foo/bar", [qos: 0]}]
       publish = %Package.Publish{topic: topic, payload: payload}
-      publish_fn = fn ^publish, state -> {:cont, state, next_actions} end
-      handler = set_state(context.handler, pid: self(), publish: publish_fn)
+      publish_fn = fn _topic_list, ^publish, state -> {:cont, state, next_actions} end
+      handler = set_state(context.handler, publish: publish_fn)
       assert {:ok, %Handler{}, ^next_actions} = Handler.execute_handle_publish(handler, publish)
-
-      # the topic will be in the form of a list making it possible to
-      # pattern match on the topic levels
-      assert_receive {:publish, topic_list, ^publish}
-      assert is_list(topic_list)
-      assert topic == Enum.join(topic_list, "/")
     end
 
     test "return continue with invalid next action", context do
@@ -244,16 +291,11 @@ defmodule Tortoise.HandlerTest do
       payload = :crypto.strong_rand_bytes(5)
       publish = %Package.Publish{topic: topic, payload: payload}
       next_actions = [{:unsubscribe, "foo/bar"}, {:invalid, "bar"}]
-      publish_fn = fn ^publish, state -> {:cont, state, next_actions} end
-      handler = set_state(context.handler, pid: self(), publish: publish_fn)
+      publish_fn = fn _topic_list, ^publish, state -> {:cont, state, next_actions} end
+      handler = set_state(context.handler, publish: publish_fn)
 
       assert {:error, {:invalid_next_action, [{:invalid, "bar"}]}} =
                Handler.execute_handle_publish(handler, publish)
-
-      # the callback is still run so lets check the received data
-      assert_receive {:publish, topic_list, ^publish}
-      assert is_list(topic_list)
-      assert topic == Enum.join(topic_list, "/")
     end
   end
 
@@ -267,12 +309,10 @@ defmodule Tortoise.HandlerTest do
       suback = %Package.Suback{identifier: 1, acks: [ok: 0]}
 
       suback_fn = fn ^subscribe, ^suback, state -> {:cont, state} end
-      handler = set_state(context.handler, pid: self(), suback: suback_fn)
+      handler = set_state(context.handler, suback: suback_fn)
 
       assert {:ok, %Handler{} = state, []} =
                Handler.execute_handle_suback(handler, subscribe, suback)
-
-      assert_receive {:suback, {^subscribe, ^suback}}
     end
 
     test "return continue with next actions", context do
@@ -286,12 +326,10 @@ defmodule Tortoise.HandlerTest do
       next_actions = [{:unsubscribe, "foo/bar"}]
 
       suback_fn = fn ^subscribe, ^suback, state -> {:cont, state, next_actions} end
-      handler = set_state(context.handler, pid: self(), suback: suback_fn)
+      handler = set_state(context.handler, suback: suback_fn)
 
       assert {:ok, %Handler{} = state, ^next_actions} =
                Handler.execute_handle_suback(handler, subscribe, suback)
-
-      assert_receive {:suback, {^subscribe, ^suback}}
     end
   end
 
@@ -300,12 +338,10 @@ defmodule Tortoise.HandlerTest do
       unsubscribe = %Package.Unsubscribe{identifier: 1, topics: ["foo"]}
       unsuback = %Package.Unsuback{identifier: 1, results: [:success]}
       unsuback_fn = fn ^unsubscribe, ^unsuback, state -> {:cont, state} end
-      handler = set_state(context.handler, pid: self(), unsuback: unsuback_fn)
+      handler = set_state(context.handler, unsuback: unsuback_fn)
 
       assert {:ok, %Handler{} = state, []} =
                Handler.execute_handle_unsuback(handler, unsubscribe, unsuback)
-
-      assert_receive {:unsuback, {^unsubscribe, ^unsuback}}
     end
 
     test "return continue with next actions", context do
@@ -313,12 +349,10 @@ defmodule Tortoise.HandlerTest do
       unsuback = %Package.Unsuback{identifier: 1, results: [:success]}
       next_actions = [{:unsubscribe, "foo/bar"}]
       unsuback_fn = fn ^unsubscribe, ^unsuback, state -> {:cont, state, next_actions} end
-      handler = set_state(context.handler, pid: self(), unsuback: unsuback_fn)
+      handler = set_state(context.handler, unsuback: unsuback_fn)
 
       assert {:ok, %Handler{} = state, ^next_actions} =
                Handler.execute_handle_unsuback(handler, unsubscribe, unsuback)
-
-      assert_receive {:unsuback, {^unsubscribe, ^unsuback}}
     end
   end
 
@@ -327,23 +361,19 @@ defmodule Tortoise.HandlerTest do
     test "return continue", context do
       puback = %Package.Puback{identifier: 1}
       puback_fn = fn ^puback, state -> {:cont, state} end
-      handler = set_state(context.handler, pid: self(), puback: puback_fn)
+      handler = set_state(context.handler, puback: puback_fn)
 
       assert {:ok, %Handler{} = state, []} = Handler.execute_handle_puback(handler, puback)
-
-      assert_receive {:puback, ^puback}
     end
 
     test "return continue with next actions", context do
       puback = %Package.Puback{identifier: 1}
       next_actions = [{:subscribe, "foo/bar", qos: 0}]
       puback_fn = fn ^puback, state -> {:cont, state, next_actions} end
-      handler = set_state(context.handler, pid: self(), puback: puback_fn)
+      handler = set_state(context.handler, puback: puback_fn)
 
       assert {:ok, %Handler{} = state, ^next_actions} =
                Handler.execute_handle_puback(handler, puback)
-
-      assert_receive {:puback, ^puback}
     end
   end
 
@@ -352,12 +382,10 @@ defmodule Tortoise.HandlerTest do
     test "return continue", context do
       pubrec = %Package.Pubrec{identifier: 1}
       pubrec_fn = fn ^pubrec, state -> {:cont, state} end
-      handler = set_state(context.handler, pid: self(), pubrec: pubrec_fn)
+      handler = set_state(context.handler, pubrec: pubrec_fn)
 
       assert {:ok, %Package.Pubrel{identifier: 1}, %Handler{}, []} =
                Handler.execute_handle_pubrec(handler, pubrec)
-
-      assert_receive {:pubrec, ^pubrec}
     end
 
     test "return continue with custom pubrel", context do
@@ -368,12 +396,10 @@ defmodule Tortoise.HandlerTest do
         {{:cont, %Package.Pubrel{identifier: 1, properties: properties}}, state}
       end
 
-      handler = set_state(context.handler, pid: self(), pubrec: pubrec_fn)
+      handler = set_state(context.handler, pubrec: pubrec_fn)
 
       assert {:ok, %Package.Pubrel{identifier: 1, properties: ^properties}, %Handler{}, []} =
                Handler.execute_handle_pubrec(handler, pubrec)
-
-      assert_receive {:pubrec, ^pubrec}
     end
 
     test "raise an error if a custom pubrel with the wrong id is returned", context do
@@ -383,25 +409,21 @@ defmodule Tortoise.HandlerTest do
         {{:cont, %Package.Pubrel{identifier: id + 1}}, state}
       end
 
-      handler = set_state(context.handler, pid: self(), pubrec: pubrec_fn)
+      handler = set_state(context.handler, pubrec: pubrec_fn)
 
       assert_raise CaseClauseError, fn ->
         Handler.execute_handle_pubrec(handler, pubrec)
       end
-
-      assert_receive {:pubrec, ^pubrec}
     end
 
     test "returning continue with a list should result in a pubrel with user props", context do
       pubrec = %Package.Pubrec{identifier: 1}
       properties = [{"foo", "bar"}]
       pubrec_fn = fn ^pubrec, state -> {{:cont, properties}, state} end
-      handler = set_state(context.handler, pid: self(), pubrec: pubrec_fn)
+      handler = set_state(context.handler, pubrec: pubrec_fn)
 
       assert {:ok, %Package.Pubrel{identifier: 1, properties: ^properties}, %Handler{}, []} =
                Handler.execute_handle_pubrec(handler, pubrec)
-
-      assert_receive {:pubrec, ^pubrec}
     end
   end
 
@@ -409,12 +431,10 @@ defmodule Tortoise.HandlerTest do
     test "return continue", context do
       pubrel = %Package.Pubrel{identifier: 1}
       pubrel_fn = fn ^pubrel, state -> {:cont, state} end
-      handler = set_state(context.handler, pid: self(), pubrel: pubrel_fn)
+      handler = set_state(context.handler, pubrel: pubrel_fn)
 
       assert {:ok, %Package.Pubcomp{identifier: 1}, %Handler{} = state, []} =
                Handler.execute_handle_pubrel(handler, pubrel)
-
-      assert_receive {:pubrel, ^pubrel}
     end
 
     test "return continue with custom pubcomp", context do
@@ -425,12 +445,10 @@ defmodule Tortoise.HandlerTest do
         {{:cont, %Package.Pubcomp{identifier: 1, properties: properties}}, state}
       end
 
-      handler = set_state(context.handler, pid: self(), pubrel: pubrel_fn)
+      handler = set_state(context.handler, pubrel: pubrel_fn)
 
       assert {:ok, %Package.Pubcomp{identifier: 1, properties: ^properties}, %Handler{} = state,
               []} = Handler.execute_handle_pubrel(handler, pubrel)
-
-      assert_receive {:pubrel, ^pubrel}
     end
 
     test "should not allow custom pubcomp with a different id", context do
@@ -440,14 +458,12 @@ defmodule Tortoise.HandlerTest do
         {{:cont, %Package.Pubcomp{identifier: id + 1}}, state}
       end
 
-      handler = set_state(context.handler, pid: self(), pubrel: pubrel_fn)
+      handler = set_state(context.handler, pubrel: pubrel_fn)
 
       # todo, consider making an IdentifierMismatchError type
       assert_raise CaseClauseError, fn ->
         Handler.execute_handle_pubrel(handler, pubrel)
       end
-
-      assert_receive {:pubrel, ^pubrel}
     end
 
     test "returning {:cont, [{string(), string()}]} become user defined properties", context do
@@ -458,12 +474,10 @@ defmodule Tortoise.HandlerTest do
         {{:cont, properties}, state}
       end
 
-      handler = set_state(context.handler, pid: self(), pubrel: pubrel_fn)
+      handler = set_state(context.handler, pubrel: pubrel_fn)
 
       assert {:ok, %Package.Pubcomp{identifier: 1, properties: ^properties}, %Handler{} = state,
               []} = Handler.execute_handle_pubrel(handler, pubrel)
-
-      assert_receive {:pubrel, ^pubrel}
     end
   end
 
@@ -471,13 +485,11 @@ defmodule Tortoise.HandlerTest do
     test "return continue", context do
       pubcomp = %Package.Pubcomp{identifier: 1}
       pubcomp_fn = fn ^pubcomp, state -> {:cont, state} end
-      handler = set_state(context.handler, pid: self(), pubcomp: pubcomp_fn)
+      handler = set_state(context.handler, pubcomp: pubcomp_fn)
 
       assert {:ok, %Handler{} = state, []} =
                handler
                |> Handler.execute_handle_pubcomp(pubcomp)
-
-      assert_receive {:pubcomp, ^pubcomp}
     end
 
     test "return continue with next actions", context do
@@ -485,12 +497,10 @@ defmodule Tortoise.HandlerTest do
       next_actions = [{:subscribe, "foo/bar", qos: 0}]
       pubcomp_fn = fn ^pubcomp, state -> {:cont, state, next_actions} end
 
-      handler = set_state(context.handler, pid: self(), pubcomp: pubcomp_fn)
+      handler = set_state(context.handler, pubcomp: pubcomp_fn)
 
       assert {:ok, %Handler{} = state, ^next_actions} =
                Handler.execute_handle_pubcomp(handler, pubcomp)
-
-      assert_receive {:pubcomp, ^pubcomp}
     end
   end
 
@@ -498,12 +508,10 @@ defmodule Tortoise.HandlerTest do
     test "return continue", context do
       disconnect = %Package.Disconnect{}
       disconnect_fn = fn ^disconnect, state -> {:cont, state} end
-      handler = set_state(context.handler, pid: self(), disconnect: disconnect_fn)
+      handler = set_state(context.handler, disconnect: disconnect_fn)
 
       assert {:ok, %Handler{} = state, []} =
                Handler.execute_handle_disconnect(handler, disconnect)
-
-      assert_receive {:disconnect, ^disconnect}
     end
 
     test "return continue with next actions", context do
@@ -511,29 +519,32 @@ defmodule Tortoise.HandlerTest do
       next_actions = [{:subscribe, "foo/bar", qos: 2}]
       disconnect_fn = fn ^disconnect, state -> {:cont, state, next_actions} end
 
-      handler = set_state(context.handler, pid: self(), disconnect: disconnect_fn)
+      handler = set_state(context.handler, disconnect: disconnect_fn)
 
       assert {:ok, %Handler{} = state, ^next_actions} =
                Handler.execute_handle_disconnect(handler, disconnect)
-
-      assert_receive {:disconnect, ^disconnect}
     end
 
     test "return stop with normal reason", context do
       disconnect = %Package.Disconnect{}
       disconnect_fn = fn ^disconnect, state -> {:stop, :normal, state} end
-      handler = set_state(context.handler, pid: self(), disconnect: disconnect_fn)
+      handler = set_state(context.handler, disconnect: disconnect_fn)
 
       assert {:stop, :normal, %Handler{} = state} =
                Handler.execute_handle_disconnect(handler, disconnect)
-
-      assert_receive {:disconnect, ^disconnect}
     end
   end
 
   describe "execute terminate/2" do
     test "return ok", context do
-      handler = set_state(context.handler, pid: self())
+      parent = self()
+
+      terminate_fn = fn reason, _state ->
+        send(parent, {:terminate, reason})
+        :ok
+      end
+
+      handler = set_state(context.handler, terminate: terminate_fn)
       assert :ok = Handler.execute_terminate(handler, :normal)
       assert_receive {:terminate, :normal}
     end

From 3638adfb3ac276271e1fc3f2ef31199244610c06 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sun, 10 Feb 2019 13:42:48 +0000
Subject: [PATCH 117/220] Allow user defined properties for unsubscribe
 packages

---
 lib/tortoise/connection.ex        |  8 +++++++-
 test/tortoise/connection_test.exs | 15 +++++++++++++--
 2 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index c7d591de..aebb91c9 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -237,7 +237,13 @@ defmodule Tortoise.Connection do
   def unsubscribe(client_id, [topic | _] = topics, opts) when is_binary(topic) do
     caller = {_, ref} = {self(), make_ref()}
     {identifier, opts} = Keyword.pop_first(opts, :identifier, nil)
-    unsubscribe = %Package.Unsubscribe{identifier: identifier, topics: topics}
+
+    unsubscribe = %Package.Unsubscribe{
+      identifier: identifier,
+      topics: topics,
+      properties: Keyword.get(opts, :properties, [])
+    }
+
     GenStateMachine.cast(via_name(client_id), {:unsubscribe, caller, unsubscribe, opts})
     {:ok, ref}
   end
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index 896b8736..a218e133 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -326,7 +326,13 @@ defmodule Tortoise.ConnectionTest do
 
       unsubscribe_foo = %Package.Unsubscribe{identifier: 2, topics: ["foo"]}
       unsuback_foo = %Package.Unsuback{results: [:success], identifier: 2}
-      unsubscribe_bar = %Package.Unsubscribe{identifier: 3, topics: ["bar"]}
+
+      unsubscribe_bar = %Package.Unsubscribe{
+        identifier: 3,
+        topics: ["bar"],
+        properties: [{"foo", "bar"}]
+      }
+
       unsuback_bar = %Package.Unsuback{results: [:success], identifier: 3}
 
       script = [
@@ -374,7 +380,12 @@ defmodule Tortoise.ConnectionTest do
       assert Map.has_key?(Tortoise.Connection.subscriptions(client_id), "bar")
 
       # and unsubscribe from bar
-      assert {:ok, ref} = Tortoise.Connection.unsubscribe(client_id, "bar", identifier: 3)
+      assert {:ok, ref} =
+               Tortoise.Connection.unsubscribe(client_id, "bar",
+                 identifier: 3,
+                 properties: [{"foo", "bar"}]
+               )
+
       assert_receive {{Tortoise, ^client_id}, ^ref, :ok}
       assert_receive {ScriptedMqttServer, {:received, ^unsubscribe_bar}}
       # handle_unsuback should get called on the callback handler

From 80ad257f84042c332f94b4069d4e8fa14d75c6b8 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sun, 10 Feb 2019 13:47:22 +0000
Subject: [PATCH 118/220] Make it possible to assign user properties on
 subscribe packages

Pass them in as options for the `Tortoise.Connection.subscribe/3`
function
---
 lib/tortoise/connection.ex        |  8 +++++++-
 test/tortoise/connection_test.exs | 10 ++++++++--
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index aebb91c9..0a2794e2 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -150,7 +150,13 @@ defmodule Tortoise.Connection do
   def subscribe(client_id, [{_, topic_opts} | _] = topics, opts) when is_list(topic_opts) do
     caller = {_, ref} = {self(), make_ref()}
     {identifier, opts} = Keyword.pop_first(opts, :identifier, nil)
-    subscribe = Enum.into(topics, %Package.Subscribe{identifier: identifier})
+
+    subscribe =
+      Enum.into(topics, %Package.Subscribe{
+        identifier: identifier,
+        properties: Keyword.get(opts, :properties, [])
+      })
+
     GenStateMachine.cast(via_name(client_id), {:subscribe, caller, subscribe, opts})
     {:ok, ref}
   end
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index a218e133..f15de65f 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -271,7 +271,7 @@ defmodule Tortoise.ConnectionTest do
       subscription_baz =
         Enum.into(
           [{"baz", [{:qos, 2} | default_subscription_opts]}],
-          %Package.Subscribe{identifier: 3}
+          %Package.Subscribe{identifier: 3, properties: [{"foo", "bar"}]}
         )
 
       suback_baz = %Package.Suback{identifier: 3, acks: [{:ok, 2}]}
@@ -303,7 +303,13 @@ defmodule Tortoise.ConnectionTest do
       assert_receive {{TestHandler, :handle_suback}, {%Package.Subscribe{}, ^suback_bar}}
 
       # subscribe to a baz
-      assert {:ok, ref} = Tortoise.Connection.subscribe(client_id, "baz", qos: 2, identifier: 3)
+      assert {:ok, ref} =
+               Tortoise.Connection.subscribe(client_id, "baz",
+                 qos: 2,
+                 identifier: 3,
+                 properties: [{"foo", "bar"}]
+               )
+
       assert_receive {{Tortoise, ^client_id}, ^ref, :ok}
       assert_receive {ScriptedMqttServer, {:received, ^subscription_baz}}
       assert_receive {{TestHandler, :handle_suback}, {%Package.Subscribe{}, ^suback_baz}}

From 6bf1b135b251a955749c58bbc4672a1959f0a430 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sun, 10 Feb 2019 16:25:43 +0000
Subject: [PATCH 119/220] The main connection init call nolonger rely on an
 5-tuple

Now we just pass the state object in; should be good.
---
 lib/tortoise/connection.ex | 26 ++++++++++++--------------
 1 file changed, 12 insertions(+), 14 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 0a2794e2..de9766bb 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -68,7 +68,14 @@ defmodule Tortoise.Connection do
       {:transport, server} | Keyword.take(connection_opts, [:client_id])
     ]
 
-    initial = {server, connect, backoff, handler, connection_opts}
+    initial = %State{
+      client_id: connect.client_id,
+      server: server,
+      connect: connect,
+      backoff: Backoff.new(backoff),
+      opts: connection_opts,
+      handler: handler
+    }
     opts = Keyword.merge(opts, name: via_name(client_id))
     GenStateMachine.start_link(__MODULE__, initial, opts)
   end
@@ -378,21 +385,12 @@ defmodule Tortoise.Connection do
 
   # Callbacks
   @impl true
-  def init({transport, connect, backoff_opts, handler, opts}) do
-    data = %State{
-      client_id: connect.client_id,
-      server: transport,
-      connect: connect,
-      backoff: Backoff.new(backoff_opts),
-      opts: opts,
-      handler: handler
-    }
-
-    case Handler.execute_init(handler) do
+  def init(%State{} = state) do
+    case Handler.execute_init(state.handler) do
       {:ok, %Handler{} = updated_handler} ->
         next_events = [{:next_event, :internal, :connect}]
-        updated_data = %State{data | handler: updated_handler}
-        {:ok, :connecting, updated_data, next_events}
+        updated_state = %State{state | handler: updated_handler}
+        {:ok, :connecting, updated_state, next_events}
 
       :ignore ->
         :ignore

From e560921dc4e46d6a817036e0370d16e0c2717f8b Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Fri, 8 Mar 2019 22:41:46 +0000
Subject: [PATCH 120/220] Fix the return expression of a couple of default
 callbacks

---
 lib/tortoise/handler.ex | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index bc12fa78..af099df6 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -149,7 +149,7 @@ defmodule Tortoise.Handler do
 
       @impl true
       def handle_puback(_puback, state) do
-        {:ok, state}
+        {:cont, state}
       end
 
       @impl true
@@ -169,12 +169,12 @@ defmodule Tortoise.Handler do
 
       @impl true
       def handle_suback(_subscribe, _suback, state) do
-        {:ok, state}
+        {:cont, state}
       end
 
       @impl true
       def handle_unsuback(_unsubscribe, _unsuback, state) do
-        {:ok, state}
+        {:cont, state}
       end
 
       @impl true

From 700d979b34d2ef14f4e75a4deb2d6991b32fccab Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Mon, 29 Apr 2019 15:11:44 +0100
Subject: [PATCH 121/220] Remove a stray IO.inspect/2

How did that get there?
---
 lib/tortoise/connection/inflight.ex | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/tortoise/connection/inflight.ex b/lib/tortoise/connection/inflight.ex
index 001b4777..b7413a99 100644
--- a/lib/tortoise/connection/inflight.ex
+++ b/lib/tortoise/connection/inflight.ex
@@ -357,7 +357,6 @@ defmodule Tortoise.Connection.Inflight do
       {:keep_state, handle_next(track, data)}
     else
       res ->
-        IO.inspect(res, label: __MODULE__)
         {:stop, res, data}
     end
   end

From d4e67503a00363b8bd0cd3ad6c587e3218262014 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Mon, 29 Apr 2019 15:21:01 +0100
Subject: [PATCH 122/220] Implement next actions for handle_connack, suback,
 and unsuback

Still need to hook it up for handle_publish and acknowledgement
packages relating to publishes; but now we can specify a list of next
actions to perform from the callback functions.
---
 lib/tortoise/connection.ex        | 38 +++++++++++++--
 lib/tortoise/handler.ex           | 32 ++++++++++--
 test/support/test_handler.ex      | 25 ++++++++++
 test/tortoise/connection_test.exs | 81 +++++++++++++++++++++++++++++++
 test/tortoise/handler_test.exs    |  4 +-
 5 files changed, 171 insertions(+), 9 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index de9766bb..e5b14755 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -76,6 +76,7 @@ defmodule Tortoise.Connection do
       opts: connection_opts,
       handler: handler
     }
+
     opts = Keyword.merge(opts, name: via_name(client_id))
     GenStateMachine.start_link(__MODULE__, initial, opts)
   end
@@ -423,8 +424,7 @@ defmodule Tortoise.Connection do
         } = data
       ) do
     case Handler.execute_handle_connack(handler, connack) do
-      {:ok, %Handler{} = updated_handler, _next_actions} ->
-        # todo, add support for next actions
+      {:ok, %Handler{} = updated_handler, next_actions} ->
         :ok = Tortoise.Registry.put_meta(via_name(client_id), data.connection)
         :ok = Events.dispatch(client_id, :connection, data.connection)
         :ok = Events.dispatch(client_id, :status, :connected)
@@ -433,6 +433,9 @@ defmodule Tortoise.Connection do
         next_actions = [
           {:state_timeout, keep_alive * 1000, :keep_alive},
           {:next_event, :internal, {:execute_handler, {:connection, :up}}}
+          | for action <- next_actions do
+              {:next_event, :internal, {:user_action, action}}
+            end
         ]
 
         {:next_state, :connected, data, next_actions}
@@ -678,7 +681,7 @@ defmodule Tortoise.Connection do
         next_actions = [
           {:next_event, :internal, {:reply, {pid, msg_ref}, :ok}}
           | for action <- next_actions do
-              {:next_event, :internal, action}
+              {:next_event, :internal, {:user_action, action}}
             end
         ]
 
@@ -754,7 +757,7 @@ defmodule Tortoise.Connection do
         next_actions = [
           {:next_event, :internal, {:reply, {pid, msg_ref}, :ok}}
           | for action <- next_actions do
-              {:next_event, :internal, action}
+              {:next_event, :internal, {:user_action, action}}
             end
         ]
 
@@ -771,6 +774,33 @@ defmodule Tortoise.Connection do
     {:keep_state_and_data, next_actions}
   end
 
+  # User actions are actions returned by the user defined callbacks;
+  # They inform the connection to perform an action, such as
+  # subscribing to a topic, and they are validated by the handler
+  # module, so there is no need to coerce here
+  def handle_event(:internal, {:user_action, action}, _, %State{client_id: client_id}) do
+    case action do
+      {:subscribe, topic, opts} when is_binary(topic) ->
+        caller = {self(), make_ref()}
+        {identifier, opts} = Keyword.pop_first(opts, :identifier, nil)
+        subscribe = %Package.Subscribe{identifier: identifier, topics: [{topic, opts}]}
+        next_actions = [{:next_event, :cast, {:subscribe, caller, subscribe, opts}}]
+        {:keep_state_and_data, next_actions}
+
+      {:unsubscribe, topic, opts} when is_binary(topic) ->
+        caller = {self(), make_ref()}
+        {identifier, opts} = Keyword.pop_first(opts, :identifier, nil)
+        subscribe = %Package.Unsubscribe{identifier: identifier, topics: [topic]}
+        next_actions = [{:next_event, :cast, {:unsubscribe, caller, subscribe, opts}}]
+        {:keep_state_and_data, next_actions}
+
+      :disconnect ->
+        :ok = Events.dispatch(client_id, :status, :terminating)
+        :ok = Inflight.drain(client_id)
+        {:stop, :normal}
+    end
+  end
+
   # dispatch to user defined handler
   def handle_event(
         :internal,
diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index af099df6..5b5e6405 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -618,7 +618,24 @@ defmodule Tortoise.Handler do
   defp transform_result({cont, updated_state, next_actions}) when is_list(next_actions) do
     case Enum.split_with(next_actions, &valid_next_action?/1) do
       {_, []} ->
-        {cont, updated_state, next_actions}
+        # add option lists to next actions that do not have them yet,
+        # this could probably be done in a smarter way; a function
+        # that both validate and coerce the arguments in one pass
+        coerced_next_actions =
+          for action <- next_actions do
+            case action do
+              {:subscribe, topic} ->
+                {:subscribe, topic, []}
+
+              {:unsubscribe, topic} ->
+                {:unsubscribe, topic, []}
+
+              otherwise ->
+                otherwise
+            end
+          end
+
+        {cont, updated_state, coerced_next_actions}
 
       {_, invalid_next_actions} ->
         {:error, {:invalid_next_action, invalid_next_actions}}
@@ -629,13 +646,22 @@ defmodule Tortoise.Handler do
     transform_result({cont, updated_state, []})
   end
 
+  # subscribe
+  defp valid_next_action?({:subscribe, topic}), do: is_binary(topic)
+
   defp valid_next_action?({:subscribe, topic, opts}) do
     is_binary(topic) and is_list(opts)
   end
 
-  defp valid_next_action?({:unsubscribe, topic}) do
-    is_binary(topic)
+  # unsubscribe
+  defp valid_next_action?({:unsubscribe, topic}), do: is_binary(topic)
+
+  defp valid_next_action?({:unsubscribe, topic, opts}) do
+    is_binary(topic) and is_list(opts)
   end
 
+  # disconnect
+  defp valid_next_action?(:disconnect), do: true
+
   defp valid_next_action?(_otherwise), do: false
 end
diff --git a/test/support/test_handler.ex b/test/support/test_handler.ex
index c9fcaddb..3b83ffe1 100644
--- a/test/support/test_handler.ex
+++ b/test/support/test_handler.ex
@@ -9,6 +9,18 @@ defmodule TestHandler do
     {:ok, state}
   end
 
+  def handle_connack(connack, %{handle_connack: fun} = state) when is_function(fun) do
+    apply(fun, [connack, state])
+  end
+
+  def handle_connack(_connack, state) do
+    {:cont, state}
+  end
+
+  def terminate(reason, %{terminate: fun} = state) when is_function(fun) do
+    apply(fun, [reason, state])
+  end
+
   def terminate(reason, state) do
     send(state[:parent], {{__MODULE__, :terminate}, reason})
     :ok
@@ -19,6 +31,10 @@ defmodule TestHandler do
     {:cont, state}
   end
 
+  def handle_disconnect(disconnect, %{handle_disconnect: fun} = state) when is_function(fun) do
+    apply(fun, [disconnect, state])
+  end
+
   def handle_disconnect(disconnect, state) do
     send(state[:parent], {{__MODULE__, :handle_disconnect}, disconnect})
     {:stop, :normal, state}
@@ -57,11 +73,20 @@ defmodule TestHandler do
     {:cont, state}
   end
 
+  def handle_suback(subscribe, suback, %{handle_suback: fun} = state) when is_function(fun) do
+    apply(fun, [subscribe, suback, state])
+  end
+
   def handle_suback(subscribe, suback, state) do
     send(state[:parent], {{__MODULE__, :handle_suback}, {subscribe, suback}})
     {:cont, state}
   end
 
+  def handle_unsuback(unsubscribe, unsuback, %{handle_unsuback: fun} = state)
+      when is_function(fun) do
+    apply(fun, [unsubscribe, unsuback, state])
+  end
+
   def handle_unsuback(unsubscribe, unsuback, state) do
     send(state[:parent], {{__MODULE__, :handle_unsuback}, {unsubscribe, unsuback}})
     {:cont, state}
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index f15de65f..7ebc85d3 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -716,6 +716,87 @@ defmodule Tortoise.ConnectionTest do
       assert_receive {{^handler_mod, :terminate}, :shutdown}
       refute_receive {{^handler_mod, _}, _}
     end
+
+    test "user next actions", context do
+      connect = %Package.Connect{client_id: context.client_id}
+      expected_connack = %Package.Connack{reason: :success, session_present: false}
+
+      subscribe = %Package.Subscribe{
+        identifier: 1,
+        properties: [],
+        topics: [
+          {"foo/bar", [qos: 1, no_local: false, retain_as_published: false, retain_handling: 1]}
+        ]
+      }
+
+      unsubscribe = %Package.Unsubscribe{
+        identifier: 2,
+        properties: [],
+        topics: ["foo/bar"]
+      }
+
+      suback = %Package.Suback{identifier: 1, acks: [{:ok, 0}]}
+      unsuback = %Package.Unsuback{identifier: 2, results: [:success]}
+      disconnect = %Package.Disconnect{}
+
+      script = [
+        {:receive, connect},
+        {:send, expected_connack},
+        {:receive, subscribe},
+        {:send, suback},
+        {:receive, unsubscribe},
+        {:send, unsuback},
+        {:receive, disconnect}
+      ]
+
+      {:ok, {ip, port}} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
+
+      # the handler will contain a handle connack that will continue
+      # and setup a subscribe command; this should result in the
+      # server receiving a subscribe package.
+      handler =
+        {TestHandler,
+         [
+           parent: self(),
+           handle_connack: fn %Package.Connack{reason: :success}, state ->
+             {:cont, state, [{:subscribe, "foo/bar", qos: 1, identifier: 1}]}
+           end,
+           handle_suback: fn %Package.Subscribe{}, %Package.Suback{}, state ->
+             {:cont, state, [{:unsubscribe, "foo/bar", identifier: 2}]}
+           end,
+           handle_unsuback: fn %Package.Unsubscribe{}, %Package.Unsuback{}, state ->
+             {:cont, state, [:disconnect]}
+           end,
+           terminate: fn reason, %{parent: parent} ->
+             send(parent, {self(), {:terminating, reason}})
+             :ok
+           end
+         ]}
+
+      opts = [
+        client_id: context.client_id,
+        server: {Tortoise.Transport.Tcp, [host: ip, port: port]},
+        handler: handler
+      ]
+
+      assert {:ok, pid} = Connection.start_link(opts)
+      assert_receive {ScriptedMqttServer, {:received, ^connect}}
+      # the handle_connack will setup a subscribe command; tortoise
+      # should subscribe to the topic
+      assert_receive {ScriptedMqttServer, {:received, ^subscribe}}
+      # the handle_suback should generate an unsubscribe command
+      assert_receive {ScriptedMqttServer, {:received, ^unsubscribe}}
+      # the handle_unsuback callback should generate a disconnect
+      # command, which should disconnect the client from the server;
+      # which will receive the disconnect message, and the client
+      # process should terminate and exit
+      assert_receive {ScriptedMqttServer, {:received, ^disconnect}}
+      assert_receive {^pid, {:terminating, :normal}}
+      refute Process.alive?(pid)
+
+      # all done
+      assert_receive {ScriptedMqttServer, :completed}
+    end
   end
 
   describe "ping" do
diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index 13ffc38a..eb981c7d 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -328,7 +328,7 @@ defmodule Tortoise.HandlerTest do
       suback_fn = fn ^subscribe, ^suback, state -> {:cont, state, next_actions} end
       handler = set_state(context.handler, suback: suback_fn)
 
-      assert {:ok, %Handler{} = state, ^next_actions} =
+      assert {:ok, %Handler{} = state, [{:unsubscribe, "foo/bar", []}]} =
                Handler.execute_handle_suback(handler, subscribe, suback)
     end
   end
@@ -351,7 +351,7 @@ defmodule Tortoise.HandlerTest do
       unsuback_fn = fn ^unsubscribe, ^unsuback, state -> {:cont, state, next_actions} end
       handler = set_state(context.handler, unsuback: unsuback_fn)
 
-      assert {:ok, %Handler{} = state, ^next_actions} =
+      assert {:ok, %Handler{} = state, [{:unsubscribe, "foo/bar", []}]} =
                Handler.execute_handle_unsuback(handler, unsubscribe, unsuback)
     end
   end

From 7a5c6b9b6ae854476a5544dd618fa7cf4c9fece5 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Thu, 2 May 2019 15:50:32 +0100
Subject: [PATCH 123/220] Increased the timeout on a assert_receive

This fixes an issue where we would not make the connection before the
timeout
---
 test/tortoise/events_test.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/tortoise/events_test.exs b/test/tortoise/events_test.exs
index a48b2edb..c10cc5b0 100644
--- a/test/tortoise/events_test.exs
+++ b/test/tortoise/events_test.exs
@@ -90,7 +90,7 @@ defmodule Tortoise.EventsTest do
 
       # the subscriber should receive the connection and it should
       # still be registered for new connections
-      assert_receive {:received, ^connection}
+      assert_receive {:received, ^connection}, 2000
       assert [:connection] = Registry.keys(Tortoise.Events, child)
 
       context = run_setup(context, :setup_connection)

From b3bdb60808d67afeb461430df6d73921095b94e9 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Fri, 3 May 2019 13:50:11 +0100
Subject: [PATCH 124/220] Introduce a eval-next action

It is now possible to execute an anonymous function in the context of
the tortoise connection by passing it in as a next action with the
form `{:eval, fn (state) -> IO.inspect(state) end}`.

The `state` given to the function is the connection state. It is not
possible to alter the state, as the return of the function is
disregarded.

This has been added to make it easier to test execution of next
actions in the callback tests.
---
 lib/tortoise/connection.ex | 9 +++++++++
 lib/tortoise/handler.ex    | 2 ++
 2 files changed, 11 insertions(+)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index e5b14755..e8e5976e 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -798,6 +798,15 @@ defmodule Tortoise.Connection do
         :ok = Events.dispatch(client_id, :status, :terminating)
         :ok = Inflight.drain(client_id)
         {:stop, :normal}
+
+      {:eval, fun} when is_function(fun, 1) ->
+        try do
+          apply(fun, [state])
+        rescue
+          _disregard -> nil
+        end
+
+        {:keep_state, state}
     end
   end
 
diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index 5b5e6405..1822de3f 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -663,5 +663,7 @@ defmodule Tortoise.Handler do
   # disconnect
   defp valid_next_action?(:disconnect), do: true
 
+  defp valid_next_action?({:eval, fun}) when is_function(fun, 1), do: true
+
   defp valid_next_action?(_otherwise), do: false
 end

From 0ce143ff0d0b765c1955d860f65b60c92115aa2d Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Fri, 3 May 2019 15:00:27 +0100
Subject: [PATCH 125/220] Make it possible to customize all callbacks in the
 TestHandler

---
 test/support/test_handler.ex      | 146 +++++++++++++++++++-----------
 test/tortoise/connection_test.exs |   1 +
 2 files changed, 93 insertions(+), 54 deletions(-)

diff --git a/test/support/test_handler.ex b/test/support/test_handler.ex
index 3b83ffe1..b8e06d53 100644
--- a/test/support/test_handler.ex
+++ b/test/support/test_handler.ex
@@ -9,86 +9,124 @@ defmodule TestHandler do
     {:ok, state}
   end
 
-  def handle_connack(connack, %{handle_connack: fun} = state) when is_function(fun) do
-    apply(fun, [connack, state])
-  end
-
-  def handle_connack(_connack, state) do
-    {:cont, state}
-  end
+  def handle_connack(%Package.Connack{} = connack, state) do
+    case state[:handle_connack] do
+      nil ->
+        send(state[:parent], {{__MODULE__, :handle_connack}, connack})
+        {:cont, state}
 
-  def terminate(reason, %{terminate: fun} = state) when is_function(fun) do
-    apply(fun, [reason, state])
+      fun when is_function(fun, 2) ->
+        apply(fun, [connack, state])
+    end
   end
 
   def terminate(reason, state) do
-    send(state[:parent], {{__MODULE__, :terminate}, reason})
-    :ok
+    case state[:terminate] do
+      nil ->
+        send(state[:parent], {{__MODULE__, :terminate}, reason})
+        :ok
+
+      fun when is_function(fun, 2) ->
+        apply(fun, [reason, state])
+    end
   end
 
   def connection(status, state) do
-    send(state[:parent], {{__MODULE__, :connection}, status})
-    {:cont, state}
-  end
+    case state[:connection] do
+      nil ->
+        send(state[:parent], {{__MODULE__, :connection}, status})
+        {:cont, state}
 
-  def handle_disconnect(disconnect, %{handle_disconnect: fun} = state) when is_function(fun) do
-    apply(fun, [disconnect, state])
+      fun when is_function(fun, 2) ->
+        apply(fun, [status, state])
+    end
   end
 
-  def handle_disconnect(disconnect, state) do
-    send(state[:parent], {{__MODULE__, :handle_disconnect}, disconnect})
-    {:stop, :normal, state}
-  end
+  def handle_disconnect(%Package.Disconnect{} = disconnect, state) do
+    case state[:handle_disconnect] do
+      nil ->
+        send(state[:parent], {{__MODULE__, :handle_disconnect}, disconnect})
+        {:stop, :normal, state}
 
-  def handle_publish(_topic, %Package.Publish{qos: 1} = publish, state) do
-    # data = %{topic: Enum.join(topic, "/"), payload: payload}
-    send(state[:parent], {{__MODULE__, :handle_publish}, publish})
-    {:cont, state}
+      fun when is_function(fun, 2) ->
+        apply(fun, [disconnect, state])
+    end
   end
 
-  def handle_publish(_topic, %Package.Publish{qos: 2} = publish, state) do
-    # data = %{topic: Enum.join(topic, "/"), payload: payload}
-    send(state[:parent], {{__MODULE__, :handle_publish}, publish})
-    {:cont, state}
-  end
+  def handle_publish(topic, %Package.Publish{} = publish, state) do
+    case state[:handle_publish] do
+      nil ->
+        send(state[:parent], {{__MODULE__, :handle_publish}, publish})
+        {:cont, state}
 
-  def handle_publish(_topic, publish, state) do
-    # data = %{topic: Enum.join(topic, "/"), payload: payload}
-    send(state[:parent], {{__MODULE__, :handle_publish}, publish})
-    {:cont, state}
+      fun when is_function(fun, 3) ->
+        apply(fun, [topic, publish, state])
+    end
   end
 
-  def handle_pubrec(pubrec, state) do
-    send(state[:parent], {{__MODULE__, :handle_pubrec}, pubrec})
-    {:cont, state}
-  end
+  def handle_puback(%Package.Puback{} = puback, state) do
+    case state[:handle_puback] do
+      nil ->
+        send(state[:parent], {{__MODULE__, :handle_puback}, puback})
+        {:cont, state}
 
-  def handle_pubrel(pubrel, state) do
-    send(state[:parent], {{__MODULE__, :handle_pubrel}, pubrel})
-    {:cont, state}
+      fun when is_function(fun, 2) ->
+        apply(fun, [puback, state])
+    end
   end
 
-  def handle_pubcomp(pubcomp, state) do
-    send(state[:parent], {{__MODULE__, :handle_pubcomp}, pubcomp})
-    {:cont, state}
+  def handle_pubrec(%Package.Pubrec{} = pubrec, state) do
+    case state[:handle_pubrec] do
+      nil ->
+        send(state[:parent], {{__MODULE__, :handle_pubrec}, pubrec})
+        {:cont, state}
+
+      fun when is_function(fun, 2) ->
+        apply(fun, [pubrec, state])
+    end
   end
 
-  def handle_suback(subscribe, suback, %{handle_suback: fun} = state) when is_function(fun) do
-    apply(fun, [subscribe, suback, state])
+  def handle_pubrel(%Package.Pubrel{} = pubrel, state) do
+    case state[:handle_pubrel] do
+      nil ->
+        send(state[:parent], {{__MODULE__, :handle_pubrel}, pubrel})
+        {:cont, state}
+
+      fun when is_function(fun, 2) ->
+        apply(fun, [pubrel, state])
+    end
   end
 
-  def handle_suback(subscribe, suback, state) do
-    send(state[:parent], {{__MODULE__, :handle_suback}, {subscribe, suback}})
-    {:cont, state}
+  def handle_pubcomp(%Package.Pubcomp{} = pubcomp, state) do
+    case state[:handle_pubcomp] do
+      nil ->
+        send(state[:parent], {{__MODULE__, :handle_pubcomp}, pubcomp})
+        {:cont, state}
+
+      fun when is_function(fun, 2) ->
+        apply(fun, [pubcomp, state])
+    end
   end
 
-  def handle_unsuback(unsubscribe, unsuback, %{handle_unsuback: fun} = state)
-      when is_function(fun) do
-    apply(fun, [unsubscribe, unsuback, state])
+  def handle_suback(%Package.Subscribe{} = subscribe, %Package.Suback{} = suback, state) do
+    case state[:handle_suback] do
+      nil ->
+        send(state[:parent], {{__MODULE__, :handle_suback}, {subscribe, suback}})
+        {:cont, state}
+
+      fun when is_function(fun, 3) ->
+        apply(fun, [subscribe, suback, state])
+    end
   end
 
-  def handle_unsuback(unsubscribe, unsuback, state) do
-    send(state[:parent], {{__MODULE__, :handle_unsuback}, {unsubscribe, unsuback}})
-    {:cont, state}
+  def handle_unsuback(%Package.Unsubscribe{} = unsubscribe, %Package.Unsuback{} = unsuback, state) do
+    case state[:handle_unsuback] do
+      nil ->
+        send(state[:parent], {{__MODULE__, :handle_unsuback}, {unsubscribe, unsuback}})
+        {:cont, state}
+
+      fun when is_function(fun, 3) ->
+        apply(fun, [unsubscribe, unsuback, state])
+    end
   end
 end
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index 7ebc85d3..ac1124b4 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -714,6 +714,7 @@ defmodule Tortoise.ConnectionTest do
       assert_receive {{^handler_mod, :init}, ^handler_init_opts}
       assert_receive {{^handler_mod, :connection}, :up}
       assert_receive {{^handler_mod, :terminate}, :shutdown}
+      assert_receive {{^handler_mod, :handle_connack}, %Package.Connack{}}
       refute_receive {{^handler_mod, _}, _}
     end
 

From 5271e772a306dde28064fd17240114ebe36e152a Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Fri, 3 May 2019 15:02:40 +0100
Subject: [PATCH 126/220] Test next actions generated by user defined callbacks

Not all are implemented and tested, but this cover some
---
 lib/tortoise/connection.ex        |  40 +++++----
 test/tortoise/connection_test.exs | 142 ++++++++++++++++++++++++++++++
 2 files changed, 163 insertions(+), 19 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index e8e5976e..97e3ad44 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -501,10 +501,9 @@ defmodule Tortoise.Connection do
     :ok = Inflight.update(client_id, {:received, puback})
 
     case Handler.execute_handle_puback(handler, puback) do
-      {:ok, %Handler{} = updated_handler, _next_actions} ->
-        # todo, handle next_actions
+      {:ok, %Handler{} = updated_handler, next_actions} ->
         updated_data = %State{data | handler: updated_handler}
-        {:keep_state, updated_data}
+        {:keep_state, updated_data, wrap_next_actions(next_actions)}
 
       {:error, reason} ->
         # todo
@@ -523,12 +522,11 @@ defmodule Tortoise.Connection do
 
     case Handler.execute_handle_pubrel(handler, pubrel) do
       {:ok, %Package.Pubcomp{identifier: ^id} = pubcomp, %Handler{} = updated_handler,
-       _next_actions} ->
-        # todo, handle next actions
+       next_actions} ->
         # dispatch the pubcomp
         :ok = Inflight.update(client_id, {:dispatch, pubcomp})
         updated_data = %State{data | handler: updated_handler}
-        {:keep_state, updated_data}
+        {:keep_state, updated_data, wrap_next_actions(next_actions)}
 
       {:error, reason} ->
         # todo
@@ -586,11 +584,10 @@ defmodule Tortoise.Connection do
     :ok = Inflight.update(client_id, {:received, pubrec})
 
     case Handler.execute_handle_pubrec(handler, pubrec) do
-      {:ok, %Package.Pubrel{identifier: ^id} = pubrel, %Handler{} = updated_handler,
-       _next_actions} ->
+      {:ok, %Package.Pubrel{identifier: ^id} = pubrel, %Handler{} = updated_handler, next_actions} ->
         updated_data = %State{data | handler: updated_handler}
         :ok = Inflight.update(client_id, {:dispatch, pubrel})
-        {:keep_state, updated_data}
+        {:keep_state, updated_data, wrap_next_actions(next_actions)}
 
       {:error, reason} ->
         # todo
@@ -607,10 +604,9 @@ defmodule Tortoise.Connection do
     :ok = Inflight.update(client_id, {:received, pubcomp})
 
     case Handler.execute_handle_pubcomp(handler, pubcomp) do
-      {:ok, %Handler{} = updated_handler, _next_actions} ->
-        # todo, handle next actions
+      {:ok, %Handler{} = updated_handler, next_actions} ->
         updated_data = %State{data | handler: updated_handler}
-        {:keep_state, updated_data}
+        {:keep_state, updated_data, wrap_next_actions(next_actions)}
 
       {:error, reason} ->
         # todo
@@ -680,9 +676,7 @@ defmodule Tortoise.Connection do
 
         next_actions = [
           {:next_event, :internal, {:reply, {pid, msg_ref}, :ok}}
-          | for action <- next_actions do
-              {:next_event, :internal, {:user_action, action}}
-            end
+          | wrap_next_actions(next_actions)
         ]
 
         {:keep_state, data, next_actions}
@@ -756,9 +750,7 @@ defmodule Tortoise.Connection do
 
         next_actions = [
           {:next_event, :internal, {:reply, {pid, msg_ref}, :ok}}
-          | for action <- next_actions do
-              {:next_event, :internal, {:user_action, action}}
-            end
+          | wrap_next_actions(next_actions)
         ]
 
         {:keep_state, data, next_actions}
@@ -778,7 +770,7 @@ defmodule Tortoise.Connection do
   # They inform the connection to perform an action, such as
   # subscribing to a topic, and they are validated by the handler
   # module, so there is no need to coerce here
-  def handle_event(:internal, {:user_action, action}, _, %State{client_id: client_id}) do
+  def handle_event(:internal, {:user_action, action}, _, %State{client_id: client_id} = state) do
     case action do
       {:subscribe, topic, opts} when is_binary(topic) ->
         caller = {self(), make_ref()}
@@ -1007,4 +999,14 @@ defmodule Tortoise.Connection do
         :ok
     end
   end
+
+  # wrapping the user specified next actions in gen_statem next actions;
+  # this is used in all the handle callback functions, so we inline it
+  defp wrap_next_actions(next_actions) do
+    for action <- next_actions do
+      {:next_event, :internal, {:user_action, action}}
+    end
+  end
+
+  @compile {:inline, wrap_next_actions: 1}
 end
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index ac1124b4..6917782d 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -1018,6 +1018,148 @@ defmodule Tortoise.ConnectionTest do
     end
   end
 
+  describe "next actions" do
+    setup [:setup_scripted_mqtt_server]
+
+    test "Handling next actions from handle_puback callback", context do
+      Process.flag(:trap_exit, true)
+      scripted_mqtt_server = context.scripted_mqtt_server
+      client_id = context.client_id
+
+      script = [
+        {:receive, %Package.Connect{client_id: client_id}},
+        {:send, %Package.Connack{reason: :success, session_present: false}}
+      ]
+
+      {:ok, {ip, port}} = ScriptedMqttServer.enact(scripted_mqtt_server, script)
+
+      handler_opts = [
+        parent: self(),
+        handle_puback: fn %Package.Puback{}, state ->
+          {:cont, state, [{:subscribe, "foo/bar", qos: 0, identifier: 2}]}
+        end
+      ]
+
+      opts = [
+        client_id: client_id,
+        server: {Tortoise.Transport.Tcp, [host: ip, port: port]},
+        handler: {TestHandler, handler_opts}
+      ]
+
+      assert {:ok, connection_pid} = Connection.start_link(opts)
+
+      assert_receive {ScriptedMqttServer, {:received, %Package.Connect{}}}
+      assert_receive {ScriptedMqttServer, :completed}
+
+      publish = %Package.Publish{identifier: 1, topic: "foo/bar", qos: 1}
+      puback = %Package.Puback{identifier: 1}
+
+      default_subscription_opts = [
+        no_local: false,
+        retain_as_published: false,
+        retain_handling: 1
+      ]
+
+      subscribe =
+        Enum.into(
+          [{"foo/bar", [{:qos, 0} | default_subscription_opts]}],
+          %Package.Subscribe{identifier: 2}
+        )
+
+      suback = %Package.Suback{identifier: 2, acks: [{:ok, 0}]}
+
+      script = [
+        {:receive, publish},
+        {:send, puback},
+        {:receive, subscribe},
+        {:send, suback}
+      ]
+
+      {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
+      pid = connection_pid
+
+      client_id = context.client_id
+      assert {:ok, ref} = Inflight.track(client_id, {:outgoing, publish})
+
+      refute_receive {:EXIT, ^pid, {:protocol_violation, {:unexpected_package, _}}}
+      assert_receive {ScriptedMqttServer, {:received, ^publish}}
+      assert_receive {ScriptedMqttServer, {:received, ^subscribe}}
+      assert_receive {ScriptedMqttServer, :completed}
+      # the caller should receive an :ok for the ref when it is published
+      assert_receive {{Tortoise, ^client_id}, {Package.Publish, ^ref}, :ok}
+      assert_receive {{TestHandler, :handle_suback}, {_subscribe, _suback}}
+    end
+
+    test "Handling next actions from handle_pubrel and handle_pubcomp callback", context do
+      Process.flag(:trap_exit, true)
+      scripted_mqtt_server = context.scripted_mqtt_server
+      client_id = context.client_id
+      expected_connack = %Package.Connack{reason: :success, session_present: false}
+
+      script = [
+        {:receive, %Package.Connect{client_id: client_id}},
+        {:send, expected_connack}
+      ]
+
+      {:ok, {ip, port}} = ScriptedMqttServer.enact(scripted_mqtt_server, script)
+
+      test_process = self()
+
+      send_back = fn package ->
+        fn _state ->
+          send(test_process, {:next_action_from, package})
+        end
+      end
+
+      handler_opts = [
+        parent: self(),
+        handle_connack: fn package, state ->
+          {:cont, state, [{:eval, send_back.(package)}]}
+        end,
+        handle_pubrec: fn package, state ->
+          {:cont, state, [{:eval, send_back.(package)}]}
+        end,
+        handle_pubcomp: fn package, state ->
+          {:cont, state, [{:eval, send_back.(package)}]}
+        end
+      ]
+
+      opts = [
+        client_id: client_id,
+        server: {Tortoise.Transport.Tcp, [host: ip, port: port]},
+        handler: {TestHandler, handler_opts}
+      ]
+
+      assert {:ok, connection_pid} = Connection.start_link(opts)
+      assert_receive {ScriptedMqttServer, {:received, %Package.Connect{}}}
+      assert_receive {ScriptedMqttServer, :completed}
+
+      publish = %Package.Publish{identifier: 1, topic: "foo/bar", qos: 2}
+      pubrec = %Package.Pubrec{identifier: 1}
+      pubrel = %Package.Pubrel{identifier: 1}
+      pubcomp = %Package.Pubcomp{identifier: 1}
+
+      script = [
+        {:receive, publish},
+        {:send, pubrec},
+        {:receive, pubrel},
+        {:send, pubcomp}
+      ]
+
+      {:ok, _} = ScriptedMqttServer.enact(scripted_mqtt_server, script)
+
+      assert {:ok, ref} = Inflight.track(context.client_id, {:outgoing, publish})
+
+      assert_receive {ScriptedMqttServer, :completed}
+      assert_receive {ScriptedMqttServer, {:received, %Package.Publish{}}}
+      assert_receive {ScriptedMqttServer, {:received, %Package.Pubrel{}}}
+
+      assert_receive {:next_action_from, ^expected_connack}
+      assert_receive {:next_action_from, ^pubrec}
+      assert_receive {:next_action_from, ^pubcomp}
+    end
+  end
+
   describe "Publish with QoS=2" do
     setup [:setup_scripted_mqtt_server, :setup_connection_and_perform_handshake]
 

From 5ada5db4973f0493509faa1fd02ad0da70ffbd3a Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Fri, 3 May 2019 15:38:27 +0100
Subject: [PATCH 127/220] Support user defined next actions for the connection
 callback

---
 lib/tortoise/connection.ex        |  5 ++---
 test/tortoise/connection_test.exs | 14 +++++++++++++-
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 97e3ad44..47d6cd1b 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -810,10 +810,9 @@ defmodule Tortoise.Connection do
         %State{handler: handler} = data
       ) do
     case Handler.execute_connection(handler, status) do
-      {:ok, %Handler{} = updated_handler, _next_actions} ->
-        # todo handle next_actions
+      {:ok, %Handler{} = updated_handler, next_actions} ->
         updated_data = %State{data | handler: updated_handler}
-        {:keep_state, updated_data}
+        {:keep_state, updated_data, wrap_next_actions(next_actions)}
 
         # handle stop
     end
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index 6917782d..3232b121 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -680,7 +680,15 @@ defmodule Tortoise.ConnectionTest do
 
       {:ok, {ip, port}} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
 
-      handler = {TestHandler, [parent: self()]}
+      handler = {TestHandler, [parent: self(),
+                               connection: fn (status, state) ->
+                                 send state.parent, {{TestHandler, :connection}, status}
+                                 fun = fn (_) ->
+                                   send state.parent, :from_connection_callback
+                                 end
+                                 {:cont, state, [{:eval, fun}]}
+                               end
+                              ]}
 
       opts = [
         client_id: client_id,
@@ -716,6 +724,10 @@ defmodule Tortoise.ConnectionTest do
       assert_receive {{^handler_mod, :terminate}, :shutdown}
       assert_receive {{^handler_mod, :handle_connack}, %Package.Connack{}}
       refute_receive {{^handler_mod, _}, _}
+
+      # make sure user defined next actions works for the connection
+      # callback
+      assert_receive :from_connection_callback
     end
 
     test "user next actions", context do

From 1f41e09a9b015dcfd2a80a438830779ad227a2c1 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Fri, 3 May 2019 17:11:38 +0100
Subject: [PATCH 128/220] mix format

---
 test/tortoise/connection_test.exs | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index 3232b121..3dce1dd3 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -680,15 +680,20 @@ defmodule Tortoise.ConnectionTest do
 
       {:ok, {ip, port}} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
 
-      handler = {TestHandler, [parent: self(),
-                               connection: fn (status, state) ->
-                                 send state.parent, {{TestHandler, :connection}, status}
-                                 fun = fn (_) ->
-                                   send state.parent, :from_connection_callback
-                                 end
-                                 {:cont, state, [{:eval, fun}]}
-                               end
-                              ]}
+      handler =
+        {TestHandler,
+         [
+           parent: self(),
+           connection: fn status, state ->
+             send(state.parent, {{TestHandler, :connection}, status})
+
+             fun = fn _ ->
+               send(state.parent, :from_connection_callback)
+             end
+
+             {:cont, state, [{:eval, fun}]}
+           end
+         ]}
 
       opts = [
         client_id: client_id,

From c51bf47dba44dbb901873cd5ac2ace27248801e9 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Fri, 3 May 2019 17:12:00 +0100
Subject: [PATCH 129/220] Made it possible to customize the callbacks in the
 disconnect tests

---
 test/support/test_handler.ex      |  2 +-
 test/tortoise/connection_test.exs | 39 +++++++++++++++++++++++++++----
 2 files changed, 36 insertions(+), 5 deletions(-)

diff --git a/test/support/test_handler.ex b/test/support/test_handler.ex
index b8e06d53..9324df07 100644
--- a/test/support/test_handler.ex
+++ b/test/support/test_handler.ex
@@ -46,7 +46,7 @@ defmodule TestHandler do
     case state[:handle_disconnect] do
       nil ->
         send(state[:parent], {{__MODULE__, :handle_disconnect}, disconnect})
-        {:stop, :normal, state}
+        {:cont, state}
 
       fun when is_function(fun, 2) ->
         apply(fun, [disconnect, state])
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index 3dce1dd3..b645bd62 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -1297,7 +1297,7 @@ defmodule Tortoise.ConnectionTest do
   end
 
   describe "Disconnect" do
-    setup [:setup_scripted_mqtt_server, :setup_connection_and_perform_handshake]
+    setup [:setup_scripted_mqtt_server]
 
     # [x] :normal_disconnection
     # [ ] :unspecified_error
@@ -1331,18 +1331,49 @@ defmodule Tortoise.ConnectionTest do
     test "normal disconnection", context do
       Process.flag(:trap_exit, true)
       disconnect = %Package.Disconnect{reason: :normal_disconnection}
-      script = [{:send, disconnect}]
 
+      callbacks = [
+        handle_disconnect: fn %Package.Disconnect{} = disconnect, state ->
+          send(state.parent, {{TestHandler, :handle_disconnect}, disconnect})
+          {:stop, :normal, state}
+        end
+      ]
+
+      {:ok, %{connection_pid: pid} = context} = connect_and_perform_handshake(context, callbacks)
+
+      script = [{:send, disconnect}]
       {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
-      pid = context.connection_pid
 
       refute_receive {:EXIT, ^pid, {:protocol_violation, {:unexpected_package, ^disconnect}}}
       assert_receive {ScriptedMqttServer, :completed}
 
       # the handle disconnect callback should have been called
       assert_receive {{TestHandler, :handle_disconnect}, ^disconnect}
-      # the callback handler will tell it to stop normally
+      # the callback tells it to stop normally
       assert_receive {:EXIT, ^pid, :normal}
     end
   end
+
+  defp connect_and_perform_handshake(%{client_id: client_id} = context, callbacks) do
+    script = [
+      {:receive, %Package.Connect{client_id: client_id}},
+      {:send, %Package.Connack{reason: :success, session_present: false}}
+    ]
+
+    {:ok, {ip, port}} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
+
+    opts = [
+      client_id: context.client_id,
+      server: {Tortoise.Transport.Tcp, [host: ip, port: port]},
+      handler: {TestHandler, Keyword.merge([parent: self()], callbacks)}
+    ]
+
+    {:ok, connection_pid} = Connection.start_link(opts)
+
+    assert_receive {{TestHandler, :init}, _}
+    assert_receive {ScriptedMqttServer, {:received, %Package.Connect{}}}
+    assert_receive {ScriptedMqttServer, :completed}
+
+    {:ok, Map.put(context, :connection_pid, connection_pid)}
+  end
 end

From 4dacd975d040a5d7d9679bdfa5e92d62c94b8638 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Sat, 4 May 2019 13:02:21 +0100
Subject: [PATCH 130/220] Support next actions in the handle_disconnect
 callback

---
 lib/tortoise/connection.ex        |  5 ++---
 test/tortoise/connection_test.exs | 23 +++++++++++++++++++++++
 2 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 47d6cd1b..4c0e41a8 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -944,9 +944,8 @@ defmodule Tortoise.Connection do
         %State{handler: handler} = data
       ) do
     case Handler.execute_handle_disconnect(handler, disconnect) do
-      {:cont, updated_handler, _next_actions} ->
-        # todo, handle next_actions
-        {:keep_state, %State{data | handler: updated_handler}}
+      {:ok, updated_handler, next_actions} ->
+        {:keep_state, %State{data | handler: updated_handler}, wrap_next_actions(next_actions)}
 
       {:stop, reason, updated_handler} ->
         {:stop, reason, %State{data | handler: updated_handler}}
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index b645bd62..ef93f88b 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -1352,6 +1352,29 @@ defmodule Tortoise.ConnectionTest do
       # the callback tells it to stop normally
       assert_receive {:EXIT, ^pid, :normal}
     end
+
+    test "handle_disconnect producing next action", context do
+      disconnect = %Package.Disconnect{reason: :normal_disconnection}
+
+      callbacks = [
+        handle_disconnect: fn %Package.Disconnect{} = disconnect, state ->
+          %{parent: parent} = state
+          send(parent, {{TestHandler, :handle_disconnect}, disconnect})
+          fun = fn _ -> send(parent, {TestHandler, :from_eval_fun}) end
+          {:cont, state, [{:eval, fun}]}
+        end
+      ]
+
+      {:ok, context} = connect_and_perform_handshake(context, callbacks)
+
+      script = [{:send, disconnect}]
+      {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
+
+      assert_receive {ScriptedMqttServer, :completed}
+      # the handle disconnect callback should have been called
+      assert_receive {{TestHandler, :handle_disconnect}, ^disconnect}
+      assert_receive {TestHandler, :from_eval_fun}
+    end
   end
 
   defp connect_and_perform_handshake(%{client_id: client_id} = context, callbacks) do

From 072f1149549bb11c3f1e6e7fe05a585f968f35a4 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Sat, 4 May 2019 13:40:39 +0100
Subject: [PATCH 131/220] Handle next actions in callback handler for QoS=0
 publish

---
 lib/tortoise/connection.ex        |  5 ++---
 test/tortoise/connection_test.exs | 20 +++++++++++++++++---
 2 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 4c0e41a8..5abeffed 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -470,10 +470,9 @@ defmodule Tortoise.Connection do
         %State{handler: handler} = data
       ) do
     case Handler.execute_handle_publish(handler, publish) do
-      {:ok, %Handler{} = updated_handler, _next_actions} ->
-        # todo, handle next_actions
+      {:ok, %Handler{} = updated_handler, next_actions} ->
         updated_data = %State{data | handler: updated_handler}
-        {:keep_state, updated_data}
+        {:keep_state, updated_data, wrap_next_actions(next_actions)}
 
         # handle stop
     end
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index ef93f88b..f30244de 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -944,15 +944,26 @@ defmodule Tortoise.ConnectionTest do
   end
 
   describe "Publish with QoS=0" do
-    setup [:setup_scripted_mqtt_server, :setup_connection_and_perform_handshake]
+    # , :setup_connection_and_perform_handshake
+    setup [:setup_scripted_mqtt_server]
 
     test "Receiving a publish", context do
       Process.flag(:trap_exit, true)
+
       publish = %Package.Publish{topic: "foo/bar", qos: 0}
 
-      script = [{:send, publish}]
+      callbacks = [
+        handle_publish: fn topic, %Package.Publish{}, %{parent: parent} = state ->
+          send(parent, {{TestHandler, :handle_publish}, publish})
+          send(parent, {{TestHandler, :altered_topic}, topic})
+          fun = fn _ -> send(parent, {TestHandler, :next_action_triggered}) end
+          {:cont, state, [{:eval, fun}]}
+        end
+      ]
+
+      {:ok, context} = connect_and_perform_handshake(context, callbacks)
 
-      {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
+      {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, [{:send, publish}])
       pid = context.connection_pid
 
       refute_receive {:EXIT, ^pid, {:protocol_violation, {:unexpected_package, ^publish}}}
@@ -960,6 +971,9 @@ defmodule Tortoise.ConnectionTest do
 
       # the handle publish callback should have been called
       assert_receive {{TestHandler, :handle_publish}, ^publish}
+      expected_topic_list = ["foo", "bar"]
+      assert_receive {{TestHandler, :altered_topic}, ^expected_topic_list}
+      assert_receive {TestHandler, :next_action_triggered}
     end
   end
 

From 9527aa92d87dae3646f0efb431e71ad523a80726 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 25 May 2019 00:07:07 +0100
Subject: [PATCH 132/220] Change the format of user properties

It is no longer possible to specify a 2-tuple of `{utf8, uft8}` when
defining user properties. Now a user property has to be specified as
`{:user_property, {utf8, utf8}}`. This might not be the most elegant
solution, but it allow us to preserve the order of the properties in
the encoded package.
---
 lib/tortoise/connection.ex          |  8 ++++++--
 lib/tortoise/package/properties.ex  | 25 +++++++++++++++++++------
 lib/tortoise/package/puback.ex      |  2 +-
 lib/tortoise/package/pubcomp.ex     |  2 +-
 lib/tortoise/package/pubrec.ex      |  2 +-
 lib/tortoise/package/pubrel.ex      |  2 +-
 lib/tortoise/package/suback.ex      |  2 +-
 lib/tortoise/package/subscribe.ex   |  2 +-
 lib/tortoise/package/unsuback.ex    |  2 +-
 lib/tortoise/package/unsubscribe.ex |  2 +-
 test/test_helper.exs                |  2 +-
 test/tortoise/connection_test.exs   |  8 ++++----
 test/tortoise_test.exs              | 13 +++++++++----
 13 files changed, 47 insertions(+), 25 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index de9766bb..0a4f1c0b 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -76,6 +76,7 @@ defmodule Tortoise.Connection do
       opts: connection_opts,
       handler: handler
     }
+
     opts = Keyword.merge(opts, name: via_name(client_id))
     GenStateMachine.start_link(__MODULE__, initial, opts)
   end
@@ -156,12 +157,14 @@ defmodule Tortoise.Connection do
 
   def subscribe(client_id, [{_, topic_opts} | _] = topics, opts) when is_list(topic_opts) do
     caller = {_, ref} = {self(), make_ref()}
+    # todo, do something with timeout, or remove it
+    {opts, properties} = Keyword.split(opts, [:identifier, :timeout])
     {identifier, opts} = Keyword.pop_first(opts, :identifier, nil)
 
     subscribe =
       Enum.into(topics, %Package.Subscribe{
         identifier: identifier,
-        properties: Keyword.get(opts, :properties, [])
+        properties: properties
       })
 
     GenStateMachine.cast(via_name(client_id), {:subscribe, caller, subscribe, opts})
@@ -249,12 +252,13 @@ defmodule Tortoise.Connection do
 
   def unsubscribe(client_id, [topic | _] = topics, opts) when is_binary(topic) do
     caller = {_, ref} = {self(), make_ref()}
+    {opts, properties} = Keyword.split(opts, [:identifier, :timeout])
     {identifier, opts} = Keyword.pop_first(opts, :identifier, nil)
 
     unsubscribe = %Package.Unsubscribe{
       identifier: identifier,
       topics: topics,
-      properties: Keyword.get(opts, :properties, [])
+      properties: properties
     }
 
     GenStateMachine.cast(via_name(client_id), {:unsubscribe, caller, unsubscribe, opts})
diff --git a/lib/tortoise/package/properties.ex b/lib/tortoise/package/properties.ex
index 910c32d2..e1781755 100644
--- a/lib/tortoise/package/properties.ex
+++ b/lib/tortoise/package/properties.ex
@@ -11,14 +11,27 @@ defmodule Tortoise.Package.Properties do
     |> Package.variable_length_encode()
   end
 
-  # User properties are special; we will allow them to be encoded as
-  # binaries to make the interface a bit cleaner to the end user
-  #
-  # Todo, revert this decision
-  defp encode_property({key, value}) when is_binary(key) do
+  # The `user_property` property should be specified as
+  # `{:user_property, {key, value}}` where both `key` and `value` are
+  # UTF-8 encoded strings. User properties with the same key are
+  # allowed, and by specifying it like this we make it possible to
+  # specify the order of the properties.
+  defp encode_property({:user_property, {<<key::binary>>, <<value::binary>>}}) do
     [0x26, length_encode(key), length_encode(value)]
   end
 
+  # We allow the user to specify a list of key/value pairs when
+  # multiple user properties are needed; all items in the list must be
+  # 2-tuples of string/string.
+  defp encode_property({:user_property, [{<<_::binary>>, <<_::binary>>} | _] = properties}) do
+    for property <- properties, do: encode_property({:user_property, property})
+  end
+
+  # Ignore the user property if an empty list is passed in
+  defp encode_property({:user_property, []}) do
+    <<>>
+  end
+
   defp encode_property({key, value}) do
     case key do
       :payload_format_indicator ->
@@ -238,7 +251,7 @@ defmodule Tortoise.Package.Properties do
     <<key::binary-size(length), rest::binary>> = rest
     <<length::integer-size(16), rest::binary>> = rest
     <<value::binary-size(length), rest::binary>> = rest
-    {{key, value}, rest}
+    {{:user_property, {key, value}}, rest}
   end
 
   defp decode_property(<<0x27, value::integer-size(32), rest::binary>>) do
diff --git a/lib/tortoise/package/puback.ex b/lib/tortoise/package/puback.ex
index 36103de2..7a40fe63 100644
--- a/lib/tortoise/package/puback.ex
+++ b/lib/tortoise/package/puback.ex
@@ -22,7 +22,7 @@ defmodule Tortoise.Package.Puback do
             __META__: Package.Meta.t(),
             identifier: Tortoise.package_identifier(),
             reason: reason(),
-            properties: [{:reason_string, String.t()}, {:user_property, String.t()}]
+            properties: [{:reason_string, String.t()}, {:user_property, {String.t(), String.t()}}]
           }
   @enforce_keys [:identifier]
   defstruct __META__: %Package.Meta{opcode: @opcode, flags: 0b0000},
diff --git a/lib/tortoise/package/pubcomp.ex b/lib/tortoise/package/pubcomp.ex
index 45a497ea..1b2114c1 100644
--- a/lib/tortoise/package/pubcomp.ex
+++ b/lib/tortoise/package/pubcomp.ex
@@ -14,7 +14,7 @@ defmodule Tortoise.Package.Pubcomp do
             __META__: Package.Meta.t(),
             identifier: Tortoise.package_identifier(),
             reason: reason(),
-            properties: [{:reason_string, String.t()}, {:user_property, String.t()}]
+            properties: [{:reason_string, String.t()}, {:user_property, {String.t(), String.t()}}]
           }
   defstruct __META__: %Package.Meta{opcode: @opcode, flags: 0},
             identifier: nil,
diff --git a/lib/tortoise/package/pubrec.ex b/lib/tortoise/package/pubrec.ex
index b01329e9..a71146db 100644
--- a/lib/tortoise/package/pubrec.ex
+++ b/lib/tortoise/package/pubrec.ex
@@ -22,7 +22,7 @@ defmodule Tortoise.Package.Pubrec do
             __META__: Package.Meta.t(),
             identifier: Tortoise.package_identifier(),
             reason: reason(),
-            properties: [{:reason_string, String.t()}, {:user_property, String.t()}]
+            properties: [{:reason_string, String.t()}, {:user_property, {String.t(), String.t()}}]
           }
   defstruct __META__: %Package.Meta{opcode: @opcode, flags: 0b0000},
             identifier: nil,
diff --git a/lib/tortoise/package/pubrel.ex b/lib/tortoise/package/pubrel.ex
index 6a5a639d..2ccaa0b9 100644
--- a/lib/tortoise/package/pubrel.ex
+++ b/lib/tortoise/package/pubrel.ex
@@ -14,7 +14,7 @@ defmodule Tortoise.Package.Pubrel do
             __META__: Package.Meta.t(),
             identifier: Tortoise.package_identifier(),
             reason: reason(),
-            properties: [{:reason_string, String.t()}, {:user_property, String.t()}]
+            properties: [{:reason_string, String.t()}, {:user_property, {String.t(), String.t()}}]
           }
   @enforce_keys [:identifier]
   defstruct __META__: %Package.Meta{opcode: @opcode, flags: 0b0010},
diff --git a/lib/tortoise/package/suback.ex b/lib/tortoise/package/suback.ex
index d60c59fb..8c67ced5 100644
--- a/lib/tortoise/package/suback.ex
+++ b/lib/tortoise/package/suback.ex
@@ -25,7 +25,7 @@ defmodule Tortoise.Package.Suback do
             __META__: Package.Meta.t(),
             identifier: Tortoise.package_identifier(),
             acks: [ack_result],
-            properties: [{:reason_string, String.t()}, {:user_property, String.t()}]
+            properties: [{:reason_string, String.t()}, {:user_property, {String.t(), String.t()}}]
           }
   @enforce_keys [:identifier]
   defstruct __META__: %Package.Meta{opcode: @opcode, flags: 0},
diff --git a/lib/tortoise/package/subscribe.ex b/lib/tortoise/package/subscribe.ex
index 11bc7fd8..7237962d 100644
--- a/lib/tortoise/package/subscribe.ex
+++ b/lib/tortoise/package/subscribe.ex
@@ -23,7 +23,7 @@ defmodule Tortoise.Package.Subscribe do
             topics: topics(),
             properties: [
               {:subscription_identifier, 0x1..0xFFFFFFF},
-              {:user_property, binary()}
+              {:user_property, {String.t(), String.t()}}
             ]
           }
   defstruct __META__: %Package.Meta{opcode: @opcode, flags: 0b0010},
diff --git a/lib/tortoise/package/unsuback.ex b/lib/tortoise/package/unsuback.ex
index d7bca837..930e8864 100644
--- a/lib/tortoise/package/unsuback.ex
+++ b/lib/tortoise/package/unsuback.ex
@@ -21,7 +21,7 @@ defmodule Tortoise.Package.Unsuback do
             __META__: Package.Meta.t(),
             identifier: Tortoise.package_identifier(),
             results: [],
-            properties: [{:reason_string, any()}, {:user_property, any()}]
+            properties: [{:reason_string, any()}, {:user_property, {String.t(), String.t()}}]
           }
   @enforce_keys [:identifier]
   defstruct __META__: %Package.Meta{opcode: @opcode, flags: 0b0000},
diff --git a/lib/tortoise/package/unsubscribe.ex b/lib/tortoise/package/unsubscribe.ex
index 76f3db83..159e7c56 100644
--- a/lib/tortoise/package/unsubscribe.ex
+++ b/lib/tortoise/package/unsubscribe.ex
@@ -13,7 +13,7 @@ defmodule Tortoise.Package.Unsubscribe do
             __META__: Package.Meta.t(),
             identifier: Tortoise.package_identifier(),
             topics: [topic],
-            properties: [{:user_property, any()}]
+            properties: [{:user_property, {String.t(), String.t()}}]
           }
   defstruct __META__: %Package.Meta{opcode: @opcode, flags: 2},
             topics: [],
diff --git a/test/test_helper.exs b/test/test_helper.exs
index cde61325..2d6c1ed1 100644
--- a/test/test_helper.exs
+++ b/test/test_helper.exs
@@ -476,7 +476,7 @@ defmodule Tortoise.TestGenerators do
       :topic_alias -> {type, choose(0x0001, 0xFFFF)}
       :maximum_qos -> {type, oneof([0, 1])}
       :retain_available -> {type, oneof([0, 1])}
-      :user_property -> {utf8(), utf8()}
+      :user_property -> {type, {utf8(), utf8()}}
       :maximum_packet_size -> {type, choose(1, 268_435_455)}
       :wildcard_subscription_available -> {type, oneof([0, 1])}
       :subscription_identifier_available -> {type, oneof([0, 1])}
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index f15de65f..df48f4bf 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -271,7 +271,7 @@ defmodule Tortoise.ConnectionTest do
       subscription_baz =
         Enum.into(
           [{"baz", [{:qos, 2} | default_subscription_opts]}],
-          %Package.Subscribe{identifier: 3, properties: [{"foo", "bar"}]}
+          %Package.Subscribe{identifier: 3, properties: [user_property: {"foo", "bar"}]}
         )
 
       suback_baz = %Package.Suback{identifier: 3, acks: [{:ok, 2}]}
@@ -307,7 +307,7 @@ defmodule Tortoise.ConnectionTest do
                Tortoise.Connection.subscribe(client_id, "baz",
                  qos: 2,
                  identifier: 3,
-                 properties: [{"foo", "bar"}]
+                 user_property: {"foo", "bar"}
                )
 
       assert_receive {{Tortoise, ^client_id}, ^ref, :ok}
@@ -336,7 +336,7 @@ defmodule Tortoise.ConnectionTest do
       unsubscribe_bar = %Package.Unsubscribe{
         identifier: 3,
         topics: ["bar"],
-        properties: [{"foo", "bar"}]
+        properties: [user_property: {"foo", "bar"}]
       }
 
       unsuback_bar = %Package.Unsuback{results: [:success], identifier: 3}
@@ -389,7 +389,7 @@ defmodule Tortoise.ConnectionTest do
       assert {:ok, ref} =
                Tortoise.Connection.unsubscribe(client_id, "bar",
                  identifier: 3,
-                 properties: [{"foo", "bar"}]
+                 user_property: {"foo", "bar"}
                )
 
       assert_receive {{Tortoise, ^client_id}, ^ref, :ok}
diff --git a/test/tortoise_test.exs b/test/tortoise_test.exs
index ec270057..3089e72d 100644
--- a/test/tortoise_test.exs
+++ b/test/tortoise_test.exs
@@ -80,7 +80,7 @@ defmodule TortoiseTest do
       transforms = [
         publish: fn %type{properties: properties}, [:init] = state ->
           send(parent, {:callback, {type, properties}, state})
-          {:ok, [{"foo", "bar"} | properties], [type | state]}
+          {:ok, [{:user_property, {"foo", "bar"}} | properties], [type | state]}
         end,
         pubrec: fn %type{properties: properties}, state ->
           send(parent, {:callback, {type, properties}, state})
@@ -88,7 +88,7 @@ defmodule TortoiseTest do
         end,
         pubrel: fn %type{properties: properties}, state ->
           send(parent, {:callback, {type, properties}, state})
-          properties = [{"hello", "world"} | properties]
+          properties = [{:user_property, {"hello", "world"}} | properties]
           {:ok, properties, [type | state]}
         end,
         pubcomp: fn %type{properties: properties}, state ->
@@ -110,7 +110,7 @@ defmodule TortoiseTest do
                topic: "foo/bar",
                qos: 2,
                payload: nil,
-               properties: [{"foo", "bar"}]
+               properties: [user_property: {"foo", "bar"}]
              } = Package.decode(data)
 
       :ok = Inflight.update(context.client_id, {:received, %Package.Pubrec{identifier: id}})
@@ -119,7 +119,12 @@ defmodule TortoiseTest do
       :ok = Inflight.update(client_id, {:dispatch, pubrel})
 
       assert {:ok, data} = :gen_tcp.recv(context.server, 0, 500)
-      expected_pubrel = %Package.Pubrel{pubrel | properties: [{"hello", "world"}]}
+
+      expected_pubrel = %Package.Pubrel{
+        pubrel
+        | properties: [user_property: {"hello", "world"}]
+      }
+
       assert expected_pubrel == Package.decode(data)
 
       :ok = Inflight.update(client_id, {:received, %Package.Pubcomp{identifier: id}})

From 98a534f0026f133f265382a6db683aa23c01553f Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sat, 25 May 2019 00:27:32 +0100
Subject: [PATCH 133/220] Need to trap exits in a test process

The user next actions test seem to blink, and it will work if I
introduce a timeout or trap exit in the test process. This is very odd
and I should investigate it.
---
 test/tortoise/connection_test.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index 80b75e9c..6931ecc6 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -736,6 +736,7 @@ defmodule Tortoise.ConnectionTest do
     end
 
     test "user next actions", context do
+      Process.flag(:trap_exit, true)
       connect = %Package.Connect{client_id: context.client_id}
       expected_connack = %Package.Connack{reason: :success, session_present: false}
 
@@ -810,7 +811,6 @@ defmodule Tortoise.ConnectionTest do
       # process should terminate and exit
       assert_receive {ScriptedMqttServer, {:received, ^disconnect}}
       assert_receive {^pid, {:terminating, :normal}}
-      refute Process.alive?(pid)
 
       # all done
       assert_receive {ScriptedMqttServer, :completed}

From 14bde6c19a51effbe34d5ead687a8cba8c9e8e9d Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sun, 26 May 2019 15:40:28 +0100
Subject: [PATCH 134/220] Update the Tortoise.Handler.Logger to follow the new
 behaviour

---
 lib/tortoise/handler/logger.ex | 75 ++++++++++++++++++++++++++--------
 1 file changed, 59 insertions(+), 16 deletions(-)

diff --git a/lib/tortoise/handler/logger.ex b/lib/tortoise/handler/logger.ex
index bb3f9765..e25ab61e 100644
--- a/lib/tortoise/handler/logger.ex
+++ b/lib/tortoise/handler/logger.ex
@@ -3,16 +3,20 @@ defmodule Tortoise.Handler.Logger do
 
   require Logger
 
+  alias Tortoise.Package
+
   use Tortoise.Handler
 
   defstruct []
   alias __MODULE__, as: State
 
+  @impl true
   def init(_opts) do
     Logger.info("Initializing handler")
     {:ok, %State{}}
   end
 
+  @impl true
   def connection(:up, state) do
     Logger.info("Connection has been established")
     {:cont, state}
@@ -28,37 +32,76 @@ defmodule Tortoise.Handler.Logger do
     {:cont, state}
   end
 
-  def subscription(:up, topic, state) do
-    Logger.info("Subscribed to #{topic}")
-    {:ok, state}
+  @impl true
+  def handle_connack(%Package.Connack{reason: :success}, state) do
+    Logger.info("Successfully connected to the server")
+    {:cont, state}
   end
 
-  def subscription({:warn, [requested: req, accepted: qos]}, topic, state) do
-    Logger.warn("Subscribed to #{topic}; requested #{req} but got accepted with QoS #{qos}")
-    {:ok, state}
+  def handle_connack(%Package.Connack{reason: {:refused, refusal}}, state) do
+    Logger.error("Server refused connection: #{inspect refusal}")
+    {:cont, state}
   end
 
-  def subscription({:error, reason}, topic, state) do
-    Logger.error("Error subscribing to #{topic}; #{inspect(reason)}")
-    {:ok, state}
+  @impl true
+  def handle_suback(%Package.Subscribe{} = subscribe, %Package.Suback{} = suback, state) do
+    for {{topic, opts}, result} <- Enum.zip(subscribe.topics, suback.acks) do
+      case {opts[:qos], result} do
+        {qos, {:ok, qos}} ->
+          Logger.info("Subscribed to #{topic} with the expected qos: #{qos}")
+
+        {req_qos, {:ok, accepted_qos}} ->
+          Logger.warn("Subscribed to #{topic} with QoS #{req_qos} but got accepted as #{accepted_qos}")
+
+        {_, {:error, reason}} ->
+          Logger.error("Failed to subscribe to topic: #{topic} reason: #{inspect reason}")
+      end
+    end
+    {:cont, state}
   end
 
-  def subscription(:down, topic, state) do
-    Logger.info("Unsubscribed from #{topic}")
-    {:ok, state}
+  @impl true
+  def handle_unsuback(%Package.Unsubscribe{} = unsubscribe, %Package.Unsuback{} = unsuback, state) do
+    for {topic, result} <- Enum.zip(unsubscribe.topics, unsuback.results) do
+      case result do
+        :success ->
+          Logger.info("Successfully unsubscribed from #{topic}")
+      end
+    end
+    {:cont, state}
   end
 
-  def handle_message(topic, publish, state) do
+  @impl true
+  def handle_publish(topic, %Package.Publish{} = publish, state) do
     Logger.info("#{Enum.join(topic, "/")} #{inspect(publish)}")
-    {:ok, state}
+    {:cont, state}
   end
 
   @impl true
-  def handle_disconnect(disconnect, state) do
+  def handle_puback(%Package.Puback{} = puback, state) do
+    Logger.info("Puback: #{puback.identifier}")
+    {:cont, state}
+  end
+
+  @impl true
+  def handle_pubrec(%Package.Pubrec{} = pubrec, state) do
+    Logger.info("Pubrec: #{pubrec.identifier}")
+    {:cont, state}
+  end
+
+  @impl true
+  def handle_pubcomp(%Package.Pubcomp{} = pubcomp, state) do
+    Logger.info("Pubcomp: #{pubcomp.identifier}")
+    {:cont, state}
+  end
+
+  @impl true
+  def handle_disconnect(%Package.Disconnect{} = disconnect, state) do
     Logger.info("Received disconnect from server #{inspect(disconnect)}")
-    {:ok, state}
+    {:cont, state}
   end
 
+  @impl true
   def terminate(reason, _state) do
     Logger.warn("Client has been terminated with reason: #{inspect(reason)}")
     :ok

From 6948c86dbfe9ed03ee869475d028b5c0bce1a91c Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Wed, 29 May 2019 13:26:11 +0100
Subject: [PATCH 135/220] Renamed the connection/2 to status_change/2

The status change event is meant to update when the client is online
and when it is dropped from the broker. There is also a "terminating"
phrase, so the user can act on this stage.

I liked the name status_change better, and it is now an optional
callback; if it is not implemented Tortoise will just skip this and
continue the life-cycle unchanged.
---
 README.md                         |  2 +-
 lib/tortoise/connection.ex        | 16 +++++++++------
 lib/tortoise/handler.ex           | 21 +++++++++----------
 lib/tortoise/handler/logger.ex    |  6 +++---
 test/support/test_handler.ex      |  6 +++---
 test/tortoise/connection_test.exs |  6 +++---
 test/tortoise/handler_test.exs    | 34 +++++++++++++++----------------
 7 files changed, 47 insertions(+), 44 deletions(-)

diff --git a/README.md b/README.md
index a8585d9c..46ac4ba5 100644
--- a/README.md
+++ b/README.md
@@ -80,7 +80,7 @@ defmodule Tortoise.Handler.Example do
     {:ok, args}
   end
 
-  def connection(status, state) do
+  def status_change(status, state) do
     # `status` will be either `:up` or `:down`; you can use this to
     # inform the rest of your system if the connection is currently
     # open or closed; tortoise should be busy reconnecting if you get
diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 3d153809..4e307348 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -811,12 +811,16 @@ defmodule Tortoise.Connection do
         :connected,
         %State{handler: handler} = data
       ) do
-    case Handler.execute_connection(handler, status) do
-      {:ok, %Handler{} = updated_handler, next_actions} ->
-        updated_data = %State{data | handler: updated_handler}
-        {:keep_state, updated_data, wrap_next_actions(next_actions)}
-
-        # handle stop
+    if function_exported?(handler.module, :status_change, 2) do
+      case Handler.execute_status_change(handler, status) do
+        {:ok, %Handler{} = updated_handler, next_actions} ->
+          updated_data = %State{data | handler: updated_handler}
+          {:keep_state, updated_data, wrap_next_actions(next_actions)}
+
+          # handle stop
+      end
+    else
+      :keep_state_and_data
     end
   end
 
diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index 1822de3f..fabf07e8 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -132,11 +132,6 @@ defmodule Tortoise.Handler do
         :ok
       end
 
-      @impl true
-      def connection(_status, state) do
-        {:cont, state}
-      end
-
       @impl true
       def handle_connack(_connack, state) do
         {:cont, state}
@@ -227,9 +222,9 @@ defmodule Tortoise.Handler do
   a list of next actions such as `{:unsubscribe, "foo/bar"}` will
   result in the state being returned and the next actions performed.
   """
-  @callback connection(status, state :: term()) ::
-              {:ok, new_state}
-              | {:ok, new_state, [next_action()]}
+  @callback status_change(status, state :: term()) ::
+              {:cont, new_state}
+              | {:cont, new_state, [next_action()]}
             when status: :up | :down,
                  new_state: term()
 
@@ -368,6 +363,8 @@ defmodule Tortoise.Handler do
             when reason: :normal | :shutdown | {:shutdown, term()},
                  ignored: term()
 
+  @optional_callbacks status_change: 2
+
   @doc false
   @spec execute_init(t) :: {:ok, t} | :ignore | {:stop, term()}
   def execute_init(handler) do
@@ -385,11 +382,13 @@ defmodule Tortoise.Handler do
     end
   end
 
+  # todo, fix the type spec here so it contain the next actions and
+  # error path as well
   @doc false
-  @spec execute_connection(t, status) :: {:ok, t}
+  @spec execute_status_change(t, status) :: {:ok, t}
         when status: :up | :down
-  def execute_connection(handler, status) do
-    apply(handler.module, :connection, [status, handler.state])
+  def execute_status_change(handler, status) do
+    apply(handler.module, :status_change, [status, handler.state])
     |> transform_result()
     |> case do
       {:cont, updated_state, next_actions} ->
diff --git a/lib/tortoise/handler/logger.ex b/lib/tortoise/handler/logger.ex
index e25ab61e..41627ef3 100644
--- a/lib/tortoise/handler/logger.ex
+++ b/lib/tortoise/handler/logger.ex
@@ -17,17 +17,17 @@ defmodule Tortoise.Handler.Logger do
   end
 
   @impl true
-  def connection(:up, state) do
+  def status_change(:up, state) do
     Logger.info("Connection has been established")
     {:cont, state}
   end
 
-  def connection(:down, state) do
+  def status_change(:down, state) do
     Logger.warn("Connection has been dropped")
     {:cont, state}
   end
 
-  def connection(:terminating, state) do
+  def status_change(:terminating, state) do
     Logger.warn("Connection is terminating")
     {:cont, state}
   end
diff --git a/test/support/test_handler.ex b/test/support/test_handler.ex
index 9324df07..316f3fac 100644
--- a/test/support/test_handler.ex
+++ b/test/support/test_handler.ex
@@ -31,10 +31,10 @@ defmodule TestHandler do
     end
   end
 
-  def connection(status, state) do
-    case state[:connection] do
+  def status_change(status, state) do
+    case state[:status_change] do
       nil ->
-        send(state[:parent], {{__MODULE__, :connection}, status})
+        send(state[:parent], {{__MODULE__, :status_change}, status})
         {:cont, state}
 
       fun when is_function(fun, 2) ->
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index 6931ecc6..f422f648 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -684,8 +684,8 @@ defmodule Tortoise.ConnectionTest do
         {TestHandler,
          [
            parent: self(),
-           connection: fn status, state ->
-             send(state.parent, {{TestHandler, :connection}, status})
+           status_change: fn status, state ->
+             send(state.parent, {{TestHandler, :status_change}, status})
 
              fun = fn _ ->
                send(state.parent, :from_connection_callback)
@@ -725,7 +725,7 @@ defmodule Tortoise.ConnectionTest do
       # triggered during this exchange
       {handler_mod, handler_init_opts} = handler
       assert_receive {{^handler_mod, :init}, ^handler_init_opts}
-      assert_receive {{^handler_mod, :connection}, :up}
+      assert_receive {{^handler_mod, :status_change}, :up}
       assert_receive {{^handler_mod, :terminate}, :shutdown}
       assert_receive {{^handler_mod, :handle_connack}, %Package.Connack{}}
       refute_receive {{^handler_mod, _}, _}
diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index eb981c7d..a8167a78 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -45,8 +45,8 @@ defmodule Tortoise.HandlerTest do
       end
     end
 
-    def connection(status, state) do
-      case state[:connection] do
+    def status_change(status, state) do
+      case state[:status_change] do
         nil ->
           {:cont, state}
 
@@ -188,41 +188,41 @@ defmodule Tortoise.HandlerTest do
     end
   end
 
-  describe "execute connection/2" do
+  describe "execute status_change/2" do
     test "return continues", context do
       parent = self()
 
-      connection_fn = fn status, state ->
-        send(parent, {:connection, status})
+      status_change_fn = fn status, state ->
+        send(parent, {:status_change, status})
         {:cont, state}
       end
 
-      handler = set_state(context.handler, connection: connection_fn)
-      assert {:ok, %Handler{}, []} = Handler.execute_connection(handler, :up)
-      assert_receive {:connection, :up}
+      handler = set_state(context.handler, status_change: status_change_fn)
+      assert {:ok, %Handler{}, []} = Handler.execute_status_change(handler, :up)
+      assert_receive {:status_change, :up}
 
-      assert {:ok, %Handler{}, []} = Handler.execute_connection(handler, :down)
-      assert_receive {:connection, :down}
+      assert {:ok, %Handler{}, []} = Handler.execute_status_change(handler, :down)
+      assert_receive {:status_change, :down}
     end
 
     test "return continue with next actions", context do
       next_actions = [{:subscribe, "foo/bar", qos: 0}]
       parent = self()
 
-      connection_fn = fn status, state ->
-        send(parent, {:connection, status})
+      status_change_fn = fn status, state ->
+        send(parent, {:status_change, status})
         {:cont, state, next_actions}
       end
 
-      handler = set_state(context.handler, connection: connection_fn)
+      handler = set_state(context.handler, status_change: status_change_fn)
 
-      assert {:ok, %Handler{}, ^next_actions} = Handler.execute_connection(handler, :up)
+      assert {:ok, %Handler{}, ^next_actions} = Handler.execute_status_change(handler, :up)
 
-      assert_receive {:connection, :up}
+      assert_receive {:status_change, :up}
 
-      assert {:ok, %Handler{}, ^next_actions} = Handler.execute_connection(handler, :down)
+      assert {:ok, %Handler{}, ^next_actions} = Handler.execute_status_change(handler, :down)
 
-      assert_receive {:connection, :down}
+      assert_receive {:status_change, :down}
     end
   end
 

From 9a85a94f928b8eaadc4f005fd5cea3cfc9e40907 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Wed, 29 May 2019 14:24:15 +0100
Subject: [PATCH 136/220] Made the handle_pub{ack|rec|rel|comp} callbacks
 optional

I also removed the default implementations of these.
---
 lib/tortoise/connection.ex | 66 +++++++++++++++++++++++---------------
 lib/tortoise/handler.ex    | 22 +------------
 2 files changed, 42 insertions(+), 46 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 4e307348..3c2142de 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -522,17 +522,23 @@ defmodule Tortoise.Connection do
       ) do
     :ok = Inflight.update(client_id, {:received, pubrel})
 
-    case Handler.execute_handle_pubrel(handler, pubrel) do
-      {:ok, %Package.Pubcomp{identifier: ^id} = pubcomp, %Handler{} = updated_handler,
-       next_actions} ->
-        # dispatch the pubcomp
-        :ok = Inflight.update(client_id, {:dispatch, pubcomp})
-        updated_data = %State{data | handler: updated_handler}
-        {:keep_state, updated_data, wrap_next_actions(next_actions)}
+    if function_exported?(handler.module, :handle_pubrel, 2) do
+      case Handler.execute_handle_pubrel(handler, pubrel) do
+        {:ok, %Package.Pubcomp{identifier: ^id} = pubcomp, %Handler{} = updated_handler,
+          next_actions} ->
+          # dispatch the pubcomp
+          :ok = Inflight.update(client_id, {:dispatch, pubcomp})
+          updated_data = %State{data | handler: updated_handler}
+          {:keep_state, updated_data, wrap_next_actions(next_actions)}
 
-      {:error, reason} ->
-        # todo
-        {:stop, reason, data}
+        {:error, reason} ->
+          # todo
+          {:stop, reason, data}
+      end
+    else
+      pubcomp = %Package.Pubcomp{identifier: id}
+      :ok = Inflight.update(client_id, {:dispatch, pubcomp})
+      :keep_state_and_data
     end
   end
 
@@ -585,15 +591,21 @@ defmodule Tortoise.Connection do
       ) do
     :ok = Inflight.update(client_id, {:received, pubrec})
 
-    case Handler.execute_handle_pubrec(handler, pubrec) do
-      {:ok, %Package.Pubrel{identifier: ^id} = pubrel, %Handler{} = updated_handler, next_actions} ->
-        updated_data = %State{data | handler: updated_handler}
-        :ok = Inflight.update(client_id, {:dispatch, pubrel})
-        {:keep_state, updated_data, wrap_next_actions(next_actions)}
+    if function_exported?(handler.module, :handle_pubrec, 2) do
+      case Handler.execute_handle_pubrec(handler, pubrec) do
+        {:ok, %Package.Pubrel{identifier: ^id} = pubrel, %Handler{} = updated_handler, next_actions} ->
+          updated_data = %State{data | handler: updated_handler}
+          :ok = Inflight.update(client_id, {:dispatch, pubrel})
+          {:keep_state, updated_data, wrap_next_actions(next_actions)}
 
-      {:error, reason} ->
-        # todo
-        {:stop, reason, data}
+        {:error, reason} ->
+          # todo
+          {:stop, reason, data}
+      end
+    else
+      pubrel = %Package.Pubrel{identifier: id}
+      :ok = Inflight.update(client_id, {:dispatch, pubrel})
+      :keep_state_and_data
     end
   end
 
@@ -605,14 +617,18 @@ defmodule Tortoise.Connection do
       ) do
     :ok = Inflight.update(client_id, {:received, pubcomp})
 
-    case Handler.execute_handle_pubcomp(handler, pubcomp) do
-      {:ok, %Handler{} = updated_handler, next_actions} ->
-        updated_data = %State{data | handler: updated_handler}
-        {:keep_state, updated_data, wrap_next_actions(next_actions)}
+    if function_exported?(handler.module, :handle_pubcomp, 2) do
+      case Handler.execute_handle_pubcomp(handler, pubcomp) do
+        {:ok, %Handler{} = updated_handler, next_actions} ->
+          updated_data = %State{data | handler: updated_handler}
+          {:keep_state, updated_data, wrap_next_actions(next_actions)}
 
-      {:error, reason} ->
-        # todo
-        {:stop, reason, data}
+        {:error, reason} ->
+          # todo
+          {:stop, reason, data}
+      end
+    else
+      :keep_state_and_data
     end
   end
 
diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index fabf07e8..9b200cc3 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -142,26 +142,6 @@ defmodule Tortoise.Handler do
         {:cont, state, []}
       end
 
-      @impl true
-      def handle_puback(_puback, state) do
-        {:cont, state}
-      end
-
-      @impl true
-      def handle_pubrec(_pubrec, state) do
-        {:cont, state, []}
-      end
-
-      @impl true
-      def handle_pubrel(_pubrel, state) do
-        {:cont, state, []}
-      end
-
-      @impl true
-      def handle_pubcomp(_pubcomp, state) do
-        {:cont, state, []}
-      end
-
       @impl true
       def handle_suback(_subscribe, _suback, state) do
         {:cont, state}
@@ -363,7 +343,7 @@ defmodule Tortoise.Handler do
             when reason: :normal | :shutdown | {:shutdown, term()},
                  ignored: term()
 
-  @optional_callbacks status_change: 2
+  @optional_callbacks status_change: 2, handle_pubrec: 2, handle_pubrel: 2, handle_pubcomp: 2, handle_puback: 2
 
   @doc false
   @spec execute_init(t) :: {:ok, t} | :ignore | {:stop, term()}

From 80f3508f5a2c32356dddec029863d485fb6cb3ef Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Wed, 29 May 2019 15:56:54 +0100
Subject: [PATCH 137/220] Make the terminate/2 callback on the Tortoise.Handler
 optional

Removed the default implementation from the __using__ macro
---
 lib/tortoise/connection.ex | 4 +++-
 lib/tortoise/handler.ex    | 7 +------
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 3c2142de..254b946d 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -406,7 +406,9 @@ defmodule Tortoise.Connection do
 
   @impl true
   def terminate(reason, _state, %State{handler: handler}) do
-    _ignored = Handler.execute_terminate(handler, reason)
+    _ignored = if function_exported?(handler.module, :terminate, 2) do
+      Handler.execute_terminate(handler, reason)
+    end
   end
 
   @impl true
diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index 9b200cc3..719783c9 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -127,11 +127,6 @@ defmodule Tortoise.Handler do
         {:ok, state}
       end
 
-      @impl true
-      def terminate(_reason, _state) do
-        :ok
-      end
-
       @impl true
       def handle_connack(_connack, state) do
         {:cont, state}
@@ -343,7 +338,7 @@ defmodule Tortoise.Handler do
             when reason: :normal | :shutdown | {:shutdown, term()},
                  ignored: term()
 
-  @optional_callbacks status_change: 2, handle_pubrec: 2, handle_pubrel: 2, handle_pubcomp: 2, handle_puback: 2
+  @optional_callbacks status_change: 2, handle_pubrec: 2, handle_pubrel: 2, handle_pubcomp: 2, handle_puback: 2, terminate: 2
 
   @doc false
   @spec execute_init(t) :: {:ok, t} | :ignore | {:stop, term()}

From ac392d2d70cbc5eac4e5da4155e8292515263a40 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Wed, 29 May 2019 16:01:04 +0100
Subject: [PATCH 138/220] Ran mix format

---
 lib/tortoise/connection.ex     | 12 +++++++-----
 lib/tortoise/handler.ex        |  7 ++++++-
 lib/tortoise/handler/logger.ex | 10 +++++++---
 3 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 254b946d..9e1ac023 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -406,9 +406,10 @@ defmodule Tortoise.Connection do
 
   @impl true
   def terminate(reason, _state, %State{handler: handler}) do
-    _ignored = if function_exported?(handler.module, :terminate, 2) do
-      Handler.execute_terminate(handler, reason)
-    end
+    _ignored =
+      if function_exported?(handler.module, :terminate, 2) do
+        Handler.execute_terminate(handler, reason)
+      end
   end
 
   @impl true
@@ -527,7 +528,7 @@ defmodule Tortoise.Connection do
     if function_exported?(handler.module, :handle_pubrel, 2) do
       case Handler.execute_handle_pubrel(handler, pubrel) do
         {:ok, %Package.Pubcomp{identifier: ^id} = pubcomp, %Handler{} = updated_handler,
-          next_actions} ->
+         next_actions} ->
           # dispatch the pubcomp
           :ok = Inflight.update(client_id, {:dispatch, pubcomp})
           updated_data = %State{data | handler: updated_handler}
@@ -595,7 +596,8 @@ defmodule Tortoise.Connection do
 
     if function_exported?(handler.module, :handle_pubrec, 2) do
       case Handler.execute_handle_pubrec(handler, pubrec) do
-        {:ok, %Package.Pubrel{identifier: ^id} = pubrel, %Handler{} = updated_handler, next_actions} ->
+        {:ok, %Package.Pubrel{identifier: ^id} = pubrel, %Handler{} = updated_handler,
+         next_actions} ->
           updated_data = %State{data | handler: updated_handler}
           :ok = Inflight.update(client_id, {:dispatch, pubrel})
           {:keep_state, updated_data, wrap_next_actions(next_actions)}
diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index 719783c9..f2f4a33d 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -338,7 +338,12 @@ defmodule Tortoise.Handler do
             when reason: :normal | :shutdown | {:shutdown, term()},
                  ignored: term()
 
-  @optional_callbacks status_change: 2, handle_pubrec: 2, handle_pubrel: 2, handle_pubcomp: 2, handle_puback: 2, terminate: 2
+  @optional_callbacks status_change: 2,
+                      handle_pubrec: 2,
+                      handle_pubrel: 2,
+                      handle_pubcomp: 2,
+                      handle_puback: 2,
+                      terminate: 2
 
   @doc false
   @spec execute_init(t) :: {:ok, t} | :ignore | {:stop, term()}
diff --git a/lib/tortoise/handler/logger.ex b/lib/tortoise/handler/logger.ex
index 41627ef3..10cd3b82 100644
--- a/lib/tortoise/handler/logger.ex
+++ b/lib/tortoise/handler/logger.ex
@@ -39,7 +39,7 @@ defmodule Tortoise.Handler.Logger do
   end
 
   def handle_connack(%Package.Connack{reason: {:refused, refusal}}, state) do
-    Logger.error("Server refused connection: #{inspect refusal}")
+    Logger.error("Server refused connection: #{inspect(refusal)}")
     {:cont, state}
   end
 
@@ -51,12 +51,15 @@ defmodule Tortoise.Handler.Logger do
           Logger.info("Subscribed to #{topic} with the expected qos: #{qos}")
 
         {req_qos, {:ok, accepted_qos}} ->
-          Logger.warn("Subscribed to #{topic} with QoS #{req_qos} but got accepted as #{accepted_qos}")
+          Logger.warn(
+            "Subscribed to #{topic} with QoS #{req_qos} but got accepted as #{accepted_qos}"
+          )
 
         {_, {:error, reason}} ->
-          Logger.error("Failed to subscribe to topic: #{topic} reason: #{inspect reason}")
+          Logger.error("Failed to subscribe to topic: #{topic} reason: #{inspect(reason)}")
       end
     end
+
     {:cont, state}
   end
 
@@ -68,6 +71,7 @@ defmodule Tortoise.Handler.Logger do
           Logger.info("Successfully unsubscribed from #{topic}")
       end
     end
+
     {:cont, state}
   end
 

From ac83845cf35e179663a6f7383656d239f23acf28 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Wed, 29 May 2019 16:58:01 +0100
Subject: [PATCH 139/220] No need to send an empty next action list for default
 handle publish

---
 lib/tortoise/handler.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index f2f4a33d..808d15ab 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -134,7 +134,7 @@ defmodule Tortoise.Handler do
 
       @impl true
       def handle_publish(_topic_list, _publish, state) do
-        {:cont, state, []}
+        {:cont, state}
       end
 
       @impl true

From bdaa16ec98f551b25f94940d683dd41b145f8bcd Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Wed, 29 May 2019 16:58:38 +0100
Subject: [PATCH 140/220] Remove some old comments no longer needed

They'll be in git if I need them later anyways.
---
 lib/tortoise/handler.ex | 45 -----------------------------------------
 1 file changed, 45 deletions(-)

diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index 808d15ab..edd5efea 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -203,57 +203,12 @@ defmodule Tortoise.Handler do
             when status: :up | :down,
                  new_state: term()
 
-  # todo, connection/2 should perhaps be handle_connack ?
-
   @callback handle_connack(connack, state :: term()) ::
               {:ok, new_state}
               | {:ok, new_state, [next_action()]}
             when connack: Package.Connack.t(),
                  new_state: term()
 
-  # todo, handle_auth
-
-  # @doc """
-  # Invoked when the subscription of a topic filter changes status.
-
-  # The `status` of a subscription can be one of:
-
-  #   - `:up`, triggered when the subscription has been accepted by the
-  #     MQTT broker with the requested quality of service
-
-  #   - `{:warn, [requested: req_qos, accepted: qos]}`, triggered when
-  #      the subscription is accepted by the MQTT broker, but with a
-  #      different quality of service `qos` than the one requested
-  #      `req_qos`
-
-  #   - `{:error, reason}`, triggered when the subscription is rejected
-  #     with the reason `reason` such as `:access_denied`
-
-  #   - `:down`, triggered when the subscription of the given topic
-  #     filter has been successfully acknowledged as unsubscribed by the
-  #     MQTT broker
-
-  # The `topic_filter` is the topic filter in question, and the `state`
-  # is the internal state being passed through transitions.
-
-  # Returning `{:ok, new_state}` will set the state for later
-  # invocations.
-
-  # Returning `{:ok, new_state, next_actions}`, where `next_actions` is
-  # a list of next actions such as `{:unsubscribe, "foo/bar"}` will
-  # result in the state being returned and the next actions performed.
-  # """
-  # @callback subscription(status, topic_filter, state :: term) ::
-  #             {:ok, new_state}
-  #             | {:ok, new_state, [next_action()]}
-  #           when status:
-  #                  :up
-  #                  | :down
-  #                  | {:warn, [requested: Tortoise.qos(), accepted: Tortoise.qos()]}
-  #                  | {:error, term()},
-  #                topic_filter: Tortoise.topic_filter(),
-  #                new_state: term
-
   @callback handle_suback(subscribe, suback, state :: term) :: {:ok, new_state}
             when subscribe: Package.Subscribe.t(),
                  suback: Package.Suback.t(),

From 2dd211cc8c40d0e9ae5ac7a57912b3934c5bccb4 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Fri, 31 May 2019 17:04:59 +0100
Subject: [PATCH 141/220] Allow the a `{:error, reason}`- return in the Publish
 QoS=0 handler

---
 lib/tortoise/connection.ex | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 9e1ac023..318f997e 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -480,7 +480,8 @@ defmodule Tortoise.Connection do
         updated_data = %State{data | handler: updated_handler}
         {:keep_state, updated_data, wrap_next_actions(next_actions)}
 
-        # handle stop
+      {:error, reason} ->
+        {:stop, reason, data}
     end
   end
 

From d1ac78d56ef10e2e02d5d6ad052b21a94f7be8ee Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sun, 2 Jun 2019 15:57:48 +0100
Subject: [PATCH 142/220] Allow a topic alias instead of a topic in
 Tortoise.publish(_sync)/4

When a topic alias is given instead of a proper topic the topic will
be set to an empty string and the given topic alias will be set in the
property list as `{:topic_alias, topic_alias}`. Tortoise will not
check if the topic alias is set, as we have no way of knowing.
---
 lib/tortoise.ex | 64 ++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 56 insertions(+), 8 deletions(-)

diff --git a/lib/tortoise.ex b/lib/tortoise.ex
index e4c76e84..09acf21c 100644
--- a/lib/tortoise.ex
+++ b/lib/tortoise.ex
@@ -147,6 +147,12 @@ defmodule Tortoise do
   """
   @type topic() :: String.t()
 
+  # todo, documentation
+  @type topic_alias :: 0x0001..0xFFFF
+
+  # todo, documentation
+  @type topic_or_topic_alias() :: topic() | topic_alias()
+
   @typedoc """
   A topic filter for a subscription.
 
@@ -246,14 +252,17 @@ defmodule Tortoise do
   with `Tortoise` so it is easy to see where the message originated
   from.
   """
-  @spec publish(client_id(), topic(), payload, [options]) ::
-          :ok | {:ok, reference()} | {:error, :unknown_connection}
+  @spec publish(client_id(), topic_or_topic_alias(), payload, [options]) ::
+          :ok | {:ok, reference()} | {:error, reason}
         when payload: binary() | nil,
              options:
                {:qos, qos()}
                | {:retain, boolean()}
-               | {:identifier, package_identifier()}
-  def publish(client_id, topic, payload \\ nil, opts \\ []) do
+               | {:identifier, package_identifier()},
+             reason: :unknown_connection | :topic_alias_specified_twice
+  def publish(client_id, topic, payload \\ nil, opts \\ [])
+
+  def publish(client_id, topic, payload, opts) when is_binary(topic) do
     {opts, properties} = Keyword.split(opts, [:retain, :qos, :transforms])
     qos = Keyword.get(opts, :qos, 0)
 
@@ -281,6 +290,24 @@ defmodule Tortoise do
     end
   end
 
+  # Support passing in a topic alias instead of a proper topic, in
+  # this case we will lift the topic alias into a topic_alias value in
+  # the property list, set the topic to an empty string and pass it on
+  # to the regular `publish/4`
+  def publish(client_id, topic_alias, payload, opts)
+      when is_integer(topic_alias) and topic_alias > 0 do
+    case Keyword.get(opts, :topic_alias) do
+      nil ->
+        publish(client_id, "", payload, Keyword.put(opts, :topic_alias, topic_alias))
+
+      ^topic_alias ->
+        publish(client_id, "", payload, opts)
+
+      _otherwise ->
+        {:error, :topic_alias_specified_twice}
+    end
+  end
+
   @doc """
   Synchronously send a message to the MQTT broker.
 
@@ -307,15 +334,18 @@ defmodule Tortoise do
 
   See the documentation for `Tortoise.publish/4` for configuration.
   """
-  @spec publish_sync(client_id(), topic(), payload, [options]) ::
-          :ok | {:error, :unknown_connection}
+  @spec publish_sync(client_id(), topic_or_topic_alias(), payload, [options]) ::
+          :ok | {:error, reason}
         when payload: binary() | nil,
              options:
                {:qos, qos()}
                | {:retain, boolean()}
                | {:identifier, package_identifier()}
-               | {:timeout, timeout()}
-  def publish_sync(client_id, topic, payload \\ nil, opts \\ []) do
+               | {:timeout, timeout()},
+             reason: :unknown_connection | :topic_alias_specified_twice
+  def publish_sync(client_id, topic, payload \\ nil, opts \\ [])
+
+  def publish_sync(client_id, topic, payload, opts) when is_binary(topic) do
     {opts, properties} = Keyword.split(opts, [:retain, :qos, :transforms])
     qos = Keyword.get(opts, :qos, 0)
     timeout = Keyword.get(opts, :timeout, :infinity)
@@ -343,4 +373,22 @@ defmodule Tortoise do
         {:error, :unknown_connection}
     end
   end
+
+  # Support passing in a topic alias instead of a proper topic, in
+  # this case we will lift the topic alias into a topic_alias value in
+  # the property list, set the topic to an empty string and pass it on
+  # to the regular `publish_sync/4`
+  def publish_sync(client_id, topic_alias, payload, opts)
+      when is_integer(topic_alias) and topic_alias > 0 do
+    case Keyword.get(opts, :topic_alias) do
+      nil ->
+        publish(client_id, "", payload, Keyword.put(opts, :topic_alias, topic_alias))
+
+      ^topic_alias ->
+        publish(client_id, "", payload, opts)
+
+      _otherwise ->
+        {:error, :topic_alias_specified_twice}
+    end
+  end
 end

From d40f17c61a3996b388862b8fdbbd44e2c3d220ae Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sun, 2 Jun 2019 21:54:51 +0100
Subject: [PATCH 143/220] Don't use the using Tortoise.Handler macro for the
 TestHandler

This should ensure we don't get the default implementation
---
 test/support/test_handler.ex | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/test/support/test_handler.ex b/test/support/test_handler.ex
index 316f3fac..207e0eea 100644
--- a/test/support/test_handler.ex
+++ b/test/support/test_handler.ex
@@ -1,14 +1,16 @@
 defmodule TestHandler do
-  use Tortoise.Handler
+  @behaviour Tortoise.Handler
 
   alias Tortoise.Package
 
+  @impl true
   def init(opts) do
     state = Enum.into(opts, %{})
     send(state[:parent], {{__MODULE__, :init}, opts})
     {:ok, state}
   end
 
+  @impl true
   def handle_connack(%Package.Connack{} = connack, state) do
     case state[:handle_connack] do
       nil ->
@@ -20,6 +22,7 @@ defmodule TestHandler do
     end
   end
 
+  @impl true
   def terminate(reason, state) do
     case state[:terminate] do
       nil ->
@@ -31,6 +34,7 @@ defmodule TestHandler do
     end
   end
 
+  @impl true
   def status_change(status, state) do
     case state[:status_change] do
       nil ->
@@ -43,6 +47,7 @@ defmodule TestHandler do
   end
 
   def handle_disconnect(%Package.Disconnect{} = disconnect, state) do
+  @impl true
     case state[:handle_disconnect] do
       nil ->
         send(state[:parent], {{__MODULE__, :handle_disconnect}, disconnect})
@@ -53,6 +58,7 @@ defmodule TestHandler do
     end
   end
 
+  @impl true
   def handle_publish(topic, %Package.Publish{} = publish, state) do
     case state[:handle_publish] do
       nil ->
@@ -64,6 +70,7 @@ defmodule TestHandler do
     end
   end
 
+  @impl true
   def handle_puback(%Package.Puback{} = puback, state) do
     case state[:handle_puback] do
       nil ->
@@ -75,6 +82,7 @@ defmodule TestHandler do
     end
   end
 
+  @impl true
   def handle_pubrec(%Package.Pubrec{} = pubrec, state) do
     case state[:handle_pubrec] do
       nil ->
@@ -86,6 +94,7 @@ defmodule TestHandler do
     end
   end
 
+  @impl true
   def handle_pubrel(%Package.Pubrel{} = pubrel, state) do
     case state[:handle_pubrel] do
       nil ->
@@ -97,6 +106,7 @@ defmodule TestHandler do
     end
   end
 
+  @impl true
   def handle_pubcomp(%Package.Pubcomp{} = pubcomp, state) do
     case state[:handle_pubcomp] do
       nil ->
@@ -108,6 +118,7 @@ defmodule TestHandler do
     end
   end
 
+  @impl true
   def handle_suback(%Package.Subscribe{} = subscribe, %Package.Suback{} = suback, state) do
     case state[:handle_suback] do
       nil ->
@@ -119,6 +130,7 @@ defmodule TestHandler do
     end
   end
 
+  @impl true
   def handle_unsuback(%Package.Unsubscribe{} = unsubscribe, %Package.Unsuback{} = unsuback, state) do
     case state[:handle_unsuback] do
       nil ->

From d8ac3c5605c331a5843ccd61abcd3d385b3fd574 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Sun, 2 Jun 2019 21:59:51 +0100
Subject: [PATCH 144/220] Made it possible to specify properties and reasons on
 disconnect

The `disconnect/1` on `Tortoise.Connection` has been extended and is
now `Tortoise.Connection.disconnect/3`. Second argument is the
disconnect reason, and the third is the properties. Both has defaults
and respectively default to `:normal_disconnection` and the empty list
`[]`.

We do not check the disconnect `reason` effectively allowing the user
to specify a disconnect normally only allowed by the server, but some
brokers might do crazy stuff with this, so let us just allow the user
to do that if they want.
---
 lib/tortoise/connection.ex                 | 28 +++++++++++++++-------
 lib/tortoise/connection/inflight.ex        | 12 ++++++----
 lib/tortoise/handler.ex                    |  9 ++++---
 test/support/test_handler.ex               |  2 +-
 test/tortoise/connection/inflight_test.exs |  3 ++-
 test/tortoise/handler_test.exs             |  8 +++----
 6 files changed, 40 insertions(+), 22 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 318f997e..5ffc4263 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -110,9 +110,18 @@ defmodule Tortoise.Connection do
   inflight messages and send the proper disconnect message to the
   broker. The session will get terminated on the server.
   """
-  @spec disconnect(Tortoise.client_id()) :: :ok
-  def disconnect(client_id) do
-    GenStateMachine.call(via_name(client_id), :disconnect)
+  @spec disconnect(Tortoise.client_id(), reason, properties) :: :ok
+        when reason: Tortoise.Package.Disconnect.reason(),
+             properties: [property],
+             property:
+               {:reason_string, String.t()}
+               | {:server_reference, String.t()}
+               | {:session_expiry_interval, 0..0xFFFFFFFF}
+               | {:user_property, {String.t(), String.t()}}
+
+  def disconnect(client_id, reason \\ :normal_disconnection, properties \\ []) do
+    disconnect = %Package.Disconnect{reason: reason, properties: properties}
+    GenStateMachine.call(via_name(client_id), {:disconnect, disconnect})
   end
 
   @doc """
@@ -811,7 +820,8 @@ defmodule Tortoise.Connection do
 
       :disconnect ->
         :ok = Events.dispatch(client_id, :status, :terminating)
-        :ok = Inflight.drain(client_id)
+        disconnect = %Package.Disconnect{reason: :normal_disconnection}
+        :ok = Inflight.drain(client_id, disconnect)
         {:stop, :normal}
 
       {:eval, fun} when is_function(fun, 1) ->
@@ -886,20 +896,20 @@ defmodule Tortoise.Connection do
   # disconnect protocol messages ---------------------------------------
   def handle_event(
         {:call, from},
-        :disconnect,
+        {:disconnect, %Package.Disconnect{} = disconnect},
         :connected,
         %State{client_id: client_id} = data
       ) do
     :ok = Events.dispatch(client_id, :status, :terminating)
 
-    :ok = Inflight.drain(client_id)
+    :ok = Inflight.drain(client_id, disconnect)
 
     {:stop_and_reply, :shutdown, [{:reply, from, :ok}], data}
   end
 
   def handle_event(
         {:call, _from},
-        :disconnect,
+        {:disconnect, _reason},
         _,
         %State{}
       ) do
@@ -963,14 +973,14 @@ defmodule Tortoise.Connection do
     {:keep_state, %State{data | ping: {:idle, []}}, next_actions}
   end
 
-  # disconnect packages
+  # server initiated disconnect packages
   def handle_event(
         :internal,
         {:received, %Package.Disconnect{} = disconnect},
         _current_state,
         %State{handler: handler} = data
       ) do
-    case Handler.execute_handle_disconnect(handler, disconnect) do
+    case Handler.execute_handle_disconnect(handler, {:server, disconnect}) do
       {:ok, updated_handler, next_actions} ->
         {:keep_state, %State{data | handler: updated_handler}, wrap_next_actions(next_actions)}
 
diff --git a/lib/tortoise/connection/inflight.ex b/lib/tortoise/connection/inflight.ex
index b7413a99..b941ed9e 100644
--- a/lib/tortoise/connection/inflight.ex
+++ b/lib/tortoise/connection/inflight.ex
@@ -33,8 +33,8 @@ defmodule Tortoise.Connection.Inflight do
   end
 
   @doc false
-  def drain(client_id) do
-    GenStateMachine.call(via_name(client_id), :drain)
+  def drain(client_id, %Package.Disconnect{} = disconnect) do
+    GenStateMachine.call(via_name(client_id), {:drain, disconnect})
   end
 
   @doc false
@@ -384,13 +384,17 @@ defmodule Tortoise.Connection.Inflight do
     end
   end
 
-  def handle_event({:call, from}, :drain, {:connected, {transport, socket}}, %State{} = data) do
+  def handle_event(
+        {:call, from},
+        {:drain, %Package.Disconnect{} = disconnect},
+        {:connected, {transport, socket}},
+        %State{} = data
+      ) do
     for {_, %Track{polarity: :negative, caller: {pid, ref}}} <- data.pending do
       send(pid, {{Tortoise, data.client_id}, ref, {:error, :canceled}})
     end
 
     data = %State{data | pending: %{}, order: []}
-    disconnect = %Package.Disconnect{}
 
     case apply(transport, :send, [socket, Package.encode(disconnect)]) do
       :ok ->
diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index edd5efea..cd3d5ea4 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -278,7 +278,8 @@ defmodule Tortoise.Handler do
                  new_state: term()
 
   @callback handle_disconnect(disconnect, state :: term()) :: {:ok, new_state}
-            when disconnect: Package.Disconnect.t(),
+            when source: :server | :network,
+                 disconnect: {source, Package.Disconnect.t()},
                  new_state: term()
 
   # todo, should we do handle_pingresp as well ?
@@ -351,8 +352,10 @@ defmodule Tortoise.Handler do
   end
 
   @doc false
-  @spec execute_handle_disconnect(t, %Package.Disconnect{}) :: {:stop, term(), t}
-  def execute_handle_disconnect(handler, %Package.Disconnect{} = disconnect) do
+  @spec execute_handle_disconnect(t, disconnect) :: {:stop, term(), t}
+        when disconnect: {:server, %Package.Disconnect{}} | {:network, atom()}
+  def execute_handle_disconnect(handler, {source, _reason} = disconnect)
+      when source in [:server, :network] do
     apply(handler.module, :handle_disconnect, [disconnect, handler.state])
     |> transform_result()
     |> case do
diff --git a/test/support/test_handler.ex b/test/support/test_handler.ex
index 207e0eea..15872161 100644
--- a/test/support/test_handler.ex
+++ b/test/support/test_handler.ex
@@ -46,8 +46,8 @@ defmodule TestHandler do
     end
   end
 
-  def handle_disconnect(%Package.Disconnect{} = disconnect, state) do
   @impl true
+  def handle_disconnect({_source, %Package.Disconnect{} = disconnect}, state) do
     case state[:handle_disconnect] do
       nil ->
         send(state[:parent], {{__MODULE__, :handle_disconnect}, disconnect})
diff --git a/test/tortoise/connection/inflight_test.exs b/test/tortoise/connection/inflight_test.exs
index 6de6cea3..1ddd34f6 100644
--- a/test/tortoise/connection/inflight_test.exs
+++ b/test/tortoise/connection/inflight_test.exs
@@ -269,7 +269,8 @@ defmodule Tortoise.Connection.InflightTest do
       expected = publish |> Package.encode() |> IO.iodata_to_binary()
       assert {:ok, ^expected} = :gen_tcp.recv(context.server, byte_size(expected), 500)
       # start draining
-      :ok = Inflight.drain(client_id)
+      disconnect = %Package.Disconnect{reason: :normal_disconnection}
+      :ok = Inflight.drain(client_id, disconnect)
       # updates should have no effect at this point
       :ok = Inflight.update(client_id, {:received, %Package.Puback{identifier: 1}})
       # the calling process should get a result response
diff --git a/test/tortoise/handler_test.exs b/test/tortoise/handler_test.exs
index a8167a78..080fe260 100644
--- a/test/tortoise/handler_test.exs
+++ b/test/tortoise/handler_test.exs
@@ -97,7 +97,7 @@ defmodule Tortoise.HandlerTest do
       make_return(pubcomp, state)
     end
 
-    def handle_disconnect(disconnect, state) do
+    def handle_disconnect({:server, disconnect}, state) do
       make_return(disconnect, state)
     end
 
@@ -511,7 +511,7 @@ defmodule Tortoise.HandlerTest do
       handler = set_state(context.handler, disconnect: disconnect_fn)
 
       assert {:ok, %Handler{} = state, []} =
-               Handler.execute_handle_disconnect(handler, disconnect)
+               Handler.execute_handle_disconnect(handler, {:server, disconnect})
     end
 
     test "return continue with next actions", context do
@@ -522,7 +522,7 @@ defmodule Tortoise.HandlerTest do
       handler = set_state(context.handler, disconnect: disconnect_fn)
 
       assert {:ok, %Handler{} = state, ^next_actions} =
-               Handler.execute_handle_disconnect(handler, disconnect)
+               Handler.execute_handle_disconnect(handler, {:server, disconnect})
     end
 
     test "return stop with normal reason", context do
@@ -531,7 +531,7 @@ defmodule Tortoise.HandlerTest do
       handler = set_state(context.handler, disconnect: disconnect_fn)
 
       assert {:stop, :normal, %Handler{} = state} =
-               Handler.execute_handle_disconnect(handler, disconnect)
+               Handler.execute_handle_disconnect(handler, {:server, disconnect})
     end
   end
 

From 8b0380e61d50bc38aa253ab72da1a9468514543a Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Mon, 13 Jan 2020 10:40:09 +0000
Subject: [PATCH 145/220] Update the client subscription state when
 unsubscribing from topics

It is possible to ask a connection for what connections it is
currently connected to. When we posted an unsubscribe package to the
server we would remove the unsubscribed topics from this internal
subscription list. Now we will only remove the topics that were
successful (or wasn't a subscription in the first place). This handle
situations where we are not allowed to unsubscribe from a topic; in
this case the client would still report that it is subscribed to the
topic.
---
 lib/tortoise/connection.ex        | 19 +++++--
 test/tortoise/connection_test.exs | 82 ++++++++++++++++++++++++++++++-
 2 files changed, 96 insertions(+), 5 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 5ffc4263..c53d7b45 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -767,10 +767,21 @@ defmodule Tortoise.Connection do
     {{pid, msg_ref}, updated_pending} = Map.pop(pending, ref)
     data = %State{data | pending_refs: updated_pending}
 
-    # todo, if the results in unsuback contain an error, such as
-    # `{:error, :no_subscription_existed}` then we would be out of
-    # sync! What should we do here?
-    subscriptions = Map.drop(data.subscriptions, unsubscribe.topics)
+    # When updating the internal subscription state tracker we will
+    # disregard the unsuccessful unsubacks, as we can assume it wasn't
+    # in the subscription list to begin with, or that we are still
+    # subscribed as we are not autorized to unsubscribe for the given
+    # topic; one exception is when the server report no subscription
+    # existed; then we will update the client state
+    to_remove =
+      for {topic, result} <- Enum.zip(unsubscribe.topics, unsuback.results),
+          match?(
+            reason when reason == :success or reason == {:error, :no_subscription_existed},
+            result
+          ),
+          do: topic
+
+    subscriptions = Map.drop(data.subscriptions, to_remove)
 
     case Handler.execute_handle_unsuback(handler, unsubscribe, unsuback) do
       {:ok, %Handler{} = updated_handler, next_actions} ->
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index f422f648..0b2cf6bd 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -407,7 +407,87 @@ defmodule Tortoise.ConnectionTest do
       assert_receive {{Tortoise, ^client_id}, ^unsub_ref, :ok}, 0
     end
 
-    # @todo unsuccessful unsubscribe
+    test "unsuccessful unsubscribe: not authorized", context do
+      client_id = context.client_id
+
+      unsubscribe_foo = %Package.Unsubscribe{identifier: 2, topics: ["foo"]}
+      unsuback_foo = %Package.Unsuback{results: [error: :not_authorized], identifier: 2}
+
+      script = [
+        {:receive,
+         %Package.Subscribe{
+           topics: [
+             {"foo", [qos: 0, no_local: false, retain_as_published: false, retain_handling: 1]}
+           ],
+           identifier: 1
+         }},
+        {:send, %Package.Suback{acks: [ok: 0], identifier: 1}},
+        # unsubscribe foo
+        {:receive, unsubscribe_foo},
+        {:send, unsuback_foo}
+      ]
+
+      {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
+
+      subscribe = %Package.Subscribe{
+        topics: [
+          {"foo", [qos: 0, no_local: false, retain_as_published: false, retain_handling: 1]}
+        ],
+        identifier: 1
+      }
+
+      {:ok, _sub_ref} = Tortoise.Connection.subscribe(client_id, subscribe.topics, identifier: 1)
+      assert_receive {ScriptedMqttServer, {:received, ^subscribe}}
+      assert_receive {{TestHandler, :handle_suback}, {_, %Package.Suback{identifier: 1}}}
+
+      subscriptions = Tortoise.Connection.subscriptions(client_id)
+      {:ok, unsub_ref} = Tortoise.Connection.unsubscribe(client_id, "foo", identifier: 2)
+      assert_receive {{Tortoise, client_id}, ^unsub_ref, :ok}
+      assert_receive {Tortoise.Integration.ScriptedMqttServer, :completed}
+      assert ^subscriptions = Tortoise.Connection.subscriptions(client_id)
+    end
+
+    test "unsuccessful unsubscribe: no subscription existed", context do
+      client_id = context.client_id
+
+      unsubscribe_foo = %Package.Unsubscribe{identifier: 2, topics: ["foo"]}
+      unsuback_foo = %Package.Unsuback{results: [error: :no_subscription_existed], identifier: 2}
+
+      script = [
+        {:receive,
+         %Package.Subscribe{
+           topics: [
+             {"foo", [qos: 0, no_local: false, retain_as_published: false, retain_handling: 1]}
+           ],
+           identifier: 1
+         }},
+        {:send, %Package.Suback{acks: [ok: 0], identifier: 1}},
+        # unsubscribe foo
+        {:receive, unsubscribe_foo},
+        {:send, unsuback_foo}
+      ]
+
+      {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
+
+      subscribe = %Package.Subscribe{
+        topics: [
+          {"foo", [qos: 0, no_local: false, retain_as_published: false, retain_handling: 1]}
+        ],
+        identifier: 1
+      }
+
+      {:ok, _sub_ref} = Tortoise.Connection.subscribe(client_id, subscribe.topics, identifier: 1)
+      assert_receive {ScriptedMqttServer, {:received, ^subscribe}}
+      assert_receive {{TestHandler, :handle_suback}, {_, %Package.Suback{identifier: 1}}}
+
+      assert Tortoise.Connection.subscriptions(client_id) |> Map.has_key?("foo")
+      {:ok, unsub_ref} = Tortoise.Connection.unsubscribe(client_id, "foo", identifier: 2)
+      assert_receive {{Tortoise, client_id}, ^unsub_ref, :ok}
+      assert_receive {Tortoise.Integration.ScriptedMqttServer, :completed}
+      # the client should update it state to not include the foo topic
+      # as the server told us that it is not subscribed
+      refute Tortoise.Connection.subscriptions(client_id) |> Map.has_key?("foo")
+    end
   end
 
   describe "encrypted connection" do

From c719fb4cd6590d196bcb6d2b0615c565117efc91 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Mon, 13 Jan 2020 20:47:44 +0000
Subject: [PATCH 146/220] Don't run handle_puback if it is not defined in the
 callback handler

---
 lib/tortoise/connection.ex | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index c53d7b45..32e73bb6 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -515,14 +515,18 @@ defmodule Tortoise.Connection do
       ) do
     :ok = Inflight.update(client_id, {:received, puback})
 
-    case Handler.execute_handle_puback(handler, puback) do
-      {:ok, %Handler{} = updated_handler, next_actions} ->
-        updated_data = %State{data | handler: updated_handler}
-        {:keep_state, updated_data, wrap_next_actions(next_actions)}
+    if function_exported?(handler.module, :handle_puback, 2) do
+      case Handler.execute_handle_puback(handler, puback) do
+        {:ok, %Handler{} = updated_handler, next_actions} ->
+          updated_data = %State{data | handler: updated_handler}
+          {:keep_state, updated_data, wrap_next_actions(next_actions)}
 
-      {:error, reason} ->
-        # todo
-        {:stop, reason, data}
+        {:error, reason} ->
+          # todo
+          {:stop, reason, data}
+      end
+    else
+      :keep_state_and_data
     end
   end
 

From 2bc696ce5cc090d9923ad259076fdc560bcc9322 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Tue, 14 Jan 2020 22:18:49 +0000
Subject: [PATCH 147/220] Pass on next actions for all of the callback handlers

---
 lib/tortoise/connection.ex | 15 +++++----------
 1 file changed, 5 insertions(+), 10 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 32e73bb6..baec89a7 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -448,9 +448,7 @@ defmodule Tortoise.Connection do
         next_actions = [
           {:state_timeout, keep_alive * 1000, :keep_alive},
           {:next_event, :internal, {:execute_handler, {:connection, :up}}}
-          | for action <- next_actions do
-              {:next_event, :internal, {:user_action, action}}
-            end
+          | wrap_next_actions(next_actions)
         ]
 
         {:next_state, :connected, data, next_actions}
@@ -569,14 +567,12 @@ defmodule Tortoise.Connection do
         %State{client_id: client_id, handler: handler} = data
       ) do
     case Handler.execute_handle_publish(handler, publish) do
-      {:ok, %Package.Puback{identifier: ^id} = puback, %Handler{} = updated_handler,
-       _next_actions} ->
-        # todo handle next actions
+      {:ok, %Package.Puback{identifier: ^id} = puback, %Handler{} = updated_handler, next_actions} ->
         # respond with a puback
         :ok = Inflight.update(client_id, {:dispatch, puback})
         # - - -
         updated_data = %State{data | handler: updated_handler}
-        {:keep_state, updated_data}
+        {:keep_state, updated_data, wrap_next_actions(next_actions)}
 
         # handle stop
     end
@@ -589,13 +585,12 @@ defmodule Tortoise.Connection do
         %State{client_id: client_id, handler: handler} = data
       ) do
     case Handler.execute_handle_publish(handler, publish) do
-      {:ok, %Package.Pubrec{identifier: ^id} = pubrec, %Handler{} = updated_handler,
-       _next_actions} ->
+      {:ok, %Package.Pubrec{identifier: ^id} = pubrec, %Handler{} = updated_handler, next_actions} ->
         # respond with pubrec
         :ok = Inflight.update(client_id, {:dispatch, pubrec})
         # - - -
         updated_data = %State{data | handler: updated_handler}
-        {:keep_state, updated_data}
+        {:keep_state, updated_data, wrap_next_actions(next_actions)}
     end
   end
 

From d833e62d43c61044ac1f2e6bf707a631f240972b Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Thu, 16 Jan 2020 22:37:39 +0000
Subject: [PATCH 148/220] Add config struct that contains the server config for
 a connection

---
 lib/tortoise/connection/config.ex | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100644 lib/tortoise/connection/config.ex

diff --git a/lib/tortoise/connection/config.ex b/lib/tortoise/connection/config.ex
new file mode 100644
index 00000000..b3a39e61
--- /dev/null
+++ b/lib/tortoise/connection/config.ex
@@ -0,0 +1,22 @@
+defmodule Tortoise.Connection.Config do
+  @moduledoc false
+
+  # will be used to keep the client/server negotiated config for the
+  # connection. The struct contain the defaults that will be used if
+  # the connect/connack messages doesn't specify values to the given
+  # configurations
+
+  @enforce_keys [:server_keep_alive]
+  defstruct session_expiry_interval: 0,
+            receive_maximum: 0xFFFF,
+            maximum_qos: 2,
+            retain_available: true,
+            # how big is it actually?
+            maximum_packet_size: :infinity,
+            assigned_client_identifier: nil,
+            topic_alias_maximum: 0,
+            wildcard_subscription_available: true,
+            subscription_identifiers_available: true,
+            shared_subscription_available: true,
+            server_keep_alive: nil
+end

From 80390b9faf3f15ea57ddc362bce871fb444b7605 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Fri, 17 Jan 2020 14:44:51 +0000
Subject: [PATCH 149/220] Have the handle_connack callback decide for all
 connack packages

Pass all connack messages to the handle_connack callback, allowing the
user to decide what action should be taken when the connection reports
success or refusal. For now I have updated the standard "no op"
callback handler to stop the tortoise instance if a refusal is given.
---
 lib/tortoise/connection.ex | 19 ++++++++-----------
 lib/tortoise/handler.ex    | 33 ++++++++++++++++++++++++++++++++-
 2 files changed, 40 insertions(+), 12 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index baec89a7..c4661e66 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -430,7 +430,7 @@ defmodule Tortoise.Connection do
   # connection acknowledgement
   def handle_event(
         :internal,
-        {:received, %Package.Connack{reason: :success} = connack},
+        {:received, %Package.Connack{} = connack},
         :connecting,
         %State{
           client_id: client_id,
@@ -452,17 +452,14 @@ defmodule Tortoise.Connection do
         ]
 
         {:next_state, :connected, data, next_actions}
-    end
-  end
 
-  def handle_event(
-        :internal,
-        {:received, %Package.Connack{reason: {:refused, reason}} = _connack},
-        :connecting,
-        %State{handler: _handler} = data
-      ) do
-    # todo, pass this through to the user defined callback handler
-    {:stop, {:connection_failed, reason}, data}
+      {:stop, reason, %Handler{} = updated_handler} ->
+        data = %State{data | handler: updated_handler}
+        {:stop, reason, data}
+
+      {:error, reason} ->
+        {:stop, reason, data}
+    end
   end
 
   def handle_event(
diff --git a/lib/tortoise/handler.ex b/lib/tortoise/handler.ex
index cd3d5ea4..5234feeb 100644
--- a/lib/tortoise/handler.ex
+++ b/lib/tortoise/handler.ex
@@ -128,10 +128,33 @@ defmodule Tortoise.Handler do
       end
 
       @impl true
-      def handle_connack(_connack, state) do
+      def handle_connack(%Package.Connack{reason: :success}, state) do
         {:cont, state}
       end
 
+      def handle_connack(%Package.Connack{reason: {:refused, reason}}, _state) do
+        # todo, we could categorize the reasons into user error,
+        # network error, etc error...
+        case reason do
+          :unsupported_protocol_version ->
+            {:error, {:connection_failed, :unsupported_protocol_version}}
+
+          :not_authorized ->
+            {:error, {:connection_failed, :not_authorized}}
+
+          :server_unavailable ->
+            {:error, {:connection_failed, :server_unavailable}}
+
+          :client_identifier_not_valid ->
+            {:error, {:connection_failed, :client_identifier_not_valid}}
+
+          :bad_user_name_or_password ->
+            {:error, {:connection_failed, :bad_user_name_or_password}}
+
+            # todo, list not exhaustive
+        end
+      end
+
       @impl true
       def handle_publish(_topic_list, _publish, state) do
         {:cont, state}
@@ -206,6 +229,7 @@ defmodule Tortoise.Handler do
   @callback handle_connack(connack, state :: term()) ::
               {:ok, new_state}
               | {:ok, new_state, [next_action()]}
+              | {:error, reason :: term()}
             when connack: Package.Connack.t(),
                  new_state: term()
 
@@ -346,6 +370,9 @@ defmodule Tortoise.Handler do
         updated_handler = %__MODULE__{handler | state: updated_state}
         {:ok, updated_handler, next_actions}
 
+      {:stop, reason, updated_state} ->
+        {:stop, reason, %__MODULE__{handler | state: updated_state}}
+
       {:error, reason} ->
         {:error, reason}
     end
@@ -552,6 +579,10 @@ defmodule Tortoise.Handler do
     {:stop, reason, updated_state}
   end
 
+  defp transform_result({:error, reason}) do
+    {:error, reason}
+  end
+
   defp transform_result({cont, updated_state, next_actions}) when is_list(next_actions) do
     case Enum.split_with(next_actions, &valid_next_action?/1) do
       {_, []} ->

From eaa58a4a2f6cae6e21bc79cdcb2844a0619d7cea Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Fri, 17 Jan 2020 16:53:51 +0000
Subject: [PATCH 150/220] Store the server set keep alive interval if present
 in connection

If the server specify a keep alive in the connack it should get used
for the keep live interval. We will not pick the server set value if
present, and keep the user defined if not.

Also, this commit introduce a `Tortoise.Connection.info/1` that will
return the connection configuration when the client is connected.
---
 lib/tortoise/connection.ex        | 35 +++++++++++++++++++---
 lib/tortoise/connection/config.ex | 15 ++++++++++
 test/tortoise/connection_test.exs | 49 ++++++++++++++++++++++++++++++-
 3 files changed, 94 insertions(+), 5 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index c4661e66..f6c542db 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -19,7 +19,8 @@ defmodule Tortoise.Connection do
             connection: nil,
             ping: {:idle, []},
             handler: nil,
-            receiver: nil
+            receiver: nil,
+            config: nil
 
   alias __MODULE__, as: State
 
@@ -357,6 +358,13 @@ defmodule Tortoise.Connection do
     end
   end
 
+  @doc """
+  Get the info on the current connection configuration
+  """
+  def info(client_id) do
+    GenStateMachine.call(via_name(client_id), :get_info)
+  end
+
   @doc false
   @spec connection(Tortoise.client_id(), [opts]) ::
           {:ok, {module(), term()}} | {:error, :unknown_connection} | {:error, :timeout}
@@ -430,11 +438,11 @@ defmodule Tortoise.Connection do
   # connection acknowledgement
   def handle_event(
         :internal,
-        {:received, %Package.Connack{} = connack},
+        {:received, %Package.Connack{properties: properties} = connack},
         :connecting,
         %State{
           client_id: client_id,
-          connect: %Package.Connect{keep_alive: keep_alive},
+          connect: %Package.Connect{keep_alive: keep_alive} = connect,
           handler: handler
         } = data
       ) do
@@ -443,7 +451,13 @@ defmodule Tortoise.Connection do
         :ok = Tortoise.Registry.put_meta(via_name(client_id), data.connection)
         :ok = Events.dispatch(client_id, :connection, data.connection)
         :ok = Events.dispatch(client_id, :status, :connected)
-        data = %State{data | backoff: Backoff.reset(data.backoff), handler: updated_handler}
+
+        data = %State{
+          data
+          | backoff: Backoff.reset(data.backoff),
+            handler: updated_handler,
+            config: Tortoise.Connection.Config.merge(connect, properties)
+        }
 
         next_actions = [
           {:state_timeout, keep_alive * 1000, :keep_alive},
@@ -472,6 +486,19 @@ defmodule Tortoise.Connection do
     {:stop, {:protocol_violation, reason}, data}
   end
 
+  # get status =========================================================
+  def handle_event({:call, from}, :get_info, :connected, data) do
+    next_actions = [{:reply, from, {:connected, data.config}}]
+
+    {:keep_state_and_data, next_actions}
+  end
+
+  def handle_event({:call, from}, :get_info, state, _data) do
+    next_actions = [{:reply, from, state}]
+
+    {:keep_state_and_data, next_actions}
+  end
+
   # publish packages ===================================================
   def handle_event(
         :internal,
diff --git a/lib/tortoise/connection/config.ex b/lib/tortoise/connection/config.ex
index b3a39e61..aa24500c 100644
--- a/lib/tortoise/connection/config.ex
+++ b/lib/tortoise/connection/config.ex
@@ -6,6 +6,8 @@ defmodule Tortoise.Connection.Config do
   # the connect/connack messages doesn't specify values to the given
   # configurations
 
+  alias Tortoise.Package.Connect
+
   @enforce_keys [:server_keep_alive]
   defstruct session_expiry_interval: 0,
             receive_maximum: 0xFFFF,
@@ -19,4 +21,17 @@ defmodule Tortoise.Connection.Config do
             subscription_identifiers_available: true,
             shared_subscription_available: true,
             server_keep_alive: nil
+
+  def merge(%Connect{keep_alive: keep_alive}, []) do
+    %__MODULE__{server_keep_alive: keep_alive}
+  end
+
+  def merge(%Connect{} = connect, [_ | _] = properties) do
+    # if no server_keep_alive is set we should use the one set by the client
+    keep_alive = Keyword.get(properties, :server_keep_alive, connect.keep_alive)
+
+    %__MODULE__{
+      server_keep_alive: keep_alive
+    }
+  end
 end
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index 0b2cf6bd..df1631cc 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -77,7 +77,7 @@ defmodule Tortoise.ConnectionTest do
       connect = %Package.Connect{client_id: client_id, clean_start: true}
       expected_connack = %Package.Connack{reason: :success, session_present: false}
 
-      script = [{:receive, connect}, {:send, expected_connack}]
+      script = [{:receive, connect}, {:send, expected_connack}, :pause]
 
       {:ok, {ip, port}} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
 
@@ -89,6 +89,18 @@ defmodule Tortoise.ConnectionTest do
 
       assert {:ok, _pid} = Connection.start_link(opts)
       assert_receive {ScriptedMqttServer, {:received, ^connect}}
+      assert_receive {ScriptedMqttServer, :paused}
+
+      # Should be able to get a connection when we have connected
+      assert {:ok, {Tortoise.Transport.Tcp, _port}} = Connection.connection(client_id)
+
+      # If the server does not specify a server_keep_alive interval we
+      # should use the one that was provided in the connect message,
+      # besides that the values of the config should be the defaults
+      expected_connection_config = %Connection.Config{server_keep_alive: connect.keep_alive}
+      assert {:connected, ^expected_connection_config} = Connection.info(client_id)
+
+      send(context.scripted_mqtt_server, :continue)
       assert_receive {ScriptedMqttServer, :completed}
     end
 
@@ -119,6 +131,41 @@ defmodule Tortoise.ConnectionTest do
       assert_receive {ScriptedMqttServer, {:received, ^reconnect}}
       assert_receive {ScriptedMqttServer, :completed}
     end
+
+    test "client should pick the servers keep alive interval if set", context do
+      client_id = context.client_id
+      connect = %Package.Connect{client_id: client_id}
+      keep_alive = 0xCAFE
+
+      script = [
+        {:receive, connect},
+        {:send, %Package.Connack{reason: :success, properties: [server_keep_alive: keep_alive]}},
+        :pause
+      ]
+
+      {:ok, {ip, port}} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
+
+      opts = [
+        client_id: client_id,
+        server: {Tortoise.Transport.Tcp, [host: ip, port: port]},
+        handler: {Tortoise.Handler.Default, []}
+      ]
+
+      assert {:ok, _pid} = Connection.start_link(opts)
+      assert_receive {ScriptedMqttServer, {:received, ^connect}}
+
+      # Should be able to get a connection when we have connected
+      assert {:ok, {Tortoise.Transport.Tcp, _port}} = Connection.connection(client_id)
+
+      # If the server does not specify a server_keep_alive interval we
+      # should use the one that was provided in the connect message,
+      # besides that the values of the config should be the defaults
+      expected_connection_config = %Connection.Config{server_keep_alive: keep_alive}
+      assert {:connected, ^expected_connection_config} = Connection.info(client_id)
+
+      send(context.scripted_mqtt_server, :continue)
+      assert_receive {ScriptedMqttServer, :completed}
+    end
   end
 
   describe "unsuccessful connect" do

From 4389a7f09c13649528f1b7a406e70a5515318f1c Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Fri, 17 Jan 2020 22:31:53 +0000
Subject: [PATCH 151/220] It is 268_435_455

---
 lib/tortoise/connection/config.ex | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/tortoise/connection/config.ex b/lib/tortoise/connection/config.ex
index aa24500c..e9ce6d4a 100644
--- a/lib/tortoise/connection/config.ex
+++ b/lib/tortoise/connection/config.ex
@@ -13,8 +13,7 @@ defmodule Tortoise.Connection.Config do
             receive_maximum: 0xFFFF,
             maximum_qos: 2,
             retain_available: true,
-            # how big is it actually?
-            maximum_packet_size: :infinity,
+            maximum_packet_size: 268_435_455,
             assigned_client_identifier: nil,
             topic_alias_maximum: 0,
             wildcard_subscription_available: true,

From e4c94fc3079d77d3d445bbf7055b4ffb539de88b Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Fri, 17 Jan 2020 22:32:58 +0000
Subject: [PATCH 152/220] Package properties now uses booleans for values where
 it makes sense

The values that are only allowed to be zero or one will now cast to
boolean values when the properties are decoded, and to byte
values (1::8 or 0::8) when encoded. This is done to make properties
easier to work with internally as they now can be used in
if-expressions.
---
 lib/tortoise/package/properties.ex | 53 ++++++++++++++++--------------
 test/test_helper.exs               | 12 +++----
 2 files changed, 34 insertions(+), 31 deletions(-)

diff --git a/lib/tortoise/package/properties.ex b/lib/tortoise/package/properties.ex
index e1781755..d63b26f4 100644
--- a/lib/tortoise/package/properties.ex
+++ b/lib/tortoise/package/properties.ex
@@ -34,7 +34,7 @@ defmodule Tortoise.Package.Properties do
 
   defp encode_property({key, value}) do
     case key do
-      :payload_format_indicator ->
+      :payload_format_indicator when value in [0, 1] ->
         [0x01, <<value::8>>]
 
       :message_expiry_interval ->
@@ -67,14 +67,14 @@ defmodule Tortoise.Package.Properties do
       :authentication_data when is_binary(value) ->
         [0x16, length_encode(value)]
 
-      :request_problem_information ->
-        [0x17, <<value::8>>]
+      :request_problem_information when is_boolean(value) ->
+        [0x17, boolean_to_byte(value)]
 
       :will_delay_interval when is_integer(value) ->
         [0x18, <<value::integer-size(32)>>]
 
-      :request_response_information ->
-        [0x19, <<value::8>>]
+      :request_response_information when is_boolean(value) ->
+        [0x19, boolean_to_byte(value)]
 
       :response_information ->
         [0x1A, length_encode(value)]
@@ -94,26 +94,29 @@ defmodule Tortoise.Package.Properties do
       :topic_alias ->
         [0x23, <<value::integer-size(16)>>]
 
-      :maximum_qos ->
+      :maximum_qos when value in [0, 1] ->
         [0x24, <<value::8>>]
 
-      :retain_available ->
-        [0x25, <<value::8>>]
+      :retain_available when is_boolean(value) ->
+        [0x25, boolean_to_byte(value)]
 
       :maximum_packet_size ->
         [0x27, <<value::integer-size(32)>>]
 
-      :wildcard_subscription_available ->
-        [0x28, <<value::8>>]
+      :wildcard_subscription_available when is_boolean(value) ->
+        [0x28, boolean_to_byte(value)]
 
-      :subscription_identifier_available ->
-        [0x29, <<value::8>>]
+      :subscription_identifier_available when is_boolean(value) ->
+        [0x29, boolean_to_byte(value)]
 
-      :shared_subscription_available ->
-        [0x2A, <<value::8>>]
+      :shared_subscription_available when is_boolean(value) ->
+        [0x2A, boolean_to_byte(value)]
     end
   end
 
+  defp boolean_to_byte(true), do: <<1::8>>
+  defp boolean_to_byte(false), do: <<0::8>>
+
   # ---
   def decode(data) do
     data
@@ -134,7 +137,7 @@ defmodule Tortoise.Package.Properties do
     {nil, <<>>}
   end
 
-  defp decode_property(<<0x01, value::8, rest::binary>>) do
+  defp decode_property(<<0x01, value::8, rest::binary>>) when value in [0, 1] do
     {{:payload_format_indicator, value}, rest}
   end
 
@@ -199,16 +202,16 @@ defmodule Tortoise.Package.Properties do
     {{:authentication_data, value}, rest}
   end
 
-  defp decode_property(<<0x17, value::8, rest::binary>>) do
-    {{:request_problem_information, value}, rest}
+  defp decode_property(<<0x17, value::8, rest::binary>>) when value in [0, 1] do
+    {{:request_problem_information, value == 1}, rest}
   end
 
   defp decode_property(<<0x18, value::integer-size(32), rest::binary>>) do
     {{:will_delay_interval, value}, rest}
   end
 
-  defp decode_property(<<0x19, value::8, rest::binary>>) do
-    {{:request_response_information, value}, rest}
+  defp decode_property(<<0x19, value::8, rest::binary>>) when value in [0, 1] do
+    {{:request_response_information, value == 1}, rest}
   end
 
   defp decode_property(<<0x1A, length::integer-size(16), rest::binary>>) do
@@ -238,12 +241,12 @@ defmodule Tortoise.Package.Properties do
     {{:topic_alias, value}, rest}
   end
 
-  defp decode_property(<<0x24, value::8, rest::binary>>) do
+  defp decode_property(<<0x24, value::8, rest::binary>>) when value in [0, 1] do
     {{:maximum_qos, value}, rest}
   end
 
-  defp decode_property(<<0x25, value::8, rest::binary>>) do
-    {{:retain_available, value}, rest}
+  defp decode_property(<<0x25, value::8, rest::binary>>) when value in [0, 1] do
+    {{:retain_available, value == 1}, rest}
   end
 
   defp decode_property(<<0x26, rest::binary>>) do
@@ -259,14 +262,14 @@ defmodule Tortoise.Package.Properties do
   end
 
   defp decode_property(<<0x28, value::8, rest::binary>>) do
-    {{:wildcard_subscription_available, value}, rest}
+    {{:wildcard_subscription_available, value == 1}, rest}
   end
 
   defp decode_property(<<0x29, value::8, rest::binary>>) do
-    {{:subscription_identifier_available, value}, rest}
+    {{:subscription_identifier_available, value == 1}, rest}
   end
 
   defp decode_property(<<0x2A, value::8, rest::binary>>) do
-    {{:shared_subscription_available, value}, rest}
+    {{:shared_subscription_available, value == 1}, rest}
   end
 end
diff --git a/test/test_helper.exs b/test/test_helper.exs
index 2d6c1ed1..e8c1ba9b 100644
--- a/test/test_helper.exs
+++ b/test/test_helper.exs
@@ -465,9 +465,9 @@ defmodule Tortoise.TestGenerators do
       :server_keep_alive -> {type, choose(0x0000, 0xFFFF)}
       :authentication_method -> {type, utf8()}
       :authentication_data -> {type, binary()}
-      :request_problem_information -> {type, oneof([0, 1])}
+      :request_problem_information -> {type, bool()}
       :will_delay_interval -> {type, choose(0, 4_294_967_295)}
-      :request_response_information -> {type, oneof([0, 1])}
+      :request_response_information -> {type, bool()}
       :response_information -> {type, utf8()}
       :server_reference -> {type, utf8()}
       :reason_string -> {type, utf8()}
@@ -475,12 +475,12 @@ defmodule Tortoise.TestGenerators do
       :topic_alias_maximum -> {type, choose(0x0000, 0xFFFF)}
       :topic_alias -> {type, choose(0x0001, 0xFFFF)}
       :maximum_qos -> {type, oneof([0, 1])}
-      :retain_available -> {type, oneof([0, 1])}
+      :retain_available -> {type, bool()}
       :user_property -> {type, {utf8(), utf8()}}
       :maximum_packet_size -> {type, choose(1, 268_435_455)}
-      :wildcard_subscription_available -> {type, oneof([0, 1])}
-      :subscription_identifier_available -> {type, oneof([0, 1])}
-      :shared_subscription_available -> {type, oneof([0, 1])}
+      :wildcard_subscription_available -> {type, bool()}
+      :subscription_identifier_available -> {type, bool()}
+      :shared_subscription_available -> {type, bool()}
     end
   end
 end

From f5be27438e4610e8a4d456174f3babc08ac03b11 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Fri, 17 Jan 2020 22:35:00 +0000
Subject: [PATCH 153/220] Simplified merging of properties into connection
 config

Now that we cast package property values into booleans it is much
easier to merge the connection config; actually it might be overkill
to check if the values exist in the config object.

Actually, this might pose a problem if the connack contain user
defined properties; but let's worry about that later.
---
 lib/tortoise/connection/config.ex | 12 ++----------
 1 file changed, 2 insertions(+), 10 deletions(-)

diff --git a/lib/tortoise/connection/config.ex b/lib/tortoise/connection/config.ex
index e9ce6d4a..bc40466f 100644
--- a/lib/tortoise/connection/config.ex
+++ b/lib/tortoise/connection/config.ex
@@ -21,16 +21,8 @@ defmodule Tortoise.Connection.Config do
             shared_subscription_available: true,
             server_keep_alive: nil
 
-  def merge(%Connect{keep_alive: keep_alive}, []) do
-    %__MODULE__{server_keep_alive: keep_alive}
-  end
-
-  def merge(%Connect{} = connect, [_ | _] = properties) do
+  def merge(%Connect{keep_alive: keep_alive}, properties) do
     # if no server_keep_alive is set we should use the one set by the client
-    keep_alive = Keyword.get(properties, :server_keep_alive, connect.keep_alive)
-
-    %__MODULE__{
-      server_keep_alive: keep_alive
-    }
+    struct!(%__MODULE__{server_keep_alive: keep_alive}, properties)
   end
 end

From 18331bcfc3c732f7531fd070a59ca55405108392 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Sat, 18 Jan 2020 11:59:40 +0000
Subject: [PATCH 154/220] Use the server set keep alive interval for keep alive
 (if set)

Changed the way we set the timeout for the keep alive scheduler; it
will now take the timeout from the connection configuration, which
will use the server set keep alive, or the user set keep alive.
---
 lib/tortoise/connection.ex | 37 ++++++++++++++++++++++++++++---------
 1 file changed, 28 insertions(+), 9 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index f6c542db..94146b88 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -442,7 +442,7 @@ defmodule Tortoise.Connection do
         :connecting,
         %State{
           client_id: client_id,
-          connect: %Package.Connect{keep_alive: keep_alive} = connect,
+          connect: %Package.Connect{} = connect,
           handler: handler
         } = data
       ) do
@@ -460,7 +460,7 @@ defmodule Tortoise.Connection do
         }
 
         next_actions = [
-          {:state_timeout, keep_alive * 1000, :keep_alive},
+          {:next_event, :internal, :setup_keep_alive_timer},
           {:next_event, :internal, {:execute_handler, {:connection, :up}}}
           | wrap_next_actions(next_actions)
         ]
@@ -954,8 +954,7 @@ defmodule Tortoise.Connection do
   def handle_event(:cast, {:ping, caller}, :connected, %State{} = data) do
     case data.ping do
       {:idle, awaiting} ->
-        # set the keep alive timeout to trigger instantly
-        next_actions = [{:state_timeout, 0, :keep_alive}]
+        next_actions = [{:next_event, :internal, :trigger_keep_alive}]
         {:keep_state, %State{data | ping: {:idle, [caller | awaiting]}}, next_actions}
 
       {{:pinging, start_time}, awaiting} ->
@@ -969,6 +968,27 @@ defmodule Tortoise.Connection do
     :keep_state_and_data
   end
 
+  # keep alive ---------------------------------------------------------
+  def handle_event(:internal, :setup_keep_alive_timer, :connected, data) do
+    timeout = data.config.server_keep_alive * 1000
+    next_actions = [{:state_timeout, timeout, :keep_alive}]
+    {:keep_state_and_data, next_actions}
+  end
+
+  def handle_event(:internal, :setup_keep_alive_timer, _other_states, _data) do
+    :keep_state_and_data
+  end
+
+  def handle_event(:internal, :trigger_keep_alive, :connected, _data) do
+    # set the keep alive timeout to trigger instantly
+    next_actions = [{:state_timeout, 0, :keep_alive}]
+    {:keep_state_and_data, next_actions}
+  end
+
+  def handle_event(:internal, :trigger_keep_alive, _other_states, _data) do
+    :keep_state_and_data
+  end
+
   def handle_event(
         :state_timeout,
         :keep_alive,
@@ -988,8 +1008,7 @@ defmodule Tortoise.Connection do
         :connected,
         %State{
           client_id: client_id,
-          ping: {{:pinging, start_time}, awaiting},
-          connect: %Package.Connect{keep_alive: keep_alive}
+          ping: {{:pinging, start_time}, awaiting}
         } = data
       ) do
     round_trip_time =
@@ -998,11 +1017,11 @@ defmodule Tortoise.Connection do
 
     :ok = Events.dispatch(client_id, :ping_response, round_trip_time)
 
-    for {caller, ref} <- awaiting do
+    Enum.each(awaiting, fn {caller, ref} ->
       send(caller, {{Tortoise, client_id}, {Package.Pingreq, ref}, round_trip_time})
-    end
+    end)
 
-    next_actions = [{:state_timeout, keep_alive * 1000, :keep_alive}]
+    next_actions = [{:next_event, :internal, :setup_keep_alive_timer}]
 
     {:keep_state, %State{data | ping: {:idle, []}}, next_actions}
   end

From d618bf91025bbbb35ceced16157b41bc881056a1 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Sat, 18 Jan 2020 14:01:06 +0000
Subject: [PATCH 155/220] Handle empty subscription topic filter list by
 sending err to caller

The user facing API has a check for an empty topic filter list in a
subscription, but for good measure, and to remove an unless-expression
I have made a special case for subscribe casts with an empty topic
filter list. If the caller is external we will send a error empty
topic filter list error back.
---
 lib/tortoise/connection.ex | 24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 94146b88..32c886eb 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -670,20 +670,30 @@ defmodule Tortoise.Connection do
   end
 
   # subscription logic
+  def handle_event(
+        :cast,
+        {:subscribe, caller, %Package.Subscribe{topics: []}, _opts},
+        :connected,
+        _data
+      ) do
+    # This should not really be able to happen as the API will not
+    # allow the user to specify an empty list, but this is added for
+    # good measure
+    reply = {:error, :empty_topic_filter_list}
+    next_actions = [{:next_event, :internal, {:reply, caller, reply}}]
+    {:keep_state_and_data, next_actions}
+  end
+
   def handle_event(
         :cast,
         {:subscribe, caller, subscribe, _opts},
         :connected,
         %State{client_id: client_id} = data
       ) do
-    unless Enum.empty?(subscribe) do
-      {:ok, ref} = Inflight.track(client_id, {:outgoing, subscribe})
-      pending = Map.put_new(data.pending_refs, ref, caller)
+    {:ok, ref} = Inflight.track(client_id, {:outgoing, subscribe})
+    pending = Map.put_new(data.pending_refs, ref, caller)
 
-      {:keep_state, %State{data | pending_refs: pending}}
-    else
-      :keep_state_and_data
-    end
+    {:keep_state, %State{data | pending_refs: pending}}
   end
 
   def handle_event(:cast, {:subscribe, _, _, _}, _state_name, _data) do

From ffc3846d9fa9bbfbe2c0574587706ab31708e765 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Mon, 20 Jan 2020 14:00:14 +0000
Subject: [PATCH 156/220] Don't allow shared subscriptions if the server
 doesn't allow them

Introducing a validator step when placing a subscription. For now it
will check if there are any shared topic filters in the subscription
topic filter list when placing a subscription on a server that is
configured to not allow subscriptions to shared topic filters. If this
is done it will result in an error send to the user.
---
 lib/tortoise/connection.ex               | 14 ++++--
 lib/tortoise/connection/config.ex        | 60 +++++++++++++++++++++++-
 lib/tortoise/package/subscribe.ex        |  2 +-
 test/tortoise/connection/config_test.exs | 24 ++++++++++
 test/tortoise/connection_test.exs        | 44 ++++++++++++++++-
 5 files changed, 137 insertions(+), 7 deletions(-)
 create mode 100644 test/tortoise/connection/config_test.exs

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 32c886eb..1b132067 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -690,10 +690,18 @@ defmodule Tortoise.Connection do
         :connected,
         %State{client_id: client_id} = data
       ) do
-    {:ok, ref} = Inflight.track(client_id, {:outgoing, subscribe})
-    pending = Map.put_new(data.pending_refs, ref, caller)
+    case State.Config.validate(data.config, subscribe) do
+      :valid ->
+        {:ok, ref} = Inflight.track(client_id, {:outgoing, subscribe})
+        pending = Map.put_new(data.pending_refs, ref, caller)
 
-    {:keep_state, %State{data | pending_refs: pending}}
+        {:keep_state, %State{data | pending_refs: pending}}
+
+      {:invalid, reasons} ->
+        reply = {:error, {:subscription_failure, reasons}}
+        next_actions = [{:next_event, :internal, {:reply, caller, reply}}]
+        {:keep_state_and_data, next_actions}
+    end
   end
 
   def handle_event(:cast, {:subscribe, _, _, _}, _state_name, _data) do
diff --git a/lib/tortoise/connection/config.ex b/lib/tortoise/connection/config.ex
index bc40466f..6dae7ed6 100644
--- a/lib/tortoise/connection/config.ex
+++ b/lib/tortoise/connection/config.ex
@@ -6,7 +6,7 @@ defmodule Tortoise.Connection.Config do
   # the connect/connack messages doesn't specify values to the given
   # configurations
 
-  alias Tortoise.Package.Connect
+  alias Tortoise.Package.{Connect, Subscribe}
 
   @enforce_keys [:server_keep_alive]
   defstruct session_expiry_interval: 0,
@@ -25,4 +25,62 @@ defmodule Tortoise.Connection.Config do
     # if no server_keep_alive is set we should use the one set by the client
     struct!(%__MODULE__{server_keep_alive: keep_alive}, properties)
   end
+
+  def validate(%__MODULE__{} = config, package) do
+    config
+    |> Map.from_struct()
+    |> Map.to_list()
+    |> do_validate(package, [])
+
+    # todo, make tests that setup connections with each of them
+    # disabled and attempt to subscribe with that feature
+  end
+
+  defp do_validate([], _, []), do: :valid
+  defp do_validate([], _, reasons), do: {:invalid, reasons}
+
+  # assigned client identifier (ignored)
+  defp do_validate([{:assigned_client_identifier, _ignore} | rest], package, acc) do
+    do_validate(rest, package, acc)
+  end
+
+  # shared subscriptions -----------------------------------------------
+  defp do_validate(
+         [{:shared_subscription_available, false} | rest],
+         %Subscribe{topics: topics} = package,
+         acc
+       ) do
+    issues =
+      for {topic, _opts} <- topics, match?("$share/" <> _, topic) do
+        {:shared_subscription_not_available, topic}
+      end
+
+    do_validate(rest, package, issues ++ acc)
+  end
+
+  defp do_validate(
+         [{:shared_subscription_available, true} | rest],
+         %Subscribe{topics: _topics} = package,
+         acc
+       ) do
+    # todo!
+
+    # The ShareName MUST NOT contain the characters "/", "+" or "#",
+    # but MUST be followed by a "/" character. This "/" character MUST
+    # be followed by a Topic Filter [MQTT-4.8.2-2] as described in
+    # section 4.7.
+
+    do_validate(rest, package, acc)
+  end
+
+  # Don't check anything for non subscribe packages
+  defp do_validate([{:shared_subscription_available, _} | rest], package, acc) do
+    do_validate(rest, package, acc)
+  end
+
+  # catch all; if an option is enabled, or not accounted for, we just
+  # assume it is okay at this point
+  defp do_validate([{_option, _value} | rest], subscribe, acc) do
+    do_validate(rest, subscribe, acc)
+  end
 end
diff --git a/lib/tortoise/package/subscribe.ex b/lib/tortoise/package/subscribe.ex
index 7237962d..0921d4bc 100644
--- a/lib/tortoise/package/subscribe.ex
+++ b/lib/tortoise/package/subscribe.ex
@@ -82,7 +82,7 @@ defmodule Tortoise.Package.Subscribe do
     def encode(
           %Package.Subscribe{
             identifier: identifier,
-            # a valid subscribe package has at least one topic/qos_opts pair
+            # a valid subscribe package has at least one topic/opts pair
             topics: [{<<_topic_filter::binary>>, opts} | _]
           } = t
         )
diff --git a/test/tortoise/connection/config_test.exs b/test/tortoise/connection/config_test.exs
new file mode 100644
index 00000000..8f9035b5
--- /dev/null
+++ b/test/tortoise/connection/config_test.exs
@@ -0,0 +1,24 @@
+defmodule Tortoise.Connection.ConfigTest do
+  use ExUnit.Case, async: true
+  doctest Tortoise.Connection.Config
+
+  alias Tortoise.Package.Subscribe
+  alias Tortoise.Connection.Config
+
+  defp create_config(properties) do
+    # server_keep_alive needs to be set
+    struct!(%Config{server_keep_alive: 60}, properties)
+  end
+
+  describe "shared_subscription_available: false" do
+    test "return error if shared subscription is placed" do
+      no_shared_subscription = create_config(shared_subscription_available: false)
+
+      shared_topic_filter = "$share/foo/bar"
+      subscribe = %Subscribe{topics: [{shared_topic_filter, qos: 0}]}
+
+      assert {:invalid, reasons} = Config.validate(no_shared_subscription, subscribe)
+      assert {:shared_subscription_not_available, shared_topic_filter} in reasons
+    end
+  end
+end
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index df1631cc..a54cdd94 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -19,9 +19,9 @@ defmodule Tortoise.ConnectionTest do
     {:ok, %{client_id: client_id}}
   end
 
-  def setup_scripted_mqtt_server(_context) do
+  def setup_scripted_mqtt_server(context) do
     {:ok, pid} = ScriptedMqttServer.start_link()
-    {:ok, %{scripted_mqtt_server: pid}}
+    {:ok, Map.put(context, :scripted_mqtt_server, pid)}
   end
 
   def setup_scripted_mqtt_server_ssl(_context) do
@@ -537,6 +537,46 @@ defmodule Tortoise.ConnectionTest do
     end
   end
 
+  describe "subscription features" do
+    setup [:setup_scripted_mqtt_server]
+
+    test "subscribing to a shared topic filter when feature is disabled", context do
+      # The client should receive an error if it attempt to subscribe
+      # to a shared topic on a server that does not allow shared
+      # topics
+      client_id = context.client_id
+
+      script = [
+        {:receive, %Package.Connect{client_id: client_id}},
+        {:send,
+         %Package.Connack{
+           reason: :success,
+           properties: [shared_subscription_available: false]
+         }}
+      ]
+
+      {:ok, {ip, port}} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
+
+      assert {:ok, connection_pid} =
+               Connection.start_link(
+                 client_id: client_id,
+                 server: {Tortoise.Transport.Tcp, [host: ip, port: port]},
+                 handler: {TestHandler, [parent: self()]}
+               )
+
+      assert_receive {ScriptedMqttServer, {:received, %Package.Connect{}}}
+
+      assert {:ok, {Tortoise.Transport.Tcp, _port}} = Connection.connection(client_id)
+
+      assert {:connected, %{shared_subscription_available: false}} = Connection.info(client_id)
+
+      assert {:error, {:subscription_failure, reasons}} =
+               Connection.subscribe_sync(client_id, {"$share/foo/bar", qos: 0})
+
+      assert {:shared_subscription_not_available, "$share/foo/bar"} in reasons
+    end
+  end
+
   describe "encrypted connection" do
     setup [:setup_scripted_mqtt_server_ssl]
 

From c98a6abefbec662fb7c1104a2a3ae401ee137a39 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Mon, 20 Jan 2020 14:53:53 +0000
Subject: [PATCH 157/220] Guard against using wildcard if the server doesn't
 support them

Tortoise will now guard against subscribing to a topic filter
containing a wildcard if the server doesn't support them.
---
 lib/tortoise/connection/config.ex        | 38 ++++++++++++++++++++--
 test/tortoise/connection/config_test.exs | 22 +++++++++++++
 test/tortoise/connection_test.exs        | 41 ++++++++++++++++++++++++
 3 files changed, 99 insertions(+), 2 deletions(-)

diff --git a/lib/tortoise/connection/config.ex b/lib/tortoise/connection/config.ex
index 6dae7ed6..5a023b0f 100644
--- a/lib/tortoise/connection/config.ex
+++ b/lib/tortoise/connection/config.ex
@@ -44,6 +44,40 @@ defmodule Tortoise.Connection.Config do
     do_validate(rest, package, acc)
   end
 
+  # wildcard subscriptions ---------------------------------------------
+  defp do_validate(
+         [{:wildcard_subscription_available, false} | rest],
+         %Subscribe{topics: topics} = package,
+         acc
+       ) do
+    issues =
+      Enum.reduce(topics, [], fn {topic, _opts}, acc ->
+        topic_list = String.split(topic, "/")
+
+        cond do
+          Enum.member?(topic_list, "+") ->
+            [{:wildcard_subscription_not_available, topic} | acc]
+
+          Enum.member?(topic_list, "#") ->
+            # multi-level wildcards are only allowed on the last
+            # position, but we test for each of the positions because
+            # we would have to iterate all the elements if we did a
+            # `List.last/1` anyways
+            [{:wildcard_subscription_not_available, topic} | acc]
+
+          true ->
+            acc
+        end
+      end)
+
+    do_validate(rest, package, issues ++ acc)
+  end
+
+  defp do_validate([{:wildcard_subscription_available, _ignored} | rest], package, acc) do
+    # This is only relevant for Subscribe packages
+    do_validate(rest, package, acc)
+  end
+
   # shared subscriptions -----------------------------------------------
   defp do_validate(
          [{:shared_subscription_available, false} | rest],
@@ -73,8 +107,8 @@ defmodule Tortoise.Connection.Config do
     do_validate(rest, package, acc)
   end
 
-  # Don't check anything for non subscribe packages
-  defp do_validate([{:shared_subscription_available, _} | rest], package, acc) do
+  defp do_validate([{:shared_subscription_available, _ignored} | rest], package, acc) do
+    # This is only relevant for Subscribe packages
     do_validate(rest, package, acc)
   end
 
diff --git a/test/tortoise/connection/config_test.exs b/test/tortoise/connection/config_test.exs
index 8f9035b5..2e3e0177 100644
--- a/test/tortoise/connection/config_test.exs
+++ b/test/tortoise/connection/config_test.exs
@@ -21,4 +21,26 @@ defmodule Tortoise.Connection.ConfigTest do
       assert {:shared_subscription_not_available, shared_topic_filter} in reasons
     end
   end
+
+  describe "wildcard_subscription_available: false" do
+    test "return error if a subscription with a wildcard is placed" do
+      no_shared_subscription = create_config(wildcard_subscription_available: false)
+
+      topic_filter_with_single_level_wildcard = "foo/+/bar"
+      topic_filter_with_multi_level_wildcard = "foo/#"
+
+      subscribe = %Subscribe{
+        topics: [
+          {topic_filter_with_single_level_wildcard, qos: 0},
+          {topic_filter_with_multi_level_wildcard, qos: 0}
+        ]
+      }
+
+      assert {:invalid, reasons} = Config.validate(no_shared_subscription, subscribe)
+
+      assert {:wildcard_subscription_not_available, topic_filter_with_single_level_wildcard} in reasons
+
+      assert {:wildcard_subscription_not_available, topic_filter_with_multi_level_wildcard} in reasons
+    end
+  end
 end
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index a54cdd94..2874ee51 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -575,6 +575,47 @@ defmodule Tortoise.ConnectionTest do
 
       assert {:shared_subscription_not_available, "$share/foo/bar"} in reasons
     end
+
+    test "subscribing to a topic filter with wildcard when feature is disabled", context do
+      # The client should receive an error if it attempt to subscribe
+      # to a topic filter containing a wildcard on a server that does
+      # not allow wildcards in topic filters
+      client_id = context.client_id
+
+      script = [
+        {:receive, %Package.Connect{client_id: client_id}},
+        {:send,
+         %Package.Connack{
+           reason: :success,
+           properties: [wildcard_subscription_available: false]
+         }}
+      ]
+
+      {:ok, {ip, port}} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
+
+      assert {:ok, connection_pid} =
+               Connection.start_link(
+                 client_id: client_id,
+                 server: {Tortoise.Transport.Tcp, [host: ip, port: port]},
+                 handler: {TestHandler, [parent: self()]}
+               )
+
+      assert_receive {ScriptedMqttServer, {:received, %Package.Connect{}}}
+
+      assert {:ok, {Tortoise.Transport.Tcp, _port}} = Connection.connection(client_id)
+
+      assert {:connected, %{wildcard_subscription_available: false}} = Connection.info(client_id)
+
+      assert {:error, {:subscription_failure, reasons}} =
+               Connection.subscribe_sync(client_id, {"foo/+/bar", qos: 0})
+
+      assert {:wildcard_subscription_not_available, "foo/+/bar"} in reasons
+
+      assert {:error, {:subscription_failure, reasons}} =
+               Connection.subscribe_sync(client_id, {"foo/#", qos: 0})
+
+      assert {:wildcard_subscription_not_available, "foo/#"} in reasons
+    end
   end
 
   describe "encrypted connection" do

From bd0fdc5af92864a1f06ff56eba5fed243b8b004f Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Mon, 20 Jan 2020 16:15:59 +0000
Subject: [PATCH 158/220] Guard against subscription identifiers if the server
 disallow them

When placing a subscription we will check if the properties list
contain a subscription identifier and error if the server doesn't
support them.
---
 lib/tortoise/connection/config.ex        | 18 +++++++++++
 lib/tortoise/package/properties.ex       |  4 +--
 test/test_helper.exs                     |  4 +--
 test/tortoise/connection/config_test.exs | 14 +++++++++
 test/tortoise/connection_test.exs        | 40 ++++++++++++++++++++++++
 5 files changed, 76 insertions(+), 4 deletions(-)

diff --git a/lib/tortoise/connection/config.ex b/lib/tortoise/connection/config.ex
index 5a023b0f..349ed14b 100644
--- a/lib/tortoise/connection/config.ex
+++ b/lib/tortoise/connection/config.ex
@@ -112,6 +112,24 @@ defmodule Tortoise.Connection.Config do
     do_validate(rest, package, acc)
   end
 
+  # subscription identifiers -------------------------------------------
+  defp do_validate(
+         [{:subscription_identifiers_available, false} | rest],
+         %Subscribe{properties: properties} = package,
+         acc
+       ) do
+    if Enum.any?(properties, &match?({:subscription_identifier, _}, &1)) do
+      do_validate(rest, package, [:subscription_identifier_not_available | acc])
+    else
+      do_validate(rest, package, acc)
+    end
+  end
+
+  defp do_validate([{:subscription_identifiers_available, _ignored} | rest], package, acc) do
+    # This is only relevant for Subscribe packages
+    do_validate(rest, package, acc)
+  end
+
   # catch all; if an option is enabled, or not accounted for, we just
   # assume it is okay at this point
   defp do_validate([{_option, _value} | rest], subscribe, acc) do
diff --git a/lib/tortoise/package/properties.ex b/lib/tortoise/package/properties.ex
index d63b26f4..9901ee7e 100644
--- a/lib/tortoise/package/properties.ex
+++ b/lib/tortoise/package/properties.ex
@@ -106,7 +106,7 @@ defmodule Tortoise.Package.Properties do
       :wildcard_subscription_available when is_boolean(value) ->
         [0x28, boolean_to_byte(value)]
 
-      :subscription_identifier_available when is_boolean(value) ->
+      :subscription_identifiers_available when is_boolean(value) ->
         [0x29, boolean_to_byte(value)]
 
       :shared_subscription_available when is_boolean(value) ->
@@ -266,7 +266,7 @@ defmodule Tortoise.Package.Properties do
   end
 
   defp decode_property(<<0x29, value::8, rest::binary>>) do
-    {{:subscription_identifier_available, value == 1}, rest}
+    {{:subscription_identifiers_available, value == 1}, rest}
   end
 
   defp decode_property(<<0x2A, value::8, rest::binary>>) do
diff --git a/test/test_helper.exs b/test/test_helper.exs
index e8c1ba9b..ace54b01 100644
--- a/test/test_helper.exs
+++ b/test/test_helper.exs
@@ -444,7 +444,7 @@ defmodule Tortoise.TestGenerators do
               :user_property,
               :maximum_packet_size,
               :wildcard_subscription_available,
-              :subscription_identifier_available,
+              :subscription_identifiers_available,
               :shared_subscription_available
             ])
           ) do
@@ -479,7 +479,7 @@ defmodule Tortoise.TestGenerators do
       :user_property -> {type, {utf8(), utf8()}}
       :maximum_packet_size -> {type, choose(1, 268_435_455)}
       :wildcard_subscription_available -> {type, bool()}
-      :subscription_identifier_available -> {type, bool()}
+      :subscription_identifiers_available -> {type, bool()}
       :shared_subscription_available -> {type, bool()}
     end
   end
diff --git a/test/tortoise/connection/config_test.exs b/test/tortoise/connection/config_test.exs
index 2e3e0177..0df64956 100644
--- a/test/tortoise/connection/config_test.exs
+++ b/test/tortoise/connection/config_test.exs
@@ -43,4 +43,18 @@ defmodule Tortoise.Connection.ConfigTest do
       assert {:wildcard_subscription_not_available, topic_filter_with_multi_level_wildcard} in reasons
     end
   end
+
+  describe "subscription_identifier_available: false" do
+    test "return error if shared subscription is placed" do
+      config = create_config(subscription_identifiers_available: false)
+
+      subscribe = %Subscribe{
+        topics: [{"foo/bar", qos: 0}],
+        properties: [subscription_identifier: 5]
+      }
+
+      assert {:invalid, reasons} = Config.validate(config, subscribe)
+      assert :subscription_identifier_not_available in reasons
+    end
+  end
 end
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index 2874ee51..6fd7956c 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -616,6 +616,46 @@ defmodule Tortoise.ConnectionTest do
 
       assert {:wildcard_subscription_not_available, "foo/#"} in reasons
     end
+
+    test "subscribing with a subscription identifier when feature is disabled", context do
+      # The client should receive an error if it attempt to subscribe
+      # to a topic filter and specifying a subscription identifier on
+      # a server that does not allow subscription identifiers in topic
+      # filters
+      client_id = context.client_id
+
+      script = [
+        {:receive, %Package.Connect{client_id: client_id}},
+        {:send,
+         %Package.Connack{
+           reason: :success,
+           properties: [subscription_identifiers_available: false]
+         }}
+      ]
+
+      {:ok, {ip, port}} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
+
+      assert {:ok, connection_pid} =
+               Connection.start_link(
+                 client_id: client_id,
+                 server: {Tortoise.Transport.Tcp, [host: ip, port: port]},
+                 handler: {TestHandler, [parent: self()]}
+               )
+
+      assert_receive {ScriptedMqttServer, {:received, %Package.Connect{}}}
+
+      assert {:ok, {Tortoise.Transport.Tcp, _port}} = Connection.connection(client_id)
+
+      assert {:connected, %{subscription_identifiers_available: false}} =
+               Connection.info(client_id)
+
+      assert {:error, {:subscription_failure, reasons}} =
+               Connection.subscribe_sync(client_id, {"foo/+/bar", qos: 0},
+                 subscription_identifier: 5
+               )
+
+      assert :subscription_identifier_not_available in reasons
+    end
   end
 
   describe "encrypted connection" do

From 33d42a391a9d6d1e3f6106546936819ce8247c20 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Tue, 21 Jan 2020 11:56:26 +0000
Subject: [PATCH 159/220] Rename config to info, and store capabilities under
 capabilities

Previously I stored the server capabilities under a key called
config. I fear that that would have led to confusion as the user would
expect to be able to change this config. In reality it is server
capabilities, so I called it that instead. Besides that I moved the
capabilities into a struct containing connection "info." This will
allow us to respond with more data when the user request info about
the connection, as we can store the client_id, keep_alive interval,
etc there.
---
 lib/tortoise/connection.ex                    |  24 ++--
 lib/tortoise/connection/config.ex             | 110 ---------------
 lib/tortoise/connection/info.ex               |  21 +++
 lib/tortoise/connection/info/capabilities.ex  | 127 ++++++++++++++++++
 .../capabilities_test.exs}                    |  14 +-
 test/tortoise/connection_test.exs             |  48 +++++--
 6 files changed, 205 insertions(+), 139 deletions(-)
 create mode 100644 lib/tortoise/connection/info.ex
 create mode 100644 lib/tortoise/connection/info/capabilities.ex
 rename test/tortoise/connection/{config_test.exs => info/capabilities_test.exs} (78%)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 1b132067..628b6726 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -20,12 +20,12 @@ defmodule Tortoise.Connection do
             ping: {:idle, []},
             handler: nil,
             receiver: nil,
-            config: nil
+            info: nil
 
   alias __MODULE__, as: State
 
   alias Tortoise.{Handler, Transport, Package, Events}
-  alias Tortoise.Connection.{Receiver, Inflight, Backoff}
+  alias Tortoise.Connection.{Info, Receiver, Inflight, Backoff}
   alias Tortoise.Package.Connect
 
   @doc """
@@ -438,7 +438,7 @@ defmodule Tortoise.Connection do
   # connection acknowledgement
   def handle_event(
         :internal,
-        {:received, %Package.Connack{properties: properties} = connack},
+        {:received, %Package.Connack{reason: connection_result} = connack},
         :connecting,
         %State{
           client_id: client_id,
@@ -447,7 +447,7 @@ defmodule Tortoise.Connection do
         } = data
       ) do
     case Handler.execute_handle_connack(handler, connack) do
-      {:ok, %Handler{} = updated_handler, next_actions} ->
+      {:ok, %Handler{} = updated_handler, next_actions} when connection_result == :success ->
         :ok = Tortoise.Registry.put_meta(via_name(client_id), data.connection)
         :ok = Events.dispatch(client_id, :connection, data.connection)
         :ok = Events.dispatch(client_id, :status, :connected)
@@ -456,7 +456,7 @@ defmodule Tortoise.Connection do
           data
           | backoff: Backoff.reset(data.backoff),
             handler: updated_handler,
-            config: Tortoise.Connection.Config.merge(connect, properties)
+            info: Info.merge(connect, connack)
         }
 
         next_actions = [
@@ -488,7 +488,7 @@ defmodule Tortoise.Connection do
 
   # get status =========================================================
   def handle_event({:call, from}, :get_info, :connected, data) do
-    next_actions = [{:reply, from, {:connected, data.config}}]
+    next_actions = [{:reply, from, {:connected, data.info}}]
 
     {:keep_state_and_data, next_actions}
   end
@@ -690,7 +690,7 @@ defmodule Tortoise.Connection do
         :connected,
         %State{client_id: client_id} = data
       ) do
-    case State.Config.validate(data.config, subscribe) do
+    case Info.Capabilities.validate(data.info.capabilities, subscribe) do
       :valid ->
         {:ok, ref} = Inflight.track(client_id, {:outgoing, subscribe})
         pending = Map.put_new(data.pending_refs, ref, caller)
@@ -987,9 +987,13 @@ defmodule Tortoise.Connection do
   end
 
   # keep alive ---------------------------------------------------------
-  def handle_event(:internal, :setup_keep_alive_timer, :connected, data) do
-    timeout = data.config.server_keep_alive * 1000
-    next_actions = [{:state_timeout, timeout, :keep_alive}]
+  def handle_event(
+        :internal,
+        :setup_keep_alive_timer,
+        :connected,
+        %State{info: %Info{keep_alive: keep_alive}}
+      ) do
+    next_actions = [{:state_timeout, keep_alive * 1000, :keep_alive}]
     {:keep_state_and_data, next_actions}
   end
 
diff --git a/lib/tortoise/connection/config.ex b/lib/tortoise/connection/config.ex
index 349ed14b..7f6451aa 100644
--- a/lib/tortoise/connection/config.ex
+++ b/lib/tortoise/connection/config.ex
@@ -25,114 +25,4 @@ defmodule Tortoise.Connection.Config do
     # if no server_keep_alive is set we should use the one set by the client
     struct!(%__MODULE__{server_keep_alive: keep_alive}, properties)
   end
-
-  def validate(%__MODULE__{} = config, package) do
-    config
-    |> Map.from_struct()
-    |> Map.to_list()
-    |> do_validate(package, [])
-
-    # todo, make tests that setup connections with each of them
-    # disabled and attempt to subscribe with that feature
-  end
-
-  defp do_validate([], _, []), do: :valid
-  defp do_validate([], _, reasons), do: {:invalid, reasons}
-
-  # assigned client identifier (ignored)
-  defp do_validate([{:assigned_client_identifier, _ignore} | rest], package, acc) do
-    do_validate(rest, package, acc)
-  end
-
-  # wildcard subscriptions ---------------------------------------------
-  defp do_validate(
-         [{:wildcard_subscription_available, false} | rest],
-         %Subscribe{topics: topics} = package,
-         acc
-       ) do
-    issues =
-      Enum.reduce(topics, [], fn {topic, _opts}, acc ->
-        topic_list = String.split(topic, "/")
-
-        cond do
-          Enum.member?(topic_list, "+") ->
-            [{:wildcard_subscription_not_available, topic} | acc]
-
-          Enum.member?(topic_list, "#") ->
-            # multi-level wildcards are only allowed on the last
-            # position, but we test for each of the positions because
-            # we would have to iterate all the elements if we did a
-            # `List.last/1` anyways
-            [{:wildcard_subscription_not_available, topic} | acc]
-
-          true ->
-            acc
-        end
-      end)
-
-    do_validate(rest, package, issues ++ acc)
-  end
-
-  defp do_validate([{:wildcard_subscription_available, _ignored} | rest], package, acc) do
-    # This is only relevant for Subscribe packages
-    do_validate(rest, package, acc)
-  end
-
-  # shared subscriptions -----------------------------------------------
-  defp do_validate(
-         [{:shared_subscription_available, false} | rest],
-         %Subscribe{topics: topics} = package,
-         acc
-       ) do
-    issues =
-      for {topic, _opts} <- topics, match?("$share/" <> _, topic) do
-        {:shared_subscription_not_available, topic}
-      end
-
-    do_validate(rest, package, issues ++ acc)
-  end
-
-  defp do_validate(
-         [{:shared_subscription_available, true} | rest],
-         %Subscribe{topics: _topics} = package,
-         acc
-       ) do
-    # todo!
-
-    # The ShareName MUST NOT contain the characters "/", "+" or "#",
-    # but MUST be followed by a "/" character. This "/" character MUST
-    # be followed by a Topic Filter [MQTT-4.8.2-2] as described in
-    # section 4.7.
-
-    do_validate(rest, package, acc)
-  end
-
-  defp do_validate([{:shared_subscription_available, _ignored} | rest], package, acc) do
-    # This is only relevant for Subscribe packages
-    do_validate(rest, package, acc)
-  end
-
-  # subscription identifiers -------------------------------------------
-  defp do_validate(
-         [{:subscription_identifiers_available, false} | rest],
-         %Subscribe{properties: properties} = package,
-         acc
-       ) do
-    if Enum.any?(properties, &match?({:subscription_identifier, _}, &1)) do
-      do_validate(rest, package, [:subscription_identifier_not_available | acc])
-    else
-      do_validate(rest, package, acc)
-    end
-  end
-
-  defp do_validate([{:subscription_identifiers_available, _ignored} | rest], package, acc) do
-    # This is only relevant for Subscribe packages
-    do_validate(rest, package, acc)
-  end
-
-  # catch all; if an option is enabled, or not accounted for, we just
-  # assume it is okay at this point
-  defp do_validate([{_option, _value} | rest], subscribe, acc) do
-    do_validate(rest, subscribe, acc)
-  end
 end
diff --git a/lib/tortoise/connection/info.ex b/lib/tortoise/connection/info.ex
new file mode 100644
index 00000000..71ab31ed
--- /dev/null
+++ b/lib/tortoise/connection/info.ex
@@ -0,0 +1,21 @@
+defmodule Tortoise.Connection.Info do
+  @enforce_keys [:keep_alive]
+  defstruct client_id: nil,
+            keep_alive: nil,
+            capabilities: nil
+
+  alias Tortoise.Package.{Connect, Connack}
+
+  def merge(
+        %Connect{keep_alive: keep_alive},
+        %Connack{reason: :success} = connack
+      ) do
+    # if no server_keep_alive is set we should use the one set by the client
+    keep_alive = Keyword.get(connack.properties, :server_keep_alive, keep_alive)
+
+    struct!(__MODULE__,
+      keep_alive: keep_alive,
+      capabilities: struct!(%__MODULE__.Capabilities{}, connack.properties)
+    )
+  end
+end
diff --git a/lib/tortoise/connection/info/capabilities.ex b/lib/tortoise/connection/info/capabilities.ex
new file mode 100644
index 00000000..fb8a417a
--- /dev/null
+++ b/lib/tortoise/connection/info/capabilities.ex
@@ -0,0 +1,127 @@
+defmodule Tortoise.Connection.Info.Capabilities do
+  @moduledoc false
+
+  alias Tortoise.Package.Subscribe
+
+  defstruct session_expiry_interval: 0,
+            receive_maximum: 0xFFFF,
+            maximum_qos: 2,
+            retain_available: true,
+            maximum_packet_size: 268_435_455,
+            assigned_client_identifier: nil,
+            topic_alias_maximum: 0,
+            wildcard_subscription_available: true,
+            subscription_identifiers_available: true,
+            shared_subscription_available: true,
+            server_keep_alive: nil
+
+  def validate(%__MODULE__{} = config, package) do
+    config
+    |> Map.from_struct()
+    |> Map.to_list()
+    |> do_validate(package, [])
+
+    # todo, make tests that setup connections with each of them
+    # disabled and attempt to subscribe with that feature
+  end
+
+  defp do_validate([], _, []), do: :valid
+  defp do_validate([], _, reasons), do: {:invalid, reasons}
+
+  # assigned client identifier (ignored)
+  defp do_validate([{:assigned_client_identifier, _ignore} | rest], package, acc) do
+    do_validate(rest, package, acc)
+  end
+
+  # wildcard subscriptions ---------------------------------------------
+  defp do_validate(
+         [{:wildcard_subscription_available, false} | rest],
+         %Subscribe{topics: topics} = package,
+         acc
+       ) do
+    issues =
+      Enum.reduce(topics, [], fn {topic, _opts}, acc ->
+        topic_list = String.split(topic, "/")
+
+        cond do
+          Enum.member?(topic_list, "+") ->
+            [{:wildcard_subscription_not_available, topic} | acc]
+
+          Enum.member?(topic_list, "#") ->
+            # multi-level wildcards are only allowed on the last
+            # position, but we test for each of the positions because
+            # we would have to iterate all the elements if we did a
+            # `List.last/1` anyways
+            [{:wildcard_subscription_not_available, topic} | acc]
+
+          true ->
+            acc
+        end
+      end)
+
+    do_validate(rest, package, issues ++ acc)
+  end
+
+  defp do_validate([{:wildcard_subscription_available, _ignored} | rest], package, acc) do
+    # This is only relevant for Subscribe packages
+    do_validate(rest, package, acc)
+  end
+
+  # shared subscriptions -----------------------------------------------
+  defp do_validate(
+         [{:shared_subscription_available, false} | rest],
+         %Subscribe{topics: topics} = package,
+         acc
+       ) do
+    issues =
+      for {topic, _opts} <- topics, match?("$share/" <> _, topic) do
+        {:shared_subscription_not_available, topic}
+      end
+
+    do_validate(rest, package, issues ++ acc)
+  end
+
+  defp do_validate(
+         [{:shared_subscription_available, true} | rest],
+         %Subscribe{topics: _topics} = package,
+         acc
+       ) do
+    # todo!
+
+    # The ShareName MUST NOT contain the characters "/", "+" or "#",
+    # but MUST be followed by a "/" character. This "/" character MUST
+    # be followed by a Topic Filter [MQTT-4.8.2-2] as described in
+    # section 4.7.
+
+    do_validate(rest, package, acc)
+  end
+
+  defp do_validate([{:shared_subscription_available, _ignored} | rest], package, acc) do
+    # This is only relevant for Subscribe packages
+    do_validate(rest, package, acc)
+  end
+
+  # subscription identifiers -------------------------------------------
+  defp do_validate(
+         [{:subscription_identifiers_available, false} | rest],
+         %Subscribe{properties: properties} = package,
+         acc
+       ) do
+    if Enum.any?(properties, &match?({:subscription_identifier, _}, &1)) do
+      do_validate(rest, package, [:subscription_identifier_not_available | acc])
+    else
+      do_validate(rest, package, acc)
+    end
+  end
+
+  defp do_validate([{:subscription_identifiers_available, _ignored} | rest], package, acc) do
+    # This is only relevant for Subscribe packages
+    do_validate(rest, package, acc)
+  end
+
+  # catch all; if an option is enabled, or not accounted for, we just
+  # assume it is okay at this point
+  defp do_validate([{_option, _value} | rest], subscribe, acc) do
+    do_validate(rest, subscribe, acc)
+  end
+end
diff --git a/test/tortoise/connection/config_test.exs b/test/tortoise/connection/info/capabilities_test.exs
similarity index 78%
rename from test/tortoise/connection/config_test.exs
rename to test/tortoise/connection/info/capabilities_test.exs
index 0df64956..e81e882a 100644
--- a/test/tortoise/connection/config_test.exs
+++ b/test/tortoise/connection/info/capabilities_test.exs
@@ -1,13 +1,13 @@
-defmodule Tortoise.Connection.ConfigTest do
+defmodule Tortoise.Connection.Info.CapabilitiesTest do
   use ExUnit.Case, async: true
-  doctest Tortoise.Connection.Config
+  doctest Tortoise.Connection.Info.Capabilities
 
   alias Tortoise.Package.Subscribe
-  alias Tortoise.Connection.Config
+  alias Tortoise.Connection.Info
 
   defp create_config(properties) do
     # server_keep_alive needs to be set
-    struct!(%Config{server_keep_alive: 60}, properties)
+    struct!(%Info.Capabilities{}, properties)
   end
 
   describe "shared_subscription_available: false" do
@@ -17,7 +17,7 @@ defmodule Tortoise.Connection.ConfigTest do
       shared_topic_filter = "$share/foo/bar"
       subscribe = %Subscribe{topics: [{shared_topic_filter, qos: 0}]}
 
-      assert {:invalid, reasons} = Config.validate(no_shared_subscription, subscribe)
+      assert {:invalid, reasons} = Info.Capabilities.validate(no_shared_subscription, subscribe)
       assert {:shared_subscription_not_available, shared_topic_filter} in reasons
     end
   end
@@ -36,7 +36,7 @@ defmodule Tortoise.Connection.ConfigTest do
         ]
       }
 
-      assert {:invalid, reasons} = Config.validate(no_shared_subscription, subscribe)
+      assert {:invalid, reasons} = Info.Capabilities.validate(no_shared_subscription, subscribe)
 
       assert {:wildcard_subscription_not_available, topic_filter_with_single_level_wildcard} in reasons
 
@@ -53,7 +53,7 @@ defmodule Tortoise.Connection.ConfigTest do
         properties: [subscription_identifier: 5]
       }
 
-      assert {:invalid, reasons} = Config.validate(config, subscribe)
+      assert {:invalid, reasons} = Info.Capabilities.validate(config, subscribe)
       assert :subscription_identifier_not_available in reasons
     end
   end
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index 6fd7956c..84548072 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -97,8 +97,14 @@ defmodule Tortoise.ConnectionTest do
       # If the server does not specify a server_keep_alive interval we
       # should use the one that was provided in the connect message,
       # besides that the values of the config should be the defaults
-      expected_connection_config = %Connection.Config{server_keep_alive: connect.keep_alive}
-      assert {:connected, ^expected_connection_config} = Connection.info(client_id)
+      expected_connection_info = %Connection.Info{
+        keep_alive: connect.keep_alive,
+        capabilities: %Connection.Info.Capabilities{
+          server_keep_alive: nil
+        }
+      }
+
+      assert {:connected, ^expected_connection_info} = Connection.info(client_id)
 
       send(context.scripted_mqtt_server, :continue)
       assert_receive {ScriptedMqttServer, :completed}
@@ -135,11 +141,12 @@ defmodule Tortoise.ConnectionTest do
     test "client should pick the servers keep alive interval if set", context do
       client_id = context.client_id
       connect = %Package.Connect{client_id: client_id}
-      keep_alive = 0xCAFE
+      server_keep_alive = 0xCAFE
 
       script = [
         {:receive, connect},
-        {:send, %Package.Connack{reason: :success, properties: [server_keep_alive: keep_alive]}},
+        {:send,
+         %Package.Connack{reason: :success, properties: [server_keep_alive: server_keep_alive]}},
         :pause
       ]
 
@@ -157,10 +164,17 @@ defmodule Tortoise.ConnectionTest do
       # Should be able to get a connection when we have connected
       assert {:ok, {Tortoise.Transport.Tcp, _port}} = Connection.connection(client_id)
 
-      # If the server does not specify a server_keep_alive interval we
-      # should use the one that was provided in the connect message,
-      # besides that the values of the config should be the defaults
-      expected_connection_config = %Connection.Config{server_keep_alive: keep_alive}
+      # If the server does specify a server_keep_alive interval we
+      # should use that one for the keep_alive instead of the user
+      # provided one in the connect message, besides that the values
+      # of the config should be the defaults
+      expected_connection_config = %Connection.Info{
+        capabilities: %Connection.Info.Capabilities{
+          server_keep_alive: server_keep_alive
+        },
+        keep_alive: server_keep_alive
+      }
+
       assert {:connected, ^expected_connection_config} = Connection.info(client_id)
 
       send(context.scripted_mqtt_server, :continue)
@@ -568,7 +582,12 @@ defmodule Tortoise.ConnectionTest do
 
       assert {:ok, {Tortoise.Transport.Tcp, _port}} = Connection.connection(client_id)
 
-      assert {:connected, %{shared_subscription_available: false}} = Connection.info(client_id)
+      assert {:connected,
+              %Connection.Info{
+                capabilities: %Connection.Info.Capabilities{
+                  shared_subscription_available: false
+                }
+              }} = Connection.info(client_id)
 
       assert {:error, {:subscription_failure, reasons}} =
                Connection.subscribe_sync(client_id, {"$share/foo/bar", qos: 0})
@@ -604,7 +623,8 @@ defmodule Tortoise.ConnectionTest do
 
       assert {:ok, {Tortoise.Transport.Tcp, _port}} = Connection.connection(client_id)
 
-      assert {:connected, %{wildcard_subscription_available: false}} = Connection.info(client_id)
+      assert {:connected, %{capabilities: %{wildcard_subscription_available: false}}} =
+               Connection.info(client_id)
 
       assert {:error, {:subscription_failure, reasons}} =
                Connection.subscribe_sync(client_id, {"foo/+/bar", qos: 0})
@@ -646,8 +666,12 @@ defmodule Tortoise.ConnectionTest do
 
       assert {:ok, {Tortoise.Transport.Tcp, _port}} = Connection.connection(client_id)
 
-      assert {:connected, %{subscription_identifiers_available: false}} =
-               Connection.info(client_id)
+      assert {:connected,
+              %Connection.Info{
+                capabilities: %Connection.Info.Capabilities{
+                  subscription_identifiers_available: false
+                }
+              }} = Connection.info(client_id)
 
       assert {:error, {:subscription_failure, reasons}} =
                Connection.subscribe_sync(client_id, {"foo/+/bar", qos: 0},

From 9d9277de91aada44b804c9bc3f2189f68302e377 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Tue, 21 Jan 2020 11:59:33 +0000
Subject: [PATCH 160/220] Get rid of unused config.ex file

---
 lib/tortoise/connection/config.ex | 28 ----------------------------
 1 file changed, 28 deletions(-)
 delete mode 100644 lib/tortoise/connection/config.ex

diff --git a/lib/tortoise/connection/config.ex b/lib/tortoise/connection/config.ex
deleted file mode 100644
index 7f6451aa..00000000
--- a/lib/tortoise/connection/config.ex
+++ /dev/null
@@ -1,28 +0,0 @@
-defmodule Tortoise.Connection.Config do
-  @moduledoc false
-
-  # will be used to keep the client/server negotiated config for the
-  # connection. The struct contain the defaults that will be used if
-  # the connect/connack messages doesn't specify values to the given
-  # configurations
-
-  alias Tortoise.Package.{Connect, Subscribe}
-
-  @enforce_keys [:server_keep_alive]
-  defstruct session_expiry_interval: 0,
-            receive_maximum: 0xFFFF,
-            maximum_qos: 2,
-            retain_available: true,
-            maximum_packet_size: 268_435_455,
-            assigned_client_identifier: nil,
-            topic_alias_maximum: 0,
-            wildcard_subscription_available: true,
-            subscription_identifiers_available: true,
-            shared_subscription_available: true,
-            server_keep_alive: nil
-
-  def merge(%Connect{keep_alive: keep_alive}, properties) do
-    # if no server_keep_alive is set we should use the one set by the client
-    struct!(%__MODULE__{server_keep_alive: keep_alive}, properties)
-  end
-end

From 1cdd80b5f986b794dfb74579aac14bf4550d425f Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Tue, 21 Jan 2020 15:25:00 +0000
Subject: [PATCH 161/220] Store connection subscription info in the info data
 structure

---
 lib/tortoise/connection.ex      | 19 +++++++++++--------
 lib/tortoise/connection/info.ex |  1 +
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 628b6726..223d4b68 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -13,7 +13,6 @@ defmodule Tortoise.Connection do
             connect: nil,
             server: nil,
             backoff: nil,
-            subscriptions: %{},
             opts: nil,
             pending_refs: %{},
             connection: nil,
@@ -723,7 +722,8 @@ defmodule Tortoise.Connection do
         :info,
         {{Tortoise, client_id}, {Package.Subscribe, ref}, {subscribe, suback}},
         _current_state,
-        %State{client_id: client_id, handler: handler, pending_refs: %{} = pending} = data
+        %State{client_id: client_id, handler: handler, pending_refs: %{} = pending, info: info} =
+          data
       ) do
     {{pid, msg_ref}, updated_pending} = Map.pop(pending, ref)
     data = %State{data | pending_refs: updated_pending}
@@ -731,7 +731,7 @@ defmodule Tortoise.Connection do
     updated_subscriptions =
       subscribe.topics
       |> Enum.zip(suback.acks)
-      |> Enum.reduce(data.subscriptions, fn
+      |> Enum.reduce(data.info.subscriptions, fn
         {{topic, opts}, {:ok, accepted_qos}}, acc ->
           Map.put(acc, topic, Keyword.replace!(opts, :qos, accepted_qos))
 
@@ -744,7 +744,7 @@ defmodule Tortoise.Connection do
         data = %State{
           data
           | handler: updated_handler,
-            subscriptions: updated_subscriptions
+            info: put_in(info.subscriptions, updated_subscriptions)
         }
 
         next_actions = [
@@ -803,7 +803,8 @@ defmodule Tortoise.Connection do
         :info,
         {{Tortoise, client_id}, {Package.Unsubscribe, ref}, {unsubscribe, unsuback}},
         _current_state,
-        %State{client_id: client_id, handler: handler, pending_refs: %{} = pending} = data
+        %State{client_id: client_id, handler: handler, pending_refs: %{} = pending, info: info} =
+          data
       ) do
     {{pid, msg_ref}, updated_pending} = Map.pop(pending, ref)
     data = %State{data | pending_refs: updated_pending}
@@ -822,14 +823,14 @@ defmodule Tortoise.Connection do
           ),
           do: topic
 
-    subscriptions = Map.drop(data.subscriptions, to_remove)
+    subscriptions = Map.drop(data.info.subscriptions, to_remove)
 
     case Handler.execute_handle_unsuback(handler, unsubscribe, unsuback) do
       {:ok, %Handler{} = updated_handler, next_actions} ->
         data = %State{
           data
           | handler: updated_handler,
-            subscriptions: subscriptions
+            info: put_in(info.subscriptions, subscriptions)
         }
 
         next_actions = [
@@ -845,7 +846,9 @@ defmodule Tortoise.Connection do
     end
   end
 
-  def handle_event({:call, from}, :subscriptions, _, %State{subscriptions: subscriptions}) do
+  def handle_event({:call, from}, :subscriptions, _, %State{
+        info: %Info{subscriptions: subscriptions}
+      }) do
     next_actions = [{:reply, from, subscriptions}]
     {:keep_state_and_data, next_actions}
   end
diff --git a/lib/tortoise/connection/info.ex b/lib/tortoise/connection/info.ex
index 71ab31ed..2d88d249 100644
--- a/lib/tortoise/connection/info.ex
+++ b/lib/tortoise/connection/info.ex
@@ -1,6 +1,7 @@
 defmodule Tortoise.Connection.Info do
   @enforce_keys [:keep_alive]
   defstruct client_id: nil,
+            subscriptions: %{},
             keep_alive: nil,
             capabilities: nil
 

From 6407eb4ea1f2ee82a8c52884266f0d89e102ae3b Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Tue, 21 Jan 2020 15:58:57 +0000
Subject: [PATCH 162/220] Out comment an unused module attribute

---
 lib/tortoise/package/connect.ex | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/lib/tortoise/package/connect.ex b/lib/tortoise/package/connect.ex
index d012c378..6ad3e93b 100644
--- a/lib/tortoise/package/connect.ex
+++ b/lib/tortoise/package/connect.ex
@@ -3,17 +3,17 @@ defmodule Tortoise.Package.Connect do
 
   @opcode 1
 
-  @allowed_properties [
-    :authentication_data,
-    :authentication_method,
-    :maximum_packet_size,
-    :receive_maximum,
-    :request_problem_information,
-    :request_response_information,
-    :session_expiry_interval,
-    :topic_alias_maximum,
-    :user_property
-  ]
+  # @allowed_properties [
+  #   :authentication_data,
+  #   :authentication_method,
+  #   :maximum_packet_size,
+  #   :receive_maximum,
+  #   :request_problem_information,
+  #   :request_response_information,
+  #   :session_expiry_interval,
+  #   :topic_alias_maximum,
+  #   :user_property
+  # ]
 
   alias Tortoise.Package
 

From 7deaa73dd3c0bf0211cad1f614564e85ae0f3d08 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Tue, 21 Jan 2020 15:59:31 +0000
Subject: [PATCH 163/220] Do not enforce client ids on connect packages

In MQTT 5 it is allowed to not specify a client id when connecting; in
these cases the server would assign one to the client and inform the
client about its client id in the connack.

We will now store the client id in the connection info.
---
 lib/tortoise/connection/info.ex   | 25 ++++++++++++++++++++++---
 lib/tortoise/package/connect.ex   |  2 +-
 test/tortoise/connection_test.exs |  4 +++-
 3 files changed, 26 insertions(+), 5 deletions(-)

diff --git a/lib/tortoise/connection/info.ex b/lib/tortoise/connection/info.ex
index 2d88d249..d0de551c 100644
--- a/lib/tortoise/connection/info.ex
+++ b/lib/tortoise/connection/info.ex
@@ -5,18 +5,37 @@ defmodule Tortoise.Connection.Info do
             keep_alive: nil,
             capabilities: nil
 
+  alias __MODULE__
   alias Tortoise.Package.{Connect, Connack}
 
   def merge(
-        %Connect{keep_alive: keep_alive},
+        %Connect{keep_alive: keep_alive} = connect,
         %Connack{reason: :success} = connack
       ) do
     # if no server_keep_alive is set we should use the one set by the client
     keep_alive = Keyword.get(connack.properties, :server_keep_alive, keep_alive)
 
-    struct!(__MODULE__,
+    struct!(Info,
       keep_alive: keep_alive,
-      capabilities: struct!(%__MODULE__.Capabilities{}, connack.properties)
+      client_id: get_client_id(connect, connack),
+      capabilities: struct!(Info.Capabilities, connack.properties)
     )
   end
+
+  # If the client does not specify a client id it should look for a
+  # server assigned client identifier in the connack properties. It
+  # would be a error if this is not specified
+  defp get_client_id(%Connect{client_id: nil}, %Connack{properties: properties}) do
+    case Keyword.get(properties, :assigned_client_identifier) do
+      nil ->
+        raise "No client id specified"
+
+      client_id when is_binary(client_id) ->
+        client_id
+    end
+  end
+
+  defp get_client_id(%Connect{client_id: client_id}, %Connack{}) do
+    client_id
+  end
 end
diff --git a/lib/tortoise/package/connect.ex b/lib/tortoise/package/connect.ex
index 6ad3e93b..7ae184c6 100644
--- a/lib/tortoise/package/connect.ex
+++ b/lib/tortoise/package/connect.ex
@@ -29,7 +29,7 @@ defmodule Tortoise.Package.Connect do
             will: Package.Publish.t() | nil,
             properties: [{any(), any()}]
           }
-  @enforce_keys [:client_id]
+
   defstruct __META__: %Package.Meta{opcode: @opcode},
             protocol: "MQTT",
             protocol_version: 0b00000101,
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index 84548072..4b54c133 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -99,6 +99,7 @@ defmodule Tortoise.ConnectionTest do
       # besides that the values of the config should be the defaults
       expected_connection_info = %Connection.Info{
         keep_alive: connect.keep_alive,
+        client_id: Atom.to_string(context.test),
         capabilities: %Connection.Info.Capabilities{
           server_keep_alive: nil
         }
@@ -172,7 +173,8 @@ defmodule Tortoise.ConnectionTest do
         capabilities: %Connection.Info.Capabilities{
           server_keep_alive: server_keep_alive
         },
-        keep_alive: server_keep_alive
+        keep_alive: server_keep_alive,
+        client_id: Atom.to_string(context.test)
       }
 
       assert {:connected, ^expected_connection_config} = Connection.info(client_id)

From 6783be953b4e0d8b93abc1f7850aefb154b3a444 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Tue, 21 Jan 2020 20:19:47 +0000
Subject: [PATCH 164/220] Introduce a session_ref and make the receiver work
 with it

We cannot use the client id as the identity of a connection anymore as
we would have to specify it up front before creating the connection;
the problem is MQTT 5 that support server assigned client ids; this is
the first step in the process of getting rid of client id as the
identity of a connection.
---
 lib/tortoise/connection.ex                 | 28 +++++++++++++-----
 lib/tortoise/connection/info.ex            |  1 +
 lib/tortoise/connection/receiver.ex        | 34 +++++++++-------------
 lib/tortoise/connection/supervisor.ex      |  2 +-
 test/tortoise/connection/receiver_test.exs | 12 ++++++--
 test/tortoise/connection_test.exs          | 33 ++++++++++-----------
 6 files changed, 60 insertions(+), 50 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 223d4b68..f43a27b3 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -9,7 +9,8 @@ defmodule Tortoise.Connection do
 
   require Logger
 
-  defstruct client_id: nil,
+  defstruct session_ref: nil,
+            client_id: nil,
             connect: nil,
             server: nil,
             backoff: nil,
@@ -69,6 +70,7 @@ defmodule Tortoise.Connection do
     ]
 
     initial = %State{
+      session_ref: make_ref(),
       client_id: connect.client_id,
       server: server,
       connect: connect,
@@ -486,8 +488,8 @@ defmodule Tortoise.Connection do
   end
 
   # get status =========================================================
-  def handle_event({:call, from}, :get_info, :connected, data) do
-    next_actions = [{:reply, from, {:connected, data.info}}]
+  def handle_event({:call, from}, :get_info, :connected, %{receiver: {receiver_pid, _}} = data) do
+    next_actions = [{:reply, from, {:connected, struct(data.info, receiver_pid: receiver_pid)}}]
 
     {:keep_state_and_data, next_actions}
   end
@@ -913,7 +915,12 @@ defmodule Tortoise.Connection do
   # connection logic ===================================================
   def handle_event(:internal, :connect, :connecting, %State{} = data) do
     :ok = Tortoise.Registry.put_meta(via_name(data.client_id), :connecting)
-    :ok = start_connection_supervisor([{:parent, self()} | data.opts])
+
+    :ok =
+      start_connection_supervisor([
+        {:session_ref, data.session_ref},
+        {:parent, self()} | data.opts
+      ])
 
     case await_and_monitor_receiver(data) do
       {:ok, data} ->
@@ -927,9 +934,12 @@ defmodule Tortoise.Connection do
         :state_timeout,
         :attempt_connection,
         :connecting,
-        %State{connect: connect} = data
+        %State{
+          connect: connect,
+          receiver: {receiver_pid, _mon_ref}
+        } = data
       ) do
-    with {:ok, {transport, socket}} <- Receiver.connect(data.client_id),
+    with {:ok, {transport, socket}} <- Receiver.connect(receiver_pid),
          :ok = transport.send(socket, Package.encode(connect)) do
       new_data = %State{
         data
@@ -1089,9 +1099,11 @@ defmodule Tortoise.Connection do
     {:next_state, :connecting, updated_data, next_actions}
   end
 
-  defp await_and_monitor_receiver(%State{client_id: client_id, receiver: nil} = data) do
+  defp await_and_monitor_receiver(%State{receiver: nil} = data) do
+    session_ref = data.session_ref
+
     receive do
-      {{Tortoise, ^client_id}, Receiver, {:ready, pid}} ->
+      {{Tortoise, ^session_ref}, Receiver, {:ready, pid}} ->
         {:ok, %State{data | receiver: {pid, Process.monitor(pid)}}}
     after
       5000 ->
diff --git a/lib/tortoise/connection/info.ex b/lib/tortoise/connection/info.ex
index d0de551c..3f638d6e 100644
--- a/lib/tortoise/connection/info.ex
+++ b/lib/tortoise/connection/info.ex
@@ -3,6 +3,7 @@ defmodule Tortoise.Connection.Info do
   defstruct client_id: nil,
             subscriptions: %{},
             keep_alive: nil,
+            receiver_pid: nil,
             capabilities: nil
 
   alias __MODULE__
diff --git a/lib/tortoise/connection/receiver.ex b/lib/tortoise/connection/receiver.ex
index d7a2eee4..eaaf497b 100644
--- a/lib/tortoise/connection/receiver.ex
+++ b/lib/tortoise/connection/receiver.ex
@@ -3,31 +3,24 @@ defmodule Tortoise.Connection.Receiver do
 
   use GenStateMachine
 
-  alias Tortoise.{Events, Transport}
+  alias Tortoise.Transport
+
+  defstruct session_ref: nil,
+            transport: nil,
+            socket: nil,
+            buffer: <<>>,
+            parent: nil
 
-  defstruct client_id: nil, transport: nil, socket: nil, buffer: <<>>, parent: nil
   alias __MODULE__, as: State
 
   def start_link(opts) do
-    client_id = Keyword.fetch!(opts, :client_id)
-
     data = %State{
-      client_id: client_id,
+      session_ref: Keyword.fetch!(opts, :session_ref),
       transport: Keyword.fetch!(opts, :transport),
       parent: Keyword.fetch!(opts, :parent)
     }
 
-    GenStateMachine.start_link(__MODULE__, data, name: via_name(client_id))
-  end
-
-  defp via_name(client_id) do
-    Tortoise.Registry.via_name(__MODULE__, client_id)
-  end
-
-  def whereis(client_id) do
-    __MODULE__
-    |> Tortoise.Registry.reg_name(client_id)
-    |> Registry.whereis_name()
+    GenStateMachine.start_link(__MODULE__, data)
   end
 
   def child_spec(opts) do
@@ -40,17 +33,18 @@ defmodule Tortoise.Connection.Receiver do
     }
   end
 
-  def connect(client_id) do
-    GenStateMachine.call(via_name(client_id), :connect)
+  def connect(pid) do
+    GenStateMachine.call(pid, :connect)
   end
 
   @impl true
   def init(%State{} = data) do
-    send(data.parent, {{Tortoise, data.client_id}, __MODULE__, {:ready, self()}})
+    send(data.parent, {{Tortoise, data.session_ref}, __MODULE__, {:ready, self()}})
     {:ok, :disconnected, data}
   end
 
-  def terminate(_reason, _state) do
+  @impl true
+  def terminate(_reason, _state, _data) do
     :ok
   end
 
diff --git a/lib/tortoise/connection/supervisor.ex b/lib/tortoise/connection/supervisor.ex
index 9d8bfdbe..4eeecdcc 100644
--- a/lib/tortoise/connection/supervisor.ex
+++ b/lib/tortoise/connection/supervisor.ex
@@ -24,7 +24,7 @@ defmodule Tortoise.Connection.Supervisor do
   def init(opts) do
     children = [
       {Inflight, Keyword.take(opts, [:client_id, :parent])},
-      {Receiver, Keyword.take(opts, [:client_id, :transport, :parent])}
+      {Receiver, Keyword.take(opts, [:session_ref, :transport, :parent])}
     ]
 
     Supervisor.init(children, strategy: :rest_for_one, max_seconds: 30, max_restarts: 10)
diff --git a/test/tortoise/connection/receiver_test.exs b/test/tortoise/connection/receiver_test.exs
index 60af9702..4977facc 100644
--- a/test/tortoise/connection/receiver_test.exs
+++ b/test/tortoise/connection/receiver_test.exs
@@ -8,18 +8,24 @@ defmodule Tortoise.Connection.ReceiverTest do
   alias Tortoise.Integration.TestTCPTunnel
 
   setup context do
-    {:ok, %{client_id: context.test}}
+    {:ok, %{session_ref: context.test}}
   end
 
   def setup_receiver(context) do
     {:ok, ref, transport} = TestTCPTunnel.new(Tortoise.Transport.Tcp)
-    opts = [client_id: context.client_id, transport: transport, parent: self()]
+
+    opts = [
+      session_ref: context.session_ref,
+      transport: transport,
+      parent: self()
+    ]
+
     {:ok, receiver_pid} = Receiver.start_link(opts)
     {:ok, %{connection_ref: ref, transport: transport, receiver_pid: receiver_pid}}
   end
 
   def setup_connection(%{connection_ref: ref} = context) when is_reference(ref) do
-    {:ok, _connection} = Receiver.connect(context.client_id)
+    {:ok, _connection} = Receiver.connect(context.receiver_pid)
     assert_receive {:server_socket, ^ref, server_socket}
     {:ok, Map.put(context, :server, server_socket)}
   end
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index 4b54c133..228681fa 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -97,15 +97,15 @@ defmodule Tortoise.ConnectionTest do
       # If the server does not specify a server_keep_alive interval we
       # should use the one that was provided in the connect message,
       # besides that the values of the config should be the defaults
-      expected_connection_info = %Connection.Info{
-        keep_alive: connect.keep_alive,
-        client_id: Atom.to_string(context.test),
-        capabilities: %Connection.Info.Capabilities{
-          server_keep_alive: nil
-        }
-      }
+      keep_alive = connect.keep_alive
 
-      assert {:connected, ^expected_connection_info} = Connection.info(client_id)
+      assert {:connected,
+              %Connection.Info{
+                keep_alive: ^keep_alive,
+                capabilities: %Connection.Info.Capabilities{
+                  server_keep_alive: nil
+                }
+              }} = Connection.info(client_id)
 
       send(context.scripted_mqtt_server, :continue)
       assert_receive {ScriptedMqttServer, :completed}
@@ -169,15 +169,7 @@ defmodule Tortoise.ConnectionTest do
       # should use that one for the keep_alive instead of the user
       # provided one in the connect message, besides that the values
       # of the config should be the defaults
-      expected_connection_config = %Connection.Info{
-        capabilities: %Connection.Info.Capabilities{
-          server_keep_alive: server_keep_alive
-        },
-        keep_alive: server_keep_alive,
-        client_id: Atom.to_string(context.test)
-      }
-
-      assert {:connected, ^expected_connection_config} = Connection.info(client_id)
+      assert {:connected, %{keep_alive: ^server_keep_alive}} = Connection.info(client_id)
 
       send(context.scripted_mqtt_server, :continue)
       assert_receive {ScriptedMqttServer, :completed}
@@ -982,7 +974,12 @@ defmodule Tortoise.ConnectionTest do
       cs_ref = Process.monitor(cs_pid)
 
       inflight_pid = Connection.Inflight.whereis(client_id)
-      receiver_pid = Connection.Receiver.whereis(client_id)
+      {:ok, {Tortoise.Transport.Tcp, _}} = Connection.connection(client_id)
+
+      {:connected,
+       %{
+         receiver_pid: receiver_pid
+       }} = Connection.info(client_id)
 
       assert :ok = Tortoise.Connection.disconnect(client_id)
 

From 45158aa73db8e466dd2f903638aa2d1808dd197b Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Tue, 21 Jan 2020 20:59:42 +0000
Subject: [PATCH 165/220] Update gen_state_machine dependency

---
 mix.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mix.lock b/mix.lock
index ccd4927d..c4dfbe03 100644
--- a/mix.lock
+++ b/mix.lock
@@ -7,7 +7,7 @@
   "ex_doc": {:hex, :ex_doc, "0.19.1", "519bb9c19526ca51d326c060cb1778d4a9056b190086a8c6c115828eaccea6cf", [:mix], [{:earmark, "~> 1.1", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.7", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"},
   "excoveralls": {:hex, :excoveralls, "0.10.0", "a4508bdd408829f38e7b2519f234b7fd5c83846099cda348efcb5291b081200c", [:mix], [{:hackney, "~> 1.13", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"},
   "exjsx": {:hex, :exjsx, "4.0.0", "60548841e0212df401e38e63c0078ec57b33e7ea49b032c796ccad8cde794b5c", [:mix], [{:jsx, "~> 2.8.0", [hex: :jsx, repo: "hexpm", optional: false]}], "hexpm"},
-  "gen_state_machine": {:hex, :gen_state_machine, "2.0.3", "477ea51b466a749ab23a0d6090e9e84073f41f9aa28c7efc40eac18f3d4a9f77", [:mix], [], "hexpm"},
+  "gen_state_machine": {:hex, :gen_state_machine, "2.0.5", "9ac15ec6e66acac994cc442dcc2c6f9796cf380ec4b08267223014be1c728a95", [:mix], [], "hexpm"},
   "hackney": {:hex, :hackney, "1.13.0", "24edc8cd2b28e1c652593833862435c80661834f6c9344e84b6a2255e7aeef03", [:rebar3], [{:certifi, "2.3.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "5.1.2", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "1.0.2", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"},
   "idna": {:hex, :idna, "5.1.2", "e21cb58a09f0228a9e0b95eaa1217f1bcfc31a1aaa6e1fdf2f53a33f7dbd9494", [:rebar3], [{:unicode_util_compat, "0.3.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm"},
   "jason": {:hex, :jason, "1.1.1", "d3ccb840dfb06f2f90a6d335b536dd074db748b3e7f5b11ab61d239506585eb2", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm"},

From b412b027a823fa149123507e7e9d69a011e11403 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Tue, 21 Jan 2020 21:22:50 +0000
Subject: [PATCH 166/220] Tortoise.Registry.delete_meta/1 was only used in a
 test

Get rid of it
---
 lib/tortoise/registry.ex        | 11 -----------
 test/tortoise/registry_test.exs |  4 +---
 2 files changed, 1 insertion(+), 14 deletions(-)

diff --git a/lib/tortoise/registry.ex b/lib/tortoise/registry.ex
index 48b69169..5eaf0b85 100644
--- a/lib/tortoise/registry.ex
+++ b/lib/tortoise/registry.ex
@@ -25,15 +25,4 @@ defmodule Tortoise.Registry do
   def put_meta(key, value) do
     :ok = Registry.put_meta(__MODULE__, key, value)
   end
-
-  @spec delete_meta(key :: term()) :: :ok | no_return
-  def delete_meta(key) do
-    try do
-      :ets.delete(__MODULE__, key)
-      :ok
-    catch
-      :error, :badarg ->
-        raise ArgumentError, "unknown registry: #{inspect(__MODULE__)}"
-    end
-  end
 end
diff --git a/test/tortoise/registry_test.exs b/test/tortoise/registry_test.exs
index fcbe1c7e..ca62e923 100644
--- a/test/tortoise/registry_test.exs
+++ b/test/tortoise/registry_test.exs
@@ -9,14 +9,12 @@ defmodule Tortoise.RegistryTest do
              Tortoise.Registry.via_name(mod, name)
   end
 
-  test "meta put, get, delete", context do
+  test "meta put, get", context do
     key = Tortoise.Registry.via_name(__MODULE__, context.test)
     value = :crypto.strong_rand_bytes(2)
 
     assert :error == Tortoise.Registry.meta(key)
     assert :ok = Tortoise.Registry.put_meta(key, value)
     assert {:ok, ^value} = Tortoise.Registry.meta(key)
-    assert :ok = Tortoise.Registry.delete_meta(key)
-    assert :error == Tortoise.Registry.meta(key)
   end
 end

From fef117b1650f80f08905019b7f91286d2c2cca73 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Wed, 22 Jan 2020 15:51:59 +0000
Subject: [PATCH 167/220] Rename Tortoise.App to Tortoise.Application

For some reason it just bugged me that I had gone with .App instead of
the regular .Application.
---
 lib/tortoise/{app.ex => application.ex} | 2 +-
 mix.exs                                 | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename lib/tortoise/{app.ex => application.ex} (93%)

diff --git a/lib/tortoise/app.ex b/lib/tortoise/application.ex
similarity index 93%
rename from lib/tortoise/app.ex
rename to lib/tortoise/application.ex
index 89320647..ac6fcb98 100644
--- a/lib/tortoise/app.ex
+++ b/lib/tortoise/application.ex
@@ -1,4 +1,4 @@
-defmodule Tortoise.App do
+defmodule Tortoise.Application do
   @moduledoc false
 
   use Application
diff --git a/mix.exs b/mix.exs
index b235a6f7..4565c643 100644
--- a/mix.exs
+++ b/mix.exs
@@ -37,7 +37,7 @@ defmodule Tortoise.MixProject do
   def application do
     [
       extra_applications: [:logger, :ssl],
-      mod: {Tortoise.App, []}
+      mod: {Tortoise.Application, []}
     ]
   end
 

From 8290f47211a2684fec744a4da7b312ca96a11878 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Tue, 11 Feb 2020 11:33:04 +0000
Subject: [PATCH 168/220] Dead code: The public interface for handle socket had
 been removed

Minor clean up. This code is no longer called.
---
 lib/tortoise/connection/receiver.ex | 23 +----------------------
 1 file changed, 1 insertion(+), 22 deletions(-)

diff --git a/lib/tortoise/connection/receiver.ex b/lib/tortoise/connection/receiver.ex
index eaaf497b..22927c9c 100644
--- a/lib/tortoise/connection/receiver.ex
+++ b/lib/tortoise/connection/receiver.ex
@@ -140,27 +140,6 @@ defmodule Tortoise.Connection.Receiver do
     :keep_state_and_data
   end
 
-  def handle_event({:call, from}, {:handle_socket, transport, socket}, :disconnected, data) do
-    new_state = {:connected, :receiving_fixed_header}
-
-    next_actions = [
-      {:reply, from, {:ok, self()}},
-      {:next_event, :internal, :activate_socket},
-      {:next_event, :internal, :consume_buffer}
-    ]
-
-    # better reset the buffer
-    new_data = %State{data | transport: transport, socket: socket, buffer: <<>>}
-
-    {:next_state, new_state, new_data, next_actions}
-  end
-
-  def handle_event({:call, from}, {:handle_socket, _transport, _socket}, current_state, data) do
-    next_actions = [{:reply, from, {:error, :not_ready}}]
-    reason = {:got_socket_in_wrong_state, current_state}
-    {:stop_and_reply, reason, next_actions, data}
-  end
-
   # connect
   def handle_event(
         {:call, from},
@@ -180,7 +159,7 @@ defmodule Tortoise.Connection.Receiver do
           {:next_event, :internal, :consume_buffer}
         ]
 
-        # better reset the buffer
+        # better make sure the buffer state is empty
         new_data = %State{data | socket: socket, buffer: <<>>}
         {:next_state, new_state, new_data, next_actions}
 

From 8ed618cd81fad398d22c833192984282079c7e35 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Tue, 11 Feb 2020 18:28:23 +0000
Subject: [PATCH 169/220] First step in getting rid of the Tortoise.Events

When the network socket is dropped we need reconnect and get a new
one; the new network socket needed to get communicated to interested
parties, such as the inflight manager. This relied on meta data in the
tortoise registry, and the meta data rely on the client_id. We need to
get away from using the client id as the instance identity, so this is
the first step in getting out of that.

Also, we need to let the user control the connection flow, so they can
add their own backoff strategies and decide on when tortoise should
connect. These changes will hopefully make this easier to pull
off. got rid of some seemingly dead code as well.
---
 lib/tortoise/connection.ex                 | 68 ++++++++++++------
 lib/tortoise/connection/inflight.ex        | 82 ++++++++--------------
 test/tortoise/connection/inflight_test.exs | 20 +++++-
 test/tortoise/pipe_test.exs                |  3 +-
 test/tortoise_test.exs                     |  6 +-
 5 files changed, 100 insertions(+), 79 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index f43a27b3..3ba51655 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -367,10 +367,16 @@ defmodule Tortoise.Connection do
   end
 
   @doc false
-  @spec connection(Tortoise.client_id(), [opts]) ::
+  @spec connection(Tortoise.client_id() | pid(), [opts]) ::
           {:ok, {module(), term()}} | {:error, :unknown_connection} | {:error, :timeout}
         when opts: {:timeout, timeout()} | {:active, boolean()}
-  def connection(client_id, opts \\ [active: false]) do
+  def connection(pid, _opts \\ [active: false])
+
+  def connection(pid, _opts) when is_pid(pid) do
+    GenStateMachine.call(pid, :get_connection)
+  end
+
+  def connection(client_id, opts) do
     # register a connection subscription in the case we are currently
     # in the connect phase; this solves a possible race condition
     # where the connection is requested while the status is
@@ -410,9 +416,19 @@ defmodule Tortoise.Connection do
   def init(%State{} = state) do
     case Handler.execute_init(state.handler) do
       {:ok, %Handler{} = updated_handler} ->
-        next_events = [{:next_event, :internal, :connect}]
         updated_state = %State{state | handler: updated_handler}
-        {:ok, :connecting, updated_state, next_events}
+        # TODO, perhaps the supervision should get reconsidered
+        :ok =
+          start_connection_supervisor([
+            {:session_ref, updated_state.session_ref},
+            {:parent, self()} | updated_state.opts
+          ])
+
+        transition_actions = [
+          {:next_event, :internal, :connect}
+        ]
+
+        {:ok, :connecting, updated_state, transition_actions}
 
       :ignore ->
         :ignore
@@ -450,7 +466,9 @@ defmodule Tortoise.Connection do
     case Handler.execute_handle_connack(handler, connack) do
       {:ok, %Handler{} = updated_handler, next_actions} when connection_result == :success ->
         :ok = Tortoise.Registry.put_meta(via_name(client_id), data.connection)
+        # todo, get rid of the connection event
         :ok = Events.dispatch(client_id, :connection, data.connection)
+        :ok = Inflight.update_connection(client_id, data.connection)
         :ok = Events.dispatch(client_id, :status, :connected)
 
         data = %State{
@@ -500,6 +518,16 @@ defmodule Tortoise.Connection do
     {:keep_state_and_data, next_actions}
   end
 
+  # a process request the connection; postpone if we are not yet connected
+  def handle_event({:call, _from}, :get_connection, :connecting, _data) do
+    {:keep_state_and_data, [:postpone]}
+  end
+
+  def handle_event({:call, from}, :get_connection, :connected, data) do
+    transition_actions = [{:reply, from, {:ok, data.connection}}]
+    {:keep_state, data, transition_actions}
+  end
+
   # publish packages ===================================================
   def handle_event(
         :internal,
@@ -916,12 +944,6 @@ defmodule Tortoise.Connection do
   def handle_event(:internal, :connect, :connecting, %State{} = data) do
     :ok = Tortoise.Registry.put_meta(via_name(data.client_id), :connecting)
 
-    :ok =
-      start_connection_supervisor([
-        {:session_ref, data.session_ref},
-        {:parent, self()} | data.opts
-      ])
-
     case await_and_monitor_receiver(data) do
       {:ok, data} ->
         {timeout, updated_data} = Map.get_and_update(data, :backoff, &Backoff.next/1)
@@ -939,22 +961,26 @@ defmodule Tortoise.Connection do
           receiver: {receiver_pid, _mon_ref}
         } = data
       ) do
-    with {:ok, {transport, socket}} <- Receiver.connect(receiver_pid),
-         :ok = transport.send(socket, Package.encode(connect)) do
-      new_data = %State{
-        data
-        | connect: %Connect{connect | clean_start: false},
-          connection: {transport, socket}
-      }
+    case Receiver.connect(receiver_pid) do
+      {:ok, {transport, socket} = connection} ->
+        # TODO: send this to the client handler allowing the user to
+        # specify a custom connect package
+        :ok = transport.send(socket, Package.encode(connect))
+
+        new_data = %State{
+          data
+          | connect: %Connect{connect | clean_start: false},
+            connection: connection
+        }
+
+        {:keep_state, new_data}
 
-      {:keep_state, new_data}
-    else
       {:error, {:stop, reason}} ->
         {:stop, reason, data}
 
       {:error, {:retry, _reason}} ->
-        next_actions = [{:next_event, :internal, :connect}]
-        {:keep_state, data, next_actions}
+        transition_actions = [{:next_event, :internal, :connect}]
+        {:keep_state, data, transition_actions}
     end
   end
 
diff --git a/lib/tortoise/connection/inflight.ex b/lib/tortoise/connection/inflight.ex
index b941ed9e..df6ae423 100644
--- a/lib/tortoise/connection/inflight.ex
+++ b/lib/tortoise/connection/inflight.ex
@@ -1,7 +1,7 @@
 defmodule Tortoise.Connection.Inflight do
   @moduledoc false
 
-  alias Tortoise.{Package, Connection}
+  alias Tortoise.Package
   alias Tortoise.Connection.Inflight.Track
 
   use GenStateMachine
@@ -32,6 +32,11 @@ defmodule Tortoise.Connection.Inflight do
     GenStateMachine.stop(via_name(client_id))
   end
 
+  @doc false
+  def update_connection(client_id, {_, _} = connection) do
+    GenStateMachine.call(via_name(client_id), {:set_connection, connection})
+  end
+
   @doc false
   def drain(client_id, %Package.Disconnect{} = disconnect) do
     GenStateMachine.call(via_name(client_id), {:drain, disconnect})
@@ -102,69 +107,37 @@ defmodule Tortoise.Connection.Inflight do
     parent_pid = Keyword.fetch!(opts, :parent)
     initial_data = %State{client_id: client_id, parent: parent_pid}
 
-    next_actions = [
-      {:next_event, :internal, :post_init}
-    ]
-
-    {:ok, _} = Tortoise.Events.register(client_id, :status)
-
-    {:ok, :disconnected, initial_data, next_actions}
+    {:ok, :disconnected, initial_data}
   end
 
   @impl true
-  def handle_event(:internal, :post_init, :disconnected, data) do
-    case Connection.connection(data.client_id, active: true) do
-      {:ok, {_transport, _socket} = connection} ->
-        {:next_state, {:connected, connection}, data}
-
-      {:error, :timeout} ->
-        {:stop, :connection_timeout}
-
-      {:error, :unknown_connection} ->
-        {:stop, :unknown_connection}
-    end
-  end
-
   # When we receive a new connection we will use that for our future
   # transmissions.
   def handle_event(
-        :info,
-        {{Tortoise, client_id}, :connection, connection},
+        {:call, from},
+        {:set_connection, connection},
         _current_state,
-        %State{client_id: client_id, pending: pending} = data
+        %State{pending: pending} = data
       ) do
-    next_actions =
-      for identifier <- Enum.reverse(data.order) do
-        case Map.get(pending, identifier, :unknown) do
-          %Track{pending: [[{:dispatch, %Package.Publish{} = publish} | action] | pending]} =
-              track ->
-            publish = %Package.Publish{publish | dup: true}
-            track = %Track{track | pending: [[{:dispatch, publish} | action] | pending]}
-            {:next_event, :internal, {:execute, track}}
-
-          %Track{} = track ->
-            {:next_event, :internal, {:execute, track}}
+    next_actions = [
+      {:reply, from, :ok}
+      | for identifier <- Enum.reverse(data.order) do
+          case Map.get(pending, identifier, :unknown) do
+            %Track{pending: [[{:dispatch, %Package.Publish{} = publish} | action] | pending]} =
+                track ->
+              publish = %Package.Publish{publish | dup: true}
+              track = %Track{track | pending: [[{:dispatch, publish} | action] | pending]}
+              {:next_event, :internal, {:execute, track}}
+
+            %Track{} = track ->
+              {:next_event, :internal, {:execute, track}}
+          end
         end
-      end
+    ]
 
     {:next_state, {:connected, connection}, data, next_actions}
   end
 
-  # Connection status events; when we go offline we should transition
-  # into the disconnected state. Everything else will get ignored.
-  def handle_event(
-        :info,
-        {{Tortoise, client_id}, :status, :down},
-        _current_state,
-        %State{client_id: client_id} = data
-      ) do
-    {:next_state, :disconnected, data}
-  end
-
-  def handle_event(:info, {{Tortoise, _}, :status, _}, _, %State{}) do
-    :keep_state_and_data
-  end
-
   # Create. Notice: we will only receive publish packages from the
   # remote; everything else is something we initiate
   def handle_event(
@@ -215,6 +188,11 @@ defmodule Tortoise.Connection.Inflight do
     end
   end
 
+  def handle_event(:cast, {:outgoing, _caller, _package}, :disconnected, %State{}) do
+    # await a connection
+    {:keep_state_and_data, [:postpone]}
+  end
+
   def handle_event(:cast, {:outgoing, {pid, ref}, _}, :draining, data) do
     send(pid, {{Tortoise, data.client_id}, ref, {:error, :terminating}})
     :keep_state_and_data
@@ -427,7 +405,7 @@ defmodule Tortoise.Connection.Inflight do
       nil ->
         {package, track}
 
-      fun when is_function(fun) ->
+      fun when is_function(fun, 2) ->
         case apply(fun, [package, track.state]) do
           {:ok, updated_state} ->
             # just update the track session state
diff --git a/test/tortoise/connection/inflight_test.exs b/test/tortoise/connection/inflight_test.exs
index 1ddd34f6..5f6b0db8 100644
--- a/test/tortoise/connection/inflight_test.exs
+++ b/test/tortoise/connection/inflight_test.exs
@@ -15,18 +15,25 @@ defmodule Tortoise.Connection.InflightTest do
     key = Tortoise.Registry.via_name(Tortoise.Connection, context.client_id)
     Tortoise.Registry.put_meta(key, connection)
     Tortoise.Events.dispatch(context.client_id, :connection, connection)
-    {:ok, Map.merge(context, %{client: client_socket, server: server_socket})}
+
+    {:ok,
+     Map.merge(context, %{client: client_socket, server: server_socket, connection: connection})}
   end
 
   defp drop_connection(%{server: server} = context) do
     :ok = :gen_tcp.close(server)
     :ok = Tortoise.Events.dispatch(context.client_id, :status, :down)
-    {:ok, Map.drop(context, [:client, :server])}
+    {:ok, Map.drop(context, [:client, :server, :connection])}
+  end
+
+  def setup_inflight(%{inflight_pid: pid} = context) when is_pid(pid) do
+    Inflight.update_connection(pid, context.connection)
+    {:ok, context}
   end
 
   def setup_inflight(context) do
     {:ok, pid} = Inflight.start_link(client_id: context.client_id, parent: self())
-    {:ok, %{inflight_pid: pid}}
+    setup_inflight(Map.put(context, :inflight_pid, pid))
   end
 
   describe "life-cycle" do
@@ -62,6 +69,7 @@ defmodule Tortoise.Connection.InflightTest do
       # drop and reestablish the connection
       {:ok, context} = drop_connection(context)
       {:ok, context} = setup_connection(context)
+      {:ok, context} = setup_inflight(context)
 
       # the inflight process should now re-transmit the publish
       assert {:ok, package} = :gen_tcp.recv(context.server, 0, 500)
@@ -92,6 +100,7 @@ defmodule Tortoise.Connection.InflightTest do
       # drop and reestablish the connection
       {:ok, context} = drop_connection(context)
       {:ok, context} = setup_connection(context)
+      {:ok, context} = setup_inflight(context)
 
       # now we should receive the same pubrec message
       assert {:ok, ^data} = :gen_tcp.recv(context.server, 0, 500)
@@ -117,6 +126,7 @@ defmodule Tortoise.Connection.InflightTest do
       # drop and reestablish the connection
       {:ok, context} = drop_connection(context)
       {:ok, context} = setup_connection(context)
+      {:ok, context} = setup_inflight(context)
       # the publish should get re-transmitted
       publish = %Package.Publish{publish | dup: true}
       assert {:ok, package} = :gen_tcp.recv(context.server, 0, 500)
@@ -134,6 +144,7 @@ defmodule Tortoise.Connection.InflightTest do
       # drop and reestablish the connection
       {:ok, context} = drop_connection(context)
       {:ok, context} = setup_connection(context)
+      {:ok, context} = setup_inflight(context)
       # re-transmit the pubrel
       assert {:ok, ^pubrel_encoded} = :gen_tcp.recv(context.server, 0, 500)
 
@@ -166,6 +177,7 @@ defmodule Tortoise.Connection.InflightTest do
       # drop and reestablish the connection
       {:ok, context} = drop_connection(context)
       {:ok, context} = setup_connection(context)
+      {:ok, context} = setup_inflight(context)
       # re-transmit the subscribe package
       assert {:ok, ^package} = :gen_tcp.recv(context.server, 0, 500)
 
@@ -198,6 +210,7 @@ defmodule Tortoise.Connection.InflightTest do
       # drop and reestablish the connection
       {:ok, context} = drop_connection(context)
       {:ok, context} = setup_connection(context)
+      {:ok, context} = setup_inflight(context)
       # re-transmit the subscribe package
       assert {:ok, ^package} = :gen_tcp.recv(context.server, 0, 500)
 
@@ -231,6 +244,7 @@ defmodule Tortoise.Connection.InflightTest do
       # drop and reestablish the connection
       {:ok, context} = drop_connection(context)
       {:ok, context} = setup_connection(context)
+      {:ok, context} = setup_inflight(context)
 
       # the in flight manager should now re-transmit the publish
       # messages in the same order they arrived
diff --git a/test/tortoise/pipe_test.exs b/test/tortoise/pipe_test.exs
index 71b75e83..51c9f78b 100644
--- a/test/tortoise/pipe_test.exs
+++ b/test/tortoise/pipe_test.exs
@@ -12,6 +12,7 @@ defmodule Tortoise.PipeTest do
   def setup_inflight(context) do
     opts = [client_id: context.client_id, parent: self()]
     {:ok, inflight_pid} = Inflight.start_link(opts)
+    :ok = Inflight.update_connection(inflight_pid, context.connection)
     {:ok, %{inflight_pid: inflight_pid}}
   end
 
@@ -26,7 +27,7 @@ defmodule Tortoise.PipeTest do
     connection = {Tortoise.Transport.Tcp, client_socket}
     key = Tortoise.Registry.via_name(Tortoise.Connection, context.client_id)
     Tortoise.Registry.put_meta(key, connection)
-    {:ok, %{client: client_socket, server: server_socket}}
+    {:ok, %{client: client_socket, server: server_socket, connection: connection}}
   end
 
   # update the context during a test run
diff --git a/test/tortoise_test.exs b/test/tortoise_test.exs
index 3089e72d..ee7d1619 100644
--- a/test/tortoise_test.exs
+++ b/test/tortoise_test.exs
@@ -12,13 +12,15 @@ defmodule TortoiseTest do
   def setup_connection(context) do
     {:ok, client_socket, server_socket} = Tortoise.Integration.TestTCPTunnel.new()
     name = Tortoise.Connection.via_name(context.client_id)
-    :ok = Tortoise.Registry.put_meta(name, {context.transport, client_socket})
-    {:ok, %{client: client_socket, server: server_socket}}
+    connection = {context.transport, client_socket}
+    :ok = Tortoise.Registry.put_meta(name, connection)
+    {:ok, %{client: client_socket, server: server_socket, connection: connection}}
   end
 
   def setup_inflight(context) do
     opts = [client_id: context.client_id, parent: self()]
     {:ok, pid} = Inflight.start_link(opts)
+    :ok = Inflight.update_connection(pid, context.connection)
     {:ok, %{inflight_pid: pid}}
   end
 

From 4899c004105277c18c48435194451842bbdb4e8f Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Wed, 12 Feb 2020 10:47:53 +0000
Subject: [PATCH 170/220] Make pings work without the need for a client id

---
 lib/tortoise/connection.ex        | 24 +++++++++++-------------
 test/tortoise/connection_test.exs | 24 ++++++++++++------------
 2 files changed, 23 insertions(+), 25 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 3ba51655..fb699bf7 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -328,10 +328,10 @@ defmodule Tortoise.Connection do
   better to listen on `:ping_response` using the `Tortoise.Events`
   PubSub.
   """
-  @spec ping(Tortoise.client_id()) :: {:ok, reference()}
-  def ping(client_id) do
+  @spec ping(pid()) :: {:ok, reference()}
+  def ping(pid) do
     ref = make_ref()
-    :ok = GenStateMachine.cast(via_name(client_id), {:ping, {self(), ref}})
+    :ok = GenStateMachine.cast(pid, {:ping, {self(), ref}})
     {:ok, ref}
   end
 
@@ -1067,20 +1067,14 @@ defmodule Tortoise.Connection do
         :internal,
         {:received, %Package.Pingresp{}},
         :connected,
-        %State{
-          client_id: client_id,
-          ping: {{:pinging, start_time}, awaiting}
-        } = data
+        %State{ping: {{:pinging, start_time}, awaiting}} = data
       ) do
     round_trip_time =
       (System.monotonic_time() - start_time)
       |> System.convert_time_unit(:native, :microsecond)
 
-    :ok = Events.dispatch(client_id, :ping_response, round_trip_time)
-
-    Enum.each(awaiting, fn {caller, ref} ->
-      send(caller, {{Tortoise, client_id}, {Package.Pingreq, ref}, round_trip_time})
-    end)
+    # reply to the clients
+    Enum.each(awaiting, &send_reply(&1, Package.Pingreq, round_trip_time))
 
     next_actions = [{:next_event, :internal, :setup_keep_alive_timer}]
 
@@ -1159,5 +1153,9 @@ defmodule Tortoise.Connection do
     end
   end
 
-  @compile {:inline, wrap_next_actions: 1}
+  defp send_reply({caller, ref}, topic, payload) when is_pid(caller) and is_reference(ref) do
+    send(caller, {{Tortoise, self()}, {topic, ref}, payload})
+  end
+
+  @compile {:inline, wrap_next_actions: 1, send_reply: 3}
 end
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index 228681fa..9bf59638 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -87,12 +87,12 @@ defmodule Tortoise.ConnectionTest do
         handler: {Tortoise.Handler.Default, []}
       ]
 
-      assert {:ok, _pid} = Connection.start_link(opts)
+      assert {:ok, pid} = Connection.start_link(opts)
       assert_receive {ScriptedMqttServer, {:received, ^connect}}
       assert_receive {ScriptedMqttServer, :paused}
 
       # Should be able to get a connection when we have connected
-      assert {:ok, {Tortoise.Transport.Tcp, _port}} = Connection.connection(client_id)
+      assert {:ok, {Tortoise.Transport.Tcp, _port}} = Connection.connection(pid)
 
       # If the server does not specify a server_keep_alive interval we
       # should use the one that was provided in the connect message,
@@ -159,17 +159,17 @@ defmodule Tortoise.ConnectionTest do
         handler: {Tortoise.Handler.Default, []}
       ]
 
-      assert {:ok, _pid} = Connection.start_link(opts)
+      assert {:ok, pid} = Connection.start_link(opts)
       assert_receive {ScriptedMqttServer, {:received, ^connect}}
 
       # Should be able to get a connection when we have connected
-      assert {:ok, {Tortoise.Transport.Tcp, _port}} = Connection.connection(client_id)
+      assert {:ok, {Tortoise.Transport.Tcp, _port}} = Connection.connection(pid)
 
       # If the server does specify a server_keep_alive interval we
       # should use that one for the keep_alive instead of the user
       # provided one in the connect message, besides that the values
       # of the config should be the defaults
-      assert {:connected, %{keep_alive: ^server_keep_alive}} = Connection.info(client_id)
+      assert {:connected, %{keep_alive: ^server_keep_alive}} = Connection.info(pid)
 
       send(context.scripted_mqtt_server, :continue)
       assert_receive {ScriptedMqttServer, :completed}
@@ -1091,19 +1091,18 @@ defmodule Tortoise.ConnectionTest do
   describe "ping" do
     setup [:setup_scripted_mqtt_server, :setup_connection_and_perform_handshake]
 
-    test "send pingreq and receive a pingresp", %{client_id: client_id} = context do
-      {:ok, _} = Tortoise.Events.register(client_id, :status)
-      assert_receive {{Tortoise, ^client_id}, :status, :connected}
-
+    test "send pingreq and receive a pingresp", %{connection_pid: connection_pid} = context do
       ping_request = %Package.Pingreq{}
       expected_pingresp = %Package.Pingresp{}
       script = [{:receive, ping_request}, {:send, expected_pingresp}]
 
       {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
+      assert_receive {{TestHandler, :handle_connack}, %Tortoise.Package.Connack{}}
 
-      {:ok, ref} = Connection.ping(context.client_id)
+      {:ok, ref} = Connection.ping(context.connection_pid)
       assert_receive {ScriptedMqttServer, {:received, ^ping_request}}
-      assert_receive {{Tortoise, ^client_id}, {Package.Pingreq, ^ref}, _}
+      assert_receive {{Tortoise, ^connection_pid}, {Package.Pingreq, ^ref}, _}
+      assert_receive {ScriptedMqttServer, :completed}
     end
 
     test "ping_sync/2", %{client_id: client_id} = context do
@@ -1119,7 +1118,8 @@ defmodule Tortoise.ConnectionTest do
       {parent, ref} = {self(), make_ref()}
 
       spawn_link(fn ->
-        send(parent, {{:child_result, ref}, Connection.ping_sync(client_id)})
+        ping_res = Connection.ping_sync(context.connection_pid)
+        send(parent, {{:child_result, ref}, ping_res})
       end)
 
       assert_receive {ScriptedMqttServer, {:received, ^ping_request}}

From 478530531ac74edef201f439d63bbe69086492cf Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Wed, 12 Feb 2020 14:05:11 +0000
Subject: [PATCH 171/220] Subscribe and unsubscribe now works without the
 client id

---
 lib/tortoise/connection.ex        | 83 +++++++++++++++-------------
 test/tortoise/connection_test.exs | 89 +++++++++++++++----------------
 2 files changed, 88 insertions(+), 84 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index fb699bf7..06eee692 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -158,15 +158,15 @@ defmodule Tortoise.Connection do
   Read the documentation for `Tortoise.Connection.subscribe_sync/3`
   for a blocking version of this call.
   """
-  @spec subscribe(Tortoise.client_id(), topic | topics, [options]) :: {:ok, reference()}
+  @spec subscribe(pid(), topic | topics, [options]) :: {:ok, reference()}
         when topics: [topic],
              topic: {Tortoise.topic_filter(), Tortoise.qos()},
              options:
                {:timeout, timeout()}
                | {:identifier, Tortoise.package_identifier()}
-  def subscribe(client_id, topics, opts \\ [])
+  def subscribe(pid, topics, opts \\ [])
 
-  def subscribe(client_id, [{_, topic_opts} | _] = topics, opts) when is_list(topic_opts) do
+  def subscribe(pid, [{_, topic_opts} | _] = topics, opts) when is_list(topic_opts) do
     caller = {_, ref} = {self(), make_ref()}
     # todo, do something with timeout, or remove it
     {opts, properties} = Keyword.split(opts, [:identifier, :timeout])
@@ -178,21 +178,21 @@ defmodule Tortoise.Connection do
         properties: properties
       })
 
-    GenStateMachine.cast(via_name(client_id), {:subscribe, caller, subscribe, opts})
+    GenStateMachine.cast(via_name(pid), {:subscribe, caller, subscribe, opts})
     {:ok, ref}
   end
 
-  def subscribe(client_id, {_, topic_opts} = topic, opts) when is_list(topic_opts) do
-    subscribe(client_id, [topic], opts)
+  def subscribe(pid, {_, topic_opts} = topic, opts) when is_list(topic_opts) do
+    subscribe(pid, [topic], opts)
   end
 
-  def subscribe(client_id, topic, opts) when is_binary(topic) do
+  def subscribe(pid, topic, opts) when is_binary(topic) do
     case Keyword.pop_first(opts, :qos) do
       {nil, _opts} ->
         throw("Please specify a quality of service for the subscription")
 
       {qos, opts} when qos in 0..2 ->
-        subscribe(client_id, [{topic, [qos: qos]}], opts)
+        subscribe(pid, [{topic, [qos: qos]}], opts)
     end
   end
 
@@ -208,38 +208,38 @@ defmodule Tortoise.Connection do
 
   See `Tortoise.Connection.subscribe/3` for configuration options.
   """
-  @spec subscribe_sync(Tortoise.client_id(), topic | topics, [options]) ::
+  @spec subscribe_sync(pid(), topic | topics, [options]) ::
           :ok | {:error, :timeout}
         when topics: [topic],
              topic: {Tortoise.topic_filter(), Tortoise.qos()},
              options:
                {:timeout, timeout()}
                | {:identifier, Tortoise.package_identifier()}
-  def subscribe_sync(client_id, topics, opts \\ [])
+  def subscribe_sync(pid, topics, opts \\ [])
 
-  def subscribe_sync(client_id, [{_, topic_opts} | _] = topics, opts) when is_list(topic_opts) do
+  def subscribe_sync(pid, [{_, topic_opts} | _] = topics, opts) when is_list(topic_opts) do
     timeout = Keyword.get(opts, :timeout, 5000)
-    {:ok, ref} = subscribe(client_id, topics, opts)
+    {:ok, ref} = subscribe(pid, topics, opts)
 
     receive do
-      {{Tortoise, ^client_id}, ^ref, result} -> result
+      {{Tortoise, ^pid}, {Package.Suback, ^ref}, result} -> result
     after
       timeout ->
         {:error, :timeout}
     end
   end
 
-  def subscribe_sync(client_id, {_, topic_opts} = topic, opts) when is_list(topic_opts) do
-    subscribe_sync(client_id, [topic], opts)
+  def subscribe_sync(pid, {_, topic_opts} = topic, opts) when is_list(topic_opts) do
+    subscribe_sync(pid, [topic], opts)
   end
 
-  def subscribe_sync(client_id, topic, opts) when is_binary(topic) do
+  def subscribe_sync(pid, topic, opts) when is_binary(topic) do
     case Keyword.pop_first(opts, :qos) do
       {nil, _opts} ->
         throw("Please specify a quality of service for the subscription")
 
       {qos, opts} ->
-        subscribe_sync(client_id, [{topic, qos: qos}], opts)
+        subscribe_sync(pid, [{topic, qos: qos}], opts)
     end
   end
 
@@ -253,15 +253,15 @@ defmodule Tortoise.Connection do
   This operation is asynchronous. When the operation is done a message
   will be received in mailbox of the originating process.
   """
-  @spec unsubscribe(Tortoise.client_id(), topic | topics, [options]) :: {:ok, reference()}
+  @spec unsubscribe(pid(), topic | topics, [options]) :: {:ok, reference()}
         when topics: [topic],
              topic: Tortoise.topic_filter(),
              options:
                {:timeout, timeout()}
                | {:identifier, Tortoise.package_identifier()}
-  def unsubscribe(client_id, topics, opts \\ [])
+  def unsubscribe(pid, topics, opts \\ [])
 
-  def unsubscribe(client_id, [topic | _] = topics, opts) when is_binary(topic) do
+  def unsubscribe(pid, [topic | _] = topics, opts) when is_binary(topic) do
     caller = {_, ref} = {self(), make_ref()}
     {opts, properties} = Keyword.split(opts, [:identifier, :timeout])
     {identifier, opts} = Keyword.pop_first(opts, :identifier, nil)
@@ -272,12 +272,12 @@ defmodule Tortoise.Connection do
       properties: properties
     }
 
-    GenStateMachine.cast(via_name(client_id), {:unsubscribe, caller, unsubscribe, opts})
+    GenStateMachine.cast(via_name(pid), {:unsubscribe, caller, unsubscribe, opts})
     {:ok, ref}
   end
 
-  def unsubscribe(client_id, topic, opts) when is_binary(topic) do
-    unsubscribe(client_id, [topic], opts)
+  def unsubscribe(pid, topic, opts) when is_binary(topic) do
+    unsubscribe(pid, [topic], opts)
   end
 
   @doc """
@@ -289,29 +289,30 @@ defmodule Tortoise.Connection do
 
   See `Tortoise.Connection.unsubscribe/3` for configuration options.
   """
-  @spec unsubscribe_sync(Tortoise.client_id(), topic | topics, [options]) ::
+  @spec unsubscribe_sync(pid(), topic | topics, [options]) ::
           :ok | {:error, :timeout}
         when topics: [topic],
              topic: Tortoise.topic_filter(),
              options:
                {:timeout, timeout()}
                | {:identifier, Tortoise.package_identifier()}
-  def unsubscribe_sync(client_id, topics, opts \\ [])
+  def unsubscribe_sync(pid, topics, opts \\ [])
 
-  def unsubscribe_sync(client_id, topics, opts) when is_list(topics) do
+  def unsubscribe_sync(pid, topics, opts) when is_list(topics) do
     timeout = Keyword.get(opts, :timeout, 5000)
-    {:ok, ref} = unsubscribe(client_id, topics, opts)
+    {:ok, ref} = unsubscribe(pid, topics, opts)
 
     receive do
-      {{Tortoise, ^client_id}, ^ref, result} -> result
+      {{Tortoise, ^pid}, {Package.Unsuback, ^ref}, result} ->
+        result
     after
       timeout ->
         {:error, :timeout}
     end
   end
 
-  def unsubscribe_sync(client_id, topic, opts) when is_binary(topic) do
-    unsubscribe_sync(client_id, [topic], opts)
+  def unsubscribe_sync(pid, topic, opts) when is_binary(topic) do
+    unsubscribe_sync(pid, [topic], opts)
   end
 
   @doc """
@@ -728,7 +729,7 @@ defmodule Tortoise.Connection do
 
       {:invalid, reasons} ->
         reply = {:error, {:subscription_failure, reasons}}
-        next_actions = [{:next_event, :internal, {:reply, caller, reply}}]
+        next_actions = [{:next_event, :internal, {:reply, caller, Package.Suback, reply}}]
         {:keep_state_and_data, next_actions}
     end
   end
@@ -778,7 +779,7 @@ defmodule Tortoise.Connection do
         }
 
         next_actions = [
-          {:next_event, :internal, {:reply, {pid, msg_ref}, :ok}}
+          {:next_event, :internal, {:reply, {pid, msg_ref}, Package.Suback, :ok}}
           | wrap_next_actions(next_actions)
         ]
 
@@ -795,16 +796,22 @@ defmodule Tortoise.Connection do
   # topic, or unsubscribe, etc.
   def handle_event(
         :internal,
-        {:reply, {pid, msg_ref}, result},
+        {:reply, {pid, _msg_ref}, _topic, _payload},
         _current_state,
-        %State{client_id: client_id}
+        %State{}
       )
-      when pid != self() do
-    send(pid, {{Tortoise, client_id}, msg_ref, result})
+      when pid == self() do
     :keep_state_and_data
   end
 
-  def handle_event(:internal, {:reply, _from, _}, _current_state, %State{}) do
+  def handle_event(
+        :internal,
+        {:reply, {caller, ref}, topic, payload},
+        _current_state,
+        %State{}
+      )
+      when caller != self() do
+    _ = send_reply({caller, ref}, topic, payload)
     :keep_state_and_data
   end
 
@@ -864,7 +871,7 @@ defmodule Tortoise.Connection do
         }
 
         next_actions = [
-          {:next_event, :internal, {:reply, {pid, msg_ref}, :ok}}
+          {:next_event, :internal, {:reply, {pid, msg_ref}, Package.Unsuback, :ok}}
           | wrap_next_actions(next_actions)
         ]
 
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index 9bf59638..db701c5d 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -298,9 +298,7 @@ defmodule Tortoise.ConnectionTest do
   describe "subscriptions" do
     setup [:setup_scripted_mqtt_server, :setup_connection_and_perform_handshake]
 
-    test "successful subscription", context do
-      client_id = context.client_id
-
+    test "successful subscription", %{connection_pid: connection} = context do
       default_subscription_opts = [
         no_local: false,
         retain_as_published: false,
@@ -346,31 +344,33 @@ defmodule Tortoise.ConnectionTest do
       {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
 
       # subscribe to a foo
-      :ok = Tortoise.Connection.subscribe_sync(client_id, {"foo", qos: 0}, identifier: 1)
+      :ok = Tortoise.Connection.subscribe_sync(connection, {"foo", qos: 0}, identifier: 1)
       assert_receive {ScriptedMqttServer, {:received, ^subscription_foo}}
-      assert Map.has_key?(Tortoise.Connection.subscriptions(client_id), "foo")
+      assert Map.has_key?(Tortoise.Connection.subscriptions(connection), "foo")
       assert_receive {{TestHandler, :handle_suback}, {%Package.Subscribe{}, ^suback_foo}}
 
       # subscribe to a bar
-      assert {:ok, ref} = Tortoise.Connection.subscribe(client_id, {"bar", qos: 1}, identifier: 2)
-      assert_receive {{Tortoise, ^client_id}, ^ref, :ok}
+      assert {:ok, ref} =
+               Tortoise.Connection.subscribe(connection, {"bar", qos: 1}, identifier: 2)
+
+      assert_receive {{Tortoise, ^connection}, {Package.Suback, ^ref}, :ok}
       assert_receive {ScriptedMqttServer, {:received, ^subscription_bar}}
       assert_receive {{TestHandler, :handle_suback}, {%Package.Subscribe{}, ^suback_bar}}
 
       # subscribe to a baz
       assert {:ok, ref} =
-               Tortoise.Connection.subscribe(client_id, "baz",
+               Tortoise.Connection.subscribe(connection, "baz",
                  qos: 2,
                  identifier: 3,
                  user_property: {"foo", "bar"}
                )
 
-      assert_receive {{Tortoise, ^client_id}, ^ref, :ok}
+      assert_receive {{Tortoise, ^connection}, {Package.Suback, ^ref}, :ok}
       assert_receive {ScriptedMqttServer, {:received, ^subscription_baz}}
       assert_receive {{TestHandler, :handle_suback}, {%Package.Subscribe{}, ^suback_baz}}
 
       # foo, bar, and baz should now be in the subscription list
-      subscriptions = Tortoise.Connection.subscriptions(client_id)
+      subscriptions = Tortoise.Connection.subscriptions(connection)
       assert Map.has_key?(subscriptions, "foo")
       assert Map.has_key?(subscriptions, "bar")
       assert Map.has_key?(subscriptions, "baz")
@@ -382,7 +382,7 @@ defmodule Tortoise.ConnectionTest do
     # @todo subscribe with a qos but have it accepted with a lower qos
     # @todo unsuccessful subscribe
 
-    test "successful unsubscribe", context do
+    test "successful unsubscribe", %{connection_pid: connection} = context do
       client_id = context.client_id
 
       unsubscribe_foo = %Package.Unsubscribe{identifier: 2, topics: ["foo"]}
@@ -424,14 +424,14 @@ defmodule Tortoise.ConnectionTest do
         identifier: 1
       }
 
-      {:ok, unsub_ref} = Tortoise.Connection.subscribe(client_id, subscribe.topics, identifier: 1)
+      {:ok, sub_ref} = Tortoise.Connection.subscribe(connection, subscribe.topics, identifier: 1)
 
       assert_receive {ScriptedMqttServer, {:received, ^subscribe}}
 
       assert_receive {{TestHandler, :handle_suback}, {_, %Package.Suback{identifier: 1}}}
 
       # now let us try to unsubscribe from foo
-      :ok = Tortoise.Connection.unsubscribe_sync(client_id, "foo", identifier: 2)
+      :ok = Tortoise.Connection.unsubscribe_sync(connection, "foo", identifier: 2)
       assert_receive {ScriptedMqttServer, {:received, ^unsubscribe_foo}}
       # handle_unsuback should get called on the callback handler
       assert_receive {{TestHandler, :handle_unsuback}, {^unsubscribe_foo, ^unsuback_foo}}
@@ -447,24 +447,22 @@ defmodule Tortoise.ConnectionTest do
                  user_property: {"foo", "bar"}
                )
 
-      assert_receive {{Tortoise, ^client_id}, ^ref, :ok}
+      assert_receive {{Tortoise, ^connection}, {Package.Unsuback, ^ref}, :ok}
       assert_receive {ScriptedMqttServer, {:received, ^unsubscribe_bar}}
       # handle_unsuback should get called on the callback handler
       assert_receive {{TestHandler, :handle_unsuback}, {^unsubscribe_bar, ^unsuback_bar}}
 
-      refute Map.has_key?(Tortoise.Connection.subscriptions(client_id), "bar")
+      refute Map.has_key?(Tortoise.Connection.subscriptions(connection), "bar")
       # there should be no subscriptions now
-      assert map_size(Tortoise.Connection.subscriptions(client_id)) == 0
+      assert map_size(Tortoise.Connection.subscriptions(connection)) == 0
       assert_receive {ScriptedMqttServer, :completed}
 
-      # the process calling the async unsubscribe should receive the
-      # result of the unsubscribe as a message
-      assert_receive {{Tortoise, ^client_id}, ^unsub_ref, :ok}, 0
+      # the process calling the async subscribe should receive the
+      # result of the subscribe as a message (suback)
+      assert_receive {{Tortoise, ^connection}, {Package.Suback, ^sub_ref}, :ok}, 0
     end
 
-    test "unsuccessful unsubscribe: not authorized", context do
-      client_id = context.client_id
-
+    test "unsuccessful unsubscribe: not authorized", %{connection_pid: connection} = context do
       unsubscribe_foo = %Package.Unsubscribe{identifier: 2, topics: ["foo"]}
       unsuback_foo = %Package.Unsuback{results: [error: :not_authorized], identifier: 2}
 
@@ -491,20 +489,19 @@ defmodule Tortoise.ConnectionTest do
         identifier: 1
       }
 
-      {:ok, _sub_ref} = Tortoise.Connection.subscribe(client_id, subscribe.topics, identifier: 1)
+      {:ok, _sub_ref} = Tortoise.Connection.subscribe(connection, subscribe.topics, identifier: 1)
       assert_receive {ScriptedMqttServer, {:received, ^subscribe}}
       assert_receive {{TestHandler, :handle_suback}, {_, %Package.Suback{identifier: 1}}}
 
-      subscriptions = Tortoise.Connection.subscriptions(client_id)
-      {:ok, unsub_ref} = Tortoise.Connection.unsubscribe(client_id, "foo", identifier: 2)
-      assert_receive {{Tortoise, client_id}, ^unsub_ref, :ok}
+      subscriptions = Tortoise.Connection.subscriptions(connection)
+      {:ok, unsub_ref} = Tortoise.Connection.unsubscribe(connection, "foo", identifier: 2)
+      assert_receive {{Tortoise, ^connection}, {Package.Unsuback, ^unsub_ref}, :ok}
       assert_receive {Tortoise.Integration.ScriptedMqttServer, :completed}
-      assert ^subscriptions = Tortoise.Connection.subscriptions(client_id)
+      assert ^subscriptions = Tortoise.Connection.subscriptions(connection)
     end
 
-    test "unsuccessful unsubscribe: no subscription existed", context do
-      client_id = context.client_id
-
+    test "unsuccessful unsubscribe: no subscription existed",
+         %{connection_pid: connection} = context do
       unsubscribe_foo = %Package.Unsubscribe{identifier: 2, topics: ["foo"]}
       unsuback_foo = %Package.Unsuback{results: [error: :no_subscription_existed], identifier: 2}
 
@@ -531,17 +528,17 @@ defmodule Tortoise.ConnectionTest do
         identifier: 1
       }
 
-      {:ok, _sub_ref} = Tortoise.Connection.subscribe(client_id, subscribe.topics, identifier: 1)
+      {:ok, _sub_ref} = Tortoise.Connection.subscribe(connection, subscribe.topics, identifier: 1)
       assert_receive {ScriptedMqttServer, {:received, ^subscribe}}
       assert_receive {{TestHandler, :handle_suback}, {_, %Package.Suback{identifier: 1}}}
 
-      assert Tortoise.Connection.subscriptions(client_id) |> Map.has_key?("foo")
-      {:ok, unsub_ref} = Tortoise.Connection.unsubscribe(client_id, "foo", identifier: 2)
-      assert_receive {{Tortoise, client_id}, ^unsub_ref, :ok}
+      assert Tortoise.Connection.subscriptions(connection) |> Map.has_key?("foo")
+      {:ok, unsub_ref} = Tortoise.Connection.unsubscribe(connection, "foo", identifier: 2)
+      assert_receive {{Tortoise, ^connection}, {Package.Unsuback, ^unsub_ref}, :ok}
       assert_receive {Tortoise.Integration.ScriptedMqttServer, :completed}
       # the client should update it state to not include the foo topic
       # as the server told us that it is not subscribed
-      refute Tortoise.Connection.subscriptions(client_id) |> Map.has_key?("foo")
+      refute Tortoise.Connection.subscriptions(connection) |> Map.has_key?("foo")
     end
   end
 
@@ -565,7 +562,7 @@ defmodule Tortoise.ConnectionTest do
 
       {:ok, {ip, port}} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
 
-      assert {:ok, connection_pid} =
+      assert {:ok, connection} =
                Connection.start_link(
                  client_id: client_id,
                  server: {Tortoise.Transport.Tcp, [host: ip, port: port]},
@@ -574,17 +571,17 @@ defmodule Tortoise.ConnectionTest do
 
       assert_receive {ScriptedMqttServer, {:received, %Package.Connect{}}}
 
-      assert {:ok, {Tortoise.Transport.Tcp, _port}} = Connection.connection(client_id)
+      assert {:ok, {Tortoise.Transport.Tcp, _port}} = Connection.connection(connection)
 
       assert {:connected,
               %Connection.Info{
                 capabilities: %Connection.Info.Capabilities{
                   shared_subscription_available: false
                 }
-              }} = Connection.info(client_id)
+              }} = Connection.info(connection)
 
       assert {:error, {:subscription_failure, reasons}} =
-               Connection.subscribe_sync(client_id, {"$share/foo/bar", qos: 0})
+               Connection.subscribe_sync(connection, {"$share/foo/bar", qos: 0})
 
       assert {:shared_subscription_not_available, "$share/foo/bar"} in reasons
     end
@@ -615,18 +612,18 @@ defmodule Tortoise.ConnectionTest do
 
       assert_receive {ScriptedMqttServer, {:received, %Package.Connect{}}}
 
-      assert {:ok, {Tortoise.Transport.Tcp, _port}} = Connection.connection(client_id)
+      assert {:ok, {Tortoise.Transport.Tcp, _port}} = Connection.connection(connection_pid)
 
       assert {:connected, %{capabilities: %{wildcard_subscription_available: false}}} =
-               Connection.info(client_id)
+               Connection.info(connection_pid)
 
       assert {:error, {:subscription_failure, reasons}} =
-               Connection.subscribe_sync(client_id, {"foo/+/bar", qos: 0})
+               Connection.subscribe_sync(connection_pid, {"foo/+/bar", qos: 0})
 
       assert {:wildcard_subscription_not_available, "foo/+/bar"} in reasons
 
       assert {:error, {:subscription_failure, reasons}} =
-               Connection.subscribe_sync(client_id, {"foo/#", qos: 0})
+               Connection.subscribe_sync(connection_pid, {"foo/#", qos: 0})
 
       assert {:wildcard_subscription_not_available, "foo/#"} in reasons
     end
@@ -658,17 +655,17 @@ defmodule Tortoise.ConnectionTest do
 
       assert_receive {ScriptedMqttServer, {:received, %Package.Connect{}}}
 
-      assert {:ok, {Tortoise.Transport.Tcp, _port}} = Connection.connection(client_id)
+      assert {:ok, {Tortoise.Transport.Tcp, _port}} = Connection.connection(connection_pid)
 
       assert {:connected,
               %Connection.Info{
                 capabilities: %Connection.Info.Capabilities{
                   subscription_identifiers_available: false
                 }
-              }} = Connection.info(client_id)
+              }} = Connection.info(connection_pid)
 
       assert {:error, {:subscription_failure, reasons}} =
-               Connection.subscribe_sync(client_id, {"foo/+/bar", qos: 0},
+               Connection.subscribe_sync(connection_pid, {"foo/+/bar", qos: 0},
                  subscription_identifier: 5
                )
 

From 045ab801bcd37dfa2cf1f6b974c1144da5c3a945 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Wed, 12 Feb 2020 14:29:23 +0000
Subject: [PATCH 172/220] Get rid of the status events from Tortoise.Events

Step towards getting rid of the Tortoise.Events entirely
---
 lib/tortoise/connection.ex                 | 4 ----
 lib/tortoise/events.ex                     | 6 +-----
 test/tortoise/connection/inflight_test.exs | 1 -
 test/tortoise/connection_test.exs          | 8 ++++----
 4 files changed, 5 insertions(+), 14 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 06eee692..802a9569 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -470,7 +470,6 @@ defmodule Tortoise.Connection do
         # todo, get rid of the connection event
         :ok = Events.dispatch(client_id, :connection, data.connection)
         :ok = Inflight.update_connection(client_id, data.connection)
-        :ok = Events.dispatch(client_id, :status, :connected)
 
         data = %State{
           data
@@ -911,7 +910,6 @@ defmodule Tortoise.Connection do
         {:keep_state_and_data, next_actions}
 
       :disconnect ->
-        :ok = Events.dispatch(client_id, :status, :terminating)
         disconnect = %Package.Disconnect{reason: :normal_disconnection}
         :ok = Inflight.drain(client_id, disconnect)
         {:stop, :normal}
@@ -998,8 +996,6 @@ defmodule Tortoise.Connection do
         :connected,
         %State{client_id: client_id} = data
       ) do
-    :ok = Events.dispatch(client_id, :status, :terminating)
-
     :ok = Inflight.drain(client_id, disconnect)
 
     {:stop_and_reply, :shutdown, [{:reply, from, :ok}], data}
diff --git a/lib/tortoise/events.ex b/lib/tortoise/events.ex
index 8214a9cc..6d10dfef 100644
--- a/lib/tortoise/events.ex
+++ b/lib/tortoise/events.ex
@@ -9,7 +9,7 @@ defmodule Tortoise.Events do
   `Tortoise.Events.unregister/2` for how to unsubscribe.
   """
 
-  @types [:connection, :status, :ping_response]
+  @types [:connection, :ping_response]
 
   @doc """
   Subscribe to messages on the client with the client id `client_id`
@@ -26,10 +26,6 @@ defmodule Tortoise.Events do
 
   Possible message types are:
 
-    - `:status` dispatched when the connection of a client changes
-      status. The value will be `:up` when the client goes online, and
-      `:down` when it goes offline.
-
     - `:ping_response` dispatched when the connection receive a
       response from a keep alive message. The value is the round trip
       time in milliseconds, and can be used to track the latency over
diff --git a/test/tortoise/connection/inflight_test.exs b/test/tortoise/connection/inflight_test.exs
index 5f6b0db8..2165013e 100644
--- a/test/tortoise/connection/inflight_test.exs
+++ b/test/tortoise/connection/inflight_test.exs
@@ -22,7 +22,6 @@ defmodule Tortoise.Connection.InflightTest do
 
   defp drop_connection(%{server: server} = context) do
     :ok = :gen_tcp.close(server)
-    :ok = Tortoise.Events.dispatch(context.client_id, :status, :down)
     {:ok, Map.drop(context, [:client, :server, :connection])}
   end
 
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index db701c5d..429e01a6 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -1102,16 +1102,16 @@ defmodule Tortoise.ConnectionTest do
       assert_receive {ScriptedMqttServer, :completed}
     end
 
-    test "ping_sync/2", %{client_id: client_id} = context do
-      {:ok, _} = Tortoise.Events.register(client_id, :status)
-      assert_receive {{Tortoise, ^client_id}, :status, :connected}
-
+    test "ping_sync/2", context do
       ping_request = %Package.Pingreq{}
       expected_pingresp = %Package.Pingresp{}
       script = [{:receive, ping_request}, {:send, expected_pingresp}]
 
       {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
 
+      # make sure the client is connected
+      assert_receive {{TestHandler, :handle_connack}, %Tortoise.Package.Connack{}}
+
       {parent, ref} = {self(), make_ref()}
 
       spawn_link(fn ->

From 7cf8f45eac7c0c2fd4e7f45a556c44de06bd93d6 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Wed, 12 Feb 2020 14:44:55 +0000
Subject: [PATCH 173/220] Get rid of the ping response event type

This is another step towards getting rid of the Tortoise.Events
system.
---
 lib/tortoise/events.ex        |  9 ++-----
 test/tortoise/events_test.exs | 46 -----------------------------------
 2 files changed, 2 insertions(+), 53 deletions(-)

diff --git a/lib/tortoise/events.ex b/lib/tortoise/events.ex
index 6d10dfef..ac3c1737 100644
--- a/lib/tortoise/events.ex
+++ b/lib/tortoise/events.ex
@@ -9,7 +9,7 @@ defmodule Tortoise.Events do
   `Tortoise.Events.unregister/2` for how to unsubscribe.
   """
 
-  @types [:connection, :ping_response]
+  @types [:connection]
 
   @doc """
   Subscribe to messages on the client with the client id `client_id`
@@ -24,12 +24,7 @@ defmodule Tortoise.Events do
   Making it possible to pattern match on multiple message types on
   multiple clients. The value depends on the message type.
 
-  Possible message types are:
-
-    - `:ping_response` dispatched when the connection receive a
-      response from a keep alive message. The value is the round trip
-      time in milliseconds, and can be used to track the latency over
-      time.
+  Possible message types are: None, don't use them.
 
   Other message types exist, but unless they are mentioned in the
   possible message types above they should be considered for internal
diff --git a/test/tortoise/events_test.exs b/test/tortoise/events_test.exs
index c10cc5b0..f193cdc7 100644
--- a/test/tortoise/events_test.exs
+++ b/test/tortoise/events_test.exs
@@ -100,50 +100,4 @@ defmodule Tortoise.EventsTest do
       assert [:connection] = Registry.keys(Tortoise.Events, child)
     end
   end
-
-  describe "ping responses" do
-    test "receive ping responses", context do
-      client_id1 = Atom.to_string(context.client_id)
-      client_id2 = client_id1 <> "2"
-      client_id3 = client_id1 <> "3"
-
-      # register retrieval of ping requests from 1 and 2
-      assert {:ok, owner} = Tortoise.Events.register(client_id1, :ping_response)
-      assert {:ok, ^owner} = Tortoise.Events.register(client_id2, :ping_response)
-
-      # dispatch ping responses; expect from 1 and 2, but not 3
-      Tortoise.Events.dispatch(client_id1, :ping_response, 500)
-      Tortoise.Events.dispatch(client_id2, :ping_response, 500)
-      Tortoise.Events.dispatch(client_id3, :ping_response, 500)
-      assert_receive {{Tortoise, ^client_id1}, :ping_response, 500}
-      assert_receive {{Tortoise, ^client_id2}, :ping_response, 500}
-      refute_receive {{Tortoise, ^client_id3}, :ping_response, 500}
-
-      # unregister 2, and register 3
-      Tortoise.Events.unregister(client_id2, :ping_response)
-      assert {:ok, ^owner} = Tortoise.Events.register(client_id3, :ping_response)
-
-      # dispatch ping responses, and expect from 1 and 3, not 2
-      Tortoise.Events.dispatch(client_id1, :ping_response, 500)
-      Tortoise.Events.dispatch(client_id2, :ping_response, 500)
-      Tortoise.Events.dispatch(client_id3, :ping_response, 500)
-      assert_receive {{Tortoise, ^client_id1}, :ping_response, 500}
-      refute_receive {{Tortoise, ^client_id2}, :ping_response, 500}
-      assert_receive {{Tortoise, ^client_id3}, :ping_response, 500}
-    end
-
-    test "Subscribing to all clients", context do
-      client_id1 = Atom.to_string(context.client_id)
-      client_id2 = client_id1 <> "2"
-
-      # :_ means every client id
-      Tortoise.Events.register(:_, :ping_response)
-
-      Tortoise.Events.dispatch(client_id1, :ping_response, 123)
-      Tortoise.Events.dispatch(client_id2, :ping_response, 234)
-
-      assert_receive {{Tortoise, ^client_id2}, :ping_response, 234}
-      assert_receive {{Tortoise, ^client_id1}, :ping_response, 123}
-    end
-  end
 end

From 0fbd4d97796f98261ae692cf5d2a95f455303117 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Wed, 12 Feb 2020 17:25:27 +0000
Subject: [PATCH 174/220] Got rid of the Events and EventsTest modules

I have skipped some tests. Will decide on how to proceed with
this. Current thinking is to put more focus on the "pipe" concept; and
perhaps make it _the way_ to publish messages to the server.

Events are gone now.
---
 lib/tortoise/connection.ex                 |  81 ++++++++--------
 lib/tortoise/events.ex                     |  61 ------------
 test/tortoise/connection/inflight_test.exs |   1 -
 test/tortoise/connection_test.exs          |  41 ++++----
 test/tortoise/events_test.exs              | 103 ---------------------
 test/tortoise/pipe_test.exs                |   7 ++
 test/tortoise_test.exs                     |   8 ++
 7 files changed, 80 insertions(+), 222 deletions(-)
 delete mode 100644 lib/tortoise/events.ex
 delete mode 100644 test/tortoise/events_test.exs

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 802a9569..387632ce 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -24,7 +24,7 @@ defmodule Tortoise.Connection do
 
   alias __MODULE__, as: State
 
-  alias Tortoise.{Handler, Transport, Package, Events}
+  alias Tortoise.{Handler, Transport, Package}
   alias Tortoise.Connection.{Info, Receiver, Inflight, Backoff}
   alias Tortoise.Package.Connect
 
@@ -373,45 +373,52 @@ defmodule Tortoise.Connection do
         when opts: {:timeout, timeout()} | {:active, boolean()}
   def connection(pid, _opts \\ [active: false])
 
-  def connection(pid, _opts) when is_pid(pid) do
-    GenStateMachine.call(pid, :get_connection)
-  end
-
-  def connection(client_id, opts) do
-    # register a connection subscription in the case we are currently
-    # in the connect phase; this solves a possible race condition
-    # where the connection is requested while the status is
-    # connecting, but will reach the receive block after the message
-    # has been dispatched from the pubsub; previously we registered
-    # for the connection message in this window.
-    {:ok, _} = Events.register(client_id, :connection)
-
-    case Tortoise.Registry.meta(via_name(client_id)) do
-      {:ok, {_transport, _socket} = connection} ->
-        {:ok, connection}
-
-      {:ok, :connecting} ->
-        timeout = Keyword.get(opts, :timeout, :infinity)
+  def connection(pid, opts) when is_pid(pid) do
+    timeout = Keyword.get(opts, :timeout, :infinity)
 
-        receive do
-          {{Tortoise, ^client_id}, :connection, {transport, socket}} ->
-            {:ok, {transport, socket}}
-        after
-          timeout ->
-            {:error, :timeout}
-        end
-
-      :error ->
-        {:error, :unknown_connection}
+    # make it possible to subscribe to a connection using "active"!
+    if Process.alive?(pid) do
+      GenStateMachine.call(pid, :get_connection, timeout)
+    else
+      {:error, :unknown_connection}
     end
-  after
-    # if the connection subscription is non-active we should remove it
-    # from the registry, so the process will not receive connection
-    # messages when the connection is reestablished.
-    active? = Keyword.get(opts, :active, false)
-    unless active?, do: Events.unregister(client_id, :connection)
   end
 
+  # def connection(client_id, opts) do
+  #   # register a connection subscription in the case we are currently
+  #   # in the connect phase; this solves a possible race condition
+  #   # where the connection is requested while the status is
+  #   # connecting, but will reach the receive block after the message
+  #   # has been dispatched from the pubsub; previously we registered
+  #   # for the connection message in this window.
+  #   {:ok, _} = Events.register(client_id, :connection)
+
+  #   case Tortoise.Registry.meta(via_name(client_id)) do
+  #     {:ok, {_transport, _socket} = connection} ->
+  #       {:ok, connection}
+
+  #     {:ok, :connecting} ->
+  #       timeout = Keyword.get(opts, :timeout, :infinity)
+
+  #       receive do
+  #         {{Tortoise, ^client_id}, :connection, {transport, socket}} ->
+  #           {:ok, {transport, socket}}
+  #       after
+  #         timeout ->
+  #           {:error, :timeout}
+  #       end
+
+  #     :error ->
+  #       {:error, :unknown_connection}
+  #   end
+  # after
+  #   # if the connection subscription is non-active we should remove it
+  #   # from the registry, so the process will not receive connection
+  #   # messages when the connection is reestablished.
+  #   active? = Keyword.get(opts, :active, false)
+  #   unless active?, do: Events.unregister(client_id, :connection)
+  # end
+
   # Callbacks
   @impl true
   def init(%State{} = state) do
@@ -467,8 +474,6 @@ defmodule Tortoise.Connection do
     case Handler.execute_handle_connack(handler, connack) do
       {:ok, %Handler{} = updated_handler, next_actions} when connection_result == :success ->
         :ok = Tortoise.Registry.put_meta(via_name(client_id), data.connection)
-        # todo, get rid of the connection event
-        :ok = Events.dispatch(client_id, :connection, data.connection)
         :ok = Inflight.update_connection(client_id, data.connection)
 
         data = %State{
diff --git a/lib/tortoise/events.ex b/lib/tortoise/events.ex
deleted file mode 100644
index ac3c1737..00000000
--- a/lib/tortoise/events.ex
+++ /dev/null
@@ -1,61 +0,0 @@
-defmodule Tortoise.Events do
-  @moduledoc """
-  A PubSub exposing various system events from a Tortoise
-  connection. This allows the user to integrate with custom metrics
-  and logging solutions.
-
-  Please read the documentation for `Tortoise.Events.register/2` for
-  information on how to subscribe to events, and
-  `Tortoise.Events.unregister/2` for how to unsubscribe.
-  """
-
-  @types [:connection]
-
-  @doc """
-  Subscribe to messages on the client with the client id `client_id`
-  of the type `type`.
-
-  When a message of the subscribed type is dispatched it will end up
-  in the mailbox of the process that placed the subscription. The
-  received message will have the format:
-
-      {{Tortoise, client_id}, type, value}
-
-  Making it possible to pattern match on multiple message types on
-  multiple clients. The value depends on the message type.
-
-  Possible message types are: None, don't use them.
-
-  Other message types exist, but unless they are mentioned in the
-  possible message types above they should be considered for internal
-  use only.
-
-  It is possible to listen on all events for a given type by
-  specifying `:_` as the `client_id`.
-  """
-  @spec register(Tortoise.client_id(), atom()) :: {:ok, pid()} | no_return()
-  def register(client_id, type) when type in @types do
-    {:ok, _pid} = Registry.register(__MODULE__, type, client_id)
-  end
-
-  @doc """
-  Unsubscribe from messages of `type` from `client_id`. This is the
-  reverse of `Tortoise.Events.register/2`.
-  """
-  @spec unregister(Tortoise.client_id(), atom()) :: :ok | no_return()
-  def unregister(client_id, type) when type in @types do
-    :ok = Registry.unregister_match(__MODULE__, type, client_id)
-  end
-
-  @doc false
-  @spec dispatch(Tortoise.client_id(), type :: atom(), value :: term()) :: :ok
-  def dispatch(client_id, type, value) when type in @types do
-    :ok =
-      Registry.dispatch(__MODULE__, type, fn subscribers ->
-        for {pid, filter} <- subscribers,
-            filter == client_id or filter == :_ do
-          Kernel.send(pid, {{Tortoise, client_id}, type, value})
-        end
-      end)
-  end
-end
diff --git a/test/tortoise/connection/inflight_test.exs b/test/tortoise/connection/inflight_test.exs
index 2165013e..9d29e067 100644
--- a/test/tortoise/connection/inflight_test.exs
+++ b/test/tortoise/connection/inflight_test.exs
@@ -14,7 +14,6 @@ defmodule Tortoise.Connection.InflightTest do
     connection = {Tortoise.Transport.Tcp, client_socket}
     key = Tortoise.Registry.via_name(Tortoise.Connection, context.client_id)
     Tortoise.Registry.put_meta(key, connection)
-    Tortoise.Events.dispatch(context.client_id, :connection, connection)
 
     {:ok,
      Map.merge(context, %{client: client_socket, server: server_socket, connection: connection})}
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index 429e01a6..01bf0d13 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -873,14 +873,14 @@ defmodule Tortoise.ConnectionTest do
   describe "socket subscription" do
     setup [:setup_scripted_mqtt_server]
 
-    test "return error if asking for a connection on an non-existent connection", context do
-      assert {:error, :unknown_connection} = Connection.connection(context.client_id)
+    test "return error if asking for a connection on an non-existent connection" do
+      {pid, ref} = spawn_monitor(fn -> nil end)
+      assert_receive {:DOWN, ^ref, :process, ^pid, :normal}
+      assert {:error, :unknown_connection} = Connection.connection(pid)
     end
 
     test "receive a socket from a connection", context do
-      client_id = context.client_id
-
-      connect = %Package.Connect{client_id: client_id, clean_start: true}
+      connect = %Package.Connect{client_id: context.client_id, clean_start: true}
       expected_connack = %Package.Connack{reason: :success, session_present: false}
 
       script = [{:receive, connect}, {:send, expected_connack}]
@@ -888,40 +888,43 @@ defmodule Tortoise.ConnectionTest do
       {:ok, {ip, port}} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
 
       opts = [
-        client_id: client_id,
+        client_id: context.client_id,
         server: {Tortoise.Transport.Tcp, [host: ip, port: port]},
         handler: {Tortoise.Handler.Default, []}
       ]
 
-      assert {:ok, _pid} = Connection.start_link(opts)
+      assert {:ok, connection} = Connection.start_link(opts)
       assert_receive {ScriptedMqttServer, {:received, ^connect}}
 
       assert {:ok, {Tortoise.Transport.Tcp, _socket}} =
-               Connection.connection(client_id, timeout: 500)
+               Connection.connection(connection, timeout: 500)
 
       assert_receive {ScriptedMqttServer, :completed}
     end
 
     test "timeout on a socket from a connection", context do
-      client_id = context.client_id
-
-      connect = %Package.Connect{client_id: client_id, clean_start: true}
+      connect = %Package.Connect{client_id: context.client_id, clean_start: true}
 
       script = [{:receive, connect}, :pause]
 
       {:ok, {ip, port}} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
 
       opts = [
-        client_id: client_id,
+        client_id: context.client_id,
         server: {Tortoise.Transport.Tcp, [host: ip, port: port]},
         handler: {Tortoise.Handler.Default, []}
       ]
 
-      assert {:ok, _pid} = Connection.start_link(opts)
+      assert {:ok, connection} = Connection.start_link(opts)
       assert_receive {ScriptedMqttServer, {:received, ^connect}}
       assert_receive {ScriptedMqttServer, :paused}
 
-      assert {:error, :timeout} = Connection.connection(client_id, timeout: 5)
+      {child_pid, mon_ref} =
+        spawn_monitor(fn ->
+          Connection.connection(connection, timeout: 5)
+        end)
+
+      assert_receive {:DOWN, ^mon_ref, :process, ^child_pid, {:timeout, _}}
 
       send(context.scripted_mqtt_server, :continue)
       assert_receive {ScriptedMqttServer, :completed}
@@ -964,24 +967,24 @@ defmodule Tortoise.ConnectionTest do
         handler: handler
       ]
 
-      assert {:ok, pid} = Connection.start_link(opts)
+      assert {:ok, connection_pid} = Connection.start_link(opts)
       assert_receive {ScriptedMqttServer, {:received, ^connect}}
 
       cs_pid = Connection.Supervisor.whereis(client_id)
       cs_ref = Process.monitor(cs_pid)
 
       inflight_pid = Connection.Inflight.whereis(client_id)
-      {:ok, {Tortoise.Transport.Tcp, _}} = Connection.connection(client_id)
+      {:ok, {Tortoise.Transport.Tcp, _}} = Connection.connection(connection_pid)
 
       {:connected,
        %{
          receiver_pid: receiver_pid
-       }} = Connection.info(client_id)
+       }} = Connection.info(connection_pid)
 
-      assert :ok = Tortoise.Connection.disconnect(client_id)
+      assert :ok = Tortoise.Connection.disconnect(connection_pid)
 
       assert_receive {ScriptedMqttServer, {:received, ^disconnect}}
-      assert_receive {:EXIT, ^pid, :shutdown}
+      assert_receive {:EXIT, ^connection_pid, :shutdown}
 
       assert_receive {ScriptedMqttServer, :completed}
 
diff --git a/test/tortoise/events_test.exs b/test/tortoise/events_test.exs
deleted file mode 100644
index f193cdc7..00000000
--- a/test/tortoise/events_test.exs
+++ /dev/null
@@ -1,103 +0,0 @@
-defmodule Tortoise.EventsTest do
-  use ExUnit.Case, async: true
-
-  setup context do
-    {:ok, %{client_id: context.test, transport: Tortoise.Transport.Tcp}}
-  end
-
-  defp via_name(client_id) do
-    Tortoise.Connection.via_name(client_id)
-  end
-
-  def run_setup(context, setup) when is_atom(setup) do
-    context_update =
-      case apply(__MODULE__, setup, [context]) do
-        {:ok, update} -> update
-        [{_, _} | _] = update -> update
-        %{} = update -> update
-      end
-
-    Enum.into(context_update, context)
-  end
-
-  def setup_connection(context) do
-    {:ok, client_socket, server_socket} = Tortoise.Integration.TestTCPTunnel.new()
-    name = via_name(context.client_id)
-    :ok = Tortoise.Registry.put_meta(name, :connecting)
-
-    {:ok, %{client: client_socket, server: server_socket}}
-  end
-
-  describe "passive connection" do
-    setup [:setup_connection]
-
-    test "get connection", context do
-      parent = self()
-
-      child =
-        spawn_link(fn ->
-          send(parent, :ready)
-          {:ok, connection} = Tortoise.Connection.connection(context.client_id)
-          send(parent, {:received, connection})
-          :timer.sleep(:infinity)
-        end)
-
-      # make sure the child process is ready
-      assert_receive :ready
-
-      # dispatch the connection
-      connection = {context.transport, context.client}
-      :ok = Tortoise.Events.dispatch(context.client_id, :connection, connection)
-      # have the process registered itself
-      assert [:connection] = Registry.keys(Tortoise.Events, child)
-
-      # the subscriber should receive the connection and unregister
-      # itself from the connection event
-      assert_receive {:received, ^connection}
-      assert [] = Registry.keys(Tortoise.Events, child)
-    end
-  end
-
-  describe "active connection" do
-    setup [:setup_connection]
-
-    test "get connection", context do
-      client_id = context.client_id
-      parent = self()
-
-      child =
-        spawn_link(fn ->
-          send(parent, :ready)
-          {:ok, connection} = Tortoise.Connection.connection(context.client_id, active: true)
-          send(parent, {:received, connection})
-          # later it should receive new sockets
-          receive do
-            {{Tortoise, ^client_id}, :connection, connection} ->
-              send(parent, {:received, connection})
-              :timer.sleep(:infinity)
-          after
-            500 ->
-              send(parent, :timeout)
-          end
-        end)
-
-      # make sure the child process is ready
-      assert_receive :ready
-
-      # dispatch the connection
-      connection = {context.transport, context.client}
-      :ok = Tortoise.Events.dispatch(context.client_id, :connection, connection)
-
-      # the subscriber should receive the connection and it should
-      # still be registered for new connections
-      assert_receive {:received, ^connection}, 2000
-      assert [:connection] = Registry.keys(Tortoise.Events, child)
-
-      context = run_setup(context, :setup_connection)
-      new_connection = {context.transport, context.client}
-      :ok = Tortoise.Events.dispatch(context.client_id, :connection, new_connection)
-      assert_receive {:received, ^new_connection}
-      assert [:connection] = Registry.keys(Tortoise.Events, child)
-    end
-  end
-end
diff --git a/test/tortoise/pipe_test.exs b/test/tortoise/pipe_test.exs
index 51c9f78b..8260fe0c 100644
--- a/test/tortoise/pipe_test.exs
+++ b/test/tortoise/pipe_test.exs
@@ -45,11 +45,13 @@ defmodule Tortoise.PipeTest do
   describe "new/2" do
     setup [:setup_registry]
 
+    @tag skip: true
     test "generating a pipe when the connection is up", context do
       context = run_setup(context, :setup_connection)
       assert %Pipe{} = Pipe.new(context.client_id)
     end
 
+    @tag skip: true
     test "generating a pipe while the connection is in connecting state", context do
       parent = self()
       client_id = context.client_id
@@ -72,6 +74,7 @@ defmodule Tortoise.PipeTest do
   describe "publish/4" do
     setup [:setup_registry, :setup_connection]
 
+    @tag skip: true
     test "publish a message", context do
       pipe = Pipe.new(context.test)
       topic = "foo/bar"
@@ -82,6 +85,7 @@ defmodule Tortoise.PipeTest do
       assert %Package.Publish{topic: ^topic, payload: ^payload} = Package.decode(package)
     end
 
+    @tag skip: true
     test "replace pipe during a publish if the socket is closed (active:false)", context do
       client_id = context.client_id
       parent = self()
@@ -126,17 +130,20 @@ defmodule Tortoise.PipeTest do
   describe "await/1" do
     setup [:setup_registry, :setup_connection, :setup_inflight]
 
+    @tag skip: true
     test "awaiting an empty pending list should complete instantly", context do
       pipe = Pipe.new(context.client_id)
       {:ok, %Pipe{pending: []}} = Pipe.await(pipe)
     end
 
+    @tag skip: true
     test "error with a timeout if given timeout is reached", context do
       pipe = Pipe.new(context.client_id)
       pipe = Pipe.publish(pipe, "foo/bar", nil, qos: 1)
       {:error, :timeout} = Pipe.await(pipe, 20)
     end
 
+    @tag skip: true
     test "block until pending packages has been acknowledged", context do
       client_id = context.client_id
       parent = self()
diff --git a/test/tortoise_test.exs b/test/tortoise_test.exs
index ee7d1619..0472ecc6 100644
--- a/test/tortoise_test.exs
+++ b/test/tortoise_test.exs
@@ -27,18 +27,21 @@ defmodule TortoiseTest do
   describe "publish/4" do
     setup [:setup_connection, :setup_inflight]
 
+    @tag skip: true
     test "publish qos=0", context do
       assert :ok = Tortoise.publish(context.client_id, "foo/bar")
       assert {:ok, data} = :gen_tcp.recv(context.server, 0, 500)
       assert %Package.Publish{topic: "foo/bar", qos: 0, payload: nil} = Package.decode(data)
     end
 
+    @tag skip: true
     test "publish qos=1", context do
       assert {:ok, _ref} = Tortoise.publish(context.client_id, "foo/bar", nil, qos: 1)
       assert {:ok, data} = :gen_tcp.recv(context.server, 0, 500)
       assert %Package.Publish{topic: "foo/bar", qos: 1, payload: nil} = Package.decode(data)
     end
 
+    @tag skip: true
     test "publish qos=1 with user defined callbacks", %{client_id: client_id} = context do
       parent = self()
 
@@ -70,12 +73,14 @@ defmodule TortoiseTest do
       assert_receive {:callback, {Package.Puback, []}, [Package.Publish, :init]}
     end
 
+    @tag skip: true
     test "publish qos=2", context do
       assert {:ok, _ref} = Tortoise.publish(context.client_id, "foo/bar", nil, qos: 2)
       assert {:ok, data} = :gen_tcp.recv(context.server, 0, 500)
       assert %Package.Publish{topic: "foo/bar", qos: 2, payload: nil} = Package.decode(data)
     end
 
+    @tag skip: true
     test "publish qos=2 with custom callbacks", %{client_id: client_id} = context do
       parent = self()
 
@@ -146,12 +151,14 @@ defmodule TortoiseTest do
   describe "publish_sync/4" do
     setup [:setup_connection, :setup_inflight]
 
+    @tag skip: true
     test "publish qos=0", context do
       assert :ok = Tortoise.publish_sync(context.client_id, "foo/bar")
       assert {:ok, data} = :gen_tcp.recv(context.server, 0, 500)
       assert %Package.Publish{topic: "foo/bar", qos: 0, payload: nil} = Package.decode(data)
     end
 
+    @tag skip: true
     test "publish qos=1", context do
       client_id = context.client_id
       parent = self()
@@ -170,6 +177,7 @@ defmodule TortoiseTest do
       assert_receive :done
     end
 
+    @tag skip: true
     test "publish qos=2", context do
       client_id = context.client_id
       parent = self()

From 5d9d0c627cdf096da59364e7a02250800dc8e709 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Sat, 9 May 2020 10:13:19 +0100
Subject: [PATCH 175/220] Update project lock file

---
 mix.lock | 36 ++++++++++++++++++------------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/mix.lock b/mix.lock
index c4dfbe03..273deb5c 100644
--- a/mix.lock
+++ b/mix.lock
@@ -1,23 +1,23 @@
 %{
-  "certifi": {:hex, :certifi, "2.3.1", "d0f424232390bf47d82da8478022301c561cf6445b5b5fb6a84d49a9e76d2639", [:rebar3], [{:parse_trans, "3.2.0", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm"},
+  "certifi": {:hex, :certifi, "2.3.1", "d0f424232390bf47d82da8478022301c561cf6445b5b5fb6a84d49a9e76d2639", [:rebar3], [{:parse_trans, "3.2.0", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm", "e12d667d042c11d130594bae2b0097e63836fe8b1e6d6b2cc48f8bb7a2cf7d68"},
   "ct_helper": {:git, "https://github.com/ninenines/ct_helper.git", "6cf0748b5ac7bd32f8d338224b843e419b1ea7c0", []},
-  "dialyxir": {:hex, :dialyxir, "1.0.0-rc.3", "774306f84973fc3f1e2e8743eeaa5f5d29b117f3916e5de74c075c02f1b8ef55", [:mix], [], "hexpm"},
-  "earmark": {:hex, :earmark, "1.2.5", "4d21980d5d2862a2e13ec3c49ad9ad783ffc7ca5769cf6ff891a4553fbaae761", [:mix], [], "hexpm"},
-  "eqc_ex": {:hex, :eqc_ex, "1.4.2", "c89322cf8fbd4f9ddcb18141fb162a871afd357c55c8c0198441ce95ffe2e105", [:mix], [], "hexpm"},
-  "ex_doc": {:hex, :ex_doc, "0.19.1", "519bb9c19526ca51d326c060cb1778d4a9056b190086a8c6c115828eaccea6cf", [:mix], [{:earmark, "~> 1.1", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.7", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"},
-  "excoveralls": {:hex, :excoveralls, "0.10.0", "a4508bdd408829f38e7b2519f234b7fd5c83846099cda348efcb5291b081200c", [:mix], [{:hackney, "~> 1.13", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"},
+  "dialyxir": {:hex, :dialyxir, "1.0.0-rc.3", "774306f84973fc3f1e2e8743eeaa5f5d29b117f3916e5de74c075c02f1b8ef55", [:mix], [], "hexpm", "ddde98a783cec47d1b0ffaf5a0d5fbe09e90e1ac2d28452eefa6f72aac072c5a"},
+  "earmark": {:hex, :earmark, "1.2.5", "4d21980d5d2862a2e13ec3c49ad9ad783ffc7ca5769cf6ff891a4553fbaae761", [:mix], [], "hexpm", "c57508ddad47dfb8038ca6de1e616e66e9b87313220ac5d9817bc4a4dc2257b9"},
+  "eqc_ex": {:hex, :eqc_ex, "1.4.2", "c89322cf8fbd4f9ddcb18141fb162a871afd357c55c8c0198441ce95ffe2e105", [:mix], [], "hexpm", "6547e68351624ca5387df7e3332136b07f1be73c5a429c1b4e40436dcad50f38"},
+  "ex_doc": {:hex, :ex_doc, "0.19.1", "519bb9c19526ca51d326c060cb1778d4a9056b190086a8c6c115828eaccea6cf", [:mix], [{:earmark, "~> 1.1", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.7", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm", "dc87f778d8260da0189a622f62790f6202af72f2f3dee6e78d91a18dd2fcd137"},
+  "excoveralls": {:hex, :excoveralls, "0.10.0", "a4508bdd408829f38e7b2519f234b7fd5c83846099cda348efcb5291b081200c", [:mix], [{:hackney, "~> 1.13", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "74d87b5642251722b94a5bcf493a409c01ebba8962ff79b47365172b11c0280d"},
   "exjsx": {:hex, :exjsx, "4.0.0", "60548841e0212df401e38e63c0078ec57b33e7ea49b032c796ccad8cde794b5c", [:mix], [{:jsx, "~> 2.8.0", [hex: :jsx, repo: "hexpm", optional: false]}], "hexpm"},
-  "gen_state_machine": {:hex, :gen_state_machine, "2.0.5", "9ac15ec6e66acac994cc442dcc2c6f9796cf380ec4b08267223014be1c728a95", [:mix], [], "hexpm"},
-  "hackney": {:hex, :hackney, "1.13.0", "24edc8cd2b28e1c652593833862435c80661834f6c9344e84b6a2255e7aeef03", [:rebar3], [{:certifi, "2.3.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "5.1.2", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "1.0.2", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"},
-  "idna": {:hex, :idna, "5.1.2", "e21cb58a09f0228a9e0b95eaa1217f1bcfc31a1aaa6e1fdf2f53a33f7dbd9494", [:rebar3], [{:unicode_util_compat, "0.3.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm"},
-  "jason": {:hex, :jason, "1.1.1", "d3ccb840dfb06f2f90a6d335b536dd074db748b3e7f5b11ab61d239506585eb2", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm"},
+  "gen_state_machine": {:hex, :gen_state_machine, "2.0.5", "9ac15ec6e66acac994cc442dcc2c6f9796cf380ec4b08267223014be1c728a95", [:mix], [], "hexpm", "5cacd405e72b2609a7e1f891bddb80c53d0b3b7b0036d1648e7382ca108c41c8"},
+  "hackney": {:hex, :hackney, "1.13.0", "24edc8cd2b28e1c652593833862435c80661834f6c9344e84b6a2255e7aeef03", [:rebar3], [{:certifi, "2.3.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "5.1.2", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "1.0.2", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "4d605d33dd07ee1b82b105033cccb02379515105fceb1850746591814b00c205"},
+  "idna": {:hex, :idna, "5.1.2", "e21cb58a09f0228a9e0b95eaa1217f1bcfc31a1aaa6e1fdf2f53a33f7dbd9494", [:rebar3], [{:unicode_util_compat, "0.3.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "8fddb3aec4692c71647d67de72536254bce9069851754e370a99f2aae69fbdf4"},
+  "jason": {:hex, :jason, "1.1.1", "d3ccb840dfb06f2f90a6d335b536dd074db748b3e7f5b11ab61d239506585eb2", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "639645cfac325e34938167b272bae0791fea3a34cf32c29525abf1d323ed4c18"},
   "jsx": {:hex, :jsx, "2.8.3", "a05252d381885240744d955fbe3cf810504eb2567164824e19303ea59eef62cf", [:mix, :rebar3], [], "hexpm"},
-  "makeup": {:hex, :makeup, "0.5.1", "966c5c2296da272d42f1de178c1d135e432662eca795d6dc12e5e8787514edf7", [:mix], [{:nimble_parsec, "~> 0.2.2", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"},
-  "makeup_elixir": {:hex, :makeup_elixir, "0.8.0", "1204a2f5b4f181775a0e456154830524cf2207cf4f9112215c05e0b76e4eca8b", [:mix], [{:makeup, "~> 0.5.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 0.2.2", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"},
-  "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm"},
-  "mimerl": {:hex, :mimerl, "1.0.2", "993f9b0e084083405ed8252b99460c4f0563e41729ab42d9074fd5e52439be88", [:rebar3], [], "hexpm"},
-  "nimble_parsec": {:hex, :nimble_parsec, "0.2.2", "d526b23bdceb04c7ad15b33c57c4526bf5f50aaa70c7c141b4b4624555c68259", [:mix], [], "hexpm"},
-  "parse_trans": {:hex, :parse_trans, "3.2.0", "2adfa4daf80c14dc36f522cf190eb5c4ee3e28008fc6394397c16f62a26258c2", [:rebar3], [], "hexpm"},
-  "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.1", "28a4d65b7f59893bc2c7de786dec1e1555bd742d336043fe644ae956c3497fbe", [:make, :rebar], [], "hexpm"},
-  "unicode_util_compat": {:hex, :unicode_util_compat, "0.3.1", "a1f612a7b512638634a603c8f401892afbf99b8ce93a45041f8aaca99cadb85e", [:rebar3], [], "hexpm"},
+  "makeup": {:hex, :makeup, "0.5.1", "966c5c2296da272d42f1de178c1d135e432662eca795d6dc12e5e8787514edf7", [:mix], [{:nimble_parsec, "~> 0.2.2", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "259748a45dfcf5f49765a7c29c9594791c82de23e22d7a3e6e59533fe8e8935b"},
+  "makeup_elixir": {:hex, :makeup_elixir, "0.8.0", "1204a2f5b4f181775a0e456154830524cf2207cf4f9112215c05e0b76e4eca8b", [:mix], [{:makeup, "~> 0.5.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 0.2.2", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "393d17c5a648e3b30522b2a4743bd1dc3533e1227c8c2823ebe8c3a8e5be5913"},
+  "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"},
+  "mimerl": {:hex, :mimerl, "1.0.2", "993f9b0e084083405ed8252b99460c4f0563e41729ab42d9074fd5e52439be88", [:rebar3], [], "hexpm", "7a4c8e1115a2732a67d7624e28cf6c9f30c66711a9e92928e745c255887ba465"},
+  "nimble_parsec": {:hex, :nimble_parsec, "0.2.2", "d526b23bdceb04c7ad15b33c57c4526bf5f50aaa70c7c141b4b4624555c68259", [:mix], [], "hexpm", "4ababf5c44164f161872704e1cfbecab3935fdebec66c72905abaad0e6e5cef6"},
+  "parse_trans": {:hex, :parse_trans, "3.2.0", "2adfa4daf80c14dc36f522cf190eb5c4ee3e28008fc6394397c16f62a26258c2", [:rebar3], [], "hexpm", "578b1d484720749499db5654091ddac818ea0b6d568f2c99c562d2a6dd4aa117"},
+  "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.1", "28a4d65b7f59893bc2c7de786dec1e1555bd742d336043fe644ae956c3497fbe", [:make, :rebar], [], "hexpm", "4f8805eb5c8a939cf2359367cb651a3180b27dfb48444846be2613d79355d65e"},
+  "unicode_util_compat": {:hex, :unicode_util_compat, "0.3.1", "a1f612a7b512638634a603c8f401892afbf99b8ce93a45041f8aaca99cadb85e", [:rebar3], [], "hexpm", "da1d9bef8a092cc7e1e51f1298037a5ddfb0f657fe862dfe7ba4c5807b551c29"},
 }

From d820c10a4647bc488ebb396a86d628cad8780f13 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Sat, 9 May 2020 10:14:11 +0100
Subject: [PATCH 176/220] Add a Nix Shell file with a known good Elixir version

It seems that the quickcheck test dependency will not play ball with
the most recent versions of Elixir, so I have down graded the Elixir
version to get the tests running again. I will consider switching
QuickCheck to something like StreamData as I am not doing state-full
testing anyways currently.
---
 shell.nix | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)
 create mode 100644 shell.nix

diff --git a/shell.nix b/shell.nix
new file mode 100644
index 00000000..e6531055
--- /dev/null
+++ b/shell.nix
@@ -0,0 +1,27 @@
+{ pkgs ? import <nixpkgs> {} }:
+
+with pkgs;
+
+let
+  inherit (lib) optional optionals;
+
+  erlang_wx = erlangR21.override {
+    wxSupport = true;
+  };
+
+  elixir = (beam.packagesWith erlangR21).elixir.override {
+    version = "1.8.2";
+      rev = "98485daab0a9f3ac2d7809d38f5e57cd73cb22ac";
+      sha256 = "1n77cpcl2b773gmj3m9s24akvj9gph9byqbmj2pvlsmby4aqwckq";
+  };
+in
+
+mkShell {
+  buildInputs = [ elixir git wxmac ]
+    ++ optional stdenv.isLinux inotify-tools # For file_system on Linux.
+    ++ optionals stdenv.isDarwin (with darwin.apple_sdk.frameworks; [
+      # For file_system on macOS.
+      CoreFoundation
+      CoreServices
+    ]);
+}

From 97a6c5ac0fec3e91089bf3b16836b8ebc737aaf9 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Sat, 9 May 2020 10:18:50 +0100
Subject: [PATCH 177/220] Connection will be referenced by their pid, not
 client_id

---
 lib/tortoise/connection.ex        | 47 ++++++++++++-------------------
 test/tortoise/connection_test.exs | 15 ++++------
 2 files changed, 24 insertions(+), 38 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 387632ce..714c8d67 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -79,17 +79,9 @@ defmodule Tortoise.Connection do
       handler: handler
     }
 
-    opts = Keyword.merge(opts, name: via_name(client_id))
     GenStateMachine.start_link(__MODULE__, initial, opts)
   end
 
-  @doc false
-  @spec via_name(Tortoise.client_id()) ::
-          pid() | {:via, Registry, {Tortoise.Registry, {atom(), Tortoise.client_id()}}}
-  def via_name(client_id) do
-    Tortoise.Registry.via_name(__MODULE__, client_id)
-  end
-
   @spec child_spec(Keyword.t()) :: %{
           id: term(),
           start: {__MODULE__, :start_link, [Keyword.t()]},
@@ -112,7 +104,7 @@ defmodule Tortoise.Connection do
   inflight messages and send the proper disconnect message to the
   broker. The session will get terminated on the server.
   """
-  @spec disconnect(Tortoise.client_id(), reason, properties) :: :ok
+  @spec disconnect(pid(), reason, properties) :: :ok
         when reason: Tortoise.Package.Disconnect.reason(),
              properties: [property],
              property:
@@ -121,9 +113,9 @@ defmodule Tortoise.Connection do
                | {:session_expiry_interval, 0..0xFFFFFFFF}
                | {:user_property, {String.t(), String.t()}}
 
-  def disconnect(client_id, reason \\ :normal_disconnection, properties \\ []) do
+  def disconnect(pid, reason \\ :normal_disconnection, properties \\ []) do
     disconnect = %Package.Disconnect{reason: reason, properties: properties}
-    GenStateMachine.call(via_name(client_id), {:disconnect, disconnect})
+    GenStateMachine.call(pid, {:disconnect, disconnect})
   end
 
   @doc """
@@ -132,9 +124,9 @@ defmodule Tortoise.Connection do
   Given the `client_id` of a running connection return its current
   subscriptions. This is helpful in a debugging situation.
   """
-  @spec subscriptions(Tortoise.client_id()) :: Tortoise.Package.Subscribe.t()
-  def subscriptions(client_id) do
-    GenStateMachine.call(via_name(client_id), :subscriptions)
+  @spec subscriptions(pid()) :: Tortoise.Package.Subscribe.t()
+  def subscriptions(pid) do
+    GenStateMachine.call(pid, :subscriptions)
   end
 
   @doc """
@@ -178,7 +170,7 @@ defmodule Tortoise.Connection do
         properties: properties
       })
 
-    GenStateMachine.cast(via_name(pid), {:subscribe, caller, subscribe, opts})
+    GenStateMachine.cast(pid, {:subscribe, caller, subscribe, opts})
     {:ok, ref}
   end
 
@@ -272,7 +264,7 @@ defmodule Tortoise.Connection do
       properties: properties
     }
 
-    GenStateMachine.cast(via_name(pid), {:unsubscribe, caller, unsubscribe, opts})
+    GenStateMachine.cast(pid, {:unsubscribe, caller, unsubscribe, opts})
     {:ok, ref}
   end
 
@@ -339,7 +331,7 @@ defmodule Tortoise.Connection do
   @doc """
   Ping the server and await the ping latency reply.
 
-  Takes a `client_id` and an optional `timeout`.
+  Takes a `pid` and an optional `timeout`.
 
   Like `ping/1` but will block the caller process until a response is
   received from the server. The response will contain the ping latency
@@ -347,12 +339,12 @@ defmodule Tortoise.Connection do
   advisable to specify a reasonable time one is willing to wait for a
   response.
   """
-  @spec ping_sync(Tortoise.client_id(), timeout()) :: {:ok, reference()} | {:error, :timeout}
-  def ping_sync(client_id, timeout \\ :infinity) do
-    {:ok, ref} = ping(client_id)
+  @spec ping_sync(pid(), timeout()) :: {:ok, reference()} | {:error, :timeout}
+  def ping_sync(pid, timeout \\ :infinity) do
+    {:ok, ref} = ping(pid)
 
     receive do
-      {{Tortoise, ^client_id}, {Package.Pingreq, ^ref}, round_trip_time} ->
+      {{Tortoise, ^pid}, {Package.Pingreq, ^ref}, round_trip_time} ->
         {:ok, round_trip_time}
     after
       timeout ->
@@ -363,12 +355,12 @@ defmodule Tortoise.Connection do
   @doc """
   Get the info on the current connection configuration
   """
-  def info(client_id) do
-    GenStateMachine.call(via_name(client_id), :get_info)
+  def info(pid) do
+    GenStateMachine.call(pid, :get_info)
   end
 
   @doc false
-  @spec connection(Tortoise.client_id() | pid(), [opts]) ::
+  @spec connection(pid(), [opts]) ::
           {:ok, {module(), term()}} | {:error, :unknown_connection} | {:error, :timeout}
         when opts: {:timeout, timeout()} | {:active, boolean()}
   def connection(pid, _opts \\ [active: false])
@@ -473,7 +465,6 @@ defmodule Tortoise.Connection do
       ) do
     case Handler.execute_handle_connack(handler, connack) do
       {:ok, %Handler{} = updated_handler, next_actions} when connection_result == :success ->
-        :ok = Tortoise.Registry.put_meta(via_name(client_id), data.connection)
         :ok = Inflight.update_connection(client_id, data.connection)
 
         data = %State{
@@ -952,8 +943,6 @@ defmodule Tortoise.Connection do
 
   # connection logic ===================================================
   def handle_event(:internal, :connect, :connecting, %State{} = data) do
-    :ok = Tortoise.Registry.put_meta(via_name(data.client_id), :connecting)
-
     case await_and_monitor_receiver(data) do
       {:ok, data} ->
         {timeout, updated_data} = Map.get_and_update(data, :backoff, &Backoff.next/1)
@@ -1028,8 +1017,8 @@ defmodule Tortoise.Connection do
   end
 
   # not connected yet
-  def handle_event(:cast, {:ping, {caller_pid, ref}}, _, %State{client_id: client_id}) do
-    send(caller_pid, {{Tortoise, client_id}, {Package.Pingreq, ref}, :not_connected})
+  def handle_event(:cast, {:ping, {caller_pid, ref}}, _, %State{}) do
+    send(caller_pid, {{Tortoise, self()}, {Package.Pingreq, ref}, :not_connected})
     :keep_state_and_data
   end
 
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index 01bf0d13..c4da7968 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -105,7 +105,7 @@ defmodule Tortoise.ConnectionTest do
                 capabilities: %Connection.Info.Capabilities{
                   server_keep_alive: nil
                 }
-              }} = Connection.info(client_id)
+              }} = Connection.info(pid)
 
       send(context.scripted_mqtt_server, :continue)
       assert_receive {ScriptedMqttServer, :completed}
@@ -383,8 +383,6 @@ defmodule Tortoise.ConnectionTest do
     # @todo unsuccessful subscribe
 
     test "successful unsubscribe", %{connection_pid: connection} = context do
-      client_id = context.client_id
-
       unsubscribe_foo = %Package.Unsubscribe{identifier: 2, topics: ["foo"]}
       unsuback_foo = %Package.Unsuback{results: [:success], identifier: 2}
 
@@ -436,13 +434,13 @@ defmodule Tortoise.ConnectionTest do
       # handle_unsuback should get called on the callback handler
       assert_receive {{TestHandler, :handle_unsuback}, {^unsubscribe_foo, ^unsuback_foo}}
 
-      refute Map.has_key?(Tortoise.Connection.subscriptions(client_id), "foo")
+      refute Map.has_key?(Tortoise.Connection.subscriptions(connection), "foo")
       # should still have bar in active subscriptions
-      assert Map.has_key?(Tortoise.Connection.subscriptions(client_id), "bar")
+      assert Map.has_key?(Tortoise.Connection.subscriptions(connection), "bar")
 
       # and unsubscribe from bar
       assert {:ok, ref} =
-               Tortoise.Connection.unsubscribe(client_id, "bar",
+               Tortoise.Connection.unsubscribe(connection, "bar",
                  identifier: 3,
                  user_property: {"foo", "bar"}
                )
@@ -800,9 +798,8 @@ defmodule Tortoise.ConnectionTest do
       # with `{:error, :econnrefused}`, and then it will finally start
       # accepting connections
       Process.flag(:trap_exit, true)
-      client_id = context.client_id
 
-      connect = %Package.Connect{client_id: client_id, clean_start: true}
+      connect = %Package.Connect{client_id: context.client_id, clean_start: true}
       expected_connack = %Package.Connack{reason: :success, session_present: false}
       refusal = {:error, :econnrefused}
 
@@ -825,7 +822,7 @@ defmodule Tortoise.ConnectionTest do
 
       assert {:ok, _pid} =
                Tortoise.Connection.start_link(
-                 client_id: client_id,
+                 client_id: context.client_id,
                  server: {ScriptedTransport, host: 'localhost', port: 1883},
                  backoff: [min_interval: 0],
                  handler: {Tortoise.Handler.Logger, []}

From 487cbb67625d3f0362ee12e0a391e80448ed0b84 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Sat, 9 May 2020 10:49:23 +0100
Subject: [PATCH 178/220] Get rid of EQC Mini, add StreamData as a dependency

Commented out all the previous property tests and inserted skipped
tests in their place. Added StreamData as a dependency.
---
 .formatter.exs                             |   3 +-
 mix.exs                                    |   2 +-
 mix.lock                                   |   1 +
 test/test_helper.exs                       | 960 ++++++++++-----------
 test/tortoise/package/auth_test.exs        |  28 +-
 test/tortoise/package/connack_test.exs     |  28 +-
 test/tortoise/package/connect_test.exs     |  28 +-
 test/tortoise/package/disconnect_test.exs  |  28 +-
 test/tortoise/package/properties_test.exs  |  30 +-
 test/tortoise/package/puback_test.exs      |  28 +-
 test/tortoise/package/pubcomp_test.exs     |  28 +-
 test/tortoise/package/publish_test.exs     |  28 +-
 test/tortoise/package/pubrec_test.exs      |  29 +-
 test/tortoise/package/pubrel_test.exs      |  28 +-
 test/tortoise/package/suback_test.exs      |  29 +-
 test/tortoise/package/subscribe_test.exs   |  87 +-
 test/tortoise/package/unsuback_test.exs    |  28 +-
 test/tortoise/package/unsubscribe_test.exs |  28 +-
 test/tortoise/package_test.exs             |  51 +-
 19 files changed, 760 insertions(+), 712 deletions(-)

diff --git a/.formatter.exs b/.formatter.exs
index 525446d4..9ba91eb7 100644
--- a/.formatter.exs
+++ b/.formatter.exs
@@ -1,4 +1,5 @@
 # Used by "mix format"
 [
-  inputs: ["mix.exs", "{config,lib,test}/**/*.{ex,exs}"]
+  inputs: ["mix.exs", "{config,lib,test}/**/*.{ex,exs}"],
+  import_deps: [:stream_data]
 ]
diff --git a/mix.exs b/mix.exs
index 4565c643..db70f452 100644
--- a/mix.exs
+++ b/mix.exs
@@ -46,7 +46,7 @@ defmodule Tortoise.MixProject do
     [
       {:gen_state_machine, "~> 2.0"},
       {:dialyxir, "~> 1.0.0-rc.3", only: [:dev], runtime: false},
-      {:eqc_ex, "~> 1.4", only: :test},
+      {:stream_data, "~> 0.5", only: [:test, :dev]},
       {:excoveralls, "~> 0.10", only: :test},
       {:ex_doc, "~> 0.19", only: :docs},
       {:ct_helper, github: "ninenines/ct_helper", only: :test}
diff --git a/mix.lock b/mix.lock
index 273deb5c..bf5f88d1 100644
--- a/mix.lock
+++ b/mix.lock
@@ -19,5 +19,6 @@
   "nimble_parsec": {:hex, :nimble_parsec, "0.2.2", "d526b23bdceb04c7ad15b33c57c4526bf5f50aaa70c7c141b4b4624555c68259", [:mix], [], "hexpm", "4ababf5c44164f161872704e1cfbecab3935fdebec66c72905abaad0e6e5cef6"},
   "parse_trans": {:hex, :parse_trans, "3.2.0", "2adfa4daf80c14dc36f522cf190eb5c4ee3e28008fc6394397c16f62a26258c2", [:rebar3], [], "hexpm", "578b1d484720749499db5654091ddac818ea0b6d568f2c99c562d2a6dd4aa117"},
   "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.1", "28a4d65b7f59893bc2c7de786dec1e1555bd742d336043fe644ae956c3497fbe", [:make, :rebar], [], "hexpm", "4f8805eb5c8a939cf2359367cb651a3180b27dfb48444846be2613d79355d65e"},
+  "stream_data": {:hex, :stream_data, "0.5.0", "b27641e58941685c75b353577dc602c9d2c12292dd84babf506c2033cd97893e", [:mix], [], "hexpm", "012bd2eec069ada4db3411f9115ccafa38540a3c78c4c0349f151fc761b9e271"},
   "unicode_util_compat": {:hex, :unicode_util_compat, "0.3.1", "a1f612a7b512638634a603c8f401892afbf99b8ce93a45041f8aaca99cadb85e", [:rebar3], [], "hexpm", "da1d9bef8a092cc7e1e51f1298037a5ddfb0f657fe862dfe7ba4c5807b551c29"},
 }
diff --git a/test/test_helper.exs b/test/test_helper.exs
index ace54b01..c572c72d 100644
--- a/test/test_helper.exs
+++ b/test/test_helper.exs
@@ -3,486 +3,486 @@ defmodule Tortoise.TestGenerators do
   EQC generators for generating variables and data structures useful
   for testing MQTT
   """
-  use EQC.ExUnit
-
-  alias Tortoise.Package
-
-  def gen_topic() do
-    let topic_list <- non_empty(list(5, gen_topic_level())) do
-      Enum.join(topic_list, "/")
-    end
-  end
-
-  def gen_topic_filter() do
-    let topic_list <- non_empty(list(5, gen_topic_level())) do
-      let {_matching?, filter} <- gen_filter_from_topic(topic_list) do
-        Enum.join(filter, "/")
-      end
-    end
-  end
-
-  defp gen_topic_level() do
-    such_that topic <- non_empty(utf8()) do
-      not String.contains?(topic, ["/", "+", "#"])
-    end
-  end
-
-  # - - -
-  defp gen_filter_from_topic(topic) do
-    {_matching?, _filter} = gen_filter(:cont, true, topic, [])
-  end
-
-  defp gen_filter(_status, matching?, _, ["#" | _] = acc) do
-    let(result <- Enum.reverse(acc), do: {matching?, result})
-  end
-
-  defp gen_filter(:stop, matching?, [t], acc) do
-    let(result <- Enum.reverse([t | acc]), do: {matching?, result})
-  end
-
-  defp gen_filter(:stop, _matching?, _topic_list, acc) do
-    let(result <- Enum.reverse(acc), do: {false, result})
-  end
-
-  defp gen_filter(status, matching?, [], acc) do
-    frequency([
-      {20, {matching?, lazy(do: Enum.reverse(acc))}},
-      {5, gen_extra_filter_topic(status, matching?, acc)}
-    ])
-  end
-
-  defp gen_filter(status, matching?, [t | ts], acc) do
-    frequency([
-      # keep
-      {15, gen_filter(status, matching?, ts, [t | acc])},
-      # one level filter
-      {20, gen_filter(status, matching?, ts, ["+" | acc])},
-      # mutate
-      {10, gen_filter(status, false, ts, [alter_topic_level(t) | acc])},
-      # multi-level filter
-      {5, gen_filter(status, matching?, [], ["#" | acc])},
-      # early bail out
-      {5, gen_filter(:stop, matching?, [t | ts], acc)}
-    ])
-  end
-
-  defp gen_extra_filter_topic(status, _matching?, acc) do
-    let extra_topic <- gen_topic_level() do
-      gen_filter(status, false, [extra_topic], acc)
-    end
-  end
-
-  # Given a specific topic level return a different one
-  defp alter_topic_level(topic_level) do
-    such_that mutation <- gen_topic_level() do
-      mutation != topic_level
-    end
-  end
-
-  # --------------------------------------------------------------------
-  def gen_identifier() do
-    choose(0x0001, 0xFFFF)
-  end
-
-  def gen_qos() do
-    choose(0, 2)
-  end
-
-  @doc """
-  Generate a valid connect message
-  """
-  def gen_connect() do
-    let will <-
-          oneof([
-            nil,
-            %Package.Publish{
-              topic: gen_topic(),
-              payload: oneof([non_empty(binary()), nil]),
-              qos: gen_qos(),
-              retain: bool(),
-              properties: [receive_maximum: 201]
-            }
-          ]) do
-      # zero byte client id is allowed, but clean session should be set to true
-      let connect <- %Package.Connect{
-            # The Server MUST allow ClientIds which are between 1 and 23
-            # UTF-8 encoded bytes in length, and that contain only the
-            # characters
-            # "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
-            # [MQTT-3.1.3-5].
-
-            # The Server MAY allow ClientId’s that contain more than 23
-            # encoded bytes. The Server MAY allow ClientId’s that contain
-            # characters not included in the list given above.
-            client_id: binary(),
-            user_name: oneof([nil, utf8()]),
-            password: oneof([nil, utf8()]),
-            clean_start: bool(),
-            keep_alive: choose(0, 65535),
-            will: will,
-            properties: [receive_maximum: 201]
-          } do
-        connect
-      end
-    end
-  end
-
-  @doc """
-  Generate a valid connack (connection acknowledgement) message
-  """
-  def gen_connack() do
-    let connack <- %Package.Connack{
-          session_present: bool(),
-          reason:
-            oneof([
-              :success,
-              {:refused, :unspecified_error},
-              {:refused, :malformed_packet},
-              {:refused, :protocol_error},
-              {:refused, :implementation_specific_error},
-              {:refused, :unsupported_protocol_version},
-              {:refused, :client_identifier_not_valid},
-              {:refused, :bad_user_name_or_password},
-              {:refused, :not_authorized},
-              {:refused, :server_unavailable},
-              {:refused, :server_busy},
-              {:refused, :banned},
-              {:refused, :bad_authentication_method},
-              {:refused, :topic_name_invalid},
-              {:refused, :packet_too_large},
-              {:refused, :quota_exceeded},
-              {:refused, :payload_format_invalid},
-              {:refused, :retain_not_supported},
-              {:refused, :qos_not_supported},
-              {:refused, :use_another_server},
-              {:refused, :server_moved},
-              {:refused, :connection_rate_exceeded}
-            ])
-        } do
-      connack
-    end
-  end
-
-  @doc """
-  Generate a valid publish message.
-
-  A publish message with a quality of zero will not have an identifier
-  or ever be a duplicate message, so we generate the quality of
-  service first and decide if we should generate values for those
-  values depending on the value of the generated QoS.
-  """
-  def gen_publish() do
-    let qos <- gen_qos() do
-      %{
-        do_gen_publish(qos)
-        | topic: gen_topic(),
-          payload: oneof([non_empty(binary()), nil]),
-          retain: bool()
-      }
-      |> gen_publish_properties()
-    end
-  end
-
-  defp do_gen_publish(0) do
-    %Package.Publish{identifier: nil, qos: 0, dup: false}
-  end
-
-  defp do_gen_publish(qos) do
-    %Package.Publish{
-      identifier: gen_identifier(),
-      qos: qos,
-      dup: bool()
-    }
-  end
-
-  defp gen_publish_properties(%Package.Publish{} = publish) do
-    allowed_properties = [
-      :payload_format_indicator,
-      :message_expiry_interval,
-      :topic_alias,
-      :response_topic,
-      :correlation_data,
-      :user_property,
-      :subscription_identifier,
-      :content_type
-    ]
-
-    let properties <- list(5, oneof(allowed_properties)) do
-      # @todo only user_properties and subscription_identifiers are allowed multiple times
-      properties = Enum.map(properties, &gen_property_value/1)
-      %Package.Publish{publish | properties: properties}
-    end
-  end
-
-  @doc """
-  Generate a valid subscribe message.
-
-  The message will get populated with one or more topic filters, each
-  with a quality of service between 0 and 2.
-  """
-  def gen_subscribe() do
-    let subscribe <- %Package.Subscribe{
-          identifier: gen_identifier(),
-          topics: non_empty(list({gen_topic_filter(), gen_subscribe_opts()})),
-          # todo, add properties
-          properties: []
-        } do
-      subscribe
-    end
-  end
-
-  # @todo improve this generator
-  def gen_subscribe_opts() do
-    let {qos, no_local, retain_as_published, retain_handling} <-
-          {gen_qos(), bool(), bool(), choose(0, 3)} do
-      [
-        qos: qos,
-        no_local: no_local,
-        retain_as_published: retain_as_published,
-        retain_handling: retain_handling
-      ]
-    end
-  end
-
-  def gen_suback() do
-    let suback <- %Package.Suback{
-          identifier: choose(0x0001, 0xFFFF),
-          acks:
-            non_empty(
-              list(
-                oneof([
-                  {:ok, gen_qos()},
-                  {:error,
-                   oneof([
-                     :unspecified_error,
-                     :implementation_specific_error,
-                     :not_authorized,
-                     :topic_filter_invalid,
-                     :packet_identifier_in_use,
-                     :quota_exceeded,
-                     :shared_subscriptions_not_supported,
-                     :subscription_identifiers_not_supported,
-                     :wildcard_subscriptions_not_supported
-                   ])}
-                ])
-              )
-            ),
-          # todo, add generators for [:reason_string, :user_property]
-          properties: []
-        } do
-      suback
-    end
-  end
-
-  @doc """
-  Generate a valid unsubscribe message.
-  """
-  def gen_unsubscribe() do
-    let unsubscribe <- %Package.Unsubscribe{
-          identifier: gen_identifier(),
-          topics: non_empty(list(gen_topic_filter())),
-          properties: []
-        } do
-      unsubscribe
-    end
-  end
-
-  def gen_unsuback() do
-    let unsuback <- %Package.Unsuback{
-          identifier: gen_identifier(),
-          results:
-            non_empty(
-              list(
-                oneof([
-                  :success,
-                  {:error,
-                   oneof([
-                     :no_subscription_existed,
-                     :unspecified_error,
-                     :implementation_specific_error,
-                     :not_authorized,
-                     :topic_filter_invalid,
-                     :packet_identifier_in_use
-                   ])}
-                ])
-              )
-            ),
-          # todo, generate :reason_string and :user_property
-          properties: []
-        } do
-      unsuback
-    end
-  end
-
-  def gen_puback() do
-    # todo, make this generator generate properties and other reasons
-    let puback <- %Package.Puback{
-          identifier: gen_identifier(),
-          reason: :success,
-          properties: []
-        } do
-      puback
-    end
-  end
-
-  def gen_pubcomp() do
-    let pubcomp <- %Package.Pubcomp{
-          identifier: gen_identifier(),
-          reason: {:refused, :packet_identifier_not_found},
-          properties: []
-        } do
-      pubcomp
-    end
-  end
-
-  def gen_pubrel() do
-    # todo, improve this generator
-    let pubrel <- %Package.Pubrel{
-          identifier: gen_identifier(),
-          reason: :success,
-          properties: []
-        } do
-      pubrel
-    end
-  end
-
-  def gen_pubrec() do
-    # todo, improve this generator
-    let pubrec <- %Package.Pubrec{
-          identifier: gen_identifier(),
-          reason: :success,
-          properties: []
-        } do
-      pubrec
-    end
-  end
-
-  def gen_disconnect() do
-    let disconnect <-
-          %Package.Disconnect{
-            reason:
-              oneof([
-                :normal_disconnection,
-                :disconnect_with_will_message,
-                :unspecified_error,
-                :malformed_packet,
-                :protocol_error,
-                :implementation_specific_error,
-                :not_authorized,
-                :server_busy,
-                :server_shutting_down,
-                :keep_alive_timeout,
-                :session_taken_over,
-                :topic_filter_invalid,
-                :topic_name_invalid,
-                :receive_maximum_exceeded,
-                :topic_alias_invalid,
-                :packet_too_large,
-                :message_rate_too_high,
-                :quota_exceeded,
-                :administrative_action,
-                :payload_format_invalid,
-                :retain_not_supported,
-                :qos_not_supported,
-                :use_another_server,
-                :server_moved,
-                :shared_subscriptions_not_supported,
-                :connection_rate_exceeded,
-                :maximum_connect_time,
-                :subscription_identifiers_not_supported,
-                :wildcard_subscriptions_not_supported
-              ])
-          } do
-      %Package.Disconnect{disconnect | properties: gen_properties(disconnect)}
-    end
-  end
-
-  def gen_auth() do
-    let auth <-
-          %Package.Auth{
-            reason: oneof([:success, :continue_authentication, :re_authenticate])
-          } do
-      %Package.Auth{auth | properties: gen_properties(auth)}
-    end
-  end
-
-  def gen_properties(%Package.Disconnect{reason: :normal_disconnection}) do
-    []
-  end
-
-  def gen_properties(%{}) do
-    []
-  end
-
-  def gen_properties() do
-    let properties <-
-          list(
-            5,
-            oneof([
-              :payload_format_indicator,
-              :message_expiry_interval,
-              :content_type,
-              :response_topic,
-              :correlation_data,
-              :subscription_identifier,
-              :session_expiry_interval,
-              :assigned_client_identifier,
-              :server_keep_alive,
-              :authentication_method,
-              :authentication_data,
-              :request_problem_information,
-              :will_delay_interval,
-              :request_response_information,
-              :response_information,
-              :server_reference,
-              :reason_string,
-              :receive_maximum,
-              :topic_alias_maximum,
-              :topic_alias,
-              :maximum_qos,
-              :retain_available,
-              :user_property,
-              :maximum_packet_size,
-              :wildcard_subscription_available,
-              :subscription_identifiers_available,
-              :shared_subscription_available
-            ])
-          ) do
-      Enum.map(properties, &gen_property_value/1)
-    end
-  end
-
-  def gen_property_value(type) do
-    case type do
-      :payload_format_indicator -> {type, oneof([0, 1])}
-      :message_expiry_interval -> {type, choose(0, 4_294_967_295)}
-      :content_type -> {type, utf8()}
-      :response_topic -> {type, gen_topic()}
-      :correlation_data -> {type, binary()}
-      :subscription_identifier -> {type, choose(1, 268_435_455)}
-      :session_expiry_interval -> {type, choose(1, 268_435_455)}
-      :assigned_client_identifier -> {type, utf8()}
-      :server_keep_alive -> {type, choose(0x0000, 0xFFFF)}
-      :authentication_method -> {type, utf8()}
-      :authentication_data -> {type, binary()}
-      :request_problem_information -> {type, bool()}
-      :will_delay_interval -> {type, choose(0, 4_294_967_295)}
-      :request_response_information -> {type, bool()}
-      :response_information -> {type, utf8()}
-      :server_reference -> {type, utf8()}
-      :reason_string -> {type, utf8()}
-      :receive_maximum -> {type, choose(0x0001, 0xFFFF)}
-      :topic_alias_maximum -> {type, choose(0x0000, 0xFFFF)}
-      :topic_alias -> {type, choose(0x0001, 0xFFFF)}
-      :maximum_qos -> {type, oneof([0, 1])}
-      :retain_available -> {type, bool()}
-      :user_property -> {type, {utf8(), utf8()}}
-      :maximum_packet_size -> {type, choose(1, 268_435_455)}
-      :wildcard_subscription_available -> {type, bool()}
-      :subscription_identifiers_available -> {type, bool()}
-      :shared_subscription_available -> {type, bool()}
-    end
-  end
+  # use EQC.ExUnit
+
+  # alias Tortoise.Package
+
+  # def gen_topic() do
+  #   let topic_list <- non_empty(list(5, gen_topic_level())) do
+  #     Enum.join(topic_list, "/")
+  #   end
+  # end
+
+  # def gen_topic_filter() do
+  #   let topic_list <- non_empty(list(5, gen_topic_level())) do
+  #     let {_matching?, filter} <- gen_filter_from_topic(topic_list) do
+  #       Enum.join(filter, "/")
+  #     end
+  #   end
+  # end
+
+  # defp gen_topic_level() do
+  #   such_that topic <- non_empty(utf8()) do
+  #     not String.contains?(topic, ["/", "+", "#"])
+  #   end
+  # end
+
+  # # - - -
+  # defp gen_filter_from_topic(topic) do
+  #   {_matching?, _filter} = gen_filter(:cont, true, topic, [])
+  # end
+
+  # defp gen_filter(_status, matching?, _, ["#" | _] = acc) do
+  #   let(result <- Enum.reverse(acc), do: {matching?, result})
+  # end
+
+  # defp gen_filter(:stop, matching?, [t], acc) do
+  #   let(result <- Enum.reverse([t | acc]), do: {matching?, result})
+  # end
+
+  # defp gen_filter(:stop, _matching?, _topic_list, acc) do
+  #   let(result <- Enum.reverse(acc), do: {false, result})
+  # end
+
+  # defp gen_filter(status, matching?, [], acc) do
+  #   frequency([
+  #     {20, {matching?, lazy(do: Enum.reverse(acc))}},
+  #     {5, gen_extra_filter_topic(status, matching?, acc)}
+  #   ])
+  # end
+
+  # defp gen_filter(status, matching?, [t | ts], acc) do
+  #   frequency([
+  #     # keep
+  #     {15, gen_filter(status, matching?, ts, [t | acc])},
+  #     # one level filter
+  #     {20, gen_filter(status, matching?, ts, ["+" | acc])},
+  #     # mutate
+  #     {10, gen_filter(status, false, ts, [alter_topic_level(t) | acc])},
+  #     # multi-level filter
+  #     {5, gen_filter(status, matching?, [], ["#" | acc])},
+  #     # early bail out
+  #     {5, gen_filter(:stop, matching?, [t | ts], acc)}
+  #   ])
+  # end
+
+  # defp gen_extra_filter_topic(status, _matching?, acc) do
+  #   let extra_topic <- gen_topic_level() do
+  #     gen_filter(status, false, [extra_topic], acc)
+  #   end
+  # end
+
+  # # Given a specific topic level return a different one
+  # defp alter_topic_level(topic_level) do
+  #   such_that mutation <- gen_topic_level() do
+  #     mutation != topic_level
+  #   end
+  # end
+
+  # # --------------------------------------------------------------------
+  # def gen_identifier() do
+  #   choose(0x0001, 0xFFFF)
+  # end
+
+  # def gen_qos() do
+  #   choose(0, 2)
+  # end
+
+  # @doc """
+  # Generate a valid connect message
+  # """
+  # def gen_connect() do
+  #   let will <-
+  #         oneof([
+  #           nil,
+  #           %Package.Publish{
+  #             topic: gen_topic(),
+  #             payload: oneof([non_empty(binary()), nil]),
+  #             qos: gen_qos(),
+  #             retain: bool(),
+  #             properties: [receive_maximum: 201]
+  #           }
+  #         ]) do
+  #     # zero byte client id is allowed, but clean session should be set to true
+  #     let connect <- %Package.Connect{
+  #           # The Server MUST allow ClientIds which are between 1 and 23
+  #           # UTF-8 encoded bytes in length, and that contain only the
+  #           # characters
+  #           # "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+  #           # [MQTT-3.1.3-5].
+
+  #           # The Server MAY allow ClientId’s that contain more than 23
+  #           # encoded bytes. The Server MAY allow ClientId’s that contain
+  #           # characters not included in the list given above.
+  #           client_id: binary(),
+  #           user_name: oneof([nil, utf8()]),
+  #           password: oneof([nil, utf8()]),
+  #           clean_start: bool(),
+  #           keep_alive: choose(0, 65535),
+  #           will: will,
+  #           properties: [receive_maximum: 201]
+  #         } do
+  #       connect
+  #     end
+  #   end
+  # end
+
+  # @doc """
+  # Generate a valid connack (connection acknowledgement) message
+  # """
+  # def gen_connack() do
+  #   let connack <- %Package.Connack{
+  #         session_present: bool(),
+  #         reason:
+  #           oneof([
+  #             :success,
+  #             {:refused, :unspecified_error},
+  #             {:refused, :malformed_packet},
+  #             {:refused, :protocol_error},
+  #             {:refused, :implementation_specific_error},
+  #             {:refused, :unsupported_protocol_version},
+  #             {:refused, :client_identifier_not_valid},
+  #             {:refused, :bad_user_name_or_password},
+  #             {:refused, :not_authorized},
+  #             {:refused, :server_unavailable},
+  #             {:refused, :server_busy},
+  #             {:refused, :banned},
+  #             {:refused, :bad_authentication_method},
+  #             {:refused, :topic_name_invalid},
+  #             {:refused, :packet_too_large},
+  #             {:refused, :quota_exceeded},
+  #             {:refused, :payload_format_invalid},
+  #             {:refused, :retain_not_supported},
+  #             {:refused, :qos_not_supported},
+  #             {:refused, :use_another_server},
+  #             {:refused, :server_moved},
+  #             {:refused, :connection_rate_exceeded}
+  #           ])
+  #       } do
+  #     connack
+  #   end
+  # end
+
+  # @doc """
+  # Generate a valid publish message.
+
+  # A publish message with a quality of zero will not have an identifier
+  # or ever be a duplicate message, so we generate the quality of
+  # service first and decide if we should generate values for those
+  # values depending on the value of the generated QoS.
+  # """
+  # def gen_publish() do
+  #   let qos <- gen_qos() do
+  #     %{
+  #       do_gen_publish(qos)
+  #       | topic: gen_topic(),
+  #         payload: oneof([non_empty(binary()), nil]),
+  #         retain: bool()
+  #     }
+  #     |> gen_publish_properties()
+  #   end
+  # end
+
+  # defp do_gen_publish(0) do
+  #   %Package.Publish{identifier: nil, qos: 0, dup: false}
+  # end
+
+  # defp do_gen_publish(qos) do
+  #   %Package.Publish{
+  #     identifier: gen_identifier(),
+  #     qos: qos,
+  #     dup: bool()
+  #   }
+  # end
+
+  # defp gen_publish_properties(%Package.Publish{} = publish) do
+  #   allowed_properties = [
+  #     :payload_format_indicator,
+  #     :message_expiry_interval,
+  #     :topic_alias,
+  #     :response_topic,
+  #     :correlation_data,
+  #     :user_property,
+  #     :subscription_identifier,
+  #     :content_type
+  #   ]
+
+  #   let properties <- list(5, oneof(allowed_properties)) do
+  #     # @todo only user_properties and subscription_identifiers are allowed multiple times
+  #     properties = Enum.map(properties, &gen_property_value/1)
+  #     %Package.Publish{publish | properties: properties}
+  #   end
+  # end
+
+  # @doc """
+  # Generate a valid subscribe message.
+
+  # The message will get populated with one or more topic filters, each
+  # with a quality of service between 0 and 2.
+  # """
+  # def gen_subscribe() do
+  #   let subscribe <- %Package.Subscribe{
+  #         identifier: gen_identifier(),
+  #         topics: non_empty(list({gen_topic_filter(), gen_subscribe_opts()})),
+  #         # todo, add properties
+  #         properties: []
+  #       } do
+  #     subscribe
+  #   end
+  # end
+
+  # # @todo improve this generator
+  # def gen_subscribe_opts() do
+  #   let {qos, no_local, retain_as_published, retain_handling} <-
+  #         {gen_qos(), bool(), bool(), choose(0, 3)} do
+  #     [
+  #       qos: qos,
+  #       no_local: no_local,
+  #       retain_as_published: retain_as_published,
+  #       retain_handling: retain_handling
+  #     ]
+  #   end
+  # end
+
+  # def gen_suback() do
+  #   let suback <- %Package.Suback{
+  #         identifier: choose(0x0001, 0xFFFF),
+  #         acks:
+  #           non_empty(
+  #             list(
+  #               oneof([
+  #                 {:ok, gen_qos()},
+  #                 {:error,
+  #                  oneof([
+  #                    :unspecified_error,
+  #                    :implementation_specific_error,
+  #                    :not_authorized,
+  #                    :topic_filter_invalid,
+  #                    :packet_identifier_in_use,
+  #                    :quota_exceeded,
+  #                    :shared_subscriptions_not_supported,
+  #                    :subscription_identifiers_not_supported,
+  #                    :wildcard_subscriptions_not_supported
+  #                  ])}
+  #               ])
+  #             )
+  #           ),
+  #         # todo, add generators for [:reason_string, :user_property]
+  #         properties: []
+  #       } do
+  #     suback
+  #   end
+  # end
+
+  # @doc """
+  # Generate a valid unsubscribe message.
+  # """
+  # def gen_unsubscribe() do
+  #   let unsubscribe <- %Package.Unsubscribe{
+  #         identifier: gen_identifier(),
+  #         topics: non_empty(list(gen_topic_filter())),
+  #         properties: []
+  #       } do
+  #     unsubscribe
+  #   end
+  # end
+
+  # def gen_unsuback() do
+  #   let unsuback <- %Package.Unsuback{
+  #         identifier: gen_identifier(),
+  #         results:
+  #           non_empty(
+  #             list(
+  #               oneof([
+  #                 :success,
+  #                 {:error,
+  #                  oneof([
+  #                    :no_subscription_existed,
+  #                    :unspecified_error,
+  #                    :implementation_specific_error,
+  #                    :not_authorized,
+  #                    :topic_filter_invalid,
+  #                    :packet_identifier_in_use
+  #                  ])}
+  #               ])
+  #             )
+  #           ),
+  #         # todo, generate :reason_string and :user_property
+  #         properties: []
+  #       } do
+  #     unsuback
+  #   end
+  # end
+
+  # def gen_puback() do
+  #   # todo, make this generator generate properties and other reasons
+  #   let puback <- %Package.Puback{
+  #         identifier: gen_identifier(),
+  #         reason: :success,
+  #         properties: []
+  #       } do
+  #     puback
+  #   end
+  # end
+
+  # def gen_pubcomp() do
+  #   let pubcomp <- %Package.Pubcomp{
+  #         identifier: gen_identifier(),
+  #         reason: {:refused, :packet_identifier_not_found},
+  #         properties: []
+  #       } do
+  #     pubcomp
+  #   end
+  # end
+
+  # def gen_pubrel() do
+  #   # todo, improve this generator
+  #   let pubrel <- %Package.Pubrel{
+  #         identifier: gen_identifier(),
+  #         reason: :success,
+  #         properties: []
+  #       } do
+  #     pubrel
+  #   end
+  # end
+
+  # def gen_pubrec() do
+  #   # todo, improve this generator
+  #   let pubrec <- %Package.Pubrec{
+  #         identifier: gen_identifier(),
+  #         reason: :success,
+  #         properties: []
+  #       } do
+  #     pubrec
+  #   end
+  # end
+
+  # def gen_disconnect() do
+  #   let disconnect <-
+  #         %Package.Disconnect{
+  #           reason:
+  #             oneof([
+  #               :normal_disconnection,
+  #               :disconnect_with_will_message,
+  #               :unspecified_error,
+  #               :malformed_packet,
+  #               :protocol_error,
+  #               :implementation_specific_error,
+  #               :not_authorized,
+  #               :server_busy,
+  #               :server_shutting_down,
+  #               :keep_alive_timeout,
+  #               :session_taken_over,
+  #               :topic_filter_invalid,
+  #               :topic_name_invalid,
+  #               :receive_maximum_exceeded,
+  #               :topic_alias_invalid,
+  #               :packet_too_large,
+  #               :message_rate_too_high,
+  #               :quota_exceeded,
+  #               :administrative_action,
+  #               :payload_format_invalid,
+  #               :retain_not_supported,
+  #               :qos_not_supported,
+  #               :use_another_server,
+  #               :server_moved,
+  #               :shared_subscriptions_not_supported,
+  #               :connection_rate_exceeded,
+  #               :maximum_connect_time,
+  #               :subscription_identifiers_not_supported,
+  #               :wildcard_subscriptions_not_supported
+  #             ])
+  #         } do
+  #     %Package.Disconnect{disconnect | properties: gen_properties(disconnect)}
+  #   end
+  # end
+
+  # def gen_auth() do
+  #   let auth <-
+  #         %Package.Auth{
+  #           reason: oneof([:success, :continue_authentication, :re_authenticate])
+  #         } do
+  #     %Package.Auth{auth | properties: gen_properties(auth)}
+  #   end
+  # end
+
+  # def gen_properties(%Package.Disconnect{reason: :normal_disconnection}) do
+  #   []
+  # end
+
+  # def gen_properties(%{}) do
+  #   []
+  # end
+
+  # def gen_properties() do
+  #   let properties <-
+  #         list(
+  #           5,
+  #           oneof([
+  #             :payload_format_indicator,
+  #             :message_expiry_interval,
+  #             :content_type,
+  #             :response_topic,
+  #             :correlation_data,
+  #             :subscription_identifier,
+  #             :session_expiry_interval,
+  #             :assigned_client_identifier,
+  #             :server_keep_alive,
+  #             :authentication_method,
+  #             :authentication_data,
+  #             :request_problem_information,
+  #             :will_delay_interval,
+  #             :request_response_information,
+  #             :response_information,
+  #             :server_reference,
+  #             :reason_string,
+  #             :receive_maximum,
+  #             :topic_alias_maximum,
+  #             :topic_alias,
+  #             :maximum_qos,
+  #             :retain_available,
+  #             :user_property,
+  #             :maximum_packet_size,
+  #             :wildcard_subscription_available,
+  #             :subscription_identifiers_available,
+  #             :shared_subscription_available
+  #           ])
+  #         ) do
+  #     Enum.map(properties, &gen_property_value/1)
+  #   end
+  # end
+
+  # def gen_property_value(type) do
+  #   case type do
+  #     :payload_format_indicator -> {type, oneof([0, 1])}
+  #     :message_expiry_interval -> {type, choose(0, 4_294_967_295)}
+  #     :content_type -> {type, utf8()}
+  #     :response_topic -> {type, gen_topic()}
+  #     :correlation_data -> {type, binary()}
+  #     :subscription_identifier -> {type, choose(1, 268_435_455)}
+  #     :session_expiry_interval -> {type, choose(1, 268_435_455)}
+  #     :assigned_client_identifier -> {type, utf8()}
+  #     :server_keep_alive -> {type, choose(0x0000, 0xFFFF)}
+  #     :authentication_method -> {type, utf8()}
+  #     :authentication_data -> {type, binary()}
+  #     :request_problem_information -> {type, bool()}
+  #     :will_delay_interval -> {type, choose(0, 4_294_967_295)}
+  #     :request_response_information -> {type, bool()}
+  #     :response_information -> {type, utf8()}
+  #     :server_reference -> {type, utf8()}
+  #     :reason_string -> {type, utf8()}
+  #     :receive_maximum -> {type, choose(0x0001, 0xFFFF)}
+  #     :topic_alias_maximum -> {type, choose(0x0000, 0xFFFF)}
+  #     :topic_alias -> {type, choose(0x0001, 0xFFFF)}
+  #     :maximum_qos -> {type, oneof([0, 1])}
+  #     :retain_available -> {type, bool()}
+  #     :user_property -> {type, {utf8(), utf8()}}
+  #     :maximum_packet_size -> {type, choose(1, 268_435_455)}
+  #     :wildcard_subscription_available -> {type, bool()}
+  #     :subscription_identifiers_available -> {type, bool()}
+  #     :shared_subscription_available -> {type, bool()}
+  #   end
+  # end
 end
 
 # make certs for tests using the SSL transport
diff --git a/test/tortoise/package/auth_test.exs b/test/tortoise/package/auth_test.exs
index 907cf487..fe777119 100644
--- a/test/tortoise/package/auth_test.exs
+++ b/test/tortoise/package/auth_test.exs
@@ -1,20 +1,22 @@
 defmodule Tortoise.Package.AuthTest do
   use ExUnit.Case
-  use EQC.ExUnit
+  # use EQC.ExUnit
   doctest Tortoise.Package.Auth
 
-  alias Tortoise.Package
+  # alias Tortoise.Package
 
-  import Tortoise.TestGenerators, only: [gen_auth: 0]
+  # import Tortoise.TestGenerators, only: [gen_auth: 0]
 
-  property "encoding and decoding auth messages" do
-    forall auth <- gen_auth() do
-      ensure(
-        auth ==
-          auth
-          |> Package.encode()
-          |> Package.decode()
-      )
-    end
-  end
+  # property "encoding and decoding auth messages" do
+  #   forall auth <- gen_auth() do
+  #     ensure(
+  #       auth ==
+  #         auth
+  #         |> Package.encode()
+  #         |> Package.decode()
+  #     )
+  #   end
+  # end
+  @tag :skip
+  test "encoding and decoding auth messages"
 end
diff --git a/test/tortoise/package/connack_test.exs b/test/tortoise/package/connack_test.exs
index 561ae32f..cb80a65b 100644
--- a/test/tortoise/package/connack_test.exs
+++ b/test/tortoise/package/connack_test.exs
@@ -1,20 +1,22 @@
 defmodule Tortoise.Package.ConnackTest do
   use ExUnit.Case
-  use EQC.ExUnit
+  # use EQC.ExUnit
   doctest Tortoise.Package.Connack
 
-  import Tortoise.TestGenerators, only: [gen_connack: 0]
+  # import Tortoise.TestGenerators, only: [gen_connack: 0]
 
-  alias Tortoise.Package
+  # alias Tortoise.Package
 
-  property "encoding and decoding connack messages" do
-    forall connack <- gen_connack() do
-      ensure(
-        connack ==
-          connack
-          |> Package.encode()
-          |> Package.decode()
-      )
-    end
-  end
+  # property "encoding and decoding connack messages" do
+  #   forall connack <- gen_connack() do
+  #     ensure(
+  #       connack ==
+  #         connack
+  #         |> Package.encode()
+  #         |> Package.decode()
+  #     )
+  #   end
+  # end
+  @tag :skip
+  test "encoding and decoding connack messages"
 end
diff --git a/test/tortoise/package/connect_test.exs b/test/tortoise/package/connect_test.exs
index 47290eca..4aa25cfb 100644
--- a/test/tortoise/package/connect_test.exs
+++ b/test/tortoise/package/connect_test.exs
@@ -1,20 +1,22 @@
 defmodule Tortoise.Package.ConnectTest do
   use ExUnit.Case
-  use EQC.ExUnit
+  # use EQC.ExUnit
   doctest Tortoise.Package.Connect
 
-  alias Tortoise.Package
+  # alias Tortoise.Package
 
-  import Tortoise.TestGenerators, only: [gen_connect: 0]
+  # import Tortoise.TestGenerators, only: [gen_connect: 0]
 
-  property "encoding and decoding connect messages" do
-    forall connect <- gen_connect() do
-      ensure(
-        connect ==
-          connect
-          |> Package.encode()
-          |> Package.decode()
-      )
-    end
-  end
+  # property "encoding and decoding connect messages" do
+  #   forall connect <- gen_connect() do
+  #     ensure(
+  #       connect ==
+  #         connect
+  #         |> Package.encode()
+  #         |> Package.decode()
+  #     )
+  #   end
+  # end
+  @tag :skip
+  test "encoding and decoding connect messages"
 end
diff --git a/test/tortoise/package/disconnect_test.exs b/test/tortoise/package/disconnect_test.exs
index 620a19f4..93b9b50b 100644
--- a/test/tortoise/package/disconnect_test.exs
+++ b/test/tortoise/package/disconnect_test.exs
@@ -1,20 +1,22 @@
 defmodule Tortoise.Package.DisconnectTest do
   use ExUnit.Case
-  use EQC.ExUnit
+  # use EQC.ExUnit
   doctest Tortoise.Package.Disconnect
 
-  alias Tortoise.Package
+  # alias Tortoise.Package
 
-  import Tortoise.TestGenerators, only: [gen_disconnect: 0]
+  # import Tortoise.TestGenerators, only: [gen_disconnect: 0]
 
-  property "encoding and decoding disconnect messages" do
-    forall disconnect <- gen_disconnect() do
-      ensure(
-        disconnect ==
-          disconnect
-          |> Package.encode()
-          |> Package.decode()
-      )
-    end
-  end
+  # property "encoding and decoding disconnect messages" do
+  #   forall disconnect <- gen_disconnect() do
+  #     ensure(
+  #       disconnect ==
+  #         disconnect
+  #         |> Package.encode()
+  #         |> Package.decode()
+  #     )
+  #   end
+  # end
+  @tag :skip
+  test "encoding and decoding disconnect messages"
 end
diff --git a/test/tortoise/package/properties_test.exs b/test/tortoise/package/properties_test.exs
index f003c1e5..294653d2 100644
--- a/test/tortoise/package/properties_test.exs
+++ b/test/tortoise/package/properties_test.exs
@@ -1,21 +1,23 @@
 defmodule Tortoise.Package.PropertiesTest do
   use ExUnit.Case
-  use EQC.ExUnit
+  # use EQC.ExUnit
   doctest Tortoise.Package.Properties
 
-  alias Tortoise.Package.Properties
+  # alias Tortoise.Package.Properties
 
-  import Tortoise.TestGenerators, only: [gen_properties: 0]
+  # import Tortoise.TestGenerators, only: [gen_properties: 0]
 
-  property "encoding and decoding properties" do
-    forall properties <- gen_properties() do
-      ensure(
-        properties ==
-          properties
-          |> Properties.encode()
-          |> IO.iodata_to_binary()
-          |> Properties.decode()
-      )
-    end
-  end
+  # property "encoding and decoding properties" do
+  #   forall properties <- gen_properties() do
+  #     ensure(
+  #       properties ==
+  #         properties
+  #         |> Properties.encode()
+  #         |> IO.iodata_to_binary()
+  #         |> Properties.decode()
+  #     )
+  #   end
+  # end
+  @tag :skip
+  test "encoding and decoding properties"
 end
diff --git a/test/tortoise/package/puback_test.exs b/test/tortoise/package/puback_test.exs
index 8082f130..f8e3441e 100644
--- a/test/tortoise/package/puback_test.exs
+++ b/test/tortoise/package/puback_test.exs
@@ -1,20 +1,22 @@
 defmodule Tortoise.Package.PubackTest do
   use ExUnit.Case
-  use EQC.ExUnit
+  # use EQC.ExUnit
   doctest Tortoise.Package.Puback
 
-  alias Tortoise.Package
+  # alias Tortoise.Package
 
-  import Tortoise.TestGenerators, only: [gen_puback: 0]
+  # import Tortoise.TestGenerators, only: [gen_puback: 0]
 
-  property "encoding and decoding puback messages" do
-    forall puback <- gen_puback() do
-      ensure(
-        puback ==
-          puback
-          |> Package.encode()
-          |> Package.decode()
-      )
-    end
-  end
+  # property "encoding and decoding puback messages" do
+  #   forall puback <- gen_puback() do
+  #     ensure(
+  #       puback ==
+  #         puback
+  #         |> Package.encode()
+  #         |> Package.decode()
+  #     )
+  #   end
+  # end
+  @tag :skip
+  test "encoding and decoding puback messages"
 end
diff --git a/test/tortoise/package/pubcomp_test.exs b/test/tortoise/package/pubcomp_test.exs
index a0df9edb..94460768 100644
--- a/test/tortoise/package/pubcomp_test.exs
+++ b/test/tortoise/package/pubcomp_test.exs
@@ -1,20 +1,22 @@
 defmodule Tortoise.Package.PubcompTest do
   use ExUnit.Case
-  use EQC.ExUnit
+  # use EQC.ExUnit
   doctest Tortoise.Package.Pubcomp
 
-  alias Tortoise.Package
+  # alias Tortoise.Package
 
-  import Tortoise.TestGenerators, only: [gen_pubcomp: 0]
+  # import Tortoise.TestGenerators, only: [gen_pubcomp: 0]
 
-  property "encoding and decoding pubcomp messages" do
-    forall pubcomp <- gen_pubcomp() do
-      ensure(
-        pubcomp ==
-          pubcomp
-          |> Package.encode()
-          |> Package.decode()
-      )
-    end
-  end
+  # property "encoding and decoding pubcomp messages" do
+  #   forall pubcomp <- gen_pubcomp() do
+  #     ensure(
+  #       pubcomp ==
+  #         pubcomp
+  #         |> Package.encode()
+  #         |> Package.decode()
+  #     )
+  #   end
+  # end
+  @tag :skip
+  test "encoding and decoding pubcomp messages"
 end
diff --git a/test/tortoise/package/publish_test.exs b/test/tortoise/package/publish_test.exs
index 43229ae8..c463e0f7 100644
--- a/test/tortoise/package/publish_test.exs
+++ b/test/tortoise/package/publish_test.exs
@@ -1,20 +1,22 @@
 defmodule Tortoise.Package.PublishTest do
   use ExUnit.Case
-  use EQC.ExUnit
+  # use EQC.ExUnit
   doctest Tortoise.Package.Publish
 
-  import Tortoise.TestGenerators, only: [gen_publish: 0]
+  # import Tortoise.TestGenerators, only: [gen_publish: 0]
 
-  alias Tortoise.Package
+  # alias Tortoise.Package
 
-  property "encoding and decoding publish messages" do
-    forall publish <- gen_publish() do
-      ensure(
-        publish ==
-          publish
-          |> Package.encode()
-          |> Package.decode()
-      )
-    end
-  end
+  # property "encoding and decoding publish messages" do
+  #   forall publish <- gen_publish() do
+  #     ensure(
+  #       publish ==
+  #         publish
+  #         |> Package.encode()
+  #         |> Package.decode()
+  #     )
+  #   end
+  # end
+  @tag :skip
+  test "encoding and decoding publish messages"
 end
diff --git a/test/tortoise/package/pubrec_test.exs b/test/tortoise/package/pubrec_test.exs
index 9d5360eb..c551d2a0 100644
--- a/test/tortoise/package/pubrec_test.exs
+++ b/test/tortoise/package/pubrec_test.exs
@@ -1,20 +1,23 @@
 defmodule Tortoise.Package.PubrecTest do
   use ExUnit.Case
-  use EQC.ExUnit
+  # use EQC.ExUnit
   doctest Tortoise.Package.Pubrec
 
-  alias Tortoise.Package
+  # alias Tortoise.Package
 
-  import Tortoise.TestGenerators, only: [gen_pubrec: 0]
+  # import Tortoise.TestGenerators, only: [gen_pubrec: 0]
 
-  property "encoding and decoding pubrec messages" do
-    forall pubrec <- gen_pubrec() do
-      ensure(
-        pubrec ==
-          pubrec
-          |> Package.encode()
-          |> Package.decode()
-      )
-    end
-  end
+  # property "encoding and decoding pubrec messages" do
+  #   forall pubrec <- gen_pubrec() do
+  #     ensure(
+  #       pubrec ==
+  #         pubrec
+  #         |> Package.encode()
+  #         |> Package.decode()
+  #     )
+  #   end
+  # end
+
+  @tag :skip
+  test "encoding and decoding pubrec messages"
 end
diff --git a/test/tortoise/package/pubrel_test.exs b/test/tortoise/package/pubrel_test.exs
index 2cba41ed..fd496432 100644
--- a/test/tortoise/package/pubrel_test.exs
+++ b/test/tortoise/package/pubrel_test.exs
@@ -1,20 +1,22 @@
 defmodule Tortoise.Package.PubrelTest do
   use ExUnit.Case
-  use EQC.ExUnit
+  # use EQC.ExUnit
   doctest Tortoise.Package.Pubrel
 
-  alias Tortoise.Package
+  # alias Tortoise.Package
 
-  import Tortoise.TestGenerators, only: [gen_pubrel: 0]
+  # import Tortoise.TestGenerators, only: [gen_pubrel: 0]
 
-  property "encoding and decoding pubrel messages" do
-    forall pubrel <- gen_pubrel() do
-      ensure(
-        pubrel ==
-          pubrel
-          |> Package.encode()
-          |> Package.decode()
-      )
-    end
-  end
+  # property "encoding and decoding pubrel messages" do
+  #   forall pubrel <- gen_pubrel() do
+  #     ensure(
+  #       pubrel ==
+  #         pubrel
+  #         |> Package.encode()
+  #         |> Package.decode()
+  #     )
+  #   end
+  # end
+  @tag :skip
+  test "encoding and decoding pubrel messages"
 end
diff --git a/test/tortoise/package/suback_test.exs b/test/tortoise/package/suback_test.exs
index 6838d042..3d7ccfcd 100644
--- a/test/tortoise/package/suback_test.exs
+++ b/test/tortoise/package/suback_test.exs
@@ -1,20 +1,23 @@
 defmodule Tortoise.Package.SubackTest do
   use ExUnit.Case
-  use EQC.ExUnit
+  # use EQC.ExUnit
   doctest Tortoise.Package.Suback
 
-  import Tortoise.TestGenerators, only: [gen_suback: 0]
+  # import Tortoise.TestGenerators, only: [gen_suback: 0]
 
-  alias Tortoise.Package
+  # alias Tortoise.Package
 
-  property "encoding and decoding suback messages" do
-    forall suback <- gen_suback() do
-      ensure(
-        suback ==
-          suback
-          |> Package.encode()
-          |> Package.decode()
-      )
-    end
-  end
+  # property "encoding and decoding suback messages" do
+  #   forall suback <- gen_suback() do
+  #     ensure(
+  #       suback ==
+  #         suback
+  #         |> Package.encode()
+  #         |> Package.decode()
+  #     )
+  #   end
+  # end
+
+  @tag :skip
+  test "encoding and decoding suback messages"
 end
diff --git a/test/tortoise/package/subscribe_test.exs b/test/tortoise/package/subscribe_test.exs
index 7cc87343..702d026e 100644
--- a/test/tortoise/package/subscribe_test.exs
+++ b/test/tortoise/package/subscribe_test.exs
@@ -1,47 +1,50 @@
 defmodule Tortoise.Package.SubscribeTest do
   use ExUnit.Case
-  use EQC.ExUnit
+  # use EQC.ExUnit
   doctest Tortoise.Package.Subscribe
 
-  import Tortoise.TestGenerators, only: [gen_subscribe: 0]
-
-  alias Tortoise.Package
-  alias Tortoise.Package.Subscribe
-
-  property "encoding and decoding subscribe messages" do
-    forall subscribe <- gen_subscribe() do
-      ensure(
-        subscribe ==
-          subscribe
-          |> Package.encode()
-          |> Package.decode()
-      )
-    end
-  end
-
-  describe "Collectable" do
-    test "Accept tuples of {binary(), opts()} as input" do
-      assert %Subscribe{topics: [{"a", [qos: 1, no_local: true]}]} =
-               [{"a", [qos: 1, no_local: true]}]
-               |> Enum.into(%Subscribe{})
-    end
-
-    test "Accept tuples of {binary(), qos()} as input" do
-      assert %Subscribe{topics: [{"a", [qos: 0]}]} =
-               [{"a", 0}]
-               |> Enum.into(%Subscribe{})
-    end
-
-    test "If no QoS is given it should default to zero" do
-      assert %Subscribe{topics: [{"a", [qos: 0]}]} =
-               ["a"]
-               |> Enum.into(%Subscribe{})
-    end
-
-    test "If two topics are the same the last write should win" do
-      assert %Subscribe{topics: [{"a", [qos: 1]}]} =
-               [{"a", qos: 2}, {"a", qos: 0}, {"a", qos: 1}]
-               |> Enum.into(%Subscribe{})
-    end
-  end
+  # import Tortoise.TestGenerators, only: [gen_subscribe: 0]
+
+  # alias Tortoise.Package
+  # alias Tortoise.Package.Subscribe
+
+  @tag :skip
+  test "encoding and decoding subscribe messages"
+
+  # property "encoding and decoding subscribe messages" do
+  #   forall subscribe <- gen_subscribe() do
+  #     ensure(
+  #       subscribe ==
+  #         subscribe
+  #         |> Package.encode()
+  #         |> Package.decode()
+  #     )
+  #   end
+  # end
+
+  # describe "Collectable" do
+  #   test "Accept tuples of {binary(), opts()} as input" do
+  #     assert %Subscribe{topics: [{"a", [qos: 1, no_local: true]}]} =
+  #              [{"a", [qos: 1, no_local: true]}]
+  #              |> Enum.into(%Subscribe{})
+  #   end
+
+  #   test "Accept tuples of {binary(), qos()} as input" do
+  #     assert %Subscribe{topics: [{"a", [qos: 0]}]} =
+  #              [{"a", 0}]
+  #              |> Enum.into(%Subscribe{})
+  #   end
+
+  #   test "If no QoS is given it should default to zero" do
+  #     assert %Subscribe{topics: [{"a", [qos: 0]}]} =
+  #              ["a"]
+  #              |> Enum.into(%Subscribe{})
+  #   end
+
+  #   test "If two topics are the same the last write should win" do
+  #     assert %Subscribe{topics: [{"a", [qos: 1]}]} =
+  #              [{"a", qos: 2}, {"a", qos: 0}, {"a", qos: 1}]
+  #              |> Enum.into(%Subscribe{})
+  #   end
+  # end
 end
diff --git a/test/tortoise/package/unsuback_test.exs b/test/tortoise/package/unsuback_test.exs
index cadf49ce..87f0318d 100644
--- a/test/tortoise/package/unsuback_test.exs
+++ b/test/tortoise/package/unsuback_test.exs
@@ -1,20 +1,22 @@
 defmodule Tortoise.Package.UnsubackTest do
   use ExUnit.Case
-  use EQC.ExUnit
+  # use EQC.ExUnit
   doctest Tortoise.Package.Unsuback
 
-  import Tortoise.TestGenerators, only: [gen_unsuback: 0]
+  # import Tortoise.TestGenerators, only: [gen_unsuback: 0]
 
-  alias Tortoise.Package
+  # alias Tortoise.Package
 
-  property "encoding and decoding unsuback messages" do
-    forall unsuback <- gen_unsuback() do
-      ensure(
-        unsuback ==
-          unsuback
-          |> Package.encode()
-          |> Package.decode()
-      )
-    end
-  end
+  # property "encoding and decoding unsuback messages" do
+  #   forall unsuback <- gen_unsuback() do
+  #     ensure(
+  #       unsuback ==
+  #         unsuback
+  #         |> Package.encode()
+  #         |> Package.decode()
+  #     )
+  #   end
+  # end
+  @tag :skip
+  test "encoding and decoding unsuback messages"
 end
diff --git a/test/tortoise/package/unsubscribe_test.exs b/test/tortoise/package/unsubscribe_test.exs
index f3059e22..234f60b7 100644
--- a/test/tortoise/package/unsubscribe_test.exs
+++ b/test/tortoise/package/unsubscribe_test.exs
@@ -1,20 +1,22 @@
 defmodule Tortoise.Package.UnsubscribeTest do
   use ExUnit.Case
-  use EQC.ExUnit
+  # use EQC.ExUnit
   doctest Tortoise.Package.Unsubscribe
 
-  import Tortoise.TestGenerators, only: [gen_unsubscribe: 0]
+  # import Tortoise.TestGenerators, only: [gen_unsubscribe: 0]
 
-  alias Tortoise.Package
+  # alias Tortoise.Package
 
-  property "encoding and decoding unsubscribe messages" do
-    forall unsubscribe <- gen_unsubscribe() do
-      ensure(
-        unsubscribe ==
-          unsubscribe
-          |> Package.encode()
-          |> Package.decode()
-      )
-    end
-  end
+  # property "encoding and decoding unsubscribe messages" do
+  #   forall unsubscribe <- gen_unsubscribe() do
+  #     ensure(
+  #       unsubscribe ==
+  #         unsubscribe
+  #         |> Package.encode()
+  #         |> Package.decode()
+  #     )
+  #   end
+  # end
+  @tag :skip
+  test "encoding and decoding unsubscribe messages"
 end
diff --git a/test/tortoise/package_test.exs b/test/tortoise/package_test.exs
index 1f6248cf..7accfad0 100644
--- a/test/tortoise/package_test.exs
+++ b/test/tortoise/package_test.exs
@@ -1,23 +1,38 @@
 defmodule Tortoise.PackageTest do
   use ExUnit.Case
-  use EQC.ExUnit
+  # use EQC.ExUnit
   doctest Tortoise.Package
 
-  alias Tortoise.Package
-
-  import Tortoise.TestGenerators,
-    only: [gen_unsuback: 0, gen_puback: 0, gen_pubcomp: 0, gen_pubrel: 0, gen_pubrec: 0]
-
-  # Test that we support encoding and decoding of all the
-  # acknowledgement and complete packages
-  property "encoding and decoding acknowledgement messages" do
-    forall ack <- oneof([gen_unsuback(), gen_puback(), gen_pubcomp(), gen_pubrel(), gen_pubrec()]) do
-      ensure(
-        ack ==
-          ack
-          |> Package.encode()
-          |> Package.decode()
-      )
-    end
-  end
+  # alias Tortoise.Package
+
+  # import Tortoise.TestGenerators,
+  #   only: [gen_unsuback: 0, gen_puback: 0, gen_pubcomp: 0, gen_pubrel: 0, gen_pubrec: 0]
+
+  # # Test that we support encoding and decoding of all the
+  # # acknowledgement and complete packages
+  # property "encoding and decoding acknowledgement messages" do
+  #   forall ack <- oneof([gen_unsuback(), gen_puback(), gen_pubcomp(), gen_pubrel(), gen_pubrec()]) do
+  #     ensure(
+  #       ack ==
+  #         ack
+  #         |> Package.encode()
+  #         |> Package.decode()
+  #     )
+  #   end
+  # end
+
+  @tag :skip
+  test "gen_unsuback/0"
+
+  @tag :skip
+  test "gen_puback/0"
+
+  @tag :skip
+  test "gen_pubcomp/0"
+
+  @tag :skip
+  test "gen_pubrel/0"
+
+  @tag :skip
+  test "gen_pubrec/0"
 end

From 151b053a5f6b3b5bf7981eaf0c2f37f10ba2449e Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Sat, 9 May 2020 10:55:25 +0100
Subject: [PATCH 179/220] Comment out a call to connection via_name

Connections are no longer registered in a name registry. The tests are
skipped anyways.

Need to get that setup working again
---
 test/tortoise_test.exs | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/test/tortoise_test.exs b/test/tortoise_test.exs
index 0472ecc6..051791e1 100644
--- a/test/tortoise_test.exs
+++ b/test/tortoise_test.exs
@@ -9,11 +9,13 @@ defmodule TortoiseTest do
     {:ok, %{client_id: context.test, transport: Tortoise.Transport.Tcp}}
   end
 
-  def setup_connection(context) do
+  def setup_connection(_context) do
     {:ok, client_socket, server_socket} = Tortoise.Integration.TestTCPTunnel.new()
-    name = Tortoise.Connection.via_name(context.client_id)
-    connection = {context.transport, client_socket}
-    :ok = Tortoise.Registry.put_meta(name, connection)
+    # TODO make this setup work again
+    # name = Tortoise.Connection.via_name(context.client_id)
+    # connection = {context.transport, client_socket}
+    # :ok = Tortoise.Registry.put_meta(name, connection)
+    connection = nil
     {:ok, %{client: client_socket, server: server_socket, connection: connection}}
   end
 

From 61682e9969afcf27c7f3e82f8ac8f9f804bcd5da Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Mon, 11 May 2020 08:31:39 +0100
Subject: [PATCH 180/220] Implemented a generator for connack messages

Added a protocol for defining structs that can be generated; I
implemented the generatable protocol for Connack messages, as they
seem fairly simple, yet not too simple. Had to figure out how to
generate properties, and it seems that my solution can error out at
times because it cannot make the list unique enough (we only allow
keys to occur once in the property list, except for the user defined
properties).

This should only load if StreamData is included in the project so we
can make StreamData an optional dependency, and hopefully this
approach can be used to test more than just Tortoise.

When generating the data one has to set the fields they want to
generate to `nil`; if they are set to any other value the generator
will pass the set value through as a constant. This allow us to only
generate successful packages like so:

```elixir
iex(1)> alias Tortoise.Package
Tortoise.Package
iex(2)> %Package.Connack{reason: :success, properties: nil} |> Package.generate() |> Enum.take(1)
[
  %Tortoise.Package.Connack{
    __META__: %Tortoise.Package.Meta{flags: 0, opcode: 2},
    properties: [maximum_qos: 0],
    reason: :success,
    session_present: false
  }
]
```

Connack encoding and decoding is now tested using this generator.
---
 lib/tortoise/generatable.ex            |   6 ++
 lib/tortoise/package.ex                |   1 +
 lib/tortoise/package/connack.ex        | 143 +++++++++++++++++++++++++
 test/tortoise/package/connack_test.exs |  27 +++--
 4 files changed, 162 insertions(+), 15 deletions(-)
 create mode 100644 lib/tortoise/generatable.ex

diff --git a/lib/tortoise/generatable.ex b/lib/tortoise/generatable.ex
new file mode 100644
index 00000000..f30c8fd8
--- /dev/null
+++ b/lib/tortoise/generatable.ex
@@ -0,0 +1,6 @@
+defprotocol Tortoise.Generatable do
+  @moduledoc false
+
+  # TODO add spec
+  def generate(data)
+end
diff --git a/lib/tortoise/package.ex b/lib/tortoise/package.ex
index bbded519..29ebd5b7 100644
--- a/lib/tortoise/package.ex
+++ b/lib/tortoise/package.ex
@@ -22,6 +22,7 @@ defmodule Tortoise.Package do
 
   defdelegate encode(data), to: Tortoise.Encodable
   defdelegate decode(data), to: Tortoise.Decodable
+  defdelegate generate(package), to: Tortoise.Generatable
 
   @doc false
   def length_encode(data) do
diff --git a/lib/tortoise/package/connack.ex b/lib/tortoise/package/connack.ex
index f0ac54f6..ffc02406 100644
--- a/lib/tortoise/package/connack.ex
+++ b/lib/tortoise/package/connack.ex
@@ -124,4 +124,147 @@ defmodule Tortoise.Package.Connack do
     defp flag(f) when f in [0, nil, false], do: 0
     defp flag(_), do: 1
   end
+
+  if Code.ensure_loaded?(StreamData) do
+    defimpl Tortoise.Generatable do
+      import StreamData
+
+      def generate(%type{__META__: _meta} = package) do
+        values = package |> Map.from_struct()
+
+        fixed_list(Enum.map(values, &constant(&1)))
+        |> bind(&gen_reason/1)
+        |> bind(&gen_session_present/1)
+        |> bind(&gen_properties/1)
+        |> bind(fn data ->
+          fixed_map([
+            {:__struct__, type}
+            | for({k, v} <- data, do: {k, constant(v)})
+          ])
+        end)
+      end
+
+      @refusals [
+        :unspecified_error,
+        :malformed_packet,
+        :protocol_error,
+        :implementation_specific_error,
+        :unsupported_protocol_version,
+        :client_identifier_not_valid,
+        :bad_user_name_or_password,
+        :not_authorized,
+        :server_unavailable,
+        :server_busy,
+        :banned,
+        :bad_authentication_method,
+        :topic_name_invalid,
+        :packet_too_large,
+        :quota_exceeded,
+        :payload_format_invalid,
+        :retain_not_supported,
+        :qos_not_supported,
+        :use_another_server,
+        :server_moved,
+        :connection_rate_exceeded
+      ]
+
+      defp gen_reason(values) do
+        case Keyword.pop(values, :reason) do
+          {nil, values} ->
+            fixed_list([
+              {
+                constant(:reason),
+                StreamData.frequency([
+                  {60, constant(:success)},
+                  {40, tuple({constant(:refused), one_of(@refusals)})}
+                ])
+              }
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {{:refused, nil}, values} ->
+            fixed_list([
+              {:reason, tuple({constant(:refused), one_of(@refusals)})}
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {:success, _} ->
+            constant(values)
+
+          {{:refused, refusal_reason}, _} when refusal_reason in @refusals ->
+            constant(values)
+        end
+      end
+
+      defp gen_session_present(values) do
+        case Keyword.get(values, :reason) do
+          :success ->
+            case Keyword.pop(values, :session_present) do
+              {nil, values} ->
+                fixed_list([
+                  {constant(:session_present), boolean()}
+                  | Enum.map(values, &constant(&1))
+                ])
+
+              {_passthrough, _} ->
+                constant(values)
+            end
+
+          {:refused, _refusal_reason} ->
+            # There will not be a session if the connection is refused
+            constant(Keyword.put(values, :session_present, false))
+        end
+      end
+
+      defp gen_properties(values) do
+        case Keyword.pop(values, :properties) do
+          {nil, values} ->
+            properties =
+              uniq_list_of(
+                one_of([
+                  # here we allow stings with a byte size of zero; don't
+                  # know if that is a problem according to the spec. Let's
+                  # handle that situation just in case:
+                  {constant(:user_property), {string(:printable), string(:printable)}},
+                  {constant(:assigned_client_identifier),
+                   string(:printable, min_length: 1, max_length: 23)},
+                  {constant(:maximum_packet_size), integer(1..0xFFFFFFFF)},
+                  {constant(:maximum_qos), integer(0..1)},
+                  {constant(:reason_string), string(:printable)},
+                  {constant(:receive_maximum), integer(0..0xFFFF)},
+                  {constant(:retain_available), boolean()},
+                  # TODO don't know if zero is a valid keep alive
+                  {constant(:server_keep_alive), integer(0..0xFFFF)},
+                  {constant(:session_expiry_interval), integer(0..0xFFFF)},
+                  {constant(:shared_subscription_available), boolean()},
+                  {constant(:subscription_identifiers_available), boolean()},
+                  {constant(:topic_alias_maximum), integer(0..0xFFFF)},
+                  {constant(:wildcard_subscription_available), boolean()}
+
+                  # TODO, generator that generate valid server references
+                  # {constant(:server_reference), boolean()},
+                  # TODO, generator that generate valid response info
+                  # {constant(:response_information), boolean()},
+                  # TODO, generate auth data and methods
+                  # {constant(:authentication_data), boolean()},
+                  # {constant(:authentication_method), boolean()}
+                ]),
+                uniq_fun: &uniq/1,
+                max_length: 5
+              )
+
+            fixed_list([
+              {constant(:properties), properties}
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {_passthrough, _} ->
+            constant(values)
+        end
+      end
+
+      defp uniq({:user_property, _v}), do: :crypto.strong_rand_bytes(2)
+      defp uniq({k, _v}), do: k
+    end
+  end
 end
diff --git a/test/tortoise/package/connack_test.exs b/test/tortoise/package/connack_test.exs
index cb80a65b..5ecb1917 100644
--- a/test/tortoise/package/connack_test.exs
+++ b/test/tortoise/package/connack_test.exs
@@ -1,22 +1,19 @@
 defmodule Tortoise.Package.ConnackTest do
   use ExUnit.Case
-  # use EQC.ExUnit
+  use ExUnitProperties
+
   doctest Tortoise.Package.Connack
 
-  # import Tortoise.TestGenerators, only: [gen_connack: 0]
+  alias Tortoise.Package
 
-  # alias Tortoise.Package
+  property "encoding and decoding connack messages" do
+    config = %Package.Connack{reason: nil, session_present: nil, properties: nil}
 
-  # property "encoding and decoding connack messages" do
-  #   forall connack <- gen_connack() do
-  #     ensure(
-  #       connack ==
-  #         connack
-  #         |> Package.encode()
-  #         |> Package.decode()
-  #     )
-  #   end
-  # end
-  @tag :skip
-  test "encoding and decoding connack messages"
+    check all connack <- Package.generate(config) do
+      assert connack ==
+               connack
+               |> Package.encode()
+               |> Package.decode()
+    end
+  end
 end

From b99e3b91d09426741b203eac7d2526a927095a72 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Mon, 11 May 2020 21:11:16 +0100
Subject: [PATCH 181/220] Add a StreamData generator for Pubrel packages

Check encoding and decoding as well
---
 lib/tortoise/package/pubrel.ex        | 96 +++++++++++++++++++++++++++
 test/tortoise/package/pubrel_test.exs | 27 ++++----
 2 files changed, 108 insertions(+), 15 deletions(-)

diff --git a/lib/tortoise/package/pubrel.ex b/lib/tortoise/package/pubrel.ex
index 2ccaa0b9..c330b4e1 100644
--- a/lib/tortoise/package/pubrel.ex
+++ b/lib/tortoise/package/pubrel.ex
@@ -75,4 +75,100 @@ defmodule Tortoise.Package.Pubrel do
     defp to_reason_code(:success), do: 0x00
     defp to_reason_code({:refused, :packet_identifier_not_found}), do: 0x92
   end
+
+  if Code.ensure_loaded?(StreamData) do
+    defimpl Tortoise.Generatable do
+      import StreamData
+
+      def generate(%type{__META__: _meta} = package) do
+        values = package |> Map.from_struct()
+
+        fixed_list(Enum.map(values, &constant(&1)))
+        |> bind(&gen_identifier/1)
+        |> bind(&gen_reason/1)
+        |> bind(&gen_properties/1)
+        |> bind(fn data ->
+          fixed_map([
+            {:__struct__, type}
+            | for({k, v} <- data, do: {k, constant(v)})
+          ])
+        end)
+      end
+
+      defp gen_identifier(values) do
+        case Keyword.pop(values, :identifier) do
+          {nil, values} ->
+            fixed_list([
+              {constant(:identifier), integer(1..0xFFFF)}
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {id, _} when is_integer(id) and id in 1..0xFFFF ->
+            constant(values)
+        end
+      end
+
+      # Might be overkill, but at least we are prepared for more
+      # refusal reasons in the future should there be more refusals in
+      # a future protocol version
+      @refusals [:packet_identifier_not_found]
+
+      defp gen_reason(values) do
+        case Keyword.pop(values, :reason) do
+          {nil, values} ->
+            fixed_list([
+              {
+                constant(:reason),
+                StreamData.frequency([
+                  {60, constant(:success)},
+                  {40, tuple({constant(:refused), one_of(@refusals)})}
+                ])
+              }
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {{:refused, nil}, values} ->
+            fixed_list([
+              {:reason, tuple({constant(:refused), one_of(@refusals)})}
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {:success, _} ->
+            constant(values)
+
+          {{:refused, refusal_reason}, _} when refusal_reason in @refusals ->
+            constant(values)
+        end
+      end
+
+      defp gen_properties(values) do
+        case Keyword.pop(values, :properties) do
+          {nil, values} ->
+            properties =
+              uniq_list_of(
+                one_of([
+                  # here we allow stings with a byte size of zero; don't
+                  # know if that is a problem according to the spec. Let's
+                  # handle that situation just in case:
+                  {constant(:user_property), {string(:printable), string(:printable)}},
+                  {constant(:reason_string), string(:printable)}
+                ]),
+                uniq_fun: &uniq/1,
+                max_length: 5
+              )
+
+            fixed_list([
+              {constant(:properties), properties}
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {_passthrough, _} ->
+            constant(values)
+        end
+      end
+
+      defp uniq({:user_property, _v}), do: :crypto.strong_rand_bytes(2)
+      defp uniq({k, _v}), do: k
+    end
+  end
 end
diff --git a/test/tortoise/package/pubrel_test.exs b/test/tortoise/package/pubrel_test.exs
index fd496432..37139f77 100644
--- a/test/tortoise/package/pubrel_test.exs
+++ b/test/tortoise/package/pubrel_test.exs
@@ -1,22 +1,19 @@
 defmodule Tortoise.Package.PubrelTest do
   use ExUnit.Case
-  # use EQC.ExUnit
+  use ExUnitProperties
+
   doctest Tortoise.Package.Pubrel
 
-  # alias Tortoise.Package
+  alias Tortoise.Package
 
-  # import Tortoise.TestGenerators, only: [gen_pubrel: 0]
+  property "encoding and decoding pubrel messages" do
+    config = %Package.Pubrel{identifier: nil, reason: nil, properties: nil}
 
-  # property "encoding and decoding pubrel messages" do
-  #   forall pubrel <- gen_pubrel() do
-  #     ensure(
-  #       pubrel ==
-  #         pubrel
-  #         |> Package.encode()
-  #         |> Package.decode()
-  #     )
-  #   end
-  # end
-  @tag :skip
-  test "encoding and decoding pubrel messages"
+    check all pubrel <- Package.generate(config) do
+      assert pubrel ==
+               pubrel
+               |> Package.encode()
+               |> Package.decode()
+    end
+  end
 end

From ae824b3ad95e1490a277fc679455d8561934431c Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Mon, 11 May 2020 21:21:55 +0100
Subject: [PATCH 182/220] Add a StreamData generator for Puback packages

Check encoding and decoding as well

Fixed a spelling mistake in a refusal atom
---
 lib/tortoise/package/puback.ex        | 108 +++++++++++++++++++++++++-
 test/tortoise/package/puback_test.exs |  27 +++----
 2 files changed, 117 insertions(+), 18 deletions(-)

diff --git a/lib/tortoise/package/puback.ex b/lib/tortoise/package/puback.ex
index 7a40fe63..46be8d27 100644
--- a/lib/tortoise/package/puback.ex
+++ b/lib/tortoise/package/puback.ex
@@ -13,7 +13,7 @@ defmodule Tortoise.Package.Puback do
           | :unspecified_error
           | :implementation_specific_error
           | :not_authorized
-          | :topic_Name_invalid
+          | :topic_name_invalid
           | :packet_identifier_in_use
           | :quota_exceeded
           | :payload_format_invalid
@@ -57,7 +57,7 @@ defmodule Tortoise.Package.Puback do
       0x80 -> {:refused, :unspecified_error}
       0x83 -> {:refused, :implementation_specific_error}
       0x87 -> {:refused, :not_authorized}
-      0x90 -> {:refused, :topic_Name_invalid}
+      0x90 -> {:refused, :topic_name_invalid}
       0x91 -> {:refused, :packet_identifier_in_use}
       0x97 -> {:refused, :quota_exceeded}
       0x99 -> {:refused, :payload_format_invalid}
@@ -98,11 +98,113 @@ defmodule Tortoise.Package.Puback do
         :unspecified_error -> 0x80
         :implementation_specific_error -> 0x83
         :not_authorized -> 0x87
-        :topic_Name_invalid -> 0x90
+        :topic_name_invalid -> 0x90
         :packet_identifier_in_use -> 0x91
         :quota_exceeded -> 0x97
         :payload_format_invalid -> 0x99
       end
     end
   end
+
+  if Code.ensure_loaded?(StreamData) do
+    defimpl Tortoise.Generatable do
+      import StreamData
+
+      def generate(%type{__META__: _meta} = package) do
+        values = package |> Map.from_struct()
+
+        fixed_list(Enum.map(values, &constant(&1)))
+        |> bind(&gen_identifier/1)
+        |> bind(&gen_reason/1)
+        |> bind(&gen_properties/1)
+        |> bind(fn data ->
+          fixed_map([
+            {:__struct__, type}
+            | for({k, v} <- data, do: {k, constant(v)})
+          ])
+        end)
+      end
+
+      defp gen_identifier(values) do
+        case Keyword.pop(values, :identifier) do
+          {nil, values} ->
+            fixed_list([
+              {constant(:identifier), integer(1..0xFFFF)}
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {id, _} when is_integer(id) and id in 1..0xFFFF ->
+            constant(values)
+        end
+      end
+
+      @refusals [
+        :no_matching_subscribers,
+        :unspecified_error,
+        :implementation_specific_error,
+        :not_authorized,
+        :topic_name_invalid,
+        :packet_identifier_in_use,
+        :quota_exceeded,
+        :payload_format_invalid
+      ]
+
+      defp gen_reason(values) do
+        case Keyword.pop(values, :reason) do
+          {nil, values} ->
+            fixed_list([
+              {
+                constant(:reason),
+                StreamData.frequency([
+                  {60, constant(:success)},
+                  {40, tuple({constant(:refused), one_of(@refusals)})}
+                ])
+              }
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {{:refused, nil}, values} ->
+            fixed_list([
+              {:reason, tuple({constant(:refused), one_of(@refusals)})}
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {:success, _} ->
+            constant(values)
+
+          {{:refused, refusal_reason}, _} when refusal_reason in @refusals ->
+            constant(values)
+        end
+      end
+
+      defp gen_properties(values) do
+        case Keyword.pop(values, :properties) do
+          {nil, values} ->
+            properties =
+              uniq_list_of(
+                one_of([
+                  # here we allow stings with a byte size of zero; don't
+                  # know if that is a problem according to the spec. Let's
+                  # handle that situation just in case:
+                  {constant(:user_property), {string(:printable), string(:printable)}},
+                  {constant(:reason_string), string(:printable)}
+                ]),
+                uniq_fun: &uniq/1,
+                max_length: 5
+              )
+
+            fixed_list([
+              {constant(:properties), properties}
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {_passthrough, _} ->
+            constant(values)
+        end
+      end
+
+      defp uniq({:user_property, _v}), do: :crypto.strong_rand_bytes(2)
+      defp uniq({k, _v}), do: k
+    end
+  end
 end
diff --git a/test/tortoise/package/puback_test.exs b/test/tortoise/package/puback_test.exs
index f8e3441e..7e17a81c 100644
--- a/test/tortoise/package/puback_test.exs
+++ b/test/tortoise/package/puback_test.exs
@@ -1,22 +1,19 @@
 defmodule Tortoise.Package.PubackTest do
   use ExUnit.Case
-  # use EQC.ExUnit
+  use ExUnitProperties
+
   doctest Tortoise.Package.Puback
 
-  # alias Tortoise.Package
+  alias Tortoise.Package
 
-  # import Tortoise.TestGenerators, only: [gen_puback: 0]
+  property "encoding and decoding puback messages" do
+    config = %Package.Puback{identifier: nil, reason: nil, properties: nil}
 
-  # property "encoding and decoding puback messages" do
-  #   forall puback <- gen_puback() do
-  #     ensure(
-  #       puback ==
-  #         puback
-  #         |> Package.encode()
-  #         |> Package.decode()
-  #     )
-  #   end
-  # end
-  @tag :skip
-  test "encoding and decoding puback messages"
+    check all puback <- Package.generate(config) do
+      assert puback ==
+               puback
+               |> Package.encode()
+               |> Package.decode()
+    end
+  end
 end

From 3ba08001b28ce57476e5adfe10628c42b1b90919 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Mon, 11 May 2020 21:29:04 +0100
Subject: [PATCH 183/220] Add a StreamData generator for Pubcomp packages

Check encoding and decoding as well
---
 lib/tortoise/package/pubcomp.ex        | 96 ++++++++++++++++++++++++++
 test/tortoise/package/pubcomp_test.exs | 27 ++++----
 2 files changed, 108 insertions(+), 15 deletions(-)

diff --git a/lib/tortoise/package/pubcomp.ex b/lib/tortoise/package/pubcomp.ex
index 1b2114c1..6795b8ee 100644
--- a/lib/tortoise/package/pubcomp.ex
+++ b/lib/tortoise/package/pubcomp.ex
@@ -72,4 +72,100 @@ defmodule Tortoise.Package.Pubcomp do
     defp to_reason_code(:success), do: 0x00
     defp to_reason_code({:refused, :packet_identifier_not_found}), do: 0x92
   end
+
+  if Code.ensure_loaded?(StreamData) do
+    defimpl Tortoise.Generatable do
+      import StreamData
+
+      def generate(%type{__META__: _meta} = package) do
+        values = package |> Map.from_struct()
+
+        fixed_list(Enum.map(values, &constant(&1)))
+        |> bind(&gen_identifier/1)
+        |> bind(&gen_reason/1)
+        |> bind(&gen_properties/1)
+        |> bind(fn data ->
+          fixed_map([
+            {:__struct__, type}
+            | for({k, v} <- data, do: {k, constant(v)})
+          ])
+        end)
+      end
+
+      defp gen_identifier(values) do
+        case Keyword.pop(values, :identifier) do
+          {nil, values} ->
+            fixed_list([
+              {constant(:identifier), integer(1..0xFFFF)}
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {id, _} when is_integer(id) and id in 1..0xFFFF ->
+            constant(values)
+        end
+      end
+
+      # Might be overkill, but at least we are prepared for more
+      # refusal reasons in the future should there be more refusals in
+      # a future protocol version
+      @refusals [:packet_identifier_not_found]
+
+      defp gen_reason(values) do
+        case Keyword.pop(values, :reason) do
+          {nil, values} ->
+            fixed_list([
+              {
+                constant(:reason),
+                StreamData.frequency([
+                  {60, constant(:success)},
+                  {40, tuple({constant(:refused), one_of(@refusals)})}
+                ])
+              }
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {{:refused, nil}, values} ->
+            fixed_list([
+              {:reason, tuple({constant(:refused), one_of(@refusals)})}
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {:success, _} ->
+            constant(values)
+
+          {{:refused, refusal_reason}, _} when refusal_reason in @refusals ->
+            constant(values)
+        end
+      end
+
+      defp gen_properties(values) do
+        case Keyword.pop(values, :properties) do
+          {nil, values} ->
+            properties =
+              uniq_list_of(
+                one_of([
+                  # here we allow stings with a byte size of zero; don't
+                  # know if that is a problem according to the spec. Let's
+                  # handle that situation just in case:
+                  {constant(:user_property), {string(:printable), string(:printable)}},
+                  {constant(:reason_string), string(:printable)}
+                ]),
+                uniq_fun: &uniq/1,
+                max_length: 5
+              )
+
+            fixed_list([
+              {constant(:properties), properties}
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {_passthrough, _} ->
+            constant(values)
+        end
+      end
+
+      defp uniq({:user_property, _v}), do: :crypto.strong_rand_bytes(2)
+      defp uniq({k, _v}), do: k
+    end
+  end
 end
diff --git a/test/tortoise/package/pubcomp_test.exs b/test/tortoise/package/pubcomp_test.exs
index 94460768..2dac0df9 100644
--- a/test/tortoise/package/pubcomp_test.exs
+++ b/test/tortoise/package/pubcomp_test.exs
@@ -1,22 +1,19 @@
 defmodule Tortoise.Package.PubcompTest do
   use ExUnit.Case
-  # use EQC.ExUnit
+  use ExUnitProperties
+
   doctest Tortoise.Package.Pubcomp
 
-  # alias Tortoise.Package
+  alias Tortoise.Package
 
-  # import Tortoise.TestGenerators, only: [gen_pubcomp: 0]
+  property "encoding and decoding pubcomp messages" do
+    config = %Package.Pubcomp{identifier: nil, reason: nil, properties: nil}
 
-  # property "encoding and decoding pubcomp messages" do
-  #   forall pubcomp <- gen_pubcomp() do
-  #     ensure(
-  #       pubcomp ==
-  #         pubcomp
-  #         |> Package.encode()
-  #         |> Package.decode()
-  #     )
-  #   end
-  # end
-  @tag :skip
-  test "encoding and decoding pubcomp messages"
+    check all pubcomp <- Package.generate(config) do
+      assert pubcomp ==
+               pubcomp
+               |> Package.encode()
+               |> Package.decode()
+    end
+  end
 end

From e582096ade0c5fe0de2c20feb5477d703f908d32 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Mon, 11 May 2020 21:38:39 +0100
Subject: [PATCH 184/220] Add a StreamData generator for Pubrec packages

Check encoding and decoding as well, and fixed an error in the coerce
reason code part of the decoder
---
 lib/tortoise/package/pubrec.ex        | 122 +++++++++++++++++++++++---
 test/tortoise/package/pubrec_test.exs |  28 +++---
 2 files changed, 124 insertions(+), 26 deletions(-)

diff --git a/lib/tortoise/package/pubrec.ex b/lib/tortoise/package/pubrec.ex
index a71146db..b8782581 100644
--- a/lib/tortoise/package/pubrec.ex
+++ b/lib/tortoise/package/pubrec.ex
@@ -13,7 +13,7 @@ defmodule Tortoise.Package.Pubrec do
           | :unspecified_error
           | :implementation_specific_error
           | :not_authorized
-          | :topic_Name_invalid
+          | :topic_name_invalid
           | :packet_identifier_in_use
           | :quota_exceeded
           | :payload_format_invalid
@@ -49,14 +49,14 @@ defmodule Tortoise.Package.Pubrec do
   defp coerce_reason_code(reason_code) do
     case reason_code do
       0x00 -> :success
-      0x10 -> :no_matching_subscribers
-      0x80 -> :unspecified_error
-      0x83 -> :implementation_specific_error
-      0x87 -> :not_authorized
-      0x90 -> :topic_Name_invalid
-      0x91 -> :packet_identifier_in_use
-      0x97 -> :quota_exceeded
-      0x99 -> :payload_format_invalid
+      0x10 -> {:refused, :no_matching_subscribers}
+      0x80 -> {:refused, :unspecified_error}
+      0x83 -> {:refused, :implementation_specific_error}
+      0x87 -> {:refused, :not_authorized}
+      0x90 -> {:refused, :topic_name_invalid}
+      0x91 -> {:refused, :packet_identifier_in_use}
+      0x97 -> {:refused, :quota_exceeded}
+      0x99 -> {:refused, :payload_format_invalid}
     end
   end
 
@@ -94,11 +94,113 @@ defmodule Tortoise.Package.Pubrec do
         :unspecified_error -> 0x80
         :implementation_specific_error -> 0x83
         :not_authorized -> 0x87
-        :topic_Name_invalid -> 0x90
+        :topic_name_invalid -> 0x90
         :packet_identifier_in_use -> 0x91
         :quota_exceeded -> 0x97
         :payload_format_invalid -> 0x99
       end
     end
   end
+
+  if Code.ensure_loaded?(StreamData) do
+    defimpl Tortoise.Generatable do
+      import StreamData
+
+      def generate(%type{__META__: _meta} = package) do
+        values = package |> Map.from_struct()
+
+        fixed_list(Enum.map(values, &constant(&1)))
+        |> bind(&gen_identifier/1)
+        |> bind(&gen_reason/1)
+        |> bind(&gen_properties/1)
+        |> bind(fn data ->
+          fixed_map([
+            {:__struct__, type}
+            | for({k, v} <- data, do: {k, constant(v)})
+          ])
+        end)
+      end
+
+      defp gen_identifier(values) do
+        case Keyword.pop(values, :identifier) do
+          {nil, values} ->
+            fixed_list([
+              {constant(:identifier), integer(1..0xFFFF)}
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {id, _} when is_integer(id) and id in 1..0xFFFF ->
+            constant(values)
+        end
+      end
+
+      @refusals [
+        :no_matching_subscribers,
+        :unspecified_error,
+        :implementation_specific_error,
+        :not_authorized,
+        :topic_name_invalid,
+        :packet_identifier_in_use,
+        :quota_exceeded,
+        :payload_format_invalid
+      ]
+
+      defp gen_reason(values) do
+        case Keyword.pop(values, :reason) do
+          {nil, values} ->
+            fixed_list([
+              {
+                constant(:reason),
+                StreamData.frequency([
+                  {60, constant(:success)},
+                  {40, tuple({constant(:refused), one_of(@refusals)})}
+                ])
+              }
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {{:refused, nil}, values} ->
+            fixed_list([
+              {:reason, tuple({constant(:refused), one_of(@refusals)})}
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {:success, _} ->
+            constant(values)
+
+          {{:refused, refusal_reason}, _} when refusal_reason in @refusals ->
+            constant(values)
+        end
+      end
+
+      defp gen_properties(values) do
+        case Keyword.pop(values, :properties) do
+          {nil, values} ->
+            properties =
+              uniq_list_of(
+                one_of([
+                  # here we allow stings with a byte size of zero; don't
+                  # know if that is a problem according to the spec. Let's
+                  # handle that situation just in case:
+                  {constant(:user_property), {string(:printable), string(:printable)}},
+                  {constant(:reason_string), string(:printable)}
+                ]),
+                uniq_fun: &uniq/1,
+                max_length: 5
+              )
+
+            fixed_list([
+              {constant(:properties), properties}
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {_passthrough, _} ->
+            constant(values)
+        end
+      end
+
+      defp uniq({:user_property, _v}), do: :crypto.strong_rand_bytes(2)
+      defp uniq({k, _v}), do: k
+    end
+  end
 end
diff --git a/test/tortoise/package/pubrec_test.exs b/test/tortoise/package/pubrec_test.exs
index c551d2a0..63744920 100644
--- a/test/tortoise/package/pubrec_test.exs
+++ b/test/tortoise/package/pubrec_test.exs
@@ -1,23 +1,19 @@
 defmodule Tortoise.Package.PubrecTest do
   use ExUnit.Case
-  # use EQC.ExUnit
-  doctest Tortoise.Package.Pubrec
+  use ExUnitProperties
 
-  # alias Tortoise.Package
+  doctest Tortoise.Package.Pubrec
 
-  # import Tortoise.TestGenerators, only: [gen_pubrec: 0]
+  alias Tortoise.Package
 
-  # property "encoding and decoding pubrec messages" do
-  #   forall pubrec <- gen_pubrec() do
-  #     ensure(
-  #       pubrec ==
-  #         pubrec
-  #         |> Package.encode()
-  #         |> Package.decode()
-  #     )
-  #   end
-  # end
+  property "encoding and decoding pubrec messages" do
+    config = %Package.Pubrec{identifier: nil, reason: nil, properties: nil}
 
-  @tag :skip
-  test "encoding and decoding pubrec messages"
+    check all pubrec <- Package.generate(config) do
+      assert pubrec ==
+               pubrec
+               |> Package.encode()
+               |> Package.decode()
+    end
+  end
 end

From ed72bf5634f7782a84c7ebe2e008df7340791780 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Mon, 11 May 2020 21:58:32 +0100
Subject: [PATCH 185/220] Add a StreamData generator for Disconnect packages

Check encoding and decoding as well
---
 lib/tortoise/package/disconnect.ex        | 96 +++++++++++++++++++++++
 test/tortoise/package/disconnect_test.exs | 27 +++----
 2 files changed, 108 insertions(+), 15 deletions(-)

diff --git a/lib/tortoise/package/disconnect.ex b/lib/tortoise/package/disconnect.ex
index df053e63..7a2f489e 100644
--- a/lib/tortoise/package/disconnect.ex
+++ b/lib/tortoise/package/disconnect.ex
@@ -157,4 +157,100 @@ defmodule Tortoise.Package.Disconnect do
       end
     end
   end
+
+  if Code.ensure_loaded?(StreamData) do
+    defimpl Tortoise.Generatable do
+      import StreamData
+
+      def generate(%type{__META__: _meta} = package) do
+        values = package |> Map.from_struct()
+
+        fixed_list(Enum.map(values, &constant(&1)))
+        |> bind(&gen_reason/1)
+        |> bind(&gen_properties/1)
+        |> bind(fn data ->
+          fixed_map([
+            {:__struct__, type}
+            | for({k, v} <- data, do: {k, constant(v)})
+          ])
+        end)
+      end
+
+      @reasons [
+        :normal_disconnection,
+        :disconnect_with_will_message,
+        :unspecified_error,
+        :malformed_packet,
+        :protocol_error,
+        :implementation_specific_error,
+        :not_authorized,
+        :server_busy,
+        :server_shutting_down,
+        :keep_alive_timeout,
+        :session_taken_over,
+        :topic_filter_invalid,
+        :topic_name_invalid,
+        :receive_maximum_exceeded,
+        :topic_alias_invalid,
+        :packet_too_large,
+        :message_rate_too_high,
+        :quota_exceeded,
+        :administrative_action,
+        :payload_format_invalid,
+        :retain_not_supported,
+        :qos_not_supported,
+        :use_another_server,
+        :server_moved,
+        :shared_subscriptions_not_supported,
+        :connection_rate_exceeded,
+        :maximum_connect_time,
+        :subscription_identifiers_not_supported,
+        :wildcard_subscriptions_not_supported
+      ]
+
+      defp gen_reason(values) do
+        case Keyword.pop(values, :reason) do
+          {nil, values} ->
+            fixed_list([
+              {constant(:reason), one_of(@reasons)}
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {reason, _} when reason in @reasons ->
+            constant(values)
+        end
+      end
+
+      defp gen_properties(values) do
+        case Keyword.pop(values, :properties) do
+          {nil, values} ->
+            properties =
+              uniq_list_of(
+                one_of([
+                  # here we allow stings with a byte size of zero; don't
+                  # know if that is a problem according to the spec. Let's
+                  # handle that situation just in case:
+                  {constant(:user_property), {string(:printable), string(:printable)}},
+                  {constant(:reason_string), string(:printable)},
+                  {constant(:session_expiry_interval), integer(0..0xFFFFFFFF)}
+                  # TODO generate valid :server_reference,
+                ]),
+                uniq_fun: &uniq/1,
+                max_length: 5
+              )
+
+            fixed_list([
+              {constant(:properties), properties}
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {_passthrough, _} ->
+            constant(values)
+        end
+      end
+
+      defp uniq({:user_property, _v}), do: :crypto.strong_rand_bytes(2)
+      defp uniq({k, _v}), do: k
+    end
+  end
 end
diff --git a/test/tortoise/package/disconnect_test.exs b/test/tortoise/package/disconnect_test.exs
index 93b9b50b..fd9d4bc1 100644
--- a/test/tortoise/package/disconnect_test.exs
+++ b/test/tortoise/package/disconnect_test.exs
@@ -1,22 +1,19 @@
 defmodule Tortoise.Package.DisconnectTest do
   use ExUnit.Case
-  # use EQC.ExUnit
+  use ExUnitProperties
+
   doctest Tortoise.Package.Disconnect
 
-  # alias Tortoise.Package
+  alias Tortoise.Package
 
-  # import Tortoise.TestGenerators, only: [gen_disconnect: 0]
+  property "encoding and decoding disconnect messages" do
+    config = %Package.Disconnect{reason: nil, properties: nil}
 
-  # property "encoding and decoding disconnect messages" do
-  #   forall disconnect <- gen_disconnect() do
-  #     ensure(
-  #       disconnect ==
-  #         disconnect
-  #         |> Package.encode()
-  #         |> Package.decode()
-  #     )
-  #   end
-  # end
-  @tag :skip
-  test "encoding and decoding disconnect messages"
+    check all disconnect <- Package.generate(config) do
+      assert disconnect ==
+               disconnect
+               |> Package.encode()
+               |> Package.decode()
+    end
+  end
 end

From 2ade95db7042e902a267458bdfe171809c34977e Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Mon, 11 May 2020 22:06:19 +0100
Subject: [PATCH 186/220] Add a StreamData generator for Pingreq packages

Check encoding and decoding as well. This is kind of silly, but done
for completeness and it will make it easy for future improvements. It
will always generate the same data as there are no fields on a pingreq
struct besides the meta.
---
 lib/tortoise/package/pingreq.ex        | 18 ++++++++++++++++++
 test/tortoise/package/pingreq_test.exs | 23 +++++++++++++++++------
 2 files changed, 35 insertions(+), 6 deletions(-)

diff --git a/lib/tortoise/package/pingreq.ex b/lib/tortoise/package/pingreq.ex
index 527f3361..8ea81d10 100644
--- a/lib/tortoise/package/pingreq.ex
+++ b/lib/tortoise/package/pingreq.ex
@@ -21,4 +21,22 @@ defmodule Tortoise.Package.Pingreq do
       [Package.Meta.encode(t.__META__), 0]
     end
   end
+
+  if Code.ensure_loaded?(StreamData) do
+    defimpl Tortoise.Generatable do
+      import StreamData
+
+      def generate(%type{__META__: _meta} = package) do
+        values = package |> Map.from_struct()
+
+        fixed_list(Enum.map(values, &constant(&1)))
+        |> bind(fn data ->
+          fixed_map([
+            {:__struct__, type}
+            | for({k, v} <- data, do: {k, constant(v)})
+          ])
+        end)
+      end
+    end
+  end
 end
diff --git a/test/tortoise/package/pingreq_test.exs b/test/tortoise/package/pingreq_test.exs
index 82c4fc30..7c54354f 100644
--- a/test/tortoise/package/pingreq_test.exs
+++ b/test/tortoise/package/pingreq_test.exs
@@ -1,15 +1,26 @@
 defmodule Tortoise.Package.PingreqTest do
   use ExUnit.Case
+  use ExUnitProperties
+
   doctest Tortoise.Package.Pingreq
 
   alias Tortoise.Package
 
-  test "encoding and decoding ping requests" do
-    pingreq = %Package.Pingreq{}
+  property "encoding and decoding pingreq messages" do
+    # as pingreqs always look the same it might be overkill to have a
+    # property based testing for this, but it is added for
+    # completeness; also in case of future changes to the protocol it
+    # might be eaiser to expand on this test, as I am hoping for user
+    # defined properties on ping request and responses (just kidding)
+    #
+    # Yeah, this is kind of silly...
+    config = %Package.Pingreq{}
 
-    assert ^pingreq =
-             pingreq
-             |> Package.encode()
-             |> Package.decode()
+    check all pingreq <- Package.generate(config) do
+      assert pingreq ==
+               pingreq
+               |> Package.encode()
+               |> Package.decode()
+    end
   end
 end

From d34219b4c4362c7e703b44ba3cfee6856402a4b1 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Mon, 11 May 2020 22:12:35 +0100
Subject: [PATCH 187/220] Add a StreamData generator for Pingresp packages

Check encoding and decoding as well. This is kind of silly, but done
for completeness and it will make it easy for future improvements. It
will always generate the same data as there are no fields on a
pingresp struct besides the meta.
---
 lib/tortoise/package/pingresp.ex        | 18 ++++++++++++++++++
 test/tortoise/package/pingresp_test.exs | 18 ++++++++++++------
 2 files changed, 30 insertions(+), 6 deletions(-)

diff --git a/lib/tortoise/package/pingresp.ex b/lib/tortoise/package/pingresp.ex
index 5f42ed6e..5d602433 100644
--- a/lib/tortoise/package/pingresp.ex
+++ b/lib/tortoise/package/pingresp.ex
@@ -21,4 +21,22 @@ defmodule Tortoise.Package.Pingresp do
       [Package.Meta.encode(t.__META__), 0]
     end
   end
+
+  if Code.ensure_loaded?(StreamData) do
+    defimpl Tortoise.Generatable do
+      import StreamData
+
+      def generate(%type{__META__: _meta} = package) do
+        values = package |> Map.from_struct()
+
+        fixed_list(Enum.map(values, &constant(&1)))
+        |> bind(fn data ->
+          fixed_map([
+            {:__struct__, type}
+            | for({k, v} <- data, do: {k, constant(v)})
+          ])
+        end)
+      end
+    end
+  end
 end
diff --git a/test/tortoise/package/pingresp_test.exs b/test/tortoise/package/pingresp_test.exs
index 2de95612..d63cabd3 100644
--- a/test/tortoise/package/pingresp_test.exs
+++ b/test/tortoise/package/pingresp_test.exs
@@ -1,15 +1,21 @@
 defmodule Tortoise.Package.PingrespTest do
   use ExUnit.Case
+  use ExUnitProperties
+
   doctest Tortoise.Package.Pingresp
 
   alias Tortoise.Package
 
-  test "encoding and decoding ping responses" do
-    pingresp = %Package.Pingresp{}
+  property "encoding and decoding pingresp messages" do
+    # I know, data will always be the same, having a property for this
+    # is kind of silly...
+    config = %Package.Pingresp{}
 
-    assert ^pingresp =
-             pingresp
-             |> Package.encode()
-             |> Package.decode()
+    check all pingresp <- Package.generate(config) do
+      assert pingresp ==
+               pingresp
+               |> Package.encode()
+               |> Package.decode()
+    end
   end
 end

From 26a64ad41400fa4ec544fb570bd4bb1abb772d15 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Wed, 13 May 2020 09:59:55 +0100
Subject: [PATCH 188/220] Use a frequency generator to make property generator
 fail less often

Using a one_of it would sometimes fail because it could not generate
an element it hadn't generated before, for the uniq_list_of. With the
frequency we can tell it to generate user defined properties more
often, and that one we are allowed to generate multiple of.
---
 lib/tortoise/package/disconnect.ex | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/lib/tortoise/package/disconnect.ex b/lib/tortoise/package/disconnect.ex
index 7a2f489e..74c0699f 100644
--- a/lib/tortoise/package/disconnect.ex
+++ b/lib/tortoise/package/disconnect.ex
@@ -226,17 +226,23 @@ defmodule Tortoise.Package.Disconnect do
           {nil, values} ->
             properties =
               uniq_list_of(
-                one_of([
+                # Use frequency to make it more likely to pick a user
+                # property as we are allowed to have multiple of them;
+                # the remaining properties may only occur once,
+                # without the weights we could end up in situations
+                # where StreamData gives up because it cannot find any
+                # candidates that hasn't been chosen before
+                frequency([
                   # here we allow stings with a byte size of zero; don't
                   # know if that is a problem according to the spec. Let's
                   # handle that situation just in case:
-                  {constant(:user_property), {string(:printable), string(:printable)}},
-                  {constant(:reason_string), string(:printable)},
-                  {constant(:session_expiry_interval), integer(0..0xFFFFFFFF)}
+                  {5, {constant(:user_property), {string(:printable), string(:printable)}}},
+                  {1, {constant(:reason_string), string(:printable)}},
+                  {1, {constant(:session_expiry_interval), integer(0..0xFFFFFFFF)}}
                   # TODO generate valid :server_reference,
                 ]),
                 uniq_fun: &uniq/1,
-                max_length: 5
+                max_length: 10
               )
 
             fixed_list([

From 0ef9073b304ce64f1aa6365df3f3c7fef48de360 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Thu, 14 May 2020 09:39:16 +0100
Subject: [PATCH 189/220] Add a data generator for suback messages and a prop
 test for them

When generating the ack list it is possible to specify `nil` which
will generate a list of acks; but it is also possible to steer the
generation such that `[nil]` will generate a list with one random ack,
and `[nil, nil]` will generate a list with two random acks. One could
also specify `[{:ok, nil}, {:error, nil}]` which will generate a list
of length two with an random ok-ack on the first position and a random
error reason on the second.
---
 lib/tortoise/package/suback.ex        | 109 ++++++++++++++++++++++++++
 test/tortoise/package/suback_test.exs |  28 +++----
 2 files changed, 121 insertions(+), 16 deletions(-)

diff --git a/lib/tortoise/package/suback.ex b/lib/tortoise/package/suback.ex
index 8c67ced5..f84b34ee 100644
--- a/lib/tortoise/package/suback.ex
+++ b/lib/tortoise/package/suback.ex
@@ -124,4 +124,113 @@ defmodule Tortoise.Package.Suback do
     defp encode_ack({:error, :subscription_identifiers_not_supported}), do: 0xA1
     defp encode_ack({:error, :wildcard_subscriptions_not_supported}), do: 0xA2
   end
+
+  if Code.ensure_loaded?(StreamData) do
+    defimpl Tortoise.Generatable do
+      import StreamData
+
+      def generate(%type{__META__: _meta} = package) do
+        values = package |> Map.from_struct()
+
+        fixed_list(Enum.map(values, &constant(&1)))
+        |> bind(&gen_identifier/1)
+        |> bind(&gen_acks/1)
+        |> bind(&gen_properties/1)
+        |> bind(&fixed_map([{:__struct__, type} | for({k, v} <- &1, do: {k, constant(v)})]))
+      end
+
+      defp gen_identifier(values) do
+        case Keyword.pop(values, :identifier) do
+          {nil, values} ->
+            fixed_list([
+              {constant(:identifier), integer(1..0xFFFF)}
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {id, _} when is_integer(id) and id in 1..0xFFFF ->
+            constant(values)
+        end
+      end
+
+      @refusals [
+        :unspecified_error,
+        :implementation_specific_error,
+        :not_authorized,
+        :topic_filter_invalid,
+        :packet_identifier_in_use,
+        :quota_exceeded,
+        :shared_subscriptions_not_supported,
+        :subscription_identifiers_not_supported,
+        :wildcard_subscriptions_not_supported
+      ]
+
+      defp gen_acks(values) do
+        # Rule: An empty ack list is not allowed; it is a protocol
+        # error to not acknowledge or rejct at least one subscription
+        case Keyword.pop(values, :acks) do
+          {nil, values} ->
+            fixed_list([
+              {constant(:acks), nonempty(list_of(do_gen_ack()))}
+              | Enum.map(values, &constant(&1))
+            ])
+
+          # Generate the acks list based on a list containing either
+          # valid ok/error tuples, or nils, where nils will get
+          # replaced with an ack generator. This allow us to generate
+          # lists with a fixed length and with specific spots filled
+          # with particular values
+          {[_ | _] = acks, values} ->
+            fixed_list([
+              {constant(:acks),
+               fixed_list(
+                 Enum.map(acks, fn
+                   nil -> do_gen_ack()
+                   {:ok, n} = value when n in 0..2 -> constant(value)
+                   {:ok, nil} -> {constant(:ok), integer(0..2)}
+                   {:error, e} = value when e in @refusals -> constant(value)
+                   {:error, nil} -> {constant(:error), one_of(@refusals)}
+                 end)
+               )}
+              | Enum.map(values, &constant(&1))
+            ])
+        end
+      end
+
+      defp do_gen_ack() do
+        frequency([
+          {60, tuple({constant(:ok), integer(0..2)})},
+          {40, tuple({constant(:error), one_of(@refusals)})}
+        ])
+      end
+
+      defp gen_properties(values) do
+        case Keyword.pop(values, :properties) do
+          {nil, values} ->
+            properties =
+              uniq_list_of(
+                frequency([
+                  # here we allow stings with a byte size of zero; don't
+                  # know if that is a problem according to the spec. Let's
+                  # handle that situation just in case:
+                  {4, {constant(:user_property), {string(:printable), string(:printable)}}},
+                  {1, {constant(:reason_string), string(:printable)}}
+                ]),
+                uniq_fun: &uniq/1,
+                max_length: 20
+              )
+
+            fixed_list([
+              {constant(:properties), properties}
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {_passthrough, _} ->
+            constant(values)
+        end
+      end
+
+      defp uniq({:user_property, _v}), do: :crypto.strong_rand_bytes(2)
+      defp uniq({k, _v}), do: k
+    end
+  end
 end
diff --git a/test/tortoise/package/suback_test.exs b/test/tortoise/package/suback_test.exs
index 3d7ccfcd..facdbbef 100644
--- a/test/tortoise/package/suback_test.exs
+++ b/test/tortoise/package/suback_test.exs
@@ -1,23 +1,19 @@
 defmodule Tortoise.Package.SubackTest do
   use ExUnit.Case
-  # use EQC.ExUnit
-  doctest Tortoise.Package.Suback
+  use ExUnitProperties
 
-  # import Tortoise.TestGenerators, only: [gen_suback: 0]
+  doctest Tortoise.Package.Suback
 
-  # alias Tortoise.Package
+  alias Tortoise.Package
 
-  # property "encoding and decoding suback messages" do
-  #   forall suback <- gen_suback() do
-  #     ensure(
-  #       suback ==
-  #         suback
-  #         |> Package.encode()
-  #         |> Package.decode()
-  #     )
-  #   end
-  # end
+  property "encoding and decoding suback messages" do
+    config = %Package.Suback{identifier: nil, acks: nil, properties: nil}
 
-  @tag :skip
-  test "encoding and decoding suback messages"
+    check all suback <- Package.generate(config) do
+      assert suback ==
+               suback
+               |> Package.encode()
+               |> Package.decode()
+    end
+  end
 end

From f6ccf1c84df415742e168b95c681f9f6de05a4d2 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Sat, 16 May 2020 11:07:36 +0100
Subject: [PATCH 190/220] Add a generator for unsubacks and test them

---
 lib/tortoise/package/unsuback.ex        | 112 +++++++++++++++++++++++-
 test/tortoise/package/unsuback_test.exs |  27 +++---
 2 files changed, 122 insertions(+), 17 deletions(-)

diff --git a/lib/tortoise/package/unsuback.ex b/lib/tortoise/package/unsuback.ex
index 930e8864..324f903d 100644
--- a/lib/tortoise/package/unsuback.ex
+++ b/lib/tortoise/package/unsuback.ex
@@ -20,8 +20,11 @@ defmodule Tortoise.Package.Unsuback do
   @opaque t :: %__MODULE__{
             __META__: Package.Meta.t(),
             identifier: Tortoise.package_identifier(),
-            results: [],
-            properties: [{:reason_string, any()}, {:user_property, {String.t(), String.t()}}]
+            results: [:success | {:error, refusal}],
+            properties: [
+              {:reason_string, String.t()}
+              | {:user_property, {String.t(), String.t()}}
+            ]
           }
   @enforce_keys [:identifier]
   defstruct __META__: %Package.Meta{opcode: @opcode, flags: 0b0000},
@@ -100,4 +103,109 @@ defmodule Tortoise.Package.Unsuback do
       end
     end
   end
+
+  if Code.ensure_loaded?(StreamData) do
+    defimpl Tortoise.Generatable do
+      import StreamData
+
+      def generate(%type{__META__: _meta} = package) do
+        values = package |> Map.from_struct()
+
+        fixed_list(Enum.map(values, &constant(&1)))
+        |> bind(&gen_identifier/1)
+        |> bind(&gen_results/1)
+        |> bind(&gen_properties/1)
+        |> bind(&fixed_map([{:__struct__, type} | for({k, v} <- &1, do: {k, constant(v)})]))
+      end
+
+      defp gen_identifier(values) do
+        case Keyword.pop(values, :identifier) do
+          {nil, values} ->
+            fixed_list([
+              {constant(:identifier), integer(1..0xFFFF)}
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {id, _} when is_integer(id) and id in 1..0xFFFF ->
+            constant(values)
+        end
+      end
+
+      @refusals [
+        :no_subscription_existed,
+        :unspecified_error,
+        :implementation_specific_error,
+        :not_authorized,
+        :topic_filter_invalid,
+        :packet_identifier_in_use
+      ]
+
+      defp gen_results(values) do
+        # Rule: An empty ack list is not allowed; it is a protocol
+        # error to not acknowledge or rejct at least one subscription
+        case Keyword.pop(values, :results) do
+          {nil, values} ->
+            fixed_list([
+              {constant(:results), nonempty(list_of(do_gen_result()))}
+              | Enum.map(values, &constant(&1))
+            ])
+
+          # Generate the results list based on a list containing either
+          # valid success/error tuples, or nils, where nils will get
+          # replaced with an result generator. This allow us to generate
+          # lists with a fixed length and with specific spots filled
+          # with particular values
+          {[_ | _] = results, values} ->
+            fixed_list([
+              {constant(:results),
+               fixed_list(
+                 Enum.map(results, fn
+                   nil -> do_gen_result()
+                   :success -> constant(:success)
+                   {:error, e} = value when e in @refusals -> constant(value)
+                   {:error, nil} -> {constant(:error), one_of(@refusals)}
+                 end)
+               )}
+              | Enum.map(values, &constant(&1))
+            ])
+        end
+      end
+
+      defp do_gen_result() do
+        frequency([
+          {60, constant(:success)},
+          {40, tuple({constant(:error), one_of(@refusals)})}
+        ])
+      end
+
+      defp gen_properties(values) do
+        case Keyword.pop(values, :properties) do
+          {nil, values} ->
+            properties =
+              uniq_list_of(
+                frequency([
+                  # here we allow stings with a byte size of zero; don't
+                  # know if that is a problem according to the spec. Let's
+                  # handle that situation just in case:
+                  {4, {constant(:user_property), {string(:printable), string(:printable)}}},
+                  {1, {constant(:reason_string), string(:printable)}}
+                ]),
+                uniq_fun: &uniq/1,
+                max_length: 20
+              )
+
+            fixed_list([
+              {constant(:properties), properties}
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {_passthrough, _} ->
+            constant(values)
+        end
+      end
+
+      defp uniq({:user_property, _v}), do: :crypto.strong_rand_bytes(2)
+      defp uniq({k, _v}), do: k
+    end
+  end
 end
diff --git a/test/tortoise/package/unsuback_test.exs b/test/tortoise/package/unsuback_test.exs
index 87f0318d..fb43810b 100644
--- a/test/tortoise/package/unsuback_test.exs
+++ b/test/tortoise/package/unsuback_test.exs
@@ -1,22 +1,19 @@
 defmodule Tortoise.Package.UnsubackTest do
   use ExUnit.Case
-  # use EQC.ExUnit
+  use ExUnitProperties
+
   doctest Tortoise.Package.Unsuback
 
-  # import Tortoise.TestGenerators, only: [gen_unsuback: 0]
+  alias Tortoise.Package
 
-  # alias Tortoise.Package
+  property "encoding and decoding unsuback messages" do
+    config = %Package.Unsuback{identifier: nil, results: nil, properties: nil}
 
-  # property "encoding and decoding unsuback messages" do
-  #   forall unsuback <- gen_unsuback() do
-  #     ensure(
-  #       unsuback ==
-  #         unsuback
-  #         |> Package.encode()
-  #         |> Package.decode()
-  #     )
-  #   end
-  # end
-  @tag :skip
-  test "encoding and decoding unsuback messages"
+    check all unsuback <- Package.generate(config) do
+      assert unsuback ==
+               unsuback
+               |> Package.encode()
+               |> Package.decode()
+    end
+  end
 end

From f6ebe70a46fb42ac71cc3f660e3dd6d78ad6c51f Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Mon, 18 May 2020 21:44:41 +0100
Subject: [PATCH 191/220] Add a generator for topics and topic filters

---
 lib/tortoise/generatable.ex | 115 ++++++++++++++++++++++++++++++++++++
 1 file changed, 115 insertions(+)

diff --git a/lib/tortoise/generatable.ex b/lib/tortoise/generatable.ex
index f30c8fd8..991561c4 100644
--- a/lib/tortoise/generatable.ex
+++ b/lib/tortoise/generatable.ex
@@ -4,3 +4,118 @@ defprotocol Tortoise.Generatable do
   # TODO add spec
   def generate(data)
 end
+
+if Code.ensure_loaded?(StreamData) do
+  defmodule Tortoise.Generatable.Topic do
+    import StreamData
+
+    # "/" is a valid topic, becomes ["", ""] in tortoise, which is a
+    # special case because topic levels are not allowed to be empty:
+    # they must be a string of at least one character. It is also
+    # allowed to start a topic with a slash "/foo" which is different
+    # from "foo". In tortoise, when the user is given the topic
+    # levels, the former if represented as `["", "foo"]`.
+
+    @doc """
+    Generate a random MQTT topic
+    """
+    def gen_topic() do
+      # generate a list of nils that will get replaced by topic level
+      # generators
+      bind(list_of(nil), &gen_topic/1)
+    end
+
+    @doc """
+    Generate a topic based on an input
+
+    TODO fix this documentation
+    """
+    # Let the smallest element that can be created be "/", which in
+    # out internal representation is represented as `["", ""]`.
+    def gen_topic([]), do: constant(["", ""])
+
+    # short circuit one level long topics
+    def gen_topic([topic_level]) do
+      fixed_list([gen_topic_level(topic_level)])
+    end
+
+    # from now on we are dealing with lists of length > 1
+    def gen_topic([_ | _] = input) do
+      bind(constant(input), fn
+        [nil | topic_list] ->
+          fixed_list([
+            frequency([
+              # the first topic level is allowed to be empty which
+              # will translate into a topic that starts with a slash
+              # ("/foo", which is different from "foo")
+              {4, gen_topic_level(nil)},
+              {1, constant("")}
+            ])
+            | for(topic_level <- topic_list, do: gen_topic_level(topic_level))
+          ])
+
+        topic_list ->
+          fixed_list(for topic_level <- topic_list, do: gen_topic_level(topic_level))
+      end)
+    end
+
+    # generate the topic level names
+    defp gen_topic_level(nil) do
+      bind(string(:ascii, min_length: 1), fn topic_level ->
+        constant(String.replace(topic_level, ["#", "+", "/"], "_"))
+      end)
+    end
+
+    defp gen_topic_level(%StreamData{} = generator), do: generator
+    defp gen_topic_level(<<literal::binary>>), do: constant(literal)
+
+    @doc """
+    Generate a random topic filter
+    """
+    def gen_filter() do
+      bind(list_of(nil), &gen_filter/1)
+    end
+
+    @doc """
+    Generate a topic filter based on a topic
+
+    The resulting topic filter will be one that matches the given
+    topic, so `["foo", "bar"]` will result in topic filters such as
+    `["foo", "+"]`, `["#"]`, `["+", "#"]`, etc.
+    """
+    def gen_filter(input) do
+      gen_topic(input)
+      |> bind(&maybe_add_multi_level_filter/1)
+      |> bind(&mutate_topic_levels/1)
+    end
+
+    defp maybe_add_multi_level_filter(topic_levels) do
+      frequency([
+        {4, constant(topic_levels)},
+        {1, bind(constant(topic_levels), &add_multi_level_filter/1)}
+      ])
+    end
+
+    defp add_multi_level_filter([_ | _] = topic_list) do
+      start = length(topic_list) * -1
+
+      bind(integer(start..-1), fn position ->
+        constant(Enum.drop(topic_list, position) ++ ["#"])
+      end)
+    end
+
+    defp mutate_topic_levels(topic_levels) do
+      fixed_list(for topic_level <- topic_levels, do: do_mutate_topic_level(topic_level))
+    end
+
+    defp do_mutate_topic_level("+"), do: constant("+")
+    defp do_mutate_topic_level("#"), do: constant("#")
+
+    defp do_mutate_topic_level(topic_level) do
+      frequency([
+        {1, constant("+")},
+        {2, constant(topic_level)}
+      ])
+    end
+  end
+end

From 9a8de243d6642b5fc757420c7f4b0472187e6497 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Sat, 23 May 2020 18:05:51 +0100
Subject: [PATCH 192/220] Add a data generator for subscribe packages and prop
 test them

---
 lib/tortoise/package/subscribe.ex        | 158 +++++++++++++++++++++++
 test/tortoise/package/subscribe_test.exs |  29 ++---
 2 files changed, 170 insertions(+), 17 deletions(-)

diff --git a/lib/tortoise/package/subscribe.ex b/lib/tortoise/package/subscribe.ex
index 0921d4bc..151282c0 100644
--- a/lib/tortoise/package/subscribe.ex
+++ b/lib/tortoise/package/subscribe.ex
@@ -167,4 +167,162 @@ defmodule Tortoise.Package.Subscribe do
        end}
     end
   end
+
+  if Code.ensure_loaded?(StreamData) do
+    defimpl Tortoise.Generatable do
+      import StreamData
+
+      alias Tortoise.Generatable.Topic
+
+      def generate(%type{__META__: _meta} = package) do
+        values = package |> Map.from_struct()
+
+        fixed_list(Enum.map(values, &constant(&1)))
+        |> bind(&gen_identifier/1)
+        |> bind(&gen_topics/1)
+        |> bind(&gen_topic_opts/1)
+        |> bind(&gen_properties/1)
+        |> bind(&fixed_map([{:__struct__, type} | for({k, v} <- &1, do: {k, constant(v)})]))
+      end
+
+      defp gen_identifier(values) do
+        case Keyword.pop(values, :identifier) do
+          {nil, values} ->
+            fixed_list([
+              {constant(:identifier), integer(1..0xFFFF)}
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {id, _} when is_integer(id) and id in 1..0xFFFF ->
+            constant(values)
+        end
+      end
+
+      defp gen_topics(values) do
+        case Keyword.pop(values, :topics) do
+          {nil, values} ->
+            fixed_list([
+              {
+                :topics,
+                list_of(
+                  {gen_topic_filter(), constant(nil)},
+                  max_length: 5,
+                  min_length: 1
+                )
+              }
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {[_ | _] = topics, values} ->
+            [
+              {:topics,
+               fixed_list(
+                 Enum.map(topics, fn
+                   nil ->
+                     {gen_topic_filter(), constant([])}
+
+                   {nil, opts} ->
+                     {gen_topic_filter(), constant(opts)}
+
+                   %StreamData{} = generator ->
+                     bind(generator, fn
+                       {<<_::binary>>, opts} = result when is_list(opts) or is_nil(opts) ->
+                         # result seems fine, pass it on
+                         constant(result)
+
+                       faulty_return ->
+                         raise ArgumentError, """
+                         Faulty result from user specified topic generator #{inspect(generator)}
+
+                         The generator should return a tuple with two elements, where the first is a binary, and the second is a `nil` or a list. Instead the generator returned:
+
+                           #{inspect(faulty_return)}
+
+                         """
+                     end)
+
+                   {%StreamData{} = generator, opts} ->
+                     {generator, constant(opts)}
+
+                   {<<_::binary>>, _} = otherwise ->
+                     constant(otherwise)
+                 end)
+               )}
+              | Enum.map(values, &constant(&1))
+            ]
+            |> fixed_list()
+        end
+      end
+
+      defp gen_topic_filter() do
+        bind(Topic.gen_filter(), &constant(Enum.join(&1, "/")))
+      end
+
+      # create random options for the topics in the topic list
+      defp gen_topic_opts(values) do
+        # at this point in time we should have a list of topics!
+        {[{_, _} | _] = topics, values} = Keyword.pop(values, :topics)
+
+        topics_with_opts =
+          Enum.map(topics, fn {topic_filter, opts} ->
+            opts = opts || []
+
+            {
+              constant(topic_filter),
+              # notice that while the order of options shouldn't
+              # matter, it kind of does in the context of the prop
+              # tests for encoding and decoding the subscribe
+              # packages, as the keyword lists will get compared for
+              # equality
+              fixed_list([
+                do_get_opts(opts, :qos, integer(0..2)),
+                do_get_opts(opts, :no_local, boolean()),
+                do_get_opts(opts, :retain_as_published, boolean()),
+                do_get_opts(opts, :retain_handling, integer(0..2))
+              ])
+            }
+          end)
+          |> fixed_list()
+
+        fixed_list([{:topics, topics_with_opts} | Enum.map(values, &constant(&1))])
+      end
+
+      defp do_get_opts(opts, key, default) do
+        generator =
+          case Keyword.get(opts, key) do
+            nil -> default
+            %StreamData{} = generator -> generator
+            otherwise -> constant(otherwise)
+          end
+
+        {key, generator}
+      end
+
+      defp gen_properties(values) do
+        case Keyword.pop(values, :properties) do
+          {nil, values} ->
+            properties =
+              uniq_list_of(
+                frequency([
+                  # here we allow stings with a byte size of zero; don't
+                  # know if that is a problem according to the spec. Let's
+                  # handle that situation just in case:
+                  {4, {:user_property, {string(:printable), string(:printable)}}},
+                  {1, {:subscription_identifier, integer(1..268_435_455)}}
+                ]),
+                uniq_fun: &uniq/1,
+                max_length: 5
+              )
+
+            fixed_list([{:properties, properties} | Enum.map(values, &constant(&1))])
+
+          {_passthrough, _} ->
+            constant(values)
+        end
+      end
+
+      defp uniq({:user_property, _v}), do: :crypto.strong_rand_bytes(2)
+      defp uniq({k, _v}), do: k
+    end
+  end
 end
diff --git a/test/tortoise/package/subscribe_test.exs b/test/tortoise/package/subscribe_test.exs
index 702d026e..7af34bf3 100644
--- a/test/tortoise/package/subscribe_test.exs
+++ b/test/tortoise/package/subscribe_test.exs
@@ -1,26 +1,21 @@
 defmodule Tortoise.Package.SubscribeTest do
   use ExUnit.Case
-  # use EQC.ExUnit
-  doctest Tortoise.Package.Subscribe
+  use ExUnitProperties
 
-  # import Tortoise.TestGenerators, only: [gen_subscribe: 0]
+  doctest Tortoise.Package.Subscribe
 
-  # alias Tortoise.Package
-  # alias Tortoise.Package.Subscribe
+  alias Tortoise.Package
 
-  @tag :skip
-  test "encoding and decoding subscribe messages"
+  property "encoding and decoding subscribe messages" do
+    config = %Package.Subscribe{identifier: nil, topics: nil, properties: nil}
 
-  # property "encoding and decoding subscribe messages" do
-  #   forall subscribe <- gen_subscribe() do
-  #     ensure(
-  #       subscribe ==
-  #         subscribe
-  #         |> Package.encode()
-  #         |> Package.decode()
-  #     )
-  #   end
-  # end
+    check all subscribe <- Package.generate(config) do
+      assert subscribe ==
+               subscribe
+               |> Package.encode()
+               |> Package.decode()
+    end
+  end
 
   # describe "Collectable" do
   #   test "Accept tuples of {binary(), opts()} as input" do

From d5169b2080ad16b15443d94d875719a1d8a2c123 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Sat, 23 May 2020 22:17:25 +0100
Subject: [PATCH 193/220] Add a data generator for unsubscribe packages and
 test them

---
 lib/tortoise/package/unsubscribe.ex        | 102 +++++++++++++++++++++
 test/tortoise/package/unsubscribe_test.exs |  27 +++---
 2 files changed, 114 insertions(+), 15 deletions(-)

diff --git a/lib/tortoise/package/unsubscribe.ex b/lib/tortoise/package/unsubscribe.ex
index 159e7c56..63222819 100644
--- a/lib/tortoise/package/unsubscribe.ex
+++ b/lib/tortoise/package/unsubscribe.ex
@@ -70,4 +70,106 @@ defmodule Tortoise.Package.Unsubscribe do
       ]
     end
   end
+
+  if Code.ensure_loaded?(StreamData) do
+    defimpl Tortoise.Generatable do
+      import StreamData
+
+      alias Tortoise.Generatable.Topic
+
+      def generate(%type{__META__: _meta} = package) do
+        values = package |> Map.from_struct()
+
+        fixed_list(Enum.map(values, &constant(&1)))
+        |> bind(&gen_identifier/1)
+        |> bind(&gen_topics/1)
+        |> bind(&gen_properties/1)
+        |> bind(&fixed_map([{:__struct__, type} | for({k, v} <- &1, do: {k, constant(v)})]))
+      end
+
+      defp gen_identifier(values) do
+        case Keyword.pop(values, :identifier) do
+          {nil, values} ->
+            fixed_list([
+              {:identifier, integer(1..0xFFFF)}
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {id, _} when is_integer(id) and id in 1..0xFFFF ->
+            constant(values)
+        end
+      end
+
+      defp gen_topics(values) do
+        case Keyword.pop(values, :topics) do
+          {nil, values} ->
+            fixed_list([
+              {
+                :topics,
+                list_of(gen_topic_filter(), min_length: 1, max_length: 5)
+              }
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {[_ | _] = topics, values} ->
+            [
+              {:topics,
+               fixed_list(
+                 Enum.map(topics, fn
+                   nil ->
+                     gen_topic_filter()
+
+                   %StreamData{} = generator ->
+                     bind(generator, fn
+                       <<topic_filter::binary>> when byte_size(topic_filter) > 1 ->
+                         constant(topic_filter)
+
+                       faulty_return ->
+                         raise ArgumentError, """
+                         Faulty result from user specified topic filter generator #{
+                           inspect(generator)
+                         }
+
+                         The generator should return a non-empty binary. Instead the generator returned:
+
+                           #{inspect(faulty_return)}
+                         """
+                     end)
+
+                   <<topic_filter::binary>> ->
+                     constant(topic_filter)
+                 end)
+               )}
+              | Enum.map(values, &constant(&1))
+            ]
+            |> fixed_list()
+        end
+      end
+
+      defp gen_topic_filter() do
+        bind(Topic.gen_filter(), &constant(Enum.join(&1, "/")))
+      end
+
+      defp gen_properties(values) do
+        # user properties are the only valid property for unsubscribe
+        # packages
+        case Keyword.pop(values, :properties) do
+          {nil, values} ->
+            properties =
+              list_of(
+                # here we allow stings with a byte size of zero; don't
+                # know if that is a problem according to the
+                # spec. Let's handle that situation just in case:
+                {:user_property, {string(:printable), string(:printable)}},
+                max_length: 5
+              )
+
+            fixed_list([{:properties, properties} | Enum.map(values, &constant(&1))])
+
+          {_passthrough, _} ->
+            constant(values)
+        end
+      end
+    end
+  end
 end
diff --git a/test/tortoise/package/unsubscribe_test.exs b/test/tortoise/package/unsubscribe_test.exs
index 234f60b7..ca12af9c 100644
--- a/test/tortoise/package/unsubscribe_test.exs
+++ b/test/tortoise/package/unsubscribe_test.exs
@@ -1,22 +1,19 @@
 defmodule Tortoise.Package.UnsubscribeTest do
   use ExUnit.Case
-  # use EQC.ExUnit
+  use ExUnitProperties
+
   doctest Tortoise.Package.Unsubscribe
 
-  # import Tortoise.TestGenerators, only: [gen_unsubscribe: 0]
+  alias Tortoise.Package
 
-  # alias Tortoise.Package
+  property "encoding and decoding unsubscribe messages" do
+    config = %Package.Unsubscribe{identifier: nil, topics: nil, properties: nil}
 
-  # property "encoding and decoding unsubscribe messages" do
-  #   forall unsubscribe <- gen_unsubscribe() do
-  #     ensure(
-  #       unsubscribe ==
-  #         unsubscribe
-  #         |> Package.encode()
-  #         |> Package.decode()
-  #     )
-  #   end
-  # end
-  @tag :skip
-  test "encoding and decoding unsubscribe messages"
+    check all unsubscribe <- Package.generate(config) do
+      assert unsubscribe ==
+               unsubscribe
+               |> Package.encode()
+               |> Package.decode()
+    end
+  end
 end

From 17ab2603b867b32396a824245368fb0d44cf06b3 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Sun, 24 May 2020 11:09:01 +0100
Subject: [PATCH 194/220] Add a auth packet generator and test them

This is a first iteration. As I read the specs the package properties
should always contain the "Authentication Method", so I need a way to
specify that it is always present/required in the generator. I think I
will reconsider the property generator in a future commit.
---
 lib/tortoise/package/auth.ex        | 65 +++++++++++++++++++++++++++++
 test/tortoise/package/auth_test.exs | 27 ++++++------
 2 files changed, 77 insertions(+), 15 deletions(-)

diff --git a/lib/tortoise/package/auth.ex b/lib/tortoise/package/auth.ex
index 32246871..b2aae0d8 100644
--- a/lib/tortoise/package/auth.ex
+++ b/lib/tortoise/package/auth.ex
@@ -65,4 +65,69 @@ defmodule Tortoise.Package.Auth do
       end
     end
   end
+
+  if Code.ensure_loaded?(StreamData) do
+    defimpl Tortoise.Generatable do
+      import StreamData
+
+      def generate(%type{__META__: _meta} = package) do
+        values = package |> Map.from_struct()
+
+        fixed_list(Enum.map(values, &constant(&1)))
+        |> bind(&gen_reason/1)
+        |> bind(&gen_properties/1)
+        |> bind(&fixed_map([{:__struct__, type} | for({k, v} <- &1, do: {k, constant(v)})]))
+      end
+
+      @reasons [
+        :success,
+        :continue_authentication,
+        :re_authenticate
+      ]
+
+      defp gen_reason(values) do
+        case Keyword.pop(values, :reason) do
+          {nil, values} ->
+            fixed_list([{:reason, one_of(@reasons)} | Enum.map(values, &constant(&1))])
+
+          {reason, _} when reason in @reasons ->
+            constant(values)
+        end
+      end
+
+      # If the initial CONNECT packet included an Authentication
+      # Method property then all AUTH packets, and any successful
+      # CONNACK packet MUST include an Authentication Method Property
+      # with the same value as in the CONNECT packet [MQTT-4.12.0-5].
+
+      defp gen_properties(values) do
+        case Keyword.pop(values, :properties) do
+          {nil, values} ->
+            properties =
+              uniq_list_of(
+                frequency([
+                  # here we allow stings with a byte size of zero; don't
+                  # know if that is a problem according to the spec. Let's
+                  # handle that situation just in case:
+                  {4, {:user_property, {string(:printable), string(:printable)}}},
+                  # TODO authentication method should always be present
+                  {1, {:authentication_method, string(:printable)}},
+                  {1, {:authentication_data, string(:printable)}},
+                  {1, {:reason_string, string(:printable)}}
+                ]),
+                uniq_fun: &uniq/1,
+                max_length: 5
+              )
+
+            fixed_list([{:properties, properties} | Enum.map(values, &constant(&1))])
+
+          {_passthrough, _} ->
+            constant(values)
+        end
+      end
+
+      defp uniq({:user_property, _v}), do: :crypto.strong_rand_bytes(2)
+      defp uniq({k, _v}), do: k
+    end
+  end
 end
diff --git a/test/tortoise/package/auth_test.exs b/test/tortoise/package/auth_test.exs
index fe777119..6e2379df 100644
--- a/test/tortoise/package/auth_test.exs
+++ b/test/tortoise/package/auth_test.exs
@@ -1,22 +1,19 @@
 defmodule Tortoise.Package.AuthTest do
   use ExUnit.Case
-  # use EQC.ExUnit
+  use ExUnitProperties
+
   doctest Tortoise.Package.Auth
 
-  # alias Tortoise.Package
+  alias Tortoise.Package
 
-  # import Tortoise.TestGenerators, only: [gen_auth: 0]
+  property "encoding and decoding auth messages" do
+    config = %Package.Auth{reason: nil, properties: nil}
 
-  # property "encoding and decoding auth messages" do
-  #   forall auth <- gen_auth() do
-  #     ensure(
-  #       auth ==
-  #         auth
-  #         |> Package.encode()
-  #         |> Package.decode()
-  #     )
-  #   end
-  # end
-  @tag :skip
-  test "encoding and decoding auth messages"
+    check all auth <- Package.generate(config) do
+      assert auth ==
+               auth
+               |> Package.encode()
+               |> Package.decode()
+    end
+  end
 end

From 0f9fe61f92095d245a2fd69ca3d5d8a13bad78af Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Sun, 24 May 2020 19:58:08 +0100
Subject: [PATCH 195/220] Add a data generator for Publish packets and test
 them

I needed a way to update the flags in the fixed packet header, and
these values had to be inferred from the content of the package, so I
added a `infer/1` function on the Meta module, which will ensure the
fixed header is correct before encoding.

After introducing this some of the tests started failing because the
fixed header was wrong; this has now been fixed.

I still need to make the user defined properties generate other
properties than the user defined ones.
---
 lib/tortoise/package/meta.ex               |  18 +++
 lib/tortoise/package/publish.ex            | 169 ++++++++++++++++++++-
 test/tortoise/connection/inflight_test.exs |  14 +-
 test/tortoise/connection_test.exs          |  52 +++++--
 test/tortoise/package/publish_test.exs     |  35 +++--
 5 files changed, 256 insertions(+), 32 deletions(-)

diff --git a/lib/tortoise/package/meta.ex b/lib/tortoise/package/meta.ex
index 4d25dc29..34b038e4 100644
--- a/lib/tortoise/package/meta.ex
+++ b/lib/tortoise/package/meta.ex
@@ -1,6 +1,8 @@
 defmodule Tortoise.Package.Meta do
   @moduledoc false
 
+  alias Tortoise.Package
+
   @opaque t() :: %__MODULE__{
             opcode: 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14,
             flags: non_neg_integer()
@@ -11,4 +13,20 @@ defmodule Tortoise.Package.Meta do
   def encode(meta) do
     <<meta.opcode::4, meta.flags::4>>
   end
+
+  @doc """
+  Infer the meta values from the package content and type
+  """
+  def infer(%Package.Publish{dup: dup, qos: qos, retain: retain} = data) do
+    <<flags::4>> = <<flag(dup)::1, qos::integer-size(2), flag(retain)::1>>
+    infered_meta = %__MODULE__{opcode: 3, flags: flags}
+    %Package.Publish{data | __META__: infered_meta}
+  end
+
+  def infer(%_type{__META__: _} = data) do
+    data
+  end
+
+  defp flag(true), do: 1
+  defp flag(false), do: 0
 end
diff --git a/lib/tortoise/package/publish.ex b/lib/tortoise/package/publish.ex
index 330f992b..a6ceb595 100644
--- a/lib/tortoise/package/publish.ex
+++ b/lib/tortoise/package/publish.ex
@@ -40,7 +40,7 @@ defmodule Tortoise.Package.Publish do
     payload = drop_length_prefix(length_prefixed_payload)
     {topic, properties, payload} = decode_message(payload)
 
-    %__MODULE__{
+    Package.Meta.infer(%__MODULE__{
       qos: 0,
       identifier: nil,
       dup: false,
@@ -48,7 +48,7 @@ defmodule Tortoise.Package.Publish do
       topic: topic,
       payload: payload,
       properties: properties
-    }
+    })
   end
 
   def decode(
@@ -57,7 +57,7 @@ defmodule Tortoise.Package.Publish do
     payload = drop_length_prefix(length_prefixed_payload)
     {topic, identifier, properties, payload} = decode_message_with_id(payload)
 
-    %__MODULE__{
+    Package.Meta.infer(%__MODULE__{
       qos: qos,
       identifier: identifier,
       dup: dup == 1,
@@ -65,7 +65,7 @@ defmodule Tortoise.Package.Publish do
       topic: topic,
       payload: payload,
       properties: properties
-    }
+    })
   end
 
   defp drop_length_prefix(payload) do
@@ -148,4 +148,165 @@ defmodule Tortoise.Package.Publish do
     defp flag(f) when f in [0, nil, false], do: 0
     defp flag(_), do: 1
   end
+
+  if Code.ensure_loaded?(StreamData) do
+    defimpl Tortoise.Generatable do
+      import StreamData
+
+      alias Tortoise.Generatable.Topic
+
+      def generate(%type{__META__: _meta} = package) do
+        values = package |> Map.from_struct()
+
+        fixed_list(Enum.map(values, &constant(&1)))
+        |> bind(&gen_topic/1)
+        |> bind(&gen_qos/1)
+        |> bind(&gen_retain/1)
+        |> bind(&gen_payload/1)
+        |> bind(&gen_identifier/1)
+        |> bind(&gen_dup/1)
+        |> bind(&gen_properties/1)
+        |> bind(&update_meta_flags/1)
+        |> bind(fn data ->
+          fixed_map([
+            {:__struct__, type}
+            | for({k, v} <- data, do: {k, constant(v)})
+          ])
+        end)
+      end
+
+      defp update_meta_flags(values) do
+        {meta, values} = Keyword.pop(values, :__META__)
+
+        qos = Keyword.get(values, :qos)
+        retain = Keyword.get(values, :retain)
+        dup = Keyword.get(values, :dup)
+
+        <<flags::4>> = <<flag(dup)::1, qos::integer-size(2), flag(retain)::1>>
+
+        fixed_list([
+          {:__META__, constant(%{meta | flags: flags})}
+          | Enum.map(values, &constant(&1))
+        ])
+      end
+
+      defp flag(true), do: 1
+      defp flag(false), do: 0
+
+      defp gen_qos(values) do
+        case Keyword.pop(values, :qos) do
+          {nil, values} ->
+            fixed_list([
+              {constant(:qos), integer(0..2)}
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {qos, _} when is_integer(qos) and qos in 0..2 ->
+            constant(values)
+        end
+      end
+
+      defp gen_identifier(values) do
+        qos = Keyword.get(values, :qos)
+
+        case Keyword.pop(values, :identifier) do
+          {nil, values} when qos > 0 ->
+            fixed_list([
+              {:identifier, integer(1..0xFFFF)}
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {nil, values} when qos == 0 ->
+            fixed_list([
+              {:identifier, nil}
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {id, _} when qos > 0 and is_integer(id) and id in 1..0xFFFF ->
+            constant(values)
+        end
+      end
+
+      defp gen_topic(values) do
+        case Keyword.pop(values, :topic) do
+          {nil, values} ->
+            bind(Topic.gen_topic(), fn topic_levels ->
+              fixed_list([
+                {:topic, constant(Enum.join(topic_levels, "/"))}
+                | Enum.map(values, &constant(&1))
+              ])
+            end)
+
+          {<<_::binary>>, _values} ->
+            constant(values)
+        end
+      end
+
+      defp gen_retain(values) do
+        case Keyword.pop(values, :retain) do
+          {nil, values} ->
+            fixed_list([{:retain, boolean()} | Enum.map(values, &constant(&1))])
+
+          {retain, _} when is_boolean(retain) ->
+            constant(values)
+        end
+      end
+
+      defp gen_payload(values) do
+        case Keyword.pop(values, :payload) do
+          {nil, values} ->
+            fixed_list([
+              {:payload, string(:ascii, min_length: 1)}
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {payload, _} when is_binary(payload) ->
+            constant(values)
+        end
+      end
+
+      defp gen_dup(values) do
+        qos = Keyword.get(values, :qos)
+
+        case Keyword.pop(values, :dup) do
+          {nil, values} when qos > 0 ->
+            fixed_list([{:dup, boolean()} | Enum.map(values, &constant(&1))])
+
+          {nil, values} when qos == 0 ->
+            fixed_list([{:dup, constant(false)} | Enum.map(values, &constant(&1))])
+
+          {dup, _} when is_boolean(dup) ->
+            constant(values)
+        end
+      end
+
+      defp gen_properties(values) do
+        case Keyword.pop(values, :properties) do
+          {nil, values} ->
+            properties =
+              uniq_list_of(
+                one_of([
+                  # here we allow stings with a byte size of zero; don't
+                  # know if that is a problem according to the spec. Let's
+                  # handle that situation just in case:
+                  {constant(:user_property), {string(:printable), string(:printable)}}
+                ]),
+                uniq_fun: &uniq/1,
+                max_length: 5
+              )
+
+            fixed_list([
+              {constant(:properties), properties}
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {_passthrough, _} ->
+            constant(values)
+        end
+      end
+
+      defp uniq({:user_property, _v}), do: :crypto.strong_rand_bytes(2)
+      defp uniq({k, _v}), do: k
+    end
+  end
 end
diff --git a/test/tortoise/connection/inflight_test.exs b/test/tortoise/connection/inflight_test.exs
index 9d29e067..f1512a57 100644
--- a/test/tortoise/connection/inflight_test.exs
+++ b/test/tortoise/connection/inflight_test.exs
@@ -59,7 +59,10 @@ defmodule Tortoise.Connection.InflightTest do
     end
 
     test "outgoing publish QoS=1", %{client_id: client_id} = context do
-      publish = %Package.Publish{identifier: 1, topic: "foo", qos: 1}
+      publish =
+        %Package.Publish{identifier: 1, topic: "foo", qos: 1}
+        |> Package.Meta.infer()
+
       {:ok, ref} = Inflight.track(client_id, {:outgoing, publish})
       assert {:ok, package} = :gen_tcp.recv(context.server, 0, 500)
       assert ^publish = Package.decode(package)
@@ -71,7 +74,7 @@ defmodule Tortoise.Connection.InflightTest do
 
       # the inflight process should now re-transmit the publish
       assert {:ok, package} = :gen_tcp.recv(context.server, 0, 500)
-      publish = %Package.Publish{publish | dup: true}
+      publish = %Package.Publish{publish | dup: true} |> Package.Meta.infer()
       assert ^publish = Package.decode(package)
 
       # simulate that we receive a puback from the server
@@ -115,7 +118,10 @@ defmodule Tortoise.Connection.InflightTest do
     end
 
     test "outgoing publish QoS=2", %{client_id: client_id} = context do
-      publish = %Package.Publish{identifier: 1, topic: "foo", qos: 2}
+      publish =
+        %Package.Publish{identifier: 1, topic: "foo", qos: 2}
+        |> Package.Meta.infer()
+
       {:ok, ref} = Inflight.track(client_id, {:outgoing, publish})
 
       # we should transmit the publish
@@ -126,7 +132,7 @@ defmodule Tortoise.Connection.InflightTest do
       {:ok, context} = setup_connection(context)
       {:ok, context} = setup_inflight(context)
       # the publish should get re-transmitted
-      publish = %Package.Publish{publish | dup: true}
+      publish = %Package.Publish{publish | dup: true} |> Package.Meta.infer()
       assert {:ok, package} = :gen_tcp.recv(context.server, 0, 500)
       assert ^publish = Package.decode(package)
 
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index c4da7968..2ed993d5 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -1250,7 +1250,11 @@ defmodule Tortoise.ConnectionTest do
 
     test "incoming publish with QoS=1", context do
       Process.flag(:trap_exit, true)
-      publish = %Package.Publish{identifier: 1, topic: "foo/bar", qos: 1}
+
+      publish =
+        %Package.Publish{identifier: 1, topic: "foo/bar", qos: 1}
+        |> Package.Meta.infer()
+
       expected_puback = %Package.Puback{identifier: 1}
 
       script = [
@@ -1267,7 +1271,11 @@ defmodule Tortoise.ConnectionTest do
 
     test "outgoing publish with QoS=1", context do
       Process.flag(:trap_exit, true)
-      publish = %Package.Publish{identifier: 1, topic: "foo/bar", qos: 1}
+
+      publish =
+        %Package.Publish{identifier: 1, topic: "foo/bar", qos: 1}
+        |> Package.Meta.infer()
+
       puback = %Package.Puback{identifier: 1}
 
       script = [
@@ -1290,7 +1298,11 @@ defmodule Tortoise.ConnectionTest do
 
     test "outgoing publish with QoS=1 (sync call)", %{client_id: client_id} = context do
       Process.flag(:trap_exit, true)
-      publish = %Package.Publish{identifier: 1, topic: "foo/bar", qos: 1}
+
+      publish =
+        %Package.Publish{identifier: 1, topic: "foo/bar", qos: 1}
+        |> Package.Meta.infer()
+
       puback = %Package.Puback{identifier: 1}
 
       script = [
@@ -1350,7 +1362,10 @@ defmodule Tortoise.ConnectionTest do
       assert_receive {ScriptedMqttServer, {:received, %Package.Connect{}}}
       assert_receive {ScriptedMqttServer, :completed}
 
-      publish = %Package.Publish{identifier: 1, topic: "foo/bar", qos: 1}
+      publish =
+        %Package.Publish{identifier: 1, topic: "foo/bar", qos: 1}
+        |> Package.Meta.infer()
+
       puback = %Package.Puback{identifier: 1}
 
       default_subscription_opts = [
@@ -1433,7 +1448,10 @@ defmodule Tortoise.ConnectionTest do
       assert_receive {ScriptedMqttServer, {:received, %Package.Connect{}}}
       assert_receive {ScriptedMqttServer, :completed}
 
-      publish = %Package.Publish{identifier: 1, topic: "foo/bar", qos: 2}
+      publish =
+        %Package.Publish{identifier: 1, topic: "foo/bar", qos: 2}
+        |> Package.Meta.infer()
+
       pubrec = %Package.Pubrec{identifier: 1}
       pubrel = %Package.Pubrel{identifier: 1}
       pubcomp = %Package.Pubcomp{identifier: 1}
@@ -1464,7 +1482,11 @@ defmodule Tortoise.ConnectionTest do
 
     test "incoming publish with QoS=2", context do
       Process.flag(:trap_exit, true)
-      publish = %Package.Publish{identifier: 1, topic: "foo/bar", qos: 2}
+
+      publish =
+        %Package.Publish{identifier: 1, topic: "foo/bar", qos: 2}
+        |> Package.Meta.infer()
+
       expected_pubrec = %Package.Pubrec{identifier: 1}
       pubrel = %Package.Pubrel{identifier: 1}
       expected_pubcomp = %Package.Pubcomp{identifier: 1}
@@ -1489,7 +1511,11 @@ defmodule Tortoise.ConnectionTest do
 
     test "incoming publish with QoS=2 with duplicate", context do
       Process.flag(:trap_exit, true)
-      publish = %Package.Publish{identifier: 1, topic: "foo/bar", qos: 2}
+
+      publish =
+        %Package.Publish{identifier: 1, topic: "foo/bar", qos: 2}
+        |> Package.Meta.infer()
+
       dup_publish = %Package.Publish{publish | dup: true}
       expected_pubrec = %Package.Pubrec{identifier: 1}
       pubrel = %Package.Pubrel{identifier: 1}
@@ -1520,7 +1546,11 @@ defmodule Tortoise.ConnectionTest do
 
     test "incoming publish with QoS=2 with first message marked as duplicate", context do
       Process.flag(:trap_exit, true)
-      publish = %Package.Publish{identifier: 1, topic: "foo/bar", qos: 2, dup: true}
+
+      publish =
+        %Package.Publish{identifier: 1, topic: "foo/bar", qos: 2, dup: true}
+        |> Package.Meta.infer()
+
       expected_pubrec = %Package.Pubrec{identifier: 1}
       pubrel = %Package.Pubrel{identifier: 1}
       expected_pubcomp = %Package.Pubcomp{identifier: 1}
@@ -1548,7 +1578,11 @@ defmodule Tortoise.ConnectionTest do
 
     test "outgoing publish with QoS=2", context do
       Process.flag(:trap_exit, true)
-      publish = %Package.Publish{identifier: 1, topic: "foo/bar", qos: 2}
+
+      publish =
+        %Package.Publish{identifier: 1, topic: "foo/bar", qos: 2}
+        |> Package.Meta.infer()
+
       pubrec = %Package.Pubrec{identifier: 1}
       pubrel = %Package.Pubrel{identifier: 1}
       pubcomp = %Package.Pubcomp{identifier: 1}
diff --git a/test/tortoise/package/publish_test.exs b/test/tortoise/package/publish_test.exs
index c463e0f7..5c9783c1 100644
--- a/test/tortoise/package/publish_test.exs
+++ b/test/tortoise/package/publish_test.exs
@@ -1,22 +1,27 @@
 defmodule Tortoise.Package.PublishTest do
   use ExUnit.Case
-  # use EQC.ExUnit
+  use ExUnitProperties
+
   doctest Tortoise.Package.Publish
 
-  # import Tortoise.TestGenerators, only: [gen_publish: 0]
+  alias Tortoise.Package
 
-  # alias Tortoise.Package
+  property "encoding and decoding publish messages" do
+    config = %Package.Publish{
+      identifier: nil,
+      topic: nil,
+      payload: nil,
+      qos: nil,
+      dup: nil,
+      retain: nil,
+      properties: nil
+    }
 
-  # property "encoding and decoding publish messages" do
-  #   forall publish <- gen_publish() do
-  #     ensure(
-  #       publish ==
-  #         publish
-  #         |> Package.encode()
-  #         |> Package.decode()
-  #     )
-  #   end
-  # end
-  @tag :skip
-  test "encoding and decoding publish messages"
+    check all publish <- Package.generate(config) do
+      assert publish ==
+               publish
+               |> Package.encode()
+               |> Package.decode()
+    end
+  end
 end

From ed44811608398339b7c54e4d1664bf51bb167f33 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Sun, 24 May 2020 20:39:26 +0100
Subject: [PATCH 196/220] Publish payload is a binary

---
 lib/tortoise/package/publish.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/tortoise/package/publish.ex b/lib/tortoise/package/publish.ex
index a6ceb595..d80eda1c 100644
--- a/lib/tortoise/package/publish.ex
+++ b/lib/tortoise/package/publish.ex
@@ -256,7 +256,7 @@ defmodule Tortoise.Package.Publish do
         case Keyword.pop(values, :payload) do
           {nil, values} ->
             fixed_list([
-              {:payload, string(:ascii, min_length: 1)}
+              {:payload, binary(min_length: 1)}
               | Enum.map(values, &constant(&1))
             ])
 

From d8e577e41ba81c2f83cbd8fee056cad1f745c658 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Sun, 24 May 2020 20:40:08 +0100
Subject: [PATCH 197/220] Generate all the available user properties on publish
 packages

---
 lib/tortoise/package/publish.ex | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/lib/tortoise/package/publish.ex b/lib/tortoise/package/publish.ex
index d80eda1c..0d132cce 100644
--- a/lib/tortoise/package/publish.ex
+++ b/lib/tortoise/package/publish.ex
@@ -285,14 +285,21 @@ defmodule Tortoise.Package.Publish do
           {nil, values} ->
             properties =
               uniq_list_of(
-                one_of([
+                frequency([
                   # here we allow stings with a byte size of zero; don't
                   # know if that is a problem according to the spec. Let's
                   # handle that situation just in case:
-                  {constant(:user_property), {string(:printable), string(:printable)}}
+                  {8, {:user_property, {string(:printable), string(:printable)}}},
+                  {1, {:payload_format_indicator, integer(0..1)}},
+                  {1, {:message_expiry_interval, integer(0..0xFFFFFFFF)}},
+                  {1, {:topic_alias, integer(1..0xFFFF)}},
+                  {1, {:response_topic, bind(Topic.gen_topic(), &constant(Enum.join(&1, "/")))}},
+                  {1, {:correlation_data, binary()}},
+                  {1, {:subscription_identifier, integer(1..268_435_455)}},
+                  {1, {:content_type, string(:printable)}}
                 ]),
                 uniq_fun: &uniq/1,
-                max_length: 5
+                max_length: 10
               )
 
             fixed_list([

From 10f39661df5af51c3645d45bcd097dc8b0051828 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Sun, 24 May 2020 21:12:59 +0100
Subject: [PATCH 198/220] Make the Publish generator generate a nil as a
 payload now and then

---
 lib/tortoise/package/publish.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/tortoise/package/publish.ex b/lib/tortoise/package/publish.ex
index 0d132cce..5efd85ed 100644
--- a/lib/tortoise/package/publish.ex
+++ b/lib/tortoise/package/publish.ex
@@ -256,7 +256,7 @@ defmodule Tortoise.Package.Publish do
         case Keyword.pop(values, :payload) do
           {nil, values} ->
             fixed_list([
-              {:payload, binary(min_length: 1)}
+              {:payload, frequency([{1, nil}, {5, binary(min_length: 1)}])}
               | Enum.map(values, &constant(&1))
             ])
 

From 09aa3e93ed214239ccbdd11ac26f437717171e01 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Mon, 25 May 2020 21:44:47 +0100
Subject: [PATCH 199/220] Make the publish generator a bit more customizable

To support generating publish messages for last will messages (will
become important for last will messages) I needed to be able to pass
in a generator for publish identifiers and properties.
---
 lib/tortoise/package/publish.ex | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/lib/tortoise/package/publish.ex b/lib/tortoise/package/publish.ex
index 5efd85ed..326ef99f 100644
--- a/lib/tortoise/package/publish.ex
+++ b/lib/tortoise/package/publish.ex
@@ -224,6 +224,25 @@ defmodule Tortoise.Package.Publish do
 
           {id, _} when qos > 0 and is_integer(id) and id in 1..0xFFFF ->
             constant(values)
+
+          {%StreamData{} = generator, values} ->
+            fixed_list([
+              {:identifier,
+               bind(generator, fn
+                 id when is_integer(id) and id in 1..0xFFFF ->
+                   constant(id)
+
+                 nil ->
+                   nil
+
+                 _otherwise ->
+                   raise ArgumentError,
+                         """
+                         User specified identifier generator should return a nil or an integer between 1 and 65535
+                         """
+               end)}
+              | Enum.map(values, &constant(&1))
+            ])
         end
       end
 
@@ -307,6 +326,14 @@ defmodule Tortoise.Package.Publish do
               | Enum.map(values, &constant(&1))
             ])
 
+          {%StreamData{} = generator, values} ->
+            bind(generator, fn properties when is_list(properties) ->
+              fixed_list([
+                {constant(:properties), constant(properties)}
+                | Enum.map(values, &constant(&1))
+              ])
+            end)
+
           {_passthrough, _} ->
             constant(values)
         end

From 26fbdac1808c067d2fbfcb7dcaf612d2f71b7cc9 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Mon, 25 May 2020 21:46:40 +0100
Subject: [PATCH 200/220] Add a todo comment to the publish packet generator

---
 lib/tortoise/package/publish.ex | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/tortoise/package/publish.ex b/lib/tortoise/package/publish.ex
index 326ef99f..80631ccd 100644
--- a/lib/tortoise/package/publish.ex
+++ b/lib/tortoise/package/publish.ex
@@ -258,6 +258,8 @@ defmodule Tortoise.Package.Publish do
 
           {<<_::binary>>, _values} ->
             constant(values)
+
+            # TODO support a user specified generator for topic
         end
       end
 

From af397d07747d315a33beec69061250e17e84f062 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Mon, 25 May 2020 21:53:44 +0100
Subject: [PATCH 201/220] Add a data generator for Connect packets and prop
 test them

---
 lib/tortoise/package/connect.ex        | 293 +++++++++++++++++++++++++
 test/tortoise/package/connect_test.exs |  35 +--
 2 files changed, 313 insertions(+), 15 deletions(-)

diff --git a/lib/tortoise/package/connect.ex b/lib/tortoise/package/connect.ex
index 7ae184c6..a7b21be5 100644
--- a/lib/tortoise/package/connect.ex
+++ b/lib/tortoise/package/connect.ex
@@ -87,6 +87,7 @@ defmodule Tortoise.Package.Connect do
             retain: will_retain == 1,
             properties: payload[:will_properties]
           }
+          |> Package.Meta.infer()
         end,
       clean_start: clean_start == 1,
       keep_alive: keep_alive,
@@ -222,4 +223,296 @@ defmodule Tortoise.Package.Connect do
     defp flag(f) when f in [0, nil, false], do: 0
     defp flag(_), do: 1
   end
+
+  if Code.ensure_loaded?(StreamData) do
+    defimpl Tortoise.Generatable do
+      import StreamData
+
+      alias Tortoise.Generatable.Topic
+
+      def generate(%type{__META__: _meta} = package) do
+        values = package |> Map.from_struct()
+
+        fixed_list(Enum.map(values, &constant(&1)))
+        |> bind(&gen_user_name/1)
+        |> bind(&gen_password/1)
+        |> bind(&gen_clean_start/1)
+        |> bind(&gen_keep_alive/1)
+        |> bind(&gen_client_id/1)
+        |> bind(&gen_will/1)
+        |> bind(&gen_properties/1)
+        |> bind(fn data ->
+          fixed_map([
+            {:__struct__, type}
+            | for({k, v} <- data, do: {k, constant(v)})
+          ])
+        end)
+      end
+
+      defp gen_clean_start(values) do
+        case Keyword.pop(values, :clean_start) do
+          {nil, values} ->
+            fixed_list([
+              {:clean_start, boolean()}
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {bool, _} when is_boolean(bool) ->
+            constant(values)
+
+          # User specified generators should produce a boolean
+          {%StreamData{} = generator, values} ->
+            bind(generator, fn
+              bool when is_boolean(bool) ->
+                fixed_list([
+                  {:clean_start, bool}
+                  | Enum.map(values, &constant(&1))
+                ])
+
+              _otherwise ->
+                raise ArgumentError, "Clean start generator should produce a boolean"
+            end)
+        end
+      end
+
+      defp gen_keep_alive(values) do
+        case Keyword.pop(values, :keep_alive) do
+          {nil, values} ->
+            fixed_list([
+              {:keep_alive,
+               frequency([
+                 # Most of the time we will produce a reasonable keep
+                 # alive value; somewhere between 1 and 5 minutes
+                 {4, integer(60..300)},
+                 # Sometimes produce a *very* long keep alive
+                 # interval, the longest the MQTT spec support is a
+                 # bit more than 18 hours!
+                 {4, integer(301..0xFFFF)},
+                 # Sometimes produce a value less than a minute
+                 {1, integer(1..59)},
+                 # A value of zero means that the client will not
+                 # send ping requests on a particular schedule, and
+                 # the server will not kick the client if it does
+                 # not; this essentially turns the keep alive off
+                 {1, constant(0)}
+               ])}
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {value, _} when is_integer(value) and value in 0..0xFFFF ->
+            constant(values)
+
+          # User specified generators should produce an integer
+          {%StreamData{} = generator, values} ->
+            bind(generator, fn
+              value when is_integer(value) and value in 0..0xFFFF ->
+                fixed_list([
+                  {:keep_alive, constant(value)}
+                  | Enum.map(values, &constant(&1))
+                ])
+
+              _otherwise ->
+                raise ArgumentError, """
+                Keep alive generator should produce an integer between 0 and 65_535
+                """
+            end)
+        end
+      end
+
+      defp gen_client_id(values) do
+        case Keyword.pop(values, :client_id) do
+          {nil, values} ->
+            fixed_list([
+              {:client_id,
+               frequency([
+                 # A server should accept a client id between 1 and 23
+                 # chars in length consisting of 0-9, a-z, and A-Z.
+                 {1, gen_client_id_string()},
+                 # The server may accept a client id longer than 23
+                 # chars, and of any chars
+                 {1, string(:printable, min_length: 1)}
+                 # If the client id is nil the server may assign a
+                 # client id and send it as a property in the connack
+                 # message
+                 # {1, nil}
+               ])}
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {%StreamData{} = generator, values} ->
+            bind(generator, fn
+              client_id when is_binary(client_id) or is_nil(client_id) ->
+                fixed_list([
+                  {:client_id, constant(client_id)}
+                  | Enum.map(values, &constant(&1))
+                ])
+
+              _otherwise ->
+                raise ArgumentError, "A client id should be nil or a binary"
+            end)
+
+          {<<_::binary>>, _} ->
+            constant(values)
+        end
+      end
+
+      defp gen_client_id_string() do
+        list_of(
+          one_of([integer(?A..?Z), integer(?a..?z), integer(?0..?9)]),
+          min_length: 1,
+          max_length: 23
+        )
+        |> bind(&constant(List.to_string(&1)))
+      end
+
+      defp gen_user_name(values) do
+        case Keyword.pop(values, :user_name) do
+          {nil, values} ->
+            fixed_list([
+              {:user_name, one_of([nil, string(:printable)])}
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {%StreamData{} = generator, values} ->
+            bind(generator, fn
+              user_name when is_binary(user_name) or is_nil(user_name) ->
+                fixed_list([
+                  {:user_name, constant(user_name)}
+                  | Enum.map(values, &constant(&1))
+                ])
+
+              _otherwise ->
+                raise ArgumentError, "User name should be nil or a binary"
+            end)
+
+          {<<_::binary>>, _} ->
+            constant(values)
+        end
+      end
+
+      defp gen_password(values) do
+        case Keyword.pop(values, :password) do
+          {nil, values} ->
+            fixed_list([
+              {:password, one_of([nil, binary()])}
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {%StreamData{} = generator, values} ->
+            bind(generator, fn
+              password when is_binary(password) or is_nil(password) ->
+                fixed_list([
+                  {:password, constant(password)}
+                  | Enum.map(values, &constant(&1))
+                ])
+
+              _otherwise ->
+                raise ArgumentError, "Password should be nil or a binary"
+            end)
+
+          {<<_::binary>>, _} ->
+            constant(values)
+        end
+      end
+
+      defp gen_will(values) do
+        case Keyword.pop(values, :will) do
+          {nil, values} ->
+            fixed_list([
+              {:will,
+               Package.generate(%Package.Publish{
+                 identifier: constant(nil),
+                 dup: false,
+                 retain: nil,
+                 qos: nil,
+                 properties: gen_will_properties()
+               })}
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {%Package.Publish{}, _values} ->
+            constant(values)
+        end
+      end
+
+      defp gen_will_properties() do
+        uniq_list_of(
+          # Use frequency to make it more likely to pick a user
+          # property as we are allowed to have multiple of them;
+          # the remaining properties may only occur once,
+          # without the weights we could end up in situations
+          # where StreamData gives up because it cannot find any
+          # candidates that hasn't been chosen before
+          frequency([
+            # here we allow stings with a byte size of zero; don't
+            # know if that is a problem according to the spec. Let's
+            # handle that situation just in case:
+            {6, {:user_property, {string(:printable), string(:printable)}}},
+            {1, {:will_delay_interval, integer(0..0xFFFFFFFF)}},
+            {1, {:payload_format_indicator, integer(0..1)}},
+            {1, {:message_expiry_interval, integer(0..0xFFFFFFFF)}},
+            {1, {:content_type, string(:printable)}},
+            {1, {:response_topic, bind(Topic.gen_topic(), &constant(Enum.join(&1, "/")))}},
+            {1, {:correlation_data, binary()}}
+          ]),
+          uniq_fun: &uniq/1,
+          max_length: 10
+        )
+      end
+
+      defp gen_properties(values) do
+        case Keyword.pop(values, :properties) do
+          {nil, values} ->
+            properties =
+              uniq_list_of(
+                # Use frequency to make it more likely to pick a user
+                # property as we are allowed to have multiple of them;
+                # the remaining properties may only occur once,
+                # without the weights we could end up in situations
+                # where StreamData gives up because it cannot find any
+                # candidates that hasn't been chosen before
+                frequency([
+                  # here we allow stings with a byte size of zero; don't
+                  # know if that is a problem according to the spec. Let's
+                  # handle that situation just in case:
+                  {8, {:user_property, {string(:printable), string(:printable)}}},
+                  {1, {:maximum_packet_size, integer(1..0xFFFFFFFF)}},
+                  {1, {:receive_maximum, integer(1..0xFFFF)}},
+                  {1, {:request_problem_information, boolean()}},
+                  {1, {:request_response_information, boolean()}},
+                  {1, {:session_expiry_interval, integer(0..0xFFFFFFFF)}},
+                  {1, {:topic_alias_maximum, integer(0..0xFFFF)}},
+                  {1, {:authentication_method, string(:printable)}}
+                ]),
+                uniq_fun: &uniq/1,
+                max_length: 10
+              )
+              |> bind(fn properties ->
+                if Keyword.has_key?(properties, :authentication_method) do
+                  one_of([
+                    constant(properties),
+                    fixed_list([
+                      {:authentication_data, binary()}
+                      | Enum.map(properties, &constant(&1))
+                    ])
+                  ])
+                else
+                  constant(properties)
+                end
+              end)
+
+            fixed_list([
+              {constant(:properties), properties}
+              | Enum.map(values, &constant(&1))
+            ])
+
+          {_passthrough, _} ->
+            constant(values)
+        end
+      end
+
+      defp uniq({:user_property, _v}), do: :crypto.strong_rand_bytes(2)
+      defp uniq({k, _v}), do: k
+    end
+  end
 end
diff --git a/test/tortoise/package/connect_test.exs b/test/tortoise/package/connect_test.exs
index 4aa25cfb..082e27d6 100644
--- a/test/tortoise/package/connect_test.exs
+++ b/test/tortoise/package/connect_test.exs
@@ -1,22 +1,27 @@
 defmodule Tortoise.Package.ConnectTest do
   use ExUnit.Case
-  # use EQC.ExUnit
+  use ExUnitProperties
+
   doctest Tortoise.Package.Connect
 
-  # alias Tortoise.Package
+  alias Tortoise.Package
 
-  # import Tortoise.TestGenerators, only: [gen_connect: 0]
+  property "encoding and decoding connect messages" do
+    config = %Package.Connect{
+      user_name: nil,
+      password: nil,
+      clean_start: nil,
+      keep_alive: nil,
+      client_id: nil,
+      will: nil,
+      properties: nil
+    }
 
-  # property "encoding and decoding connect messages" do
-  #   forall connect <- gen_connect() do
-  #     ensure(
-  #       connect ==
-  #         connect
-  #         |> Package.encode()
-  #         |> Package.decode()
-  #     )
-  #   end
-  # end
-  @tag :skip
-  test "encoding and decoding connect messages"
+    check all connect <- Package.generate(config) do
+      assert connect ==
+               connect
+               |> Package.encode()
+               |> Package.decode()
+    end
+  end
 end

From 1d96fc310e655bee376d5ed7c7ee84d16624f56b Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Thu, 9 Jul 2020 21:39:28 +0100
Subject: [PATCH 202/220] Remove some cruft from the nix shell file

---
 shell.nix | 12 ++----------
 1 file changed, 2 insertions(+), 10 deletions(-)

diff --git a/shell.nix b/shell.nix
index e6531055..2c93b324 100644
--- a/shell.nix
+++ b/shell.nix
@@ -5,19 +5,11 @@ with pkgs;
 let
   inherit (lib) optional optionals;
 
-  erlang_wx = erlangR21.override {
-    wxSupport = true;
-  };
-
-  elixir = (beam.packagesWith erlangR21).elixir.override {
-    version = "1.8.2";
-      rev = "98485daab0a9f3ac2d7809d38f5e57cd73cb22ac";
-      sha256 = "1n77cpcl2b773gmj3m9s24akvj9gph9byqbmj2pvlsmby4aqwckq";
-  };
+  elixir = (beam.packagesWith erlangR22).elixir;
 in
 
 mkShell {
-  buildInputs = [ elixir git wxmac ]
+  buildInputs = [ elixir ]
     ++ optional stdenv.isLinux inotify-tools # For file_system on Linux.
     ++ optionals stdenv.isDarwin (with darwin.apple_sdk.frameworks; [
       # For file_system on macOS.

From 900bbfc009e84a1b19bc098289b164def71c80e6 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Tue, 1 Sep 2020 12:14:09 +0100
Subject: [PATCH 203/220] Get rid of the Tortoise.Events registry

We are moving away from the push based parts that required the
registry.
---
 lib/tortoise/application.ex | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/tortoise/application.ex b/lib/tortoise/application.ex
index ac6fcb98..330dbccd 100644
--- a/lib/tortoise/application.ex
+++ b/lib/tortoise/application.ex
@@ -10,7 +10,6 @@ defmodule Tortoise.Application do
 
     children = [
       {Registry, [keys: :unique, name: Tortoise.Registry]},
-      {Registry, [keys: :duplicate, name: Tortoise.Events]},
       {Tortoise.Supervisor, [strategy: :one_for_one]}
     ]
 

From a039df7df87fee19f10018f165895705ced238db Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Thu, 17 Sep 2020 14:15:36 +0100
Subject: [PATCH 204/220] Replace the inflight system with an ETS based one

This is part of the work to get rid of the client id as the identifier
for the connection. The inflight messages are now stored in a ETS
table; outside of the connection; so it can survive a connection
rolling over, and eventually it would support loading a session into
memory and start from that.

It is not 100% complete.

Also, it has a notion of backends, and the ETS is default, but it
would be possible to back it with external services such as an SQL
server or a Redis, which would allow us to tear down an entire node
and continue on another.
---
 lib/tortoise/application.ex       |   1 +
 lib/tortoise/connection.ex        | 527 ++++++++++++++++++------------
 lib/tortoise/session.ex           |  95 ++++++
 lib/tortoise/session/ets.ex       |  98 ++++++
 test/tortoise/connection_test.exs |  28 +-
 test/tortoise/session_test.exs    | 134 ++++++++
 6 files changed, 656 insertions(+), 227 deletions(-)
 create mode 100644 lib/tortoise/session.ex
 create mode 100644 lib/tortoise/session/ets.ex
 create mode 100644 test/tortoise/session_test.exs

diff --git a/lib/tortoise/application.ex b/lib/tortoise/application.ex
index 330dbccd..590562db 100644
--- a/lib/tortoise/application.ex
+++ b/lib/tortoise/application.ex
@@ -10,6 +10,7 @@ defmodule Tortoise.Application do
 
     children = [
       {Registry, [keys: :unique, name: Tortoise.Registry]},
+      {Tortoise.Session, [backend: Tortoise.Session.Ets]},
       {Tortoise.Supervisor, [strategy: :one_for_one]}
     ]
 
diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 714c8d67..27c5dc20 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -11,6 +11,7 @@ defmodule Tortoise.Connection do
 
   defstruct session_ref: nil,
             client_id: nil,
+            session: nil,
             connect: nil,
             server: nil,
             backoff: nil,
@@ -24,8 +25,8 @@ defmodule Tortoise.Connection do
 
   alias __MODULE__, as: State
 
-  alias Tortoise.{Handler, Transport, Package}
-  alias Tortoise.Connection.{Info, Receiver, Inflight, Backoff}
+  alias Tortoise.{Handler, Transport, Package, Session}
+  alias Tortoise.Connection.{Info, Receiver, Backoff}
   alias Tortoise.Package.Connect
 
   @doc """
@@ -72,6 +73,7 @@ defmodule Tortoise.Connection do
     initial = %State{
       session_ref: make_ref(),
       client_id: connect.client_id,
+      session: %Session{client_id: connect.client_id},
       server: server,
       connect: connect,
       backoff: Backoff.new(backoff),
@@ -307,6 +309,28 @@ defmodule Tortoise.Connection do
     unsubscribe_sync(pid, [topic], opts)
   end
 
+  @doc """
+  Publish a message, but go through the connection
+
+  In most circumstances it would be preferable to go through the
+  publish function on the Tortoise module instead.
+  """
+  def publish(pid, %Package.Publish{} = publish) do
+    GenStateMachine.call(pid, {:publish, publish})
+  end
+
+  def publish_sync(pid, %Package.Publish{} = publish, timeout \\ 5000) do
+    {:ok, ref} = publish(pid, publish)
+
+    receive do
+      {{Tortoise, ^pid}, {Package.Publish, ^ref}, result} ->
+        result
+    after
+      timeout ->
+        {:error, :timeout}
+    end
+  end
+
   @doc """
   Ping the broker.
 
@@ -458,15 +482,12 @@ defmodule Tortoise.Connection do
         {:received, %Package.Connack{reason: connection_result} = connack},
         :connecting,
         %State{
-          client_id: client_id,
           connect: %Package.Connect{} = connect,
           handler: handler
         } = data
       ) do
     case Handler.execute_handle_connack(handler, connack) do
       {:ok, %Handler{} = updated_handler, next_actions} when connection_result == :success ->
-        :ok = Inflight.update_connection(client_id, data.connection)
-
         data = %State{
           data
           | backoff: Backoff.reset(data.backoff),
@@ -541,58 +562,130 @@ defmodule Tortoise.Connection do
     end
   end
 
-  # incoming publish QoS>0 ---------------------------------------------
+  # incoming publish QoS=1 ---------------------------------------------
   def handle_event(
         :internal,
-        {:received, %Package.Publish{qos: qos} = publish},
+        {:received, %Package.Publish{identifier: id, qos: 1} = publish},
         _,
-        %State{client_id: client_id}
-      )
-      when qos in 1..2 do
-    :ok = Inflight.track(client_id, {:incoming, publish})
-    :keep_state_and_data
+        %State{
+          connection: {transport, socket},
+          handler: handler,
+          session: session
+        } = data
+      ) do
+    case Session.track(session, {:incoming, publish}) do
+      {{:cont, publish}, session} ->
+        case Handler.execute_handle_publish(handler, publish) do
+          {:ok, %Package.Puback{identifier: ^id} = puback, %Handler{} = updated_handler,
+           next_actions} ->
+            # respond with a puback
+            Session.progress(session, {:outgoing, puback})
+            :ok = transport.send(socket, Package.encode(puback))
+            {:ok, session} = Session.release(session, id)
+            # - - -
+            updated_data = %State{data | handler: updated_handler, session: session}
+            {:keep_state, updated_data, wrap_next_actions(next_actions)}
+
+            # handle stop
+        end
+    end
   end
 
   # outgoing publish QoS=1 ---------------------------------------------
   def handle_event(
-        :internal,
-        {:received, %Package.Puback{} = puback},
+        {:call, {_caller_pid, ref} = from},
+        {:publish, %Package.Publish{qos: 1} = publish},
         _,
-        %State{client_id: client_id, handler: handler} = data
+        %State{
+          connection: {transport, socket},
+          session: session,
+          pending_refs: pending
+        } = data
       ) do
-    :ok = Inflight.update(client_id, {:received, puback})
-
-    if function_exported?(handler.module, :handle_puback, 2) do
-      case Handler.execute_handle_puback(handler, puback) do
-        {:ok, %Handler{} = updated_handler, next_actions} ->
-          updated_data = %State{data | handler: updated_handler}
-          {:keep_state, updated_data, wrap_next_actions(next_actions)}
+    case Session.track(session, {:outgoing, publish}) do
+      {{:cont, %Package.Publish{identifier: id} = publish}, session} ->
+        :ok = transport.send(socket, Package.encode(publish))
+        next_actions = [{:reply, from, {:ok, ref}}]
+        data = %State{data | session: session, pending_refs: Map.put_new(pending, id, from)}
+        {:keep_state, data, next_actions}
+    end
+  end
 
-        {:error, reason} ->
-          # todo
-          {:stop, reason, data}
-      end
-    else
-      :keep_state_and_data
+  def handle_event(
+        :internal,
+        {:received, %Package.Puback{identifier: id} = puback},
+        _,
+        %State{session: session, handler: handler, pending_refs: pending} = data
+      ) do
+    case Map.pop(pending, id) do
+      {caller, pending} ->
+        {{:cont, puback}, session} = Session.progress(session, {:incoming, puback})
+        data = %State{data | pending_refs: pending, session: session}
+
+        if function_exported?(handler.module, :handle_puback, 2) do
+          case Handler.execute_handle_puback(handler, puback) do
+            {:ok, %Handler{} = updated_handler, next_actions} ->
+              next_actions = [
+                {:next_event, :internal, {:reply, caller, Package.Publish, :ok}}
+                | wrap_next_actions(next_actions)
+              ]
+
+              {:ok, session} = Session.release(session, id)
+              updated_data = %State{data | handler: updated_handler, session: session}
+              {:keep_state, updated_data, next_actions}
+
+            {:error, reason} ->
+              # todo
+              updated_data = %State{data | session: session}
+              {:stop, reason, updated_data}
+          end
+        else
+          {:ok, session} = Session.release(session, id)
+          next_actions = [{:next_event, :internal, {:reply, caller, Package.Publish, :ok}}]
+          {:keep_state, %State{data | session: session}, next_actions}
+        end
     end
   end
 
   # incoming publish QoS=2 ---------------------------------------------
+  # TODO handle duplicate messages
+  def handle_event(
+        :internal,
+        {:received, %Package.Publish{qos: 2, identifier: id} = publish},
+        _,
+        %State{connection: {transport, socket}, handler: handler, session: session} = data
+      ) do
+    case Session.track(session, {:incoming, publish}) do
+      {{:cont, publish}, session} ->
+        case Handler.execute_handle_publish(handler, publish) do
+          {:ok, %Package.Pubrec{identifier: ^id} = pubrec, %Handler{} = updated_handler,
+           next_actions} ->
+            # respond with pubrec
+            {{:cont, pubrec}, session} = Session.progress(session, {:outgoing, pubrec})
+            :ok = transport.send(socket, Package.encode(pubrec))
+            # - - -
+            updated_data = %State{data | handler: updated_handler, session: session}
+            {:keep_state, updated_data, wrap_next_actions(next_actions)}
+        end
+    end
+  end
+
   def handle_event(
         :internal,
         {:received, %Package.Pubrel{identifier: id} = pubrel},
         _,
-        %State{client_id: client_id, handler: handler} = data
+        %State{connection: {transport, socket}, handler: handler, session: session} = data
       ) do
-    :ok = Inflight.update(client_id, {:received, pubrel})
+    {{:cont, pubrel}, session} = Session.progress(session, {:incoming, pubrel})
 
     if function_exported?(handler.module, :handle_pubrel, 2) do
       case Handler.execute_handle_pubrel(handler, pubrel) do
         {:ok, %Package.Pubcomp{identifier: ^id} = pubcomp, %Handler{} = updated_handler,
          next_actions} ->
           # dispatch the pubcomp
-          :ok = Inflight.update(client_id, {:dispatch, pubcomp})
-          updated_data = %State{data | handler: updated_handler}
+          {{:cont, pubcomp}, session} = Session.progress(session, {:outgoing, pubcomp})
+          :ok = transport.send(socket, Package.encode(pubcomp))
+          updated_data = %State{data | handler: updated_handler, session: session}
           {:keep_state, updated_data, wrap_next_actions(next_actions)}
 
         {:error, reason} ->
@@ -601,64 +694,50 @@ defmodule Tortoise.Connection do
       end
     else
       pubcomp = %Package.Pubcomp{identifier: id}
-      :ok = Inflight.update(client_id, {:dispatch, pubcomp})
-      :keep_state_and_data
-    end
-  end
-
-  # an incoming publish with QoS>0 will get parked in the inflight
-  # manager process, which will onward it to the controller, making
-  # sure we will only dispatch it once to the publish-handler.
-  def handle_event(
-        :info,
-        {{Inflight, client_id}, %Package.Publish{identifier: id, qos: 1} = publish},
-        _,
-        %State{client_id: client_id, handler: handler} = data
-      ) do
-    case Handler.execute_handle_publish(handler, publish) do
-      {:ok, %Package.Puback{identifier: ^id} = puback, %Handler{} = updated_handler, next_actions} ->
-        # respond with a puback
-        :ok = Inflight.update(client_id, {:dispatch, puback})
-        # - - -
-        updated_data = %State{data | handler: updated_handler}
-        {:keep_state, updated_data, wrap_next_actions(next_actions)}
-
-        # handle stop
+      {{:cont, pubcomp}, session} = Session.progress(session, {:outgoing, pubcomp})
+      :ok = transport.send(socket, Package.encode(pubcomp))
+      updated_data = %State{data | session: session}
+      {:keep_state, updated_data}
     end
   end
 
+  # outgoing publish QoS=2 ---------------------------------------------
   def handle_event(
-        :info,
-        {{Inflight, client_id}, %Package.Publish{identifier: id, qos: 2} = publish},
+        {:call, {_caller_pid, ref} = from},
+        {:publish, %Package.Publish{qos: 2, dup: false} = publish},
         _,
-        %State{client_id: client_id, handler: handler} = data
+        %State{
+          connection: {transport, socket},
+          session: session,
+          pending_refs: pending
+        } = data
       ) do
-    case Handler.execute_handle_publish(handler, publish) do
-      {:ok, %Package.Pubrec{identifier: ^id} = pubrec, %Handler{} = updated_handler, next_actions} ->
-        # respond with pubrec
-        :ok = Inflight.update(client_id, {:dispatch, pubrec})
-        # - - -
-        updated_data = %State{data | handler: updated_handler}
-        {:keep_state, updated_data, wrap_next_actions(next_actions)}
+    case Session.track(session, {:outgoing, publish}) do
+      {{:cont, %Package.Publish{identifier: id} = publish}, session} ->
+        :ok = transport.send(socket, Package.encode(publish))
+        next_actions = [{:reply, from, {:ok, ref}}]
+        data = %State{data | session: session, pending_refs: Map.put_new(pending, id, from)}
+        {:keep_state, data, next_actions}
     end
   end
 
-  # outgoing publish QoS=2 ---------------------------------------------
   def handle_event(
         :internal,
         {:received, %Package.Pubrec{identifier: id} = pubrec},
         _,
-        %State{client_id: client_id, handler: handler} = data
+        %State{connection: {transport, socket}, session: session, handler: handler} = data
       ) do
-    :ok = Inflight.update(client_id, {:received, pubrec})
+    {{:cont, pubrec}, session} = Session.progress(session, {:incoming, pubrec})
+    data = %State{data | session: session}
 
     if function_exported?(handler.module, :handle_pubrec, 2) do
       case Handler.execute_handle_pubrec(handler, pubrec) do
         {:ok, %Package.Pubrel{identifier: ^id} = pubrel, %Handler{} = updated_handler,
          next_actions} ->
-          updated_data = %State{data | handler: updated_handler}
-          :ok = Inflight.update(client_id, {:dispatch, pubrel})
-          {:keep_state, updated_data, wrap_next_actions(next_actions)}
+          {{:cont, pubrel}, session} = Session.progress(session, {:outgoing, pubrel})
+          :ok = transport.send(socket, Package.encode(pubrel))
+          data = %State{data | session: session, handler: updated_handler}
+          {:keep_state, data, wrap_next_actions(next_actions)}
 
         {:error, reason} ->
           # todo
@@ -666,31 +745,48 @@ defmodule Tortoise.Connection do
       end
     else
       pubrel = %Package.Pubrel{identifier: id}
-      :ok = Inflight.update(client_id, {:dispatch, pubrel})
-      :keep_state_and_data
+      {{:cont, pubrel}, session} = Session.progress(session, {:outgoing, pubrel})
+      :ok = transport.send(socket, Package.encode(pubrel))
+      data = %State{data | session: session}
+      {:keep_state, data}
     end
   end
 
   def handle_event(
         :internal,
-        {:received, %Package.Pubcomp{} = pubcomp},
+        {:received, %Package.Pubcomp{identifier: id} = pubcomp},
         :connected,
-        %State{client_id: client_id, handler: handler} = data
+        %State{session: session, pending_refs: pending, handler: handler} = data
       ) do
-    :ok = Inflight.update(client_id, {:received, pubcomp})
-
-    if function_exported?(handler.module, :handle_pubcomp, 2) do
-      case Handler.execute_handle_pubcomp(handler, pubcomp) do
-        {:ok, %Handler{} = updated_handler, next_actions} ->
-          updated_data = %State{data | handler: updated_handler}
-          {:keep_state, updated_data, wrap_next_actions(next_actions)}
-
-        {:error, reason} ->
-          # todo
-          {:stop, reason, data}
-      end
-    else
-      :keep_state_and_data
+    case Map.pop(pending, id) do
+      {caller, pending} ->
+        {{:cont, pubcomp}, session} = Session.progress(session, {:incoming, pubcomp})
+        data = %State{data | pending_refs: pending, session: session}
+
+        if function_exported?(handler.module, :handle_pubcomp, 2) do
+          case Handler.execute_handle_pubcomp(handler, pubcomp) do
+            {:ok, %Handler{} = updated_handler, next_actions} ->
+              {:ok, session} = Session.release(session, id)
+
+              data = %State{data | session: session, handler: updated_handler}
+
+              next_actions = [
+                {:next_event, :internal, {:reply, caller, Package.Publish, :ok}}
+                | wrap_next_actions(next_actions)
+              ]
+
+              {:keep_state, data, next_actions}
+
+            {:error, reason} ->
+              # todo
+              {:stop, reason, data}
+          end
+        else
+          {:ok, session} = Session.release(session, id)
+          data = %State{data | session: session}
+          next_actions = [{:next_event, :internal, {:reply, caller, Package.Publish, :ok}}]
+          {:keep_state, data, next_actions}
+        end
     end
   end
 
@@ -711,16 +807,21 @@ defmodule Tortoise.Connection do
 
   def handle_event(
         :cast,
-        {:subscribe, caller, subscribe, _opts},
+        {:subscribe, caller, %Package.Subscribe{} = subscribe, _opts},
         :connected,
-        %State{client_id: client_id} = data
+        %State{
+          connection: {transport, socket},
+          session: session
+        } = data
       ) do
     case Info.Capabilities.validate(data.info.capabilities, subscribe) do
       :valid ->
-        {:ok, ref} = Inflight.track(client_id, {:outgoing, subscribe})
-        pending = Map.put_new(data.pending_refs, ref, caller)
-
-        {:keep_state, %State{data | pending_refs: pending}}
+        case Session.track(session, {:outgoing, subscribe}) do
+          {{:cont, %Package.Subscribe{identifier: id} = subscribe}, session} ->
+            :ok = transport.send(socket, Package.encode(subscribe))
+            pending = Map.put_new(data.pending_refs, id, {caller, subscribe})
+            {:keep_state, %State{data | pending_refs: pending, session: session}}
+        end
 
       {:invalid, reasons} ->
         reply = {:error, {:subscription_failure, reasons}}
@@ -735,146 +836,143 @@ defmodule Tortoise.Connection do
 
   def handle_event(
         :internal,
-        {:received, %Package.Suback{} = suback},
+        {:received, %Package.Suback{identifier: id} = suback},
         :connected,
-        data
+        %State{session: session, handler: handler, pending_refs: pending, info: info} = data
       ) do
-    :ok = Inflight.update(data.client_id, {:received, suback})
+    case Map.pop(pending, id) do
+      {{caller, %Package.Subscribe{identifier: ^id} = subscribe}, pending} ->
+        {pid, msg_ref} = caller
 
-    :keep_state_and_data
-  end
+        {{:cont, suback}, session} = Session.progress(session, {:incoming, suback})
 
-  def handle_event(
-        :info,
-        {{Tortoise, client_id}, {Package.Subscribe, ref}, {subscribe, suback}},
-        _current_state,
-        %State{client_id: client_id, handler: handler, pending_refs: %{} = pending, info: info} =
-          data
-      ) do
-    {{pid, msg_ref}, updated_pending} = Map.pop(pending, ref)
-    data = %State{data | pending_refs: updated_pending}
+        data = %State{data | pending_refs: pending, session: session}
 
-    updated_subscriptions =
-      subscribe.topics
-      |> Enum.zip(suback.acks)
-      |> Enum.reduce(data.info.subscriptions, fn
-        {{topic, opts}, {:ok, accepted_qos}}, acc ->
-          Map.put(acc, topic, Keyword.replace!(opts, :qos, accepted_qos))
+        updated_subscriptions =
+          subscribe.topics
+          |> Enum.zip(suback.acks)
+          |> Enum.reduce(data.info.subscriptions, fn
+            {{topic, opts}, {:ok, accepted_qos}}, acc ->
+              Map.put(acc, topic, Keyword.replace!(opts, :qos, accepted_qos))
 
-        {_, {:error, _}}, acc ->
-          acc
-      end)
+            {_, {:error, _}}, acc ->
+              acc
+          end)
 
-    case Handler.execute_handle_suback(handler, subscribe, suback) do
-      {:ok, %Handler{} = updated_handler, next_actions} ->
-        data = %State{
-          data
-          | handler: updated_handler,
-            info: put_in(info.subscriptions, updated_subscriptions)
-        }
+        case Handler.execute_handle_suback(handler, subscribe, suback) do
+          {:ok, %Handler{} = updated_handler, next_actions} ->
+            data = %State{
+              data
+              | handler: updated_handler,
+                info: put_in(info.subscriptions, updated_subscriptions)
+            }
 
-        next_actions = [
-          {:next_event, :internal, {:reply, {pid, msg_ref}, Package.Suback, :ok}}
-          | wrap_next_actions(next_actions)
-        ]
+            next_actions = [
+              {:next_event, :internal, {:reply, {pid, msg_ref}, Package.Suback, :ok}}
+              | wrap_next_actions(next_actions)
+            ]
 
-        {:keep_state, data, next_actions}
+            {:ok, session} = Session.release(session, id)
 
-      {:error, reason} ->
-        # todo
-        {:stop, reason, data}
-    end
-  end
+            {:keep_state, %State{data | session: session}, next_actions}
 
-  # Pass on the result of an operation if we have a calling pid. This
-  # can happen if a process order the connection to subscribe to a
-  # topic, or unsubscribe, etc.
-  def handle_event(
-        :internal,
-        {:reply, {pid, _msg_ref}, _topic, _payload},
-        _current_state,
-        %State{}
-      )
-      when pid == self() do
-    :keep_state_and_data
+          {:error, reason} ->
+            # todo
+            {:stop, reason, data}
+        end
+    end
   end
 
   def handle_event(
-        :internal,
-        {:reply, {caller, ref}, topic, payload},
-        _current_state,
-        %State{}
-      )
-      when caller != self() do
-    _ = send_reply({caller, ref}, topic, payload)
-    :keep_state_and_data
-  end
-
-  def handle_event(:cast, {:unsubscribe, caller, unsubscribe, opts}, :connected, data) do
-    client_id = data.client_id
+        :cast,
+        {:unsubscribe, caller, unsubscribe, opts},
+        :connected,
+        %State{
+          session: session,
+          connection: {transport, socket},
+          pending_refs: pending
+        } = data
+      ) do
     _timeout = Keyword.get(opts, :timeout, 5000)
 
-    {:ok, ref} = Inflight.track(client_id, {:outgoing, unsubscribe})
-    pending = Map.put_new(data.pending_refs, ref, caller)
-
-    {:keep_state, %State{data | pending_refs: pending}}
+    case Session.track(session, {:outgoing, unsubscribe}) do
+      {{:cont, %Package.Unsubscribe{identifier: id} = unsubscribe}, session} ->
+        :ok = transport.send(socket, Package.encode(unsubscribe))
+        pending = Map.put_new(pending, id, {caller, unsubscribe})
+        {:keep_state, %State{data | pending_refs: pending, session: session}}
+    end
   end
 
   def handle_event(
         :internal,
-        {:received, %Package.Unsuback{results: [_ | _]} = unsuback},
+        {:received, %Package.Unsuback{identifier: id, results: [_ | _]} = unsuback},
         :connected,
-        data
+        %State{session: session, handler: handler, pending_refs: pending, info: info} = data
       ) do
-    :ok = Inflight.update(data.client_id, {:received, unsuback})
-    :keep_state_and_data
+    case Map.pop(pending, id) do
+      {{caller, %Package.Unsubscribe{identifier: ^id} = unsubscribe}, pending} ->
+        {pid, msg_ref} = caller
+        {{:cont, unsuback}, session} = Session.progress(session, {:incoming, unsuback})
+        data = %State{data | pending_refs: pending, session: session}
+
+        # When updating the internal subscription state tracker we will
+        # disregard the unsuccessful unsubacks, as we can assume it wasn't
+        # in the subscription list to begin with, or that we are still
+        # subscribed as we are not autorized to unsubscribe for the given
+        # topic; one exception is when the server report no subscription
+        # existed; then we will update the client state
+        to_remove =
+          for {topic, result} <- Enum.zip(unsubscribe.topics, unsuback.results),
+              match?(
+                reason when reason == :success or reason == {:error, :no_subscription_existed},
+                result
+              ),
+              do: topic
+
+        # TODO handle the unsuback error cases !
+        subscriptions = Map.drop(data.info.subscriptions, to_remove)
+
+        case Handler.execute_handle_unsuback(handler, unsubscribe, unsuback) do
+          {:ok, %Handler{} = updated_handler, next_actions} ->
+            {:ok, session} = Session.release(session, id)
+
+            data = %State{
+              data
+              | handler: updated_handler,
+                session: session,
+                info: put_in(info.subscriptions, subscriptions)
+            }
+
+            next_actions = [
+              {:next_event, :internal, {:reply, {pid, msg_ref}, Package.Unsuback, :ok}}
+              | wrap_next_actions(next_actions)
+            ]
+
+            {:keep_state, data, next_actions}
+
+          {:error, reason} ->
+            # todo
+            {:stop, reason, data}
+        end
+    end
   end
 
-  # todo; handle the unsuback error cases !
+  # Pass on the result of an operation if we have a calling pid. This
+  # can happen if a process order the connection to subscribe to a
+  # topic, or unsubscribe, etc.
   def handle_event(
-        :info,
-        {{Tortoise, client_id}, {Package.Unsubscribe, ref}, {unsubscribe, unsuback}},
+        :internal,
+        {:reply, caller, topic, payload},
         _current_state,
-        %State{client_id: client_id, handler: handler, pending_refs: %{} = pending, info: info} =
-          data
+        %State{}
       ) do
-    {{pid, msg_ref}, updated_pending} = Map.pop(pending, ref)
-    data = %State{data | pending_refs: updated_pending}
-
-    # When updating the internal subscription state tracker we will
-    # disregard the unsuccessful unsubacks, as we can assume it wasn't
-    # in the subscription list to begin with, or that we are still
-    # subscribed as we are not autorized to unsubscribe for the given
-    # topic; one exception is when the server report no subscription
-    # existed; then we will update the client state
-    to_remove =
-      for {topic, result} <- Enum.zip(unsubscribe.topics, unsuback.results),
-          match?(
-            reason when reason == :success or reason == {:error, :no_subscription_existed},
-            result
-          ),
-          do: topic
-
-    subscriptions = Map.drop(data.info.subscriptions, to_remove)
-
-    case Handler.execute_handle_unsuback(handler, unsubscribe, unsuback) do
-      {:ok, %Handler{} = updated_handler, next_actions} ->
-        data = %State{
-          data
-          | handler: updated_handler,
-            info: put_in(info.subscriptions, subscriptions)
-        }
-
-        next_actions = [
-          {:next_event, :internal, {:reply, {pid, msg_ref}, Package.Unsuback, :ok}}
-          | wrap_next_actions(next_actions)
-        ]
+    case caller do
+      {pid, _ref} when pid != self() ->
+        _ = send_reply(caller, topic, payload)
+        :keep_state_and_data
 
-        {:keep_state, data, next_actions}
-
-      {:error, reason} ->
-        # todo
-        {:stop, reason, data}
+      _otherwise ->
+        :keep_state_and_data
     end
   end
 
@@ -889,7 +987,12 @@ defmodule Tortoise.Connection do
   # They inform the connection to perform an action, such as
   # subscribing to a topic, and they are validated by the handler
   # module, so there is no need to coerce here
-  def handle_event(:internal, {:user_action, action}, _, %State{client_id: client_id} = state) do
+  def handle_event(
+        :internal,
+        {:user_action, action},
+        _,
+        %State{connection: {transport, socket}} = state
+      ) do
     case action do
       {:subscribe, topic, opts} when is_binary(topic) ->
         caller = {self(), make_ref()}
@@ -907,7 +1010,8 @@ defmodule Tortoise.Connection do
 
       :disconnect ->
         disconnect = %Package.Disconnect{reason: :normal_disconnection}
-        :ok = Inflight.drain(client_id, disconnect)
+        # TODO consider draining messages with qos
+        :ok = transport.send(socket, Package.encode(disconnect))
         {:stop, :normal}
 
       {:eval, fun} when is_function(fun, 1) ->
@@ -988,9 +1092,10 @@ defmodule Tortoise.Connection do
         {:call, from},
         {:disconnect, %Package.Disconnect{} = disconnect},
         :connected,
-        %State{client_id: client_id} = data
+        %State{connection: {transport, socket}} = data
       ) do
-    :ok = Inflight.drain(client_id, disconnect)
+    # TODO consider draining messages with QoS
+    :ok = transport.send(socket, Package.encode(disconnect))
 
     {:stop_and_reply, :shutdown, [{:reply, from, :ok}], data}
   end
diff --git a/lib/tortoise/session.ex b/lib/tortoise/session.ex
new file mode 100644
index 00000000..8833ed70
--- /dev/null
+++ b/lib/tortoise/session.ex
@@ -0,0 +1,95 @@
+defmodule Tortoise.Session do
+  @moduledoc """
+  Keep track of inflight message for a session
+  """
+
+  alias __MODULE__
+  alias Tortoise.Package
+
+  @enforce_keys [:client_id]
+  defstruct backend: {Tortoise.Session.Ets, Tortoise.Session},
+            client_id: nil
+
+  def child_spec(opts) do
+    mod = Keyword.fetch!(opts, :backend)
+
+    %{
+      id: mod,
+      start: {mod, :start_link, [opts]},
+      type: :worker,
+      restart: :permanent,
+      shutdown: 500
+    }
+  end
+
+  @doc """
+
+  """
+  def track(
+        %Session{client_id: client_id} = session,
+        {:incoming, %Package.Publish{identifier: id, qos: qos, dup: _} = package}
+      )
+      when not is_nil(id) and qos in 1..2 do
+    {backend, ref} = session.backend
+
+    case backend.create(ref, client_id, {:incoming, package}) do
+      {:ok, %Package.Publish{identifier: ^id} = package} ->
+        {{:cont, package}, session}
+
+      {:error, _reason} = error ->
+        error
+    end
+  end
+
+  def track(
+        %Session{client_id: client_id} = session,
+        {:outgoing, %type{identifier: _hopefully_nil} = package}
+      )
+      when type in [Package.Publish, Package.Subscribe, Package.Unsubscribe] do
+    {backend, ref} = session.backend
+
+    case backend.create(ref, client_id, {:outgoing, package}) do
+      {:ok, %Package.Publish{qos: qos} = package} when qos in 1..2 ->
+        # By passing back the package we can allow the backend to
+        # monkey with the user defined properties, and set a unique id
+        {{:cont, package}, session}
+
+      {:ok, %Package.Subscribe{} = package} ->
+        {{:cont, package}, session}
+
+      {:ok, %Package.Unsubscribe{} = package} ->
+        {{:cont, package}, session}
+    end
+  end
+
+  @doc """
+
+  """
+  def progress(
+        %Session{client_id: client_id} = session,
+        {direction, %_type{identifier: id} = package}
+      )
+      when direction in [:incoming, :outgoing] and id in 0x0001..0xFFFF do
+    {backend, ref} = session.backend
+
+    case backend.update(ref, client_id, {direction, package}) do
+      {:ok, package} ->
+        {{:cont, package}, session}
+
+      {:error, :not_found} = error ->
+        error
+    end
+  end
+
+  @doc """
+
+  """
+  def release(
+        %Session{backend: {backend, ref}, client_id: client_id} = session,
+        id
+      )
+      when id in 0x0001..0xFFFF do
+    :ok = backend.release(ref, client_id, id)
+    {:ok, session}
+  end
+end
diff --git a/lib/tortoise/session/ets.ex b/lib/tortoise/session/ets.ex
new file mode 100644
index 00000000..71f5b0bc
--- /dev/null
+++ b/lib/tortoise/session/ets.ex
@@ -0,0 +1,98 @@
+defmodule Tortoise.Session.Ets do
+  use GenServer
+
+  @name Tortoise.Session
+
+  alias Tortoise.Package
+
+  # Client API
+  def start_link(opts) do
+    name = Keyword.get(opts, :name, @name)
+    GenServer.start_link(__MODULE__, opts, name: name)
+  end
+
+  def create(session \\ @name, client_id, package)
+
+  def create(session, client_id, {:incoming, %Package.Publish{identifier: id} = package})
+      when not is_nil(id) do
+    do_create(session, client_id, {:incoming, package})
+  end
+
+  def create(session, client_id, {:outgoing, %_type{} = package}) do
+    do_create(session, client_id, {:outgoing, package})
+  end
+
+  # attempt to create an id if none is present
+  defp do_create(session, client_id, package, attempt \\ 0)
+
+  defp do_create(session, client_id, {:outgoing, %type{identifier: nil} = package}, attempt)
+       when attempt < 10 do
+    <<id::integer-size(16)>> = :crypto.strong_rand_bytes(2)
+    data = {:outgoing, %{package | identifier: id}}
+
+    case do_create(session, client_id, data, attempt) do
+      {:ok, %^type{} = package} ->
+        {:ok, package}
+
+      {:error, :non_unique_package_identifier} ->
+        do_create(session, client_id, package, attempt + 1)
+    end
+  end
+
+  defp do_create(_, _, {:outgoing, %_type{identifier: nil}}, _attempt) do
+    {:error, :could_not_create_unique_identifier}
+  end
+
+  defp do_create(session, client_id, {direction, %{identifier: id} = package}, _attempt)
+       when direction in [:incoming, :outgoing] and id in 1..0xFFFF do
+    now = System.monotonic_time()
+
+    case :ets.insert_new(session, {{client_id, id}, {now, direction, package}}) do
+      true ->
+        {:ok, package}
+
+      false ->
+        {:error, :non_unique_package_identifier}
+    end
+  end
+
+  def read(session \\ @name, client_id, package_id) do
+    case :ets.lookup(session, {client_id, package_id}) do
+      [{{^client_id, ^package_id}, {_, direction, %_type{identifier: ^package_id} = package}}] ->
+        {:ok, {direction, package}}
+
+      [] ->
+        {:error, :not_found}
+    end
+  end
+
+  def update(session \\ @name, client_id, package)
+
+  def update(session, client_id, {direction, %_type{identifier: package_id} = package}) do
+    now = System.monotonic_time()
+    key = {client_id, package_id}
+
+    case :ets.update_element(session, key, {2, {now, direction, package}}) do
+      true ->
+        {:ok, package}
+
+      false ->
+        {:error, :not_found}
+    end
+  end
+
+  def release(session \\ @name, client_id, package_id) do
+    true = :ets.delete(session, {client_id, package_id})
+    :ok
+  end
+
+  # Server callbacks
+  def init(opts) do
+    # do as little as possible, making it really hard to crash the
+    # session state
+    name = Keyword.get(opts, :name, @name)
+    ref = :ets.new(name, [:named_table, :public, {:write_concurrency, true}])
+
+    {:ok, ref}
+  end
+end
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index 2ed993d5..510354fd 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -4,7 +4,6 @@ defmodule Tortoise.ConnectionTest do
 
   alias Tortoise.Integration.{ScriptedMqttServer, ScriptedTransport}
   alias Tortoise.Connection
-  alias Tortoise.Connection.Inflight
   alias Tortoise.Package
 
   setup context do
@@ -970,7 +969,6 @@ defmodule Tortoise.ConnectionTest do
       cs_pid = Connection.Supervisor.whereis(client_id)
       cs_ref = Process.monitor(cs_pid)
 
-      inflight_pid = Connection.Inflight.whereis(client_id)
       {:ok, {Tortoise.Transport.Tcp, _}} = Connection.connection(connection_pid)
 
       {:connected,
@@ -986,7 +984,6 @@ defmodule Tortoise.ConnectionTest do
       assert_receive {ScriptedMqttServer, :completed}
 
       assert_receive {:DOWN, ^cs_ref, :process, ^cs_pid, :shutdown}
-      refute Process.alive?(inflight_pid)
       refute Process.alive?(receiver_pid)
 
       # The user defined handler should have the following callbacks
@@ -1286,17 +1283,16 @@ defmodule Tortoise.ConnectionTest do
       {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
       pid = context.connection_pid
 
-      client_id = context.client_id
-      assert {:ok, ref} = Inflight.track(client_id, {:outgoing, publish})
+      assert {:ok, ref} = Tortoise.Connection.publish(pid, publish)
 
       refute_receive {:EXIT, ^pid, {:protocol_violation, {:unexpected_package, _}}}
       assert_receive {ScriptedMqttServer, {:received, ^publish}}
       assert_receive {ScriptedMqttServer, :completed}
       # the caller should receive an :ok for the ref when it is published
-      assert_receive {{Tortoise, ^client_id}, {Package.Publish, ^ref}, :ok}
+      assert_receive {{Tortoise, ^pid}, {Package.Publish, ^ref}, :ok}
     end
 
-    test "outgoing publish with QoS=1 (sync call)", %{client_id: client_id} = context do
+    test "outgoing publish with QoS=1 (sync call)", context do
       Process.flag(:trap_exit, true)
 
       publish =
@@ -1314,13 +1310,13 @@ defmodule Tortoise.ConnectionTest do
 
       # setup a blocking call
       {parent, test_ref} = {self(), make_ref()}
+      pid = context.connection_pid
 
       spawn_link(fn ->
-        test_result = Inflight.track_sync(client_id, {:outgoing, publish})
+        test_result = Tortoise.Connection.publish_sync(pid, publish)
         send(parent, {:sync_call_result, test_ref, test_result})
       end)
 
-      pid = context.connection_pid
       refute_receive {:EXIT, ^pid, {:protocol_violation, {:unexpected_package, _}}}
       assert_receive {ScriptedMqttServer, {:received, ^publish}}
       assert_receive {ScriptedMqttServer, :completed}
@@ -1392,15 +1388,14 @@ defmodule Tortoise.ConnectionTest do
       {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
       pid = connection_pid
 
-      client_id = context.client_id
-      assert {:ok, ref} = Inflight.track(client_id, {:outgoing, publish})
+      assert {:ok, ref} = Tortoise.Connection.publish(pid, publish)
 
       refute_receive {:EXIT, ^pid, {:protocol_violation, {:unexpected_package, _}}}
       assert_receive {ScriptedMqttServer, {:received, ^publish}}
       assert_receive {ScriptedMqttServer, {:received, ^subscribe}}
       assert_receive {ScriptedMqttServer, :completed}
       # the caller should receive an :ok for the ref when it is published
-      assert_receive {{Tortoise, ^client_id}, {Package.Publish, ^ref}, :ok}
+      assert_receive {{Tortoise, ^pid}, {Package.Publish, ^ref}, :ok}
       assert_receive {{TestHandler, :handle_suback}, {_subscribe, _suback}}
     end
 
@@ -1465,7 +1460,7 @@ defmodule Tortoise.ConnectionTest do
 
       {:ok, _} = ScriptedMqttServer.enact(scripted_mqtt_server, script)
 
-      assert {:ok, ref} = Inflight.track(context.client_id, {:outgoing, publish})
+      assert {:ok, ref} = Tortoise.Connection.publish(connection_pid, publish)
 
       assert_receive {ScriptedMqttServer, :completed}
       assert_receive {ScriptedMqttServer, {:received, %Package.Publish{}}}
@@ -1509,6 +1504,7 @@ defmodule Tortoise.ConnectionTest do
       assert_receive {{TestHandler, :handle_publish}, ^publish}
     end
 
+    @tag skip: true
     test "incoming publish with QoS=2 with duplicate", context do
       Process.flag(:trap_exit, true)
 
@@ -1544,6 +1540,7 @@ defmodule Tortoise.ConnectionTest do
       refute_receive {{TestHandler, :handle_publish}, ^dup_publish}
     end
 
+    @tag skip: true
     test "incoming publish with QoS=2 with first message marked as duplicate", context do
       Process.flag(:trap_exit, true)
 
@@ -1597,14 +1594,13 @@ defmodule Tortoise.ConnectionTest do
       {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
       pid = context.connection_pid
 
-      client_id = context.client_id
-      assert {:ok, ref} = Inflight.track(client_id, {:outgoing, publish})
+      assert {:ok, ref} = Tortoise.Connection.publish(pid, publish)
 
       refute_receive {:EXIT, ^pid, {:protocol_violation, {:unexpected_package, _}}}
       assert_receive {ScriptedMqttServer, {:received, ^publish}}
       assert_receive {ScriptedMqttServer, {:received, ^pubrel}}
       assert_receive {ScriptedMqttServer, :completed}
-      assert_receive {{Tortoise, ^client_id}, {Package.Publish, ^ref}, :ok}
+      assert_receive {{Tortoise, ^pid}, {Package.Publish, ^ref}, :ok}
 
       # the handle_pubrec callback should have been called
       assert_receive {{TestHandler, :handle_pubrec}, ^pubrec}
diff --git a/test/tortoise/session_test.exs b/test/tortoise/session_test.exs
new file mode 100644
index 00000000..1b96f561
--- /dev/null
+++ b/test/tortoise/session_test.exs
@@ -0,0 +1,134 @@
+defmodule Tortoise.SessionTest do
+  use ExUnit.Case, async: true
+  doctest Tortoise.Session
+
+  alias Tortoise.{Session, Package}
+
+  test "track an incoming publish qos=1" do
+    # The connection will receive the publish, store that in the
+    # session, as the backend might be of a kind that doesn't throw
+    # away packages. We will dispatch this publish message to the user
+    # defined connection handler, and use the puback it produces to
+    # progress the state of the session; then we will complete the
+    # session.
+
+    session = %Tortoise.Session{client_id: "foo"}
+
+    publish_package =
+      %Package.Publish{identifier: id} = %Package.Publish{qos: 1, identifier: 123, dup: false}
+
+    # The track command will send back the publish message, this will
+    # allow the backend to insert values to the user defined
+    # properties (which doesn't make sense in the incoming scenario,
+    # but will make sense in the outgoing cases)
+    assert {{:cont, %Package.Publish{identifier: ^id} = publish_package}, %Session{}} =
+             Session.track(session, {:incoming, publish_package})
+
+    puback_package = %Package.Puback{identifier: id}
+
+    assert {{:cont, %Package.Puback{identifier: ^id} = puback_package}, %Session{}} =
+             Session.progress(session, {:outgoing, puback_package})
+
+    assert {:ok, _} = Session.release(session, id)
+  end
+
+  test "track an outgoing publish qos=1" do
+    session = %Tortoise.Session{client_id: "foo"}
+
+    publish_package = %Package.Publish{qos: 1, identifier: nil, dup: false}
+
+    assert {{:cont, %Package.Publish{identifier: id} = publish_package}, %Session{}} =
+             Session.track(session, {:outgoing, publish_package})
+
+    puback_package = %Package.Puback{identifier: id}
+
+    assert {{:cont, %Package.Puback{identifier: ^id} = puback_package}, %Session{}} =
+             Session.progress(session, {:incoming, puback_package})
+
+    assert {:ok, _} = Session.release(session, id)
+  end
+
+  test "track an outgoing publish qos=2" do
+    session = %Tortoise.Session{client_id: "foo"}
+
+    publish_package = %Package.Publish{qos: 2, identifier: nil, dup: false}
+
+    assert {{:cont, %Package.Publish{identifier: id} = publish_package}, %Session{}} =
+             Session.track(session, {:outgoing, publish_package})
+
+    pubrec_package = %Package.Pubrec{identifier: id}
+
+    assert {{:cont, %Package.Pubrec{identifier: ^id} = pubrec_package}, %Session{}} =
+             Session.progress(session, {:incoming, pubrec_package})
+
+    pubrel_package = %Package.Pubrel{identifier: id}
+
+    assert {{:cont, %Package.Pubrel{identifier: ^id} = pubrel_package}, %Session{}} =
+             Session.progress(session, {:outgoing, pubrel_package})
+
+    pubcomp_package = %Package.Pubcomp{identifier: id}
+
+    assert {{:cont, %Package.Pubcomp{identifier: ^id} = pubcomp_package}, %Session{}} =
+             Session.progress(session, {:incoming, pubcomp_package})
+
+    assert {:ok, _} = Session.release(session, id)
+  end
+
+  test "track an incoming publish qos=2" do
+    session = %Tortoise.Session{client_id: "foo"}
+
+    publish_package = %Package.Publish{qos: 2, identifier: 125, dup: false}
+
+    assert {{:cont, %Package.Publish{identifier: id} = publish_package}, %Session{}} =
+             Session.track(session, {:incoming, publish_package})
+
+    pubrec_package = %Package.Pubrec{identifier: id}
+
+    assert {{:cont, %Package.Pubrec{identifier: ^id} = pubrec_package}, %Session{}} =
+             Session.progress(session, {:outgoing, pubrec_package})
+
+    pubrel_package = %Package.Pubrel{identifier: id}
+
+    assert {{:cont, %Package.Pubrel{identifier: ^id} = pubrel_package}, %Session{}} =
+             Session.progress(session, {:incoming, pubrel_package})
+
+    pubcomp_package = %Package.Pubcomp{identifier: id}
+
+    assert {{:cont, %Package.Pubcomp{identifier: ^id} = pubcomp_package}, %Session{}} =
+             Session.progress(session, {:outgoing, pubcomp_package})
+
+    assert {:ok, _} = Session.release(session, id)
+  end
+
+  test "track an outgoing subscribe package" do
+    session = %Tortoise.Session{client_id: "foo"}
+
+    subscribe_package = %Package.Subscribe{}
+
+    assert {{:cont, %Package.Subscribe{identifier: id} = subscribe_package}, %Session{}} =
+             Session.track(session, {:outgoing, subscribe_package})
+
+    suback_package = %Package.Suback{identifier: id}
+
+    assert {{:cont, %Package.Suback{identifier: id} = suback_package}, %Session{}} =
+             Session.progress(session, {:incoming, suback_package})
+
+    assert {:ok, _} = Session.release(session, id)
+  end
+
+  test "track an outgoing unsubscribe package" do
+    session = %Tortoise.Session{client_id: "foo"}
+
+    unsubscribe_package = %Package.Unsubscribe{}
+
+    assert {{:cont, %Package.Unsubscribe{identifier: id} = unsubscribe_package}, %Session{}} =
+             Session.track(session, {:outgoing, unsubscribe_package})
+
+    unsuback_package = %Package.Unsuback{identifier: id}
+
+    assert {{:cont, %Package.Unsuback{identifier: id} = suback_package}, %Session{}} =
+             Session.progress(session, {:incoming, unsuback_package})
+
+    assert {:ok, _} = Session.release(session, id)
+  end
+end

From 58109b454b0aecdb5e8cfdccabd4e3b2e3c91da5 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Thu, 17 Sep 2020 19:46:01 +0100
Subject: [PATCH 205/220] Always pass the session data back from the session
 callback

This will allow us to implement a backend that is backed by a data
structure instead of a reference; such as a Map based backend that
will be very useful for testing.
---
 lib/tortoise/session.ex     | 27 +++++++--------
 lib/tortoise/session/ets.ex | 68 ++++++++++++++++++++++---------------
 2 files changed, 54 insertions(+), 41 deletions(-)

diff --git a/lib/tortoise/session.ex b/lib/tortoise/session.ex
index 8833ed70..bb3bad1a 100644
--- a/lib/tortoise/session.ex
+++ b/lib/tortoise/session.ex
@@ -26,14 +26,14 @@ defmodule Tortoise.Session do
 
   """
   def track(
-        %Session{client_id: client_id} = session,
+        %Session{} = session,
         {:incoming, %Package.Publish{identifier: id, qos: qos, dup: _} = package}
       )
       when not is_nil(id) and qos in 1..2 do
     {backend, ref} = session.backend
 
-    case backend.create(ref, client_id, {:incoming, package}) do
-      {:ok, %Package.Publish{identifier: ^id} = package} ->
+    case backend.create(ref, session, {:incoming, package}) do
+      {:ok, %Package.Publish{identifier: ^id} = package, session} ->
         {{:cont, package}, session}
 
       {:error, _reason} = error ->
@@ -42,22 +42,22 @@ defmodule Tortoise.Session do
   end
 
   def track(
-        %Session{client_id: client_id} = session,
+        %Session{} = session,
         {:outgoing, %type{identifier: _hopefully_nil} = package}
       )
       when type in [Package.Publish, Package.Subscribe, Package.Unsubscribe] do
     {backend, ref} = session.backend
 
-    case backend.create(ref, client_id, {:outgoing, package}) do
-      {:ok, %Package.Publish{qos: qos} = package} when qos in 1..2 ->
+    case backend.create(ref, session, {:outgoing, package}) do
+      {:ok, %Package.Publish{qos: qos} = package, session} when qos in 1..2 ->
         # By passing back the package we can allow the backend to
         # monkey with the user defined properties, and set a unique id
         {{:cont, package}, session}
 
-      {:ok, %Package.Subscribe{} = package} ->
+      {:ok, %Package.Subscribe{} = package, session} ->
         {{:cont, package}, session}
 
-      {:ok, %Package.Unsubscribe{} = package} ->
+      {:ok, %Package.Unsubscribe{} = package, session} ->
         {{:cont, package}, session}
     end
   end
@@ -66,14 +66,14 @@ defmodule Tortoise.Session do
 
   """
   def progress(
-        %Session{client_id: client_id} = session,
+        %Session{} = session,
         {direction, %_type{identifier: id} = package}
       )
       when direction in [:incoming, :outgoing] and id in 0x0001..0xFFFF do
     {backend, ref} = session.backend
 
-    case backend.update(ref, client_id, {direction, package}) do
-      {:ok, package} ->
+    case backend.update(ref, session, {direction, package}) do
+      {:ok, package, session} ->
         {{:cont, package}, session}
 
       {:error, :not_found} = error ->
@@ -85,11 +85,10 @@ defmodule Tortoise.Session do
 
   """
   def release(
-        %Session{backend: {backend, ref}, client_id: client_id} = session,
+        %Session{backend: {backend, ref}} = session,
         id
       )
       when id in 0x0001..0xFFFF do
-    :ok = backend.release(ref, client_id, id)
-    {:ok, session}
+    {:ok, %Session{}} = backend.release(ref, session, id)
   end
 end
diff --git a/lib/tortoise/session/ets.ex b/lib/tortoise/session/ets.ex
index 71f5b0bc..c243b08a 100644
--- a/lib/tortoise/session/ets.ex
+++ b/lib/tortoise/session/ets.ex
@@ -3,7 +3,7 @@ defmodule Tortoise.Session.Ets do
 
   @name Tortoise.Session
 
-  alias Tortoise.Package
+  alias Tortoise.{Session, Package}
 
   # Client API
   def start_link(opts) do
@@ -11,31 +11,36 @@ defmodule Tortoise.Session.Ets do
     GenServer.start_link(__MODULE__, opts, name: name)
   end
 
-  def create(session \\ @name, client_id, package)
+  def create(instance \\ @name, session, package)
 
-  def create(session, client_id, {:incoming, %Package.Publish{identifier: id} = package})
+  def create(instance, session, {:incoming, %Package.Publish{identifier: id} = package})
       when not is_nil(id) do
-    do_create(session, client_id, {:incoming, package})
+    do_create(instance, session, {:incoming, package})
   end
 
-  def create(session, client_id, {:outgoing, %_type{} = package}) do
-    do_create(session, client_id, {:outgoing, package})
+  def create(instance, session, {:outgoing, %_type{} = package}) do
+    do_create(instance, session, {:outgoing, package})
   end
 
   # attempt to create an id if none is present
-  defp do_create(session, client_id, package, attempt \\ 0)
-
-  defp do_create(session, client_id, {:outgoing, %type{identifier: nil} = package}, attempt)
+  defp do_create(instance, session, package, attempt \\ 0)
+
+  defp do_create(
+         instance,
+         %Session{} = session,
+         {:outgoing, %type{identifier: nil} = package},
+         attempt
+       )
        when attempt < 10 do
     <<id::integer-size(16)>> = :crypto.strong_rand_bytes(2)
     data = {:outgoing, %{package | identifier: id}}
 
-    case do_create(session, client_id, data, attempt) do
-      {:ok, %^type{} = package} ->
-        {:ok, package}
+    case do_create(instance, session, data, attempt) do
+      {:ok, %^type{} = package, session} ->
+        {:ok, package, session}
 
       {:error, :non_unique_package_identifier} ->
-        do_create(session, client_id, package, attempt + 1)
+        do_create(instance, session, package, attempt + 1)
     end
   end
 
@@ -43,53 +48,62 @@ defmodule Tortoise.Session.Ets do
     {:error, :could_not_create_unique_identifier}
   end
 
-  defp do_create(session, client_id, {direction, %{identifier: id} = package}, _attempt)
+  defp do_create(
+         instance,
+         %Session{client_id: client_id} = session,
+         {direction, %{identifier: id} = package},
+         _attempt
+       )
        when direction in [:incoming, :outgoing] and id in 1..0xFFFF do
     now = System.monotonic_time()
 
-    case :ets.insert_new(session, {{client_id, id}, {now, direction, package}}) do
+    case :ets.insert_new(instance, {{client_id, id}, {now, direction, package}}) do
       true ->
-        {:ok, package}
+        {:ok, package, session}
 
       false ->
         {:error, :non_unique_package_identifier}
     end
   end
 
-  def read(session \\ @name, client_id, package_id) do
-    case :ets.lookup(session, {client_id, package_id}) do
+  def read(instance \\ @name, %Session{client_id: client_id} = session, package_id) do
+    case :ets.lookup(instance, {client_id, package_id}) do
       [{{^client_id, ^package_id}, {_, direction, %_type{identifier: ^package_id} = package}}] ->
-        {:ok, {direction, package}}
+        {:ok, {direction, package}, session}
 
       [] ->
         {:error, :not_found}
     end
   end
 
-  def update(session \\ @name, client_id, package)
+  def update(instance \\ @name, session, package)
 
-  def update(session, client_id, {direction, %_type{identifier: package_id} = package}) do
+  def update(
+        instance,
+        %Session{client_id: client_id} = session,
+        {direction, %_type{identifier: package_id} = package}
+      ) do
     now = System.monotonic_time()
     key = {client_id, package_id}
 
-    case :ets.update_element(session, key, {2, {now, direction, package}}) do
+    case :ets.update_element(instance, key, {2, {now, direction, package}}) do
       true ->
-        {:ok, package}
+        {:ok, package, session}
 
       false ->
         {:error, :not_found}
     end
   end
 
-  def release(session \\ @name, client_id, package_id) do
-    true = :ets.delete(session, {client_id, package_id})
-    :ok
+  def release(instance \\ @name, %Session{client_id: client_id} = session, package_id) do
+    true = :ets.delete(instance, {client_id, package_id})
+    {:ok, session}
   end
 
   # Server callbacks
   def init(opts) do
     # do as little as possible, making it really hard to crash the
-    # session state
+    # instance state
     name = Keyword.get(opts, :name, @name)
     ref = :ets.new(name, [:named_table, :public, {:write_concurrency, true}])
 

From 9463ce69f9dc1cfe438b6e903d7959992a853a48 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Fri, 18 Sep 2020 17:01:46 +0100
Subject: [PATCH 206/220] Make sure we release the id in the session state on
 outgoing pubcomp

---
 lib/tortoise/connection.ex | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 27c5dc20..3372a316 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -685,6 +685,7 @@ defmodule Tortoise.Connection do
           # dispatch the pubcomp
           {{:cont, pubcomp}, session} = Session.progress(session, {:outgoing, pubcomp})
           :ok = transport.send(socket, Package.encode(pubcomp))
+          {:ok, session} = Session.release(session, id)
           updated_data = %State{data | handler: updated_handler, session: session}
           {:keep_state, updated_data, wrap_next_actions(next_actions)}
 
@@ -696,6 +697,7 @@ defmodule Tortoise.Connection do
       pubcomp = %Package.Pubcomp{identifier: id}
       {{:cont, pubcomp}, session} = Session.progress(session, {:outgoing, pubcomp})
       :ok = transport.send(socket, Package.encode(pubcomp))
+      {:ok, session} = Session.release(session, id)
       updated_data = %State{data | session: session}
       {:keep_state, updated_data}
     end

From 5378cece6108be0e314dfe9a3a61f606055aca0f Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Tue, 22 Sep 2020 16:00:59 +0100
Subject: [PATCH 207/220] Make the Tortoise.publish function work with the
 connection pid

---
 lib/tortoise.ex            | 28 ++++++++++++++--------------
 lib/tortoise/connection.ex |  8 ++++----
 2 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/lib/tortoise.ex b/lib/tortoise.ex
index 09acf21c..a1dc9019 100644
--- a/lib/tortoise.ex
+++ b/lib/tortoise.ex
@@ -252,7 +252,7 @@ defmodule Tortoise do
   with `Tortoise` so it is easy to see where the message originated
   from.
   """
-  @spec publish(client_id(), topic_or_topic_alias(), payload, [options]) ::
+  @spec publish(Process.dest(), topic_or_topic_alias(), payload, [options]) ::
           :ok | {:ok, reference()} | {:error, reason}
         when payload: binary() | nil,
              options:
@@ -260,9 +260,9 @@ defmodule Tortoise do
                | {:retain, boolean()}
                | {:identifier, package_identifier()},
              reason: :unknown_connection | :topic_alias_specified_twice
-  def publish(client_id, topic, payload \\ nil, opts \\ [])
+  def publish(name_or_pid, topic, payload \\ nil, opts \\ [])
 
-  def publish(client_id, topic, payload, opts) when is_binary(topic) do
+  def publish(name_or_pid, topic, payload, opts) when is_binary(topic) do
     {opts, properties} = Keyword.split(opts, [:retain, :qos, :transforms])
     qos = Keyword.get(opts, :qos, 0)
 
@@ -274,19 +274,19 @@ defmodule Tortoise do
       properties: properties
     }
 
-    with {:ok, {transport, socket}} <- Connection.connection(client_id) do
-      case publish do
-        %Package.Publish{qos: 0} ->
+    case publish do
+      %Package.Publish{qos: 0} ->
+        with {:ok, {transport, socket}} <- Connection.connection(name_or_pid) do
           encoded_publish = Package.encode(publish)
           apply(transport, :send, [socket, encoded_publish])
-
-        %Package.Publish{qos: qos} when qos in [1, 2] ->
-          transforms = Keyword.get(opts, :transforms, {[], nil})
-          Inflight.track(client_id, {:outgoing, publish, transforms})
-      end
-    else
-      {:error, :unknown_connection} ->
-        {:error, :unknown_connection}
+        else
+          {:error, :unknown_connection} ->
+            {:error, :unknown_connection}
+        end
+
+      %Package.Publish{qos: qos} when qos in [1, 2] ->
+        # transforms = Keyword.get(opts, :transforms, {[], nil})
+        Connection.publish(name_or_pid, publish)
     end
   end
 
diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 3372a316..51aa6915 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -389,12 +389,12 @@ defmodule Tortoise.Connection do
         when opts: {:timeout, timeout()} | {:active, boolean()}
   def connection(pid, _opts \\ [active: false])
 
-  def connection(pid, opts) when is_pid(pid) do
+  def connection(name_or_pid, opts) do
     timeout = Keyword.get(opts, :timeout, :infinity)
 
-    # make it possible to subscribe to a connection using "active"!
-    if Process.alive?(pid) do
-      GenStateMachine.call(pid, :get_connection, timeout)
+    # TODO make it possible to subscribe to a connection using "active"!
+    if GenServer.whereis(name_or_pid) |> Process.alive?() do
+      GenStateMachine.call(name_or_pid, :get_connection, timeout)
     else
       {:error, :unknown_connection}
     end

From 8db7f12890838d043ce06efb3798c7a6eec9a4df Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Wed, 23 Sep 2020 15:57:06 +0100
Subject: [PATCH 208/220] Make the public interface use the pid as identity for
 a connection

---
 lib/tortoise.ex                   |  35 +++---
 lib/tortoise/connection.ex        | 176 ++++++++++++++++++++----------
 test/tortoise/connection_test.exs |  72 +++++++-----
 3 files changed, 179 insertions(+), 104 deletions(-)

diff --git a/lib/tortoise.ex b/lib/tortoise.ex
index a1dc9019..ec6aea71 100644
--- a/lib/tortoise.ex
+++ b/lib/tortoise.ex
@@ -62,7 +62,6 @@ defmodule Tortoise do
 
   alias Tortoise.Package
   alias Tortoise.Connection
-  alias Tortoise.Connection.Inflight
 
   @typedoc """
   An identifier used to identify the client on the server.
@@ -334,7 +333,7 @@ defmodule Tortoise do
 
   See the documentation for `Tortoise.publish/4` for configuration.
   """
-  @spec publish_sync(client_id(), topic_or_topic_alias(), payload, [options]) ::
+  @spec publish_sync(Process.dest(), topic_or_topic_alias(), payload, [options]) ::
           :ok | {:error, reason}
         when payload: binary() | nil,
              options:
@@ -343,12 +342,11 @@ defmodule Tortoise do
                | {:identifier, package_identifier()}
                | {:timeout, timeout()},
              reason: :unknown_connection | :topic_alias_specified_twice
-  def publish_sync(client_id, topic, payload \\ nil, opts \\ [])
+  def publish_sync(pid_or_name, topic, payload \\ nil, opts \\ [])
 
-  def publish_sync(client_id, topic, payload, opts) when is_binary(topic) do
-    {opts, properties} = Keyword.split(opts, [:retain, :qos, :transforms])
+  def publish_sync(pid_or_name, topic, payload, opts) when is_binary(topic) do
+    {opts, properties} = Keyword.split(opts, [:retain, :qos, :transforms, :timeout])
     qos = Keyword.get(opts, :qos, 0)
-    timeout = Keyword.get(opts, :timeout, :infinity)
 
     publish = %Package.Publish{
       topic: topic,
@@ -358,19 +356,20 @@ defmodule Tortoise do
       properties: properties
     }
 
-    with {:ok, {transport, socket}} <- Connection.connection(client_id) do
-      case publish do
-        %Package.Publish{qos: 0} ->
-          encoded_publish = Package.encode(publish)
-          apply(transport, :send, [socket, encoded_publish])
+    case publish do
+      # %Package.Publish{qos: 0} ->
+      # with {:ok, {transport, socket}} <- Connection.connection(client_id) do
+      #   encoded_publish = Package.encode(publish)
+      #   apply(transport, :send, [socket, encoded_publish])
+      # else
+      #   {:error, :unknown_connection} ->
+      #     {:error, :unknown_connection}
+      # end
 
-        %Package.Publish{qos: qos} when qos in [1, 2] ->
-          transforms = Keyword.get(opts, :transforms, {[], nil})
-          Inflight.track_sync(client_id, {:outgoing, publish, transforms}, timeout)
-      end
-    else
-      {:error, :unknown_connection} ->
-        {:error, :unknown_connection}
+      %Package.Publish{qos: qos} when qos in [1, 2] ->
+        # transforms = Keyword.get(opts, :transforms, {[], nil})
+        timeout = Keyword.get(opts, :timeout, :infinity)
+        Tortoise.Connection.publish_sync(pid_or_name, publish, timeout)
     end
   end
 
diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 51aa6915..0c5432df 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -102,9 +102,9 @@ defmodule Tortoise.Connection do
   @doc """
   Close the connection to the broker.
 
-  Given the `client_id` of a running connection it will cancel the
-  inflight messages and send the proper disconnect message to the
-  broker. The session will get terminated on the server.
+  Given the `pid` of a running connection it will cancel the inflight
+  messages and send the proper disconnect message to the broker. The
+  session will get terminated on the server.
   """
   @spec disconnect(pid(), reason, properties) :: :ok
         when reason: Tortoise.Package.Disconnect.reason(),
@@ -152,7 +152,7 @@ defmodule Tortoise.Connection do
   Read the documentation for `Tortoise.Connection.subscribe_sync/3`
   for a blocking version of this call.
   """
-  @spec subscribe(pid(), topic | topics, [options]) :: {:ok, reference()}
+  @spec subscribe(pid(), topic | topics, [options]) :: {:ok, {Tortoise.client_id(), reference()}}
         when topics: [topic],
              topic: {Tortoise.topic_filter(), Tortoise.qos()},
              options:
@@ -161,7 +161,6 @@ defmodule Tortoise.Connection do
   def subscribe(pid, topics, opts \\ [])
 
   def subscribe(pid, [{_, topic_opts} | _] = topics, opts) when is_list(topic_opts) do
-    caller = {_, ref} = {self(), make_ref()}
     # todo, do something with timeout, or remove it
     {opts, properties} = Keyword.split(opts, [:identifier, :timeout])
     {identifier, opts} = Keyword.pop_first(opts, :identifier, nil)
@@ -172,8 +171,7 @@ defmodule Tortoise.Connection do
         properties: properties
       })
 
-    GenStateMachine.cast(pid, {:subscribe, caller, subscribe, opts})
-    {:ok, ref}
+    GenStateMachine.call(pid, {:subscribe, subscribe, opts})
   end
 
   def subscribe(pid, {_, topic_opts} = topic, opts) when is_list(topic_opts) do
@@ -212,14 +210,19 @@ defmodule Tortoise.Connection do
   def subscribe_sync(pid, topics, opts \\ [])
 
   def subscribe_sync(pid, [{_, topic_opts} | _] = topics, opts) when is_list(topic_opts) do
-    timeout = Keyword.get(opts, :timeout, 5000)
-    {:ok, ref} = subscribe(pid, topics, opts)
+    case subscribe(pid, topics, opts) do
+      {:ok, {client_id, ref}} ->
+        timeout = Keyword.get(opts, :timeout, 5000)
+
+        receive do
+          {{Tortoise, ^client_id}, {Package.Suback, ^ref}, result} -> result
+        after
+          timeout ->
+            {:error, :timeout}
+        end
 
-    receive do
-      {{Tortoise, ^pid}, {Package.Suback, ^ref}, result} -> result
-    after
-      timeout ->
-        {:error, :timeout}
+      {:error, _reason} = error ->
+        error
     end
   end
 
@@ -247,7 +250,8 @@ defmodule Tortoise.Connection do
   This operation is asynchronous. When the operation is done a message
   will be received in mailbox of the originating process.
   """
-  @spec unsubscribe(pid(), topic | topics, [options]) :: {:ok, reference()}
+  @spec unsubscribe(pid(), topic | topics, [options]) ::
+          {:ok, {Tortoise.client_id(), reference()}}
         when topics: [topic],
              topic: Tortoise.topic_filter(),
              options:
@@ -256,7 +260,6 @@ defmodule Tortoise.Connection do
   def unsubscribe(pid, topics, opts \\ [])
 
   def unsubscribe(pid, [topic | _] = topics, opts) when is_binary(topic) do
-    caller = {_, ref} = {self(), make_ref()}
     {opts, properties} = Keyword.split(opts, [:identifier, :timeout])
     {identifier, opts} = Keyword.pop_first(opts, :identifier, nil)
 
@@ -266,8 +269,7 @@ defmodule Tortoise.Connection do
       properties: properties
     }
 
-    GenStateMachine.cast(pid, {:unsubscribe, caller, unsubscribe, opts})
-    {:ok, ref}
+    GenStateMachine.call(pid, {:unsubscribe, unsubscribe, opts})
   end
 
   def unsubscribe(pid, topic, opts) when is_binary(topic) do
@@ -294,10 +296,10 @@ defmodule Tortoise.Connection do
 
   def unsubscribe_sync(pid, topics, opts) when is_list(topics) do
     timeout = Keyword.get(opts, :timeout, 5000)
-    {:ok, ref} = unsubscribe(pid, topics, opts)
+    {:ok, {client_id, ref}} = unsubscribe(pid, topics, opts)
 
     receive do
-      {{Tortoise, ^pid}, {Package.Unsuback, ^ref}, result} ->
+      {{Tortoise, ^client_id}, {Package.Unsuback, ^ref}, result} ->
         result
     after
       timeout ->
@@ -319,11 +321,11 @@ defmodule Tortoise.Connection do
     GenStateMachine.call(pid, {:publish, publish})
   end
 
-  def publish_sync(pid, %Package.Publish{} = publish, timeout \\ 5000) do
-    {:ok, ref} = publish(pid, publish)
+  def publish_sync(pid, %Package.Publish{} = publish, timeout \\ :infinity) do
+    {:ok, {client_id, ref}} = publish(pid, publish)
 
     receive do
-      {{Tortoise, ^pid}, {Package.Publish, ^ref}, result} ->
+      {{Tortoise, ^client_id}, {Package.Publish, ^ref}, result} ->
         result
     after
       timeout ->
@@ -345,11 +347,9 @@ defmodule Tortoise.Connection do
   better to listen on `:ping_response` using the `Tortoise.Events`
   PubSub.
   """
-  @spec ping(pid()) :: {:ok, reference()}
-  def ping(pid) do
-    ref = make_ref()
-    :ok = GenStateMachine.cast(pid, {:ping, {self(), ref}})
-    {:ok, ref}
+  @spec ping(pid(), timeout()) :: {:ok, reference()}
+  def ping(pid, timeout \\ :infinity) do
+    {:ok, {_client_id, _ref}} = GenStateMachine.call(pid, :ping, timeout)
   end
 
   @doc """
@@ -365,10 +365,10 @@ defmodule Tortoise.Connection do
   """
   @spec ping_sync(pid(), timeout()) :: {:ok, reference()} | {:error, :timeout}
   def ping_sync(pid, timeout \\ :infinity) do
-    {:ok, ref} = ping(pid)
+    {:ok, {client_id, ref}} = ping(pid, timeout)
 
     receive do
-      {{Tortoise, ^pid}, {Package.Pingreq, ^ref}, round_trip_time} ->
+      {{Tortoise, ^client_id}, {Package.Pingreq, ^ref}, round_trip_time} ->
         {:ok, round_trip_time}
     after
       timeout ->
@@ -605,7 +605,7 @@ defmodule Tortoise.Connection do
     case Session.track(session, {:outgoing, publish}) do
       {{:cont, %Package.Publish{identifier: id} = publish}, session} ->
         :ok = transport.send(socket, Package.encode(publish))
-        next_actions = [{:reply, from, {:ok, ref}}]
+        next_actions = [{:reply, from, {:ok, {session.client_id, ref}}}]
         data = %State{data | session: session, pending_refs: Map.put_new(pending, id, from)}
         {:keep_state, data, next_actions}
     end
@@ -717,7 +717,7 @@ defmodule Tortoise.Connection do
     case Session.track(session, {:outgoing, publish}) do
       {{:cont, %Package.Publish{identifier: id} = publish}, session} ->
         :ok = transport.send(socket, Package.encode(publish))
-        next_actions = [{:reply, from, {:ok, ref}}]
+        next_actions = [{:reply, from, {:ok, {session.client_id, ref}}}]
         data = %State{data | session: session, pending_refs: Map.put_new(pending, id, from)}
         {:keep_state, data, next_actions}
     end
@@ -794,22 +794,21 @@ defmodule Tortoise.Connection do
 
   # subscription logic
   def handle_event(
-        :cast,
-        {:subscribe, caller, %Package.Subscribe{topics: []}, _opts},
+        {:call, from},
+        {:subscribe, %Package.Subscribe{topics: []}, _opts},
         :connected,
         _data
       ) do
     # This should not really be able to happen as the API will not
     # allow the user to specify an empty list, but this is added for
     # good measure
-    reply = {:error, :empty_topic_filter_list}
-    next_actions = [{:next_event, :internal, {:reply, caller, reply}}]
+    next_actions = [{:reply, from, {:error, :empty_topic_filter_list}}]
     {:keep_state_and_data, next_actions}
   end
 
   def handle_event(
-        :cast,
-        {:subscribe, caller, %Package.Subscribe{} = subscribe, _opts},
+        {:call, {_, ref} = from},
+        {:subscribe, %Package.Subscribe{} = subscribe, _opts},
         :connected,
         %State{
           connection: {transport, socket},
@@ -820,19 +819,50 @@ defmodule Tortoise.Connection do
       :valid ->
         case Session.track(session, {:outgoing, subscribe}) do
           {{:cont, %Package.Subscribe{identifier: id} = subscribe}, session} ->
+            :ok = transport.send(socket, Package.encode(subscribe))
+            pending = Map.put_new(data.pending_refs, id, {from, subscribe})
+            state = %State{data | pending_refs: pending, session: session}
+            next_actions = [{:reply, from, {:ok, {session.client_id, ref}}}]
+            {:keep_state, state, next_actions}
+        end
+
+      {:invalid, reasons} ->
+        reply = {:error, {:subscription_failure, reasons}}
+        next_actions = [{:reply, from, reply}]
+        {:keep_state_and_data, next_actions}
+    end
+  end
+
+  def handle_event({:call, _}, {:subscribe, _, _}, _state_name, _data) do
+    {:keep_state_and_data, [:postpone]}
+  end
+
+  def handle_event(
+        :internal,
+        {:subscribe, %Package.Subscribe{} = subscribe, _opts},
+        :connected,
+        %State{
+          connection: {transport, socket},
+          session: session
+        } = data
+      ) do
+    case Info.Capabilities.validate(data.info.capabilities, subscribe) do
+      :valid ->
+        case Session.track(session, {:outgoing, subscribe}) do
+          {{:cont, %Package.Subscribe{identifier: id} = subscribe}, session} ->
+            caller = {self(), make_ref()}
             :ok = transport.send(socket, Package.encode(subscribe))
             pending = Map.put_new(data.pending_refs, id, {caller, subscribe})
             {:keep_state, %State{data | pending_refs: pending, session: session}}
         end
 
       {:invalid, reasons} ->
-        reply = {:error, {:subscription_failure, reasons}}
-        next_actions = [{:next_event, :internal, {:reply, caller, Package.Suback, reply}}]
+        next_actions = [{:error, {:subscription_failure, reasons}}]
         {:keep_state_and_data, next_actions}
     end
   end
 
-  def handle_event(:cast, {:subscribe, _, _, _}, _state_name, _data) do
+  def handle_event(:internal, {:subscribe, _, _}, _state_name, _data) do
     {:keep_state_and_data, [:postpone]}
   end
 
@@ -886,8 +916,29 @@ defmodule Tortoise.Connection do
   end
 
   def handle_event(
-        :cast,
-        {:unsubscribe, caller, unsubscribe, opts},
+        {:call, {_pid, ref} = from},
+        {:unsubscribe, unsubscribe, opts},
+        :connected,
+        %State{
+          session: session,
+          connection: {transport, socket},
+          pending_refs: pending
+        } = data
+      ) do
+    _timeout = Keyword.get(opts, :timeout, 5000)
+
+    case Session.track(session, {:outgoing, unsubscribe}) do
+      {{:cont, %Package.Unsubscribe{identifier: id} = unsubscribe}, session} ->
+        :ok = transport.send(socket, Package.encode(unsubscribe))
+        pending = Map.put_new(pending, id, {from, unsubscribe})
+        next_actions = [{:reply, from, {:ok, {session.client_id, ref}}}]
+        {:keep_state, %State{data | pending_refs: pending, session: session}, next_actions}
+    end
+  end
+
+  def handle_event(
+        :internal,
+        {:unsubscribe, unsubscribe, opts},
         :connected,
         %State{
           session: session,
@@ -900,6 +951,7 @@ defmodule Tortoise.Connection do
     case Session.track(session, {:outgoing, unsubscribe}) do
       {{:cont, %Package.Unsubscribe{identifier: id} = unsubscribe}, session} ->
         :ok = transport.send(socket, Package.encode(unsubscribe))
+        caller = {self(), make_ref()}
         pending = Map.put_new(pending, id, {caller, unsubscribe})
         {:keep_state, %State{data | pending_refs: pending, session: session}}
     end
@@ -966,11 +1018,11 @@ defmodule Tortoise.Connection do
         :internal,
         {:reply, caller, topic, payload},
         _current_state,
-        %State{}
+        %State{session: session}
       ) do
     case caller do
       {pid, _ref} when pid != self() ->
-        _ = send_reply(caller, topic, payload)
+        _ = send_reply(session.client_id, caller, topic, payload)
         :keep_state_and_data
 
       _otherwise ->
@@ -997,17 +1049,15 @@ defmodule Tortoise.Connection do
       ) do
     case action do
       {:subscribe, topic, opts} when is_binary(topic) ->
-        caller = {self(), make_ref()}
         {identifier, opts} = Keyword.pop_first(opts, :identifier, nil)
         subscribe = %Package.Subscribe{identifier: identifier, topics: [{topic, opts}]}
-        next_actions = [{:next_event, :cast, {:subscribe, caller, subscribe, opts}}]
+        next_actions = [{:next_event, :internal, {:subscribe, subscribe, opts}}]
         {:keep_state_and_data, next_actions}
 
       {:unsubscribe, topic, opts} when is_binary(topic) ->
-        caller = {self(), make_ref()}
         {identifier, opts} = Keyword.pop_first(opts, :identifier, nil)
         subscribe = %Package.Unsubscribe{identifier: identifier, topics: [topic]}
-        next_actions = [{:next_event, :cast, {:unsubscribe, caller, subscribe, opts}}]
+        next_actions = [{:next_event, :internal, {:unsubscribe, subscribe, opts}}]
         {:keep_state_and_data, next_actions}
 
       :disconnect ->
@@ -1112,19 +1162,26 @@ defmodule Tortoise.Connection do
   end
 
   # ping handling ------------------------------------------------------
-  def handle_event(:cast, {:ping, caller}, :connected, %State{} = data) do
+  def handle_event({:call, {_, ref} = from}, :ping, :connected, %State{session: session} = data) do
+    next_actions = [{:reply, from, {:ok, {session.client_id, ref}}}]
+
     case data.ping do
       {:idle, awaiting} ->
-        next_actions = [{:next_event, :internal, :trigger_keep_alive}]
-        {:keep_state, %State{data | ping: {:idle, [caller | awaiting]}}, next_actions}
+        next_actions = [
+          {:next_event, :internal, :trigger_keep_alive}
+          | next_actions
+        ]
+
+        {:keep_state, %State{data | ping: {:idle, [from | awaiting]}}, next_actions}
 
       {{:pinging, start_time}, awaiting} ->
-        {:keep_state, %State{data | ping: {{:pinging, start_time}, [caller | awaiting]}}}
+        {:keep_state, %State{data | ping: {{:pinging, start_time}, [from | awaiting]}},
+         next_actions}
     end
   end
 
   # not connected yet
-  def handle_event(:cast, {:ping, {caller_pid, ref}}, _, %State{}) do
+  def handle_event({:call, {caller_pid, ref} = _from}, :ping, _, %State{}) do
     send(caller_pid, {{Tortoise, self()}, {Package.Pingreq, ref}, :not_connected})
     :keep_state_and_data
   end
@@ -1171,14 +1228,14 @@ defmodule Tortoise.Connection do
         :internal,
         {:received, %Package.Pingresp{}},
         :connected,
-        %State{ping: {{:pinging, start_time}, awaiting}} = data
+        %State{ping: {{:pinging, start_time}, awaiting}, session: session} = data
       ) do
     round_trip_time =
       (System.monotonic_time() - start_time)
       |> System.convert_time_unit(:native, :microsecond)
 
     # reply to the clients
-    Enum.each(awaiting, &send_reply(&1, Package.Pingreq, round_trip_time))
+    Enum.each(awaiting, &send_reply(session.client_id, &1, Package.Pingreq, round_trip_time))
 
     next_actions = [{:next_event, :internal, :setup_keep_alive_timer}]
 
@@ -1257,9 +1314,10 @@ defmodule Tortoise.Connection do
     end
   end
 
-  defp send_reply({caller, ref}, topic, payload) when is_pid(caller) and is_reference(ref) do
-    send(caller, {{Tortoise, self()}, {topic, ref}, payload})
+  defp send_reply(client_id, {caller, ref}, topic, payload)
+       when is_pid(caller) and is_reference(ref) do
+    send(caller, {{Tortoise, client_id}, {topic, ref}, payload})
   end
 
-  @compile {:inline, wrap_next_actions: 1, send_reply: 3}
+  @compile {:inline, wrap_next_actions: 1, send_reply: 4}
 end
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index 510354fd..16988347 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -298,6 +298,8 @@ defmodule Tortoise.ConnectionTest do
     setup [:setup_scripted_mqtt_server, :setup_connection_and_perform_handshake]
 
     test "successful subscription", %{connection_pid: connection} = context do
+      client_id = context.client_id
+
       default_subscription_opts = [
         no_local: false,
         retain_as_published: false,
@@ -349,22 +351,22 @@ defmodule Tortoise.ConnectionTest do
       assert_receive {{TestHandler, :handle_suback}, {%Package.Subscribe{}, ^suback_foo}}
 
       # subscribe to a bar
-      assert {:ok, ref} =
+      assert {:ok, {^client_id, ref}} =
                Tortoise.Connection.subscribe(connection, {"bar", qos: 1}, identifier: 2)
 
-      assert_receive {{Tortoise, ^connection}, {Package.Suback, ^ref}, :ok}
+      assert_receive {{Tortoise, ^client_id}, {Package.Suback, ^ref}, :ok}
       assert_receive {ScriptedMqttServer, {:received, ^subscription_bar}}
       assert_receive {{TestHandler, :handle_suback}, {%Package.Subscribe{}, ^suback_bar}}
 
       # subscribe to a baz
-      assert {:ok, ref} =
+      assert {:ok, {^client_id, ref}} =
                Tortoise.Connection.subscribe(connection, "baz",
                  qos: 2,
                  identifier: 3,
                  user_property: {"foo", "bar"}
                )
 
-      assert_receive {{Tortoise, ^connection}, {Package.Suback, ^ref}, :ok}
+      assert_receive {{Tortoise, ^client_id}, {Package.Suback, ^ref}, :ok}
       assert_receive {ScriptedMqttServer, {:received, ^subscription_baz}}
       assert_receive {{TestHandler, :handle_suback}, {%Package.Subscribe{}, ^suback_baz}}
 
@@ -382,6 +384,7 @@ defmodule Tortoise.ConnectionTest do
     # @todo unsuccessful subscribe
 
     test "successful unsubscribe", %{connection_pid: connection} = context do
+      client_id = context.client_id
       unsubscribe_foo = %Package.Unsubscribe{identifier: 2, topics: ["foo"]}
       unsuback_foo = %Package.Unsuback{results: [:success], identifier: 2}
 
@@ -421,7 +424,8 @@ defmodule Tortoise.ConnectionTest do
         identifier: 1
       }
 
-      {:ok, sub_ref} = Tortoise.Connection.subscribe(connection, subscribe.topics, identifier: 1)
+      {:ok, {^client_id, sub_ref}} =
+        Tortoise.Connection.subscribe(connection, subscribe.topics, identifier: 1)
 
       assert_receive {ScriptedMqttServer, {:received, ^subscribe}}
 
@@ -438,13 +442,13 @@ defmodule Tortoise.ConnectionTest do
       assert Map.has_key?(Tortoise.Connection.subscriptions(connection), "bar")
 
       # and unsubscribe from bar
-      assert {:ok, ref} =
+      assert {:ok, {^client_id, ref}} =
                Tortoise.Connection.unsubscribe(connection, "bar",
                  identifier: 3,
                  user_property: {"foo", "bar"}
                )
 
-      assert_receive {{Tortoise, ^connection}, {Package.Unsuback, ^ref}, :ok}
+      assert_receive {{Tortoise, ^client_id}, {Package.Unsuback, ^ref}, :ok}
       assert_receive {ScriptedMqttServer, {:received, ^unsubscribe_bar}}
       # handle_unsuback should get called on the callback handler
       assert_receive {{TestHandler, :handle_unsuback}, {^unsubscribe_bar, ^unsuback_bar}}
@@ -456,10 +460,11 @@ defmodule Tortoise.ConnectionTest do
 
       # the process calling the async subscribe should receive the
       # result of the subscribe as a message (suback)
-      assert_receive {{Tortoise, ^connection}, {Package.Suback, ^sub_ref}, :ok}, 0
+      assert_receive {{Tortoise, ^client_id}, {Package.Suback, ^sub_ref}, :ok}, 0
     end
 
     test "unsuccessful unsubscribe: not authorized", %{connection_pid: connection} = context do
+      client_id = context.client_id
       unsubscribe_foo = %Package.Unsubscribe{identifier: 2, topics: ["foo"]}
       unsuback_foo = %Package.Unsuback{results: [error: :not_authorized], identifier: 2}
 
@@ -486,19 +491,25 @@ defmodule Tortoise.ConnectionTest do
         identifier: 1
       }
 
-      {:ok, _sub_ref} = Tortoise.Connection.subscribe(connection, subscribe.topics, identifier: 1)
+      {:ok, {^client_id, _sub_ref}} =
+        Tortoise.Connection.subscribe(connection, subscribe.topics, identifier: 1)
+
       assert_receive {ScriptedMqttServer, {:received, ^subscribe}}
       assert_receive {{TestHandler, :handle_suback}, {_, %Package.Suback{identifier: 1}}}
 
       subscriptions = Tortoise.Connection.subscriptions(connection)
-      {:ok, unsub_ref} = Tortoise.Connection.unsubscribe(connection, "foo", identifier: 2)
-      assert_receive {{Tortoise, ^connection}, {Package.Unsuback, ^unsub_ref}, :ok}
+
+      {:ok, {^client_id, unsub_ref}} =
+        Tortoise.Connection.unsubscribe(connection, "foo", identifier: 2)
+
+      assert_receive {{Tortoise, ^client_id}, {Package.Unsuback, ^unsub_ref}, :ok}
       assert_receive {Tortoise.Integration.ScriptedMqttServer, :completed}
       assert ^subscriptions = Tortoise.Connection.subscriptions(connection)
     end
 
     test "unsuccessful unsubscribe: no subscription existed",
          %{connection_pid: connection} = context do
+      client_id = context.client_id
       unsubscribe_foo = %Package.Unsubscribe{identifier: 2, topics: ["foo"]}
       unsuback_foo = %Package.Unsuback{results: [error: :no_subscription_existed], identifier: 2}
 
@@ -525,13 +536,18 @@ defmodule Tortoise.ConnectionTest do
         identifier: 1
       }
 
-      {:ok, _sub_ref} = Tortoise.Connection.subscribe(connection, subscribe.topics, identifier: 1)
+      {:ok, {^client_id, _sub_ref}} =
+        Tortoise.Connection.subscribe(connection, subscribe.topics, identifier: 1)
+
       assert_receive {ScriptedMqttServer, {:received, ^subscribe}}
       assert_receive {{TestHandler, :handle_suback}, {_, %Package.Suback{identifier: 1}}}
 
       assert Tortoise.Connection.subscriptions(connection) |> Map.has_key?("foo")
-      {:ok, unsub_ref} = Tortoise.Connection.unsubscribe(connection, "foo", identifier: 2)
-      assert_receive {{Tortoise, ^connection}, {Package.Unsuback, ^unsub_ref}, :ok}
+
+      {:ok, {^client_id, unsub_ref}} =
+        Tortoise.Connection.unsubscribe(connection, "foo", identifier: 2)
+
+      assert_receive {{Tortoise, ^client_id}, {Package.Unsuback, ^unsub_ref}, :ok}
       assert_receive {Tortoise.Integration.ScriptedMqttServer, :completed}
       # the client should update it state to not include the foo topic
       # as the server told us that it is not subscribed
@@ -1002,7 +1018,8 @@ defmodule Tortoise.ConnectionTest do
 
     test "user next actions", context do
       Process.flag(:trap_exit, true)
-      connect = %Package.Connect{client_id: context.client_id}
+      client_id = context.client_id
+      connect = %Package.Connect{client_id: client_id}
       expected_connack = %Package.Connack{reason: :success, session_present: false}
 
       subscribe = %Package.Subscribe{
@@ -1058,7 +1075,7 @@ defmodule Tortoise.ConnectionTest do
          ]}
 
       opts = [
-        client_id: context.client_id,
+        client_id: client_id,
         server: {Tortoise.Transport.Tcp, [host: ip, port: port]},
         handler: handler
       ]
@@ -1085,7 +1102,8 @@ defmodule Tortoise.ConnectionTest do
   describe "ping" do
     setup [:setup_scripted_mqtt_server, :setup_connection_and_perform_handshake]
 
-    test "send pingreq and receive a pingresp", %{connection_pid: connection_pid} = context do
+    test "send pingreq and receive a pingresp", context do
+      client_id = context.client_id
       ping_request = %Package.Pingreq{}
       expected_pingresp = %Package.Pingresp{}
       script = [{:receive, ping_request}, {:send, expected_pingresp}]
@@ -1093,9 +1111,9 @@ defmodule Tortoise.ConnectionTest do
       {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
       assert_receive {{TestHandler, :handle_connack}, %Tortoise.Package.Connack{}}
 
-      {:ok, ref} = Connection.ping(context.connection_pid)
+      {:ok, {^client_id, ref}} = Connection.ping(context.connection_pid)
       assert_receive {ScriptedMqttServer, {:received, ^ping_request}}
-      assert_receive {{Tortoise, ^connection_pid}, {Package.Pingreq, ^ref}, _}
+      assert_receive {{Tortoise, ^client_id}, {Package.Pingreq, ^ref}, _}
       assert_receive {ScriptedMqttServer, :completed}
     end
 
@@ -1266,7 +1284,7 @@ defmodule Tortoise.ConnectionTest do
       assert_receive {{TestHandler, :handle_publish}, ^publish}
     end
 
-    test "outgoing publish with QoS=1", context do
+    test "outgoing publish with QoS=1", %{client_id: client_id} = context do
       Process.flag(:trap_exit, true)
 
       publish =
@@ -1283,13 +1301,13 @@ defmodule Tortoise.ConnectionTest do
       {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
       pid = context.connection_pid
 
-      assert {:ok, ref} = Tortoise.Connection.publish(pid, publish)
+      assert {:ok, {^client_id, ref}} = Tortoise.Connection.publish(pid, publish)
 
       refute_receive {:EXIT, ^pid, {:protocol_violation, {:unexpected_package, _}}}
       assert_receive {ScriptedMqttServer, {:received, ^publish}}
       assert_receive {ScriptedMqttServer, :completed}
       # the caller should receive an :ok for the ref when it is published
-      assert_receive {{Tortoise, ^pid}, {Package.Publish, ^ref}, :ok}
+      assert_receive {{Tortoise, ^client_id}, {Package.Publish, ^ref}, :ok}
     end
 
     test "outgoing publish with QoS=1 (sync call)", context do
@@ -1388,14 +1406,14 @@ defmodule Tortoise.ConnectionTest do
       {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
       pid = connection_pid
 
-      assert {:ok, ref} = Tortoise.Connection.publish(pid, publish)
+      assert {:ok, {^client_id, ref}} = Tortoise.Connection.publish(pid, publish)
 
       refute_receive {:EXIT, ^pid, {:protocol_violation, {:unexpected_package, _}}}
       assert_receive {ScriptedMqttServer, {:received, ^publish}}
       assert_receive {ScriptedMqttServer, {:received, ^subscribe}}
       assert_receive {ScriptedMqttServer, :completed}
       # the caller should receive an :ok for the ref when it is published
-      assert_receive {{Tortoise, ^pid}, {Package.Publish, ^ref}, :ok}
+      assert_receive {{Tortoise, ^client_id}, {Package.Publish, ^ref}, :ok}
       assert_receive {{TestHandler, :handle_suback}, {_subscribe, _suback}}
     end
 
@@ -1573,7 +1591,7 @@ defmodule Tortoise.ConnectionTest do
       assert_receive {{TestHandler, :handle_publish}, ^non_dup_publish}
     end
 
-    test "outgoing publish with QoS=2", context do
+    test "outgoing publish with QoS=2", %{client_id: client_id} = context do
       Process.flag(:trap_exit, true)
 
       publish =
@@ -1594,13 +1612,13 @@ defmodule Tortoise.ConnectionTest do
       {:ok, _} = ScriptedMqttServer.enact(context.scripted_mqtt_server, script)
       pid = context.connection_pid
 
-      assert {:ok, ref} = Tortoise.Connection.publish(pid, publish)
+      assert {:ok, {^client_id, ref}} = Tortoise.Connection.publish(pid, publish)
 
       refute_receive {:EXIT, ^pid, {:protocol_violation, {:unexpected_package, _}}}
       assert_receive {ScriptedMqttServer, {:received, ^publish}}
       assert_receive {ScriptedMqttServer, {:received, ^pubrel}}
       assert_receive {ScriptedMqttServer, :completed}
-      assert_receive {{Tortoise, ^pid}, {Package.Publish, ^ref}, :ok}
+      assert_receive {{Tortoise, ^client_id}, {Package.Publish, ^ref}, :ok}
 
       # the handle_pubrec callback should have been called
       assert_receive {{TestHandler, :handle_pubrec}, ^pubrec}

From 78a5855cc2c420726b6e663cdf8927bd8ca85550 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Wed, 23 Sep 2020 15:57:52 +0100
Subject: [PATCH 209/220] Don't start a inflight process for a connection

This has been replaced with another inflight tracking system
---
 lib/tortoise/connection/supervisor.ex | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/lib/tortoise/connection/supervisor.ex b/lib/tortoise/connection/supervisor.ex
index 4eeecdcc..8b592bc3 100644
--- a/lib/tortoise/connection/supervisor.ex
+++ b/lib/tortoise/connection/supervisor.ex
@@ -3,7 +3,7 @@ defmodule Tortoise.Connection.Supervisor do
 
   use Supervisor
 
-  alias Tortoise.Connection.{Receiver, Inflight}
+  alias Tortoise.Connection.Receiver
 
   def start_link(opts) do
     client_id = Keyword.fetch!(opts, :client_id)
@@ -23,10 +23,9 @@ defmodule Tortoise.Connection.Supervisor do
   @impl true
   def init(opts) do
     children = [
-      {Inflight, Keyword.take(opts, [:client_id, :parent])},
       {Receiver, Keyword.take(opts, [:session_ref, :transport, :parent])}
     ]
 
-    Supervisor.init(children, strategy: :rest_for_one, max_seconds: 30, max_restarts: 10)
+    Supervisor.init(children, strategy: :one_for_one, max_seconds: 30, max_restarts: 10)
   end
 end

From e0a48ffd250ccbcc7812e3bfc011bd50a6a80012 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Wed, 23 Sep 2020 16:01:24 +0100
Subject: [PATCH 210/220] Reply directly to a ping request if the connection is
 not up yet

---
 lib/tortoise/connection.ex | 26 +++++++++++++++-----------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 0c5432df..227c3d7c 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -349,7 +349,7 @@ defmodule Tortoise.Connection do
   """
   @spec ping(pid(), timeout()) :: {:ok, reference()}
   def ping(pid, timeout \\ :infinity) do
-    {:ok, {_client_id, _ref}} = GenStateMachine.call(pid, :ping, timeout)
+    GenStateMachine.call(pid, :ping, timeout)
   end
 
   @doc """
@@ -365,14 +365,18 @@ defmodule Tortoise.Connection do
   """
   @spec ping_sync(pid(), timeout()) :: {:ok, reference()} | {:error, :timeout}
   def ping_sync(pid, timeout \\ :infinity) do
-    {:ok, {client_id, ref}} = ping(pid, timeout)
+    case ping(pid, timeout) do
+      {:ok, {client_id, ref}} ->
+        receive do
+          {{Tortoise, ^client_id}, {Package.Pingreq, ^ref}, round_trip_time} ->
+            {:ok, round_trip_time}
+        after
+          timeout ->
+            {:error, :timeout}
+        end
 
-    receive do
-      {{Tortoise, ^client_id}, {Package.Pingreq, ^ref}, round_trip_time} ->
-        {:ok, round_trip_time}
-    after
-      timeout ->
-        {:error, :timeout}
+      {:error, _reason} = error ->
+        error
     end
   end
 
@@ -1181,9 +1185,9 @@ defmodule Tortoise.Connection do
   end
 
   # not connected yet
-  def handle_event({:call, {caller_pid, ref} = _from}, :ping, _, %State{}) do
-    send(caller_pid, {{Tortoise, self()}, {Package.Pingreq, ref}, :not_connected})
-    :keep_state_and_data
+  def handle_event({:call, from}, :ping, _, %State{}) do
+    next_actions = [{:reply, from, {:error, :not_connected}}]
+    {:keep_state_and_data, next_actions}
   end
 
   # keep alive ---------------------------------------------------------

From 1b62650535486f8dbd2a6cb4d9a6ae0b91b028b0 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Wed, 23 Sep 2020 16:30:42 +0100
Subject: [PATCH 211/220] Remember to update the session state when progressing
 the state

For some session implementations the inflight ids would be tracked in
the session data itself, so it is important to pass the session along.
---
 lib/tortoise/connection.ex | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 227c3d7c..f1e84e57 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -580,10 +580,9 @@ defmodule Tortoise.Connection do
     case Session.track(session, {:incoming, publish}) do
       {{:cont, publish}, session} ->
         case Handler.execute_handle_publish(handler, publish) do
-          {:ok, %Package.Puback{identifier: ^id} = puback, %Handler{} = updated_handler,
-           next_actions} ->
+          {:ok, %Package.Puback{identifier: ^id} = puback, updated_handler, next_actions} ->
             # respond with a puback
-            Session.progress(session, {:outgoing, puback})
+            {{:cont, _puback}, session} = Session.progress(session, {:outgoing, puback})
             :ok = transport.send(socket, Package.encode(puback))
             {:ok, session} = Session.release(session, id)
             # - - -

From 8acf78ed5c2ec1be1ba26dedf165f1a86c8895d6 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Mon, 28 Sep 2020 10:45:34 +0100
Subject: [PATCH 212/220] Narrow the allowed success reasons when processing a
 pubrec

There are two success cases for a pubrec, that it succeeded, or that
it "succeeded" with the reason {:refused, :no_matching_subscribers}
where the latter signify that the server received the publish, but did
not onward to any clients; the callback will now accept this case as a
success if the user decide to respond with a pubrel.
---
 lib/tortoise/connection.ex | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index f1e84e57..4ca1bd9a 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -728,7 +728,7 @@ defmodule Tortoise.Connection do
 
   def handle_event(
         :internal,
-        {:received, %Package.Pubrec{identifier: id} = pubrec},
+        {:received, %Package.Pubrec{identifier: id, reason: reason} = pubrec},
         _,
         %State{connection: {transport, socket}, session: session, handler: handler} = data
       ) do
@@ -738,12 +738,24 @@ defmodule Tortoise.Connection do
     if function_exported?(handler.module, :handle_pubrec, 2) do
       case Handler.execute_handle_pubrec(handler, pubrec) do
         {:ok, %Package.Pubrel{identifier: ^id} = pubrel, %Handler{} = updated_handler,
-         next_actions} ->
+         next_actions}
+        when reason == :success or reason == {:refused, :no_matching_subscribers} ->
+          # NOTICE that we do allow the "no matching subscribers"
+          # reason as a success; "...in the case of QoS 2 PUBLISH it
+          # is PUBCOMP or a PUBREC with a Reason Code of 128 or
+          # greater"
           {{:cont, pubrel}, session} = Session.progress(session, {:outgoing, pubrel})
           :ok = transport.send(socket, Package.encode(pubrel))
           data = %State{data | session: session, handler: updated_handler}
           {:keep_state, data, wrap_next_actions(next_actions)}
 
+        {:ok, %Package.Pubrel{}, _updated_handler, _} ->
+          # user error; should not respond with pubrel if the publish failed
+          # TODO add a test case returning ok-pubrel on rejected pubrec
+          {:ok, session} = Session.release(session, id)
+          data = %State{data | session: session}
+          {:stop, reason, data}
+
         {:error, reason} ->
           # todo
           {:stop, reason, data}

From 115cfae5f28cfb9fbef78d80d9603a42a6d566a4 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Mon, 28 Sep 2020 16:50:57 +0100
Subject: [PATCH 213/220] Release the package id if pubrec returned failure and
 has no cb

In the case that there is no handle_pubrec callback implementation and
the publish was rejected (not including the no subscribers reason) we
will remove the id from the inflight monitor, as we should, per the
spec.
---
 lib/tortoise/connection.ex | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 4ca1bd9a..2e04797d 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -761,11 +761,21 @@ defmodule Tortoise.Connection do
           {:stop, reason, data}
       end
     else
-      pubrel = %Package.Pubrel{identifier: id}
-      {{:cont, pubrel}, session} = Session.progress(session, {:outgoing, pubrel})
-      :ok = transport.send(socket, Package.encode(pubrel))
-      data = %State{data | session: session}
-      {:keep_state, data}
+      if reason in [:success, {:refused, :no_matching_subscribers}] do
+        pubrel = %Package.Pubrel{identifier: id}
+        {{:cont, pubrel}, session} = Session.progress(session, {:outgoing, pubrel})
+        :ok = transport.send(socket, Package.encode(pubrel))
+        data = %State{data | session: session}
+        {:keep_state, data}
+      else
+        # "The Packet Identifier becomes available for reuse once the
+        # sender has received the PUBCOMP packet *or a PUBREC with a
+        # Reason Code of 0x80 or greater.*"
+        {:ok, session} = Session.release(session, id)
+        data = %State{data | session: session}
+        # TODO; Reply an error-tuple to the caller
+        {:keep_state, data}
+      end
     end
   end
 

From 5820e8d4daa2b8fb57fcffe303dd57e3c96ac2c4 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Tue, 13 Oct 2020 12:36:15 +0100
Subject: [PATCH 214/220] Code style change

This way of checking the reason is similar to the one used a few lines
below, so let's go with one over the other
---
 lib/tortoise/connection.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 2e04797d..14cf1a12 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -739,7 +739,7 @@ defmodule Tortoise.Connection do
       case Handler.execute_handle_pubrec(handler, pubrec) do
         {:ok, %Package.Pubrel{identifier: ^id} = pubrel, %Handler{} = updated_handler,
          next_actions}
-        when reason == :success or reason == {:refused, :no_matching_subscribers} ->
+        when reason in [:success, {:refused, :no_matching_subscribers}] ->
           # NOTICE that we do allow the "no matching subscribers"
           # reason as a success; "...in the case of QoS 2 PUBLISH it
           # is PUBCOMP or a PUBREC with a Reason Code of 128 or

From 592debdb2e1e85467936091f9372fda4f92a528b Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Thu, 15 Oct 2020 12:17:12 +0100
Subject: [PATCH 215/220] Restructure the transmitter supervision for the
 connections

Introducing a transmitter supervisor that is managed by the tortoise
application; when tortoise start a connection it will start a
connection using this (dynamic) supervisor, and both the transmitter
and the connection will monitor each other. Previously the connection
would spawn link a supervisor that held the connection, and that was
because we used to store more data in multiple processes; but after
the session supervisor has been introduced we only kept track of the
transmitter in the connection supervisor, and at this point we might
as well simplify things and pull it out. Because it is monitored it
should clean up as expected when one of them terminates.
---
 lib/tortoise/application.ex            |  1 +
 lib/tortoise/connection.ex             | 67 +++++++++-----------------
 lib/tortoise/connection/receiver.ex    | 28 ++++++++---
 lib/tortoise/connection/supervisor.ex  | 31 ------------
 lib/tortoise/transmitter_supervisor.ex | 22 +++++++++
 test/tortoise/connection_test.exs      |  9 ++--
 6 files changed, 72 insertions(+), 86 deletions(-)
 delete mode 100644 lib/tortoise/connection/supervisor.ex
 create mode 100644 lib/tortoise/transmitter_supervisor.ex

diff --git a/lib/tortoise/application.ex b/lib/tortoise/application.ex
index 590562db..c112dfdd 100644
--- a/lib/tortoise/application.ex
+++ b/lib/tortoise/application.ex
@@ -10,6 +10,7 @@ defmodule Tortoise.Application do
 
     children = [
       {Registry, [keys: :unique, name: Tortoise.Registry]},
+      {Tortoise.TransmitterSupervisor, []},
       {Tortoise.Session, [backend: Tortoise.Session.Ets]},
       {Tortoise.Supervisor, [strategy: :one_for_one]}
     ]
diff --git a/lib/tortoise/connection.ex b/lib/tortoise/connection.ex
index 14cf1a12..813a8f19 100644
--- a/lib/tortoise/connection.ex
+++ b/lib/tortoise/connection.ex
@@ -9,8 +9,7 @@ defmodule Tortoise.Connection do
 
   require Logger
 
-  defstruct session_ref: nil,
-            client_id: nil,
+  defstruct client_id: nil,
             session: nil,
             connect: nil,
             server: nil,
@@ -26,7 +25,7 @@ defmodule Tortoise.Connection do
   alias __MODULE__, as: State
 
   alias Tortoise.{Handler, Transport, Package, Session}
-  alias Tortoise.Connection.{Info, Receiver, Backoff}
+  alias Tortoise.Connection.{Info, Backoff}
   alias Tortoise.Package.Connect
 
   @doc """
@@ -71,7 +70,6 @@ defmodule Tortoise.Connection do
     ]
 
     initial = %State{
-      session_ref: make_ref(),
       client_id: connect.client_id,
       session: %Session{client_id: connect.client_id},
       server: server,
@@ -445,12 +443,6 @@ defmodule Tortoise.Connection do
     case Handler.execute_init(state.handler) do
       {:ok, %Handler{} = updated_handler} ->
         updated_state = %State{state | handler: updated_handler}
-        # TODO, perhaps the supervision should get reconsidered
-        :ok =
-          start_connection_supervisor([
-            {:session_ref, updated_state.session_ref},
-            {:parent, self()} | updated_state.opts
-          ])
 
         transition_actions = [
           {:next_event, :internal, :connect}
@@ -1124,12 +1116,24 @@ defmodule Tortoise.Connection do
 
   # connection logic ===================================================
   def handle_event(:internal, :connect, :connecting, %State{} = data) do
-    case await_and_monitor_receiver(data) do
-      {:ok, data} ->
-        {timeout, updated_data} = Map.get_and_update(data, :backoff, &Backoff.next/1)
-        next_actions = [{:state_timeout, timeout, :attempt_connection}]
-        {:keep_state, updated_data, next_actions}
-    end
+    transport = Keyword.get(data.opts, :transport)
+    # We cannot use the client_id to identify the connection because
+    # it could be the case that the server assign the client_id
+    {:ok, t_pid} =
+      Tortoise.TransmitterSupervisor.start_transmitter(
+        parent: self(),
+        transport: transport
+      )
+
+    data = %State{data | receiver: {t_pid, Process.monitor(t_pid)}}
+
+    # setup connection loop
+
+    # TODO make backoff a user defined callback with a default
+    {timeout, data} = Map.get_and_update(data, :backoff, &Backoff.next/1)
+    next_actions = [{:state_timeout, timeout, :attempt_connection}]
+
+    {:keep_state, data, next_actions}
   end
 
   def handle_event(
@@ -1141,7 +1145,7 @@ defmodule Tortoise.Connection do
           receiver: {receiver_pid, _mon_ref}
         } = data
       ) do
-    case Receiver.connect(receiver_pid) do
+    case Tortoise.Connection.Receiver.connect(receiver_pid) do
       {:ok, {transport, socket} = connection} ->
         # TODO: send this to the client handler allowing the user to
         # specify a custom connect package
@@ -1276,6 +1280,9 @@ defmodule Tortoise.Connection do
       ) do
     case Handler.execute_handle_disconnect(handler, {:server, disconnect}) do
       {:ok, updated_handler, next_actions} ->
+        # we should close the network connection now (or assume that
+        # the server will shutdown the connection immediately); from
+        # here we could reconnect, or stop
         {:keep_state, %State{data | handler: updated_handler}, wrap_next_actions(next_actions)}
 
       {:stop, reason, updated_handler} ->
@@ -1305,32 +1312,6 @@ defmodule Tortoise.Connection do
     {:next_state, :connecting, updated_data, next_actions}
   end
 
-  defp await_and_monitor_receiver(%State{receiver: nil} = data) do
-    session_ref = data.session_ref
-
-    receive do
-      {{Tortoise, ^session_ref}, Receiver, {:ready, pid}} ->
-        {:ok, %State{data | receiver: {pid, Process.monitor(pid)}}}
-    after
-      5000 ->
-        {:error, :receiver_timeout}
-    end
-  end
-
-  defp await_and_monitor_receiver(data) do
-    {:ok, data}
-  end
-
-  defp start_connection_supervisor(opts) do
-    case Tortoise.Connection.Supervisor.start_link(opts) do
-      {:ok, _pid} ->
-        :ok
-
-      {:error, {:already_started, _pid}} ->
-        :ok
-    end
-  end
-
   # wrapping the user specified next actions in gen_statem next actions;
   # this is used in all the handle callback functions, so we inline it
   defp wrap_next_actions(next_actions) do
diff --git a/lib/tortoise/connection/receiver.ex b/lib/tortoise/connection/receiver.ex
index 22927c9c..a413e128 100644
--- a/lib/tortoise/connection/receiver.ex
+++ b/lib/tortoise/connection/receiver.ex
@@ -5,17 +5,16 @@ defmodule Tortoise.Connection.Receiver do
 
   alias Tortoise.Transport
 
-  defstruct session_ref: nil,
-            transport: nil,
+  defstruct transport: nil,
             socket: nil,
             buffer: <<>>,
-            parent: nil
+            parent: nil,
+            parent_mon: nil
 
   alias __MODULE__, as: State
 
   def start_link(opts) do
     data = %State{
-      session_ref: Keyword.fetch!(opts, :session_ref),
       transport: Keyword.fetch!(opts, :transport),
       parent: Keyword.fetch!(opts, :parent)
     }
@@ -28,7 +27,9 @@ defmodule Tortoise.Connection.Receiver do
       id: __MODULE__,
       start: {__MODULE__, :start_link, [opts]},
       type: :worker,
-      restart: :permanent,
+      # We will let the connection process monitor and start a new
+      # receiver process if the current one should crash
+      restart: :temporary,
       shutdown: 500
     }
   end
@@ -39,8 +40,8 @@ defmodule Tortoise.Connection.Receiver do
 
   @impl true
   def init(%State{} = data) do
-    send(data.parent, {{Tortoise, data.session_ref}, __MODULE__, {:ready, self()}})
-    {:ok, :disconnected, data}
+    parent_mon = Process.monitor(data.parent)
+    {:ok, :disconnected, %State{data | parent_mon: parent_mon}}
   end
 
   @impl true
@@ -61,6 +62,19 @@ defmodule Tortoise.Connection.Receiver do
     {:keep_state, new_data, next_actions}
   end
 
+  def handle_event(
+        :info,
+        {:DOWN, ref, :process, pid, reason},
+        _,
+        %State{parent: pid, parent_mon: ref} = data
+      ) do
+    # our parent process is shutting down
+    case reason do
+      :shutdown ->
+        {:stop, :normal, data}
+    end
+  end
+
   def handle_event(:info, unknown_info, _, data) do
     {:stop, {:unknown_info, unknown_info}, data}
   end
diff --git a/lib/tortoise/connection/supervisor.ex b/lib/tortoise/connection/supervisor.ex
deleted file mode 100644
index 8b592bc3..00000000
--- a/lib/tortoise/connection/supervisor.ex
+++ /dev/null
@@ -1,31 +0,0 @@
-defmodule Tortoise.Connection.Supervisor do
-  @moduledoc false
-
-  use Supervisor
-
-  alias Tortoise.Connection.Receiver
-
-  def start_link(opts) do
-    client_id = Keyword.fetch!(opts, :client_id)
-    Supervisor.start_link(__MODULE__, opts, name: via_name(client_id))
-  end
-
-  defp via_name(client_id) do
-    Tortoise.Registry.via_name(__MODULE__, client_id)
-  end
-
-  def whereis(client_id) do
-    __MODULE__
-    |> Tortoise.Registry.reg_name(client_id)
-    |> Registry.whereis_name()
-  end
-
-  @impl true
-  def init(opts) do
-    children = [
-      {Receiver, Keyword.take(opts, [:session_ref, :transport, :parent])}
-    ]
-
-    Supervisor.init(children, strategy: :one_for_one, max_seconds: 30, max_restarts: 10)
-  end
-end
diff --git a/lib/tortoise/transmitter_supervisor.ex b/lib/tortoise/transmitter_supervisor.ex
new file mode 100644
index 00000000..ebb02b8a
--- /dev/null
+++ b/lib/tortoise/transmitter_supervisor.ex
@@ -0,0 +1,22 @@
+defmodule Tortoise.TransmitterSupervisor do
+  @moduledoc false
+
+  alias Tortoise.Connection.Receiver
+
+  use DynamicSupervisor
+
+  def start_link(init_arg) do
+    DynamicSupervisor.start_link(__MODULE__, init_arg, name: __MODULE__)
+  end
+
+  def start_transmitter(sup \\ __MODULE__, opts) do
+    opts = Keyword.put(opts, :parent, self())
+    spec = {Receiver, Keyword.take(opts, [:transport, :parent])}
+    DynamicSupervisor.start_child(sup, spec)
+  end
+
+  @impl true
+  def init(_init_arg) do
+    DynamicSupervisor.init(strategy: :one_for_one)
+  end
+end
diff --git a/test/tortoise/connection_test.exs b/test/tortoise/connection_test.exs
index 16988347..f6f3a4e8 100644
--- a/test/tortoise/connection_test.exs
+++ b/test/tortoise/connection_test.exs
@@ -982,9 +982,6 @@ defmodule Tortoise.ConnectionTest do
       assert {:ok, connection_pid} = Connection.start_link(opts)
       assert_receive {ScriptedMqttServer, {:received, ^connect}}
 
-      cs_pid = Connection.Supervisor.whereis(client_id)
-      cs_ref = Process.monitor(cs_pid)
-
       {:ok, {Tortoise.Transport.Tcp, _}} = Connection.connection(connection_pid)
 
       {:connected,
@@ -992,6 +989,8 @@ defmodule Tortoise.ConnectionTest do
          receiver_pid: receiver_pid
        }} = Connection.info(connection_pid)
 
+      receiver_mon = Process.monitor(receiver_pid)
+
       assert :ok = Tortoise.Connection.disconnect(connection_pid)
 
       assert_receive {ScriptedMqttServer, {:received, ^disconnect}}
@@ -999,8 +998,8 @@ defmodule Tortoise.ConnectionTest do
 
       assert_receive {ScriptedMqttServer, :completed}
 
-      assert_receive {:DOWN, ^cs_ref, :process, ^cs_pid, :shutdown}
-      refute Process.alive?(receiver_pid)
+      # make sure the transmitter terminates as well
+      assert_receive {:DOWN, ^receiver_mon, :process, ^receiver_pid, :normal}
 
       # The user defined handler should have the following callbacks
       # triggered during this exchange

From cd03ac940c4ee7123ee5ffedf9b9630ffc7eb119 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Fri, 16 Oct 2020 13:43:28 +0100
Subject: [PATCH 216/220] Clean up the old inflight tracking

Get rid of some modules that are no longer needed because they have
been replaced with new implementations
---
 lib/tortoise/connection/inflight.ex           | 472 ------------------
 lib/tortoise/connection/inflight/track.ex     | 261 ----------
 .../connection/inflight/track_test.exs        | 163 ------
 test/tortoise/connection/inflight_test.exs    | 305 -----------
 test/tortoise/pipe_test.exs                   |  21 +-
 test/tortoise_test.exs                        |  26 +-
 6 files changed, 14 insertions(+), 1234 deletions(-)
 delete mode 100644 lib/tortoise/connection/inflight.ex
 delete mode 100644 lib/tortoise/connection/inflight/track.ex
 delete mode 100644 test/tortoise/connection/inflight/track_test.exs
 delete mode 100644 test/tortoise/connection/inflight_test.exs

diff --git a/lib/tortoise/connection/inflight.ex b/lib/tortoise/connection/inflight.ex
deleted file mode 100644
index df6ae423..00000000
--- a/lib/tortoise/connection/inflight.ex
+++ /dev/null
@@ -1,472 +0,0 @@
-defmodule Tortoise.Connection.Inflight do
-  @moduledoc false
-
-  alias Tortoise.Package
-  alias Tortoise.Connection.Inflight.Track
-
-  use GenStateMachine
-
-  @enforce_keys [:client_id, :parent]
-  defstruct client_id: nil, parent: nil, pending: %{}, order: []
-
-  alias __MODULE__, as: State
-
-  # Client API
-  def start_link(opts) do
-    client_id = Keyword.fetch!(opts, :client_id)
-
-    GenStateMachine.start_link(__MODULE__, opts, name: via_name(client_id))
-  end
-
-  defp via_name(client_id) do
-    Tortoise.Registry.via_name(__MODULE__, client_id)
-  end
-
-  def whereis(client_id) do
-    __MODULE__
-    |> Tortoise.Registry.reg_name(client_id)
-    |> Registry.whereis_name()
-  end
-
-  def stop(client_id) do
-    GenStateMachine.stop(via_name(client_id))
-  end
-
-  @doc false
-  def update_connection(client_id, {_, _} = connection) do
-    GenStateMachine.call(via_name(client_id), {:set_connection, connection})
-  end
-
-  @doc false
-  def drain(client_id, %Package.Disconnect{} = disconnect) do
-    GenStateMachine.call(via_name(client_id), {:drain, disconnect})
-  end
-
-  @doc false
-  def track(client_id, {:incoming, %Package.Publish{qos: qos} = publish})
-      when qos in 1..2 do
-    :ok = GenStateMachine.cast(via_name(client_id), {:incoming, publish})
-  end
-
-  def track(client_id, {:outgoing, package}) do
-    # no transforms and nil track session state
-    track(client_id, {:outgoing, package, {[], nil}})
-  end
-
-  def track(client_id, {:outgoing, package, opts}) do
-    caller = {_, ref} = {self(), make_ref()}
-
-    case package do
-      %Package.Publish{qos: qos} when qos in [1, 2] ->
-        :ok = GenStateMachine.cast(via_name(client_id), {:outgoing, caller, {package, opts}})
-        {:ok, ref}
-
-      %Package.Subscribe{} ->
-        :ok = GenStateMachine.cast(via_name(client_id), {:outgoing, caller, package})
-        {:ok, ref}
-
-      %Package.Unsubscribe{} ->
-        :ok = GenStateMachine.cast(via_name(client_id), {:outgoing, caller, package})
-        {:ok, ref}
-    end
-  end
-
-  @doc false
-  def track_sync(client_id, command, timeout \\ :infinity)
-
-  def track_sync(client_id, {:outgoing, %Package.Publish{}} = command, timeout) do
-    # add no transforms and a nil track state
-    track_sync(client_id, Tuple.append(command, {[], nil}), timeout)
-  end
-
-  def track_sync(client_id, {:outgoing, %Package.Publish{}, _transforms} = command, timeout) do
-    {:ok, ref} = track(client_id, command)
-
-    receive do
-      {{Tortoise, ^client_id}, {Package.Publish, ^ref}, result} ->
-        result
-    after
-      timeout -> {:error, :timeout}
-    end
-  end
-
-  @doc false
-  def update(client_id, {_, %{__struct__: _, identifier: _identifier}} = event) do
-    :ok = GenStateMachine.cast(via_name(client_id), {:update, event})
-  end
-
-  @doc false
-  def reset(client_id) do
-    :ok = GenStateMachine.cast(via_name(client_id), :reset)
-  end
-
-  # Server callbacks
-  @impl true
-  def init(opts) do
-    client_id = Keyword.fetch!(opts, :client_id)
-    parent_pid = Keyword.fetch!(opts, :parent)
-    initial_data = %State{client_id: client_id, parent: parent_pid}
-
-    {:ok, :disconnected, initial_data}
-  end
-
-  @impl true
-  # When we receive a new connection we will use that for our future
-  # transmissions.
-  def handle_event(
-        {:call, from},
-        {:set_connection, connection},
-        _current_state,
-        %State{pending: pending} = data
-      ) do
-    next_actions = [
-      {:reply, from, :ok}
-      | for identifier <- Enum.reverse(data.order) do
-          case Map.get(pending, identifier, :unknown) do
-            %Track{pending: [[{:dispatch, %Package.Publish{} = publish} | action] | pending]} =
-                track ->
-              publish = %Package.Publish{publish | dup: true}
-              track = %Track{track | pending: [[{:dispatch, publish} | action] | pending]}
-              {:next_event, :internal, {:execute, track}}
-
-            %Track{} = track ->
-              {:next_event, :internal, {:execute, track}}
-          end
-        end
-    ]
-
-    {:next_state, {:connected, connection}, data, next_actions}
-  end
-
-  # Create. Notice: we will only receive publish packages from the
-  # remote; everything else is something we initiate
-  def handle_event(
-        :cast,
-        {:incoming, %Package.Publish{dup: false} = package},
-        _state,
-        %State{pending: pending} = data
-      ) do
-    track = Track.create(:positive, package)
-
-    data = %State{
-      data
-      | pending: Map.put_new(pending, track.identifier, track),
-        order: [track.identifier | data.order]
-    }
-
-    next_actions = [
-      {:next_event, :internal, {:onward_publish, package}}
-    ]
-
-    {:keep_state, data, next_actions}
-  end
-
-  # possible duplicate
-  def handle_event(:cast, {:incoming, _}, :draining, %State{}) do
-    :keep_state_and_data
-  end
-
-  def handle_event(
-        :cast,
-        {:incoming, %Package.Publish{identifier: identifier, dup: true} = publish},
-        _state,
-        %State{pending: pending} = data
-      ) do
-    case Map.get(pending, identifier) do
-      nil ->
-        next_actions = [
-          {:next_event, :cast, {:incoming, %Package.Publish{publish | dup: false}}}
-        ]
-
-        {:keep_state_and_data, next_actions}
-
-      %Track{polarity: :positive, status: [{:received, %{__struct__: Package.Publish}}]} ->
-        :keep_state_and_data
-
-      _otherwise ->
-        {:stop, :state_out_of_sync, data}
-    end
-  end
-
-  def handle_event(:cast, {:outgoing, _caller, _package}, :disconnected, %State{}) do
-    # await a connection
-    {:keep_state_and_data, [:postpone]}
-  end
-
-  def handle_event(:cast, {:outgoing, {pid, ref}, _}, :draining, data) do
-    send(pid, {{Tortoise, data.client_id}, ref, {:error, :terminating}})
-    :keep_state_and_data
-  end
-
-  def handle_event(:cast, {:outgoing, caller, {package, opts}}, _state, data) do
-    {:ok, package} = assign_identifier(package, data.pending)
-    track = Track.create({:negative, caller}, {package, opts})
-
-    next_actions = [
-      {:next_event, :internal, {:execute, track}}
-    ]
-
-    data = %State{
-      data
-      | pending: Map.put_new(data.pending, track.identifier, track),
-        order: [track.identifier | data.order]
-    }
-
-    {:keep_state, data, next_actions}
-  end
-
-  def handle_event(:cast, {:outgoing, caller, package}, _state, data) do
-    {:ok, package} = assign_identifier(package, data.pending)
-    track = Track.create({:negative, caller}, package)
-
-    next_actions = [
-      {:next_event, :internal, {:execute, track}}
-    ]
-
-    data = %State{
-      data
-      | pending: Map.put_new(data.pending, track.identifier, track),
-        order: [track.identifier | data.order]
-    }
-
-    {:keep_state, data, next_actions}
-  end
-
-  # update
-  def handle_event(:cast, {:update, _}, :draining, _data) do
-    :keep_state_and_data
-  end
-
-  def handle_event(
-        :cast,
-        {:update, {:received, %{identifier: identifier} = package} = update},
-        _state,
-        %State{pending: pending} = data
-      ) do
-    with {:ok, track} <- Map.fetch(pending, identifier),
-         {_package, track} = apply_transform(package, track),
-         {:ok, track} <- Track.resolve(track, update) do
-      data = %State{
-        data
-        | pending: Map.put(pending, identifier, track),
-          order: [identifier | data.order -- [identifier]]
-      }
-
-      case track do
-        %Track{pending: [[{:respond, _}, _] | _]} ->
-          next_actions = [
-            {:next_event, :internal, {:execute, track}}
-          ]
-
-          {:keep_state, data, next_actions}
-
-        # to support user defined properties we need to await a
-        # dispatch command from the controller before we can
-        # progress.
-        %Track{pending: [[{:dispatch, _}, _] | _]} ->
-          {:keep_state, data}
-      end
-    else
-      :error ->
-        {:stop, {:protocol_violation, :unknown_identifier}, data}
-
-      {:error, reason} ->
-        {:stop, reason, data}
-    end
-  end
-
-  def handle_event(
-        :cast,
-        {:update, {:dispatch, %{identifier: identifier}} = update},
-        _state,
-        %State{pending: pending} = data
-      ) do
-    with {:ok, track} <- Map.fetch(pending, identifier),
-         {:ok, track} <- Track.resolve(track, update) do
-      data = %State{
-        data
-        | pending: Map.put(pending, identifier, track),
-          order: [identifier | data.order -- [identifier]]
-      }
-
-      next_actions = [
-        {:next_event, :internal, {:execute, track}}
-      ]
-
-      {:keep_state, data, next_actions}
-    end
-  end
-
-  def handle_event(:cast, :reset, _, %State{pending: pending} = data) do
-    # cancel all currently outgoing messages
-    for {_, %Track{polarity: :negative, caller: {pid, ref}}} <- pending do
-      send(pid, {{Tortoise, data.client_id}, ref, {:error, :canceled}})
-    end
-
-    {:keep_state, %State{data | pending: %{}, order: []}}
-  end
-
-  # We trap the incoming QoS>0 packages in the inflight manager so we
-  # can make sure we will not onward the same publish to the user
-  # defined callback more than once.
-  def handle_event(
-        :internal,
-        {:onward_publish, %Package.Publish{qos: qos} = publish},
-        _,
-        %State{client_id: client_id, parent: parent_pid}
-      )
-      when qos in 1..2 do
-    send(parent_pid, {{__MODULE__, client_id}, publish})
-    :keep_state_and_data
-  end
-
-  def handle_event(:internal, {:execute, _}, :draining, _) do
-    :keep_state_and_data
-  end
-
-  def handle_event(
-        :internal,
-        {:execute, %Track{pending: [[{:dispatch, package}, _] | _]} = track},
-        {:connected, {transport, socket}},
-        %State{} = data
-      ) do
-    with {package, track} <- apply_transform(package, track),
-         :ok = apply(transport, :send, [socket, Package.encode(package)]) do
-      {:keep_state, handle_next(track, data)}
-    else
-      res ->
-        {:stop, res, data}
-    end
-  end
-
-  def handle_event(
-        :internal,
-        {:execute, %Track{pending: [[{:dispatch, _}, _] | _]}},
-        :disconnected,
-        %State{}
-      ) do
-    # the dispatch will get re-queued when we regain the connection
-    :keep_state_and_data
-  end
-
-  def handle_event(
-        :internal,
-        {:execute, %Track{pending: [[{:respond, {pid, ref}}, _] | _]} = track},
-        _state,
-        %State{client_id: client_id} = data
-      ) do
-    case Track.result(track) do
-      {:ok, result} ->
-        send(pid, {{Tortoise, client_id}, {track.type, ref}, result})
-        {:keep_state, handle_next(track, data)}
-    end
-  end
-
-  def handle_event(
-        {:call, from},
-        {:drain, %Package.Disconnect{} = disconnect},
-        {:connected, {transport, socket}},
-        %State{} = data
-      ) do
-    for {_, %Track{polarity: :negative, caller: {pid, ref}}} <- data.pending do
-      send(pid, {{Tortoise, data.client_id}, ref, {:error, :canceled}})
-    end
-
-    data = %State{data | pending: %{}, order: []}
-
-    case apply(transport, :send, [socket, Package.encode(disconnect)]) do
-      :ok ->
-        :ok = transport.close(socket)
-        reply = {:reply, from, :ok}
-
-        {:next_state, :draining, data, reply}
-    end
-  end
-
-  # helpers ------------------------------------------------------------
-  @type_to_cb %{
-    Package.Publish => :publish,
-    Package.Pubrec => :pubrec,
-    Package.Pubrel => :pubrel,
-    Package.Pubcomp => :pubcomp,
-    Package.Puback => :puback
-  }
-
-  defp apply_transform(package, %Track{transforms: []} = track) do
-    # noop, no transforms defined for any package type, so just pass
-    # through
-    {package, track}
-  end
-
-  defp apply_transform(%type{identifier: identifier} = package, %Track{} = track) do
-    # see if the callback for the given type is defined; if so, apply
-    # it and update the track state
-    case track.transforms[Map.get(@type_to_cb, type)] do
-      nil ->
-        {package, track}
-
-      fun when is_function(fun, 2) ->
-        case apply(fun, [package, track.state]) do
-          {:ok, updated_state} ->
-            # just update the track session state
-            updated_track_state = %Track{track | state: updated_state}
-            {package, updated_track_state}
-
-          {:ok, properties, updated_state} when is_list(properties) ->
-            # Update the user defined properties on the package with a
-            # list of `[{string, string}]`
-            updated_package = %{package | properties: properties}
-            updated_track_state = %Track{track | state: updated_state}
-            {updated_package, updated_track_state}
-
-          {:ok, %^type{identifier: ^identifier} = updated_package, updated_state} ->
-            # overwrite the package with a package of the same type
-            # and identifier; allows us to alter properties besides
-            # the user defined properties
-            updated_track_state = %Track{track | state: updated_state}
-            {updated_package, updated_track_state}
-        end
-    end
-  end
-
-  defp handle_next(
-         %Track{pending: [[_, :cleanup]], identifier: identifier},
-         %State{pending: pending} = state
-       ) do
-    order = state.order -- [identifier]
-    %State{state | pending: Map.delete(pending, identifier), order: order}
-  end
-
-  defp handle_next(%Track{identifier: identifier} = track, %State{pending: pending} = state) do
-    %State{state | pending: Map.replace!(pending, identifier, track)}
-  end
-
-  # Assign a random identifier to the tracked package; this will make
-  # sure we pick a random number that is not in use
-  defp assign_identifier(%{identifier: nil} = package, pending) do
-    case :crypto.strong_rand_bytes(2) do
-      <<0, 0>> ->
-        # an identifier cannot be zero
-        assign_identifier(package, pending)
-
-      <<identifier::integer-size(16)>> ->
-        unless Map.has_key?(pending, identifier) do
-          {:ok, %{package | identifier: identifier}}
-        else
-          assign_identifier(package, pending)
-        end
-    end
-  end
-
-  # ...as such we should let the in-flight process assign identifiers,
-  # but the possibility to pass one in has been kept so we can make
-  # deterministic unit tests
-  defp assign_identifier(%{identifier: identifier} = package, pending)
-       when identifier in 0x0001..0xFFFF do
-    unless Map.has_key?(pending, identifier) do
-      {:ok, package}
-    else
-      {:error, {:identifier_already_in_use, identifier}}
-    end
-  end
-end
diff --git a/lib/tortoise/connection/inflight/track.ex b/lib/tortoise/connection/inflight/track.ex
deleted file mode 100644
index 47724f93..00000000
--- a/lib/tortoise/connection/inflight/track.ex
+++ /dev/null
@@ -1,261 +0,0 @@
-defmodule Tortoise.Connection.Inflight.Track do
-  @moduledoc false
-
-  # A data structure implementing state machines tracking the state of a
-  # message in flight.
-
-  # Messages can have two polarities, positive and negative, describing
-  # what direction they are going. A positive polarity is messages
-  # coming from the server to the client (us), a negative polarity is
-  # messages send from the client (us) to the server.
-
-  # For now we care about tracking the state of a handful of message
-  # kinds: the publish control packages with a quality of service above
-  # 0 and subscribe and unsubscribe control packages. We do not track
-  # the in-flight state of a QoS 0 control packet because there is no
-  # state to track.
-
-  # For negative polarity we need to track the caller, which is the
-  # process that instantiated the publish control package. This process
-  # will wait for a message to get passed to it when the ownership of
-  # the control package has been transferred to the server. Messages
-  # with a positive polarity will get passed to the callback module
-  # attached to the Controller module, so in that case there will be no
-  # caller.
-
-  @type package :: Package.Publish | Package.Subscribe | Package.Unsubscribe
-
-  @type caller :: {pid(), reference()} | nil
-  @type polarity :: :positive | {:negative, caller()}
-  @type next_action :: {:dispatch | :expect, Tortoise.Encodable.t()}
-  @type status_update :: {:received | :dispatched, Tortoise.Encodable.t()}
-
-  @opaque t :: %__MODULE__{
-            polarity: :positive | :negative,
-            type: package,
-            caller: {pid(), reference()} | nil,
-            identifier: Tortoise.package_identifier(),
-            status: [status_update()],
-            pending: [next_action()],
-            transforms: [function()],
-            state: any()
-          }
-  @enforce_keys [:type, :identifier, :polarity, :pending]
-  defstruct type: nil,
-            polarity: nil,
-            caller: nil,
-            identifier: nil,
-            status: [],
-            pending: [],
-            transforms: [],
-            state: nil
-
-  alias __MODULE__, as: State
-  alias Tortoise.Package
-
-  def next(%State{pending: [[next_action, resolution] | _]}) do
-    {next_action, resolution}
-  end
-
-  def resolve(%State{pending: [[action, :cleanup]]} = state, :cleanup) do
-    {:ok, %State{state | pending: [], status: [action | state.status]}}
-  end
-
-  def resolve(
-        %State{pending: [[action, {:received, %{__struct__: t, identifier: id}}] | rest]} = state,
-        {:received, %{__struct__: t, identifier: id}} = expected
-      ) do
-    {:ok, %State{state | pending: rest, status: [expected, action | state.status]}}
-  end
-
-  # When we are awaiting a package to dispatch, replace the package
-  # with the message given by the user; this allow us to support user
-  # defined properties on packages such as pubrec, pubrel, pubcomp,
-  # etc.
-  def resolve(
-        %State{pending: [[{:dispatch, %{__struct__: t, identifier: id}}, resolution] | rest]} =
-          state,
-        {:dispatch, %{__struct__: t, identifier: id}} = dispatch
-      ) do
-    {:ok, %State{state | pending: [[dispatch, resolution] | rest]}}
-  end
-
-  # the value has previously been received; here we should stay where
-  # we are at and retry the transmission
-  def resolve(
-        %State{status: [{same, %{__struct__: t, identifier: id}} | _]} = state,
-        {same, %{__struct__: t, identifier: id}}
-      ) do
-    {:ok, state}
-  end
-
-  def resolve(%State{pending: []} = state, :cleanup) do
-    {:ok, state}
-  end
-
-  def resolve(%State{}, {:received, package}) do
-    {:error, {:protocol_violation, {:unexpected_package_from_remote, package}}}
-  end
-
-  @type trackable :: Tortoise.Encodable
-
-  @doc """
-  Set up a data structure that will track the status of a control
-  packet
-  """
-  # @todo, enable this when I've figured out what is wrong with this spec
-  # @spec create(polarity :: polarity(), package :: trackable()) :: __MODULE__.t()
-  def create(:positive, %Package.Publish{qos: 1, identifier: id} = publish) do
-    %State{
-      type: Package.Publish,
-      polarity: :positive,
-      identifier: id,
-      status: [{:received, publish}],
-      pending: [
-        [
-          {:dispatch, %Package.Puback{identifier: id}},
-          :cleanup
-        ]
-      ]
-    }
-  end
-
-  def create(
-        {:negative, {pid, ref}},
-        {%Package.Publish{qos: 1, identifier: id} = publish, {transforms, initial_state}}
-      )
-      when is_pid(pid) and is_reference(ref) do
-    %State{
-      type: Package.Publish,
-      polarity: :negative,
-      caller: {pid, ref},
-      identifier: id,
-      transforms: transforms,
-      state: initial_state,
-      pending: [
-        [
-          {:dispatch, publish},
-          {:received, %Package.Puback{identifier: id}}
-        ],
-        [
-          {:respond, {pid, ref}},
-          :cleanup
-        ]
-      ]
-    }
-  end
-
-  def create(:positive, %Package.Publish{identifier: id, qos: 2} = publish) do
-    %State{
-      type: Package.Publish,
-      polarity: :positive,
-      identifier: id,
-      status: [{:received, publish}],
-      pending: [
-        [
-          {:dispatch, %Package.Pubrec{identifier: id}},
-          {:received, %Package.Pubrel{identifier: id}}
-        ],
-        [
-          {:dispatch, %Package.Pubcomp{identifier: id}},
-          :cleanup
-        ]
-      ]
-    }
-  end
-
-  def create(
-        {:negative, {pid, ref}},
-        {%Package.Publish{identifier: id, qos: 2} = publish, {transforms, initial_state}}
-      )
-      when is_pid(pid) and is_reference(ref) do
-    %State{
-      type: Package.Publish,
-      polarity: :negative,
-      caller: {pid, ref},
-      identifier: id,
-      transforms: transforms,
-      state: initial_state,
-      pending: [
-        [
-          {:dispatch, publish},
-          {:received, %Package.Pubrec{identifier: id}}
-        ],
-        [
-          {:dispatch, %Package.Pubrel{identifier: id}},
-          {:received, %Package.Pubcomp{identifier: id}}
-        ],
-        [
-          {:respond, {pid, ref}},
-          :cleanup
-        ]
-      ]
-    }
-  end
-
-  # subscription
-  def create({:negative, {pid, ref}}, %Package.Subscribe{identifier: id} = subscribe)
-      when is_pid(pid) and is_reference(ref) do
-    %State{
-      type: Package.Subscribe,
-      polarity: :negative,
-      caller: {pid, ref},
-      identifier: id,
-      pending: [
-        [
-          {:dispatch, subscribe},
-          {:received, %Package.Suback{identifier: id}}
-        ],
-        [
-          {:respond, {pid, ref}},
-          :cleanup
-        ]
-      ]
-    }
-  end
-
-  def create({:negative, {pid, ref}}, %Package.Unsubscribe{identifier: id} = unsubscribe)
-      when is_pid(pid) and is_reference(ref) do
-    %State{
-      type: Package.Unsubscribe,
-      polarity: :negative,
-      caller: {pid, ref},
-      identifier: id,
-      pending: [
-        [
-          {:dispatch, unsubscribe},
-          {:received, %Package.Unsuback{identifier: id}}
-        ],
-        [
-          {:respond, {pid, ref}},
-          :cleanup
-        ]
-      ]
-    }
-  end
-
-  # calculate result
-  def result(%State{type: Package.Publish}) do
-    {:ok, :ok}
-  end
-
-  def result(%State{
-        type: Package.Unsubscribe,
-        status: [
-          {:received, %Package.Unsuback{} = unsuback},
-          {:dispatch, %Package.Unsubscribe{} = unsubscribe} | _other
-        ]
-      }) do
-    {:ok, {unsubscribe, unsuback}}
-  end
-
-  def result(%State{
-        type: Package.Subscribe,
-        status: [
-          {:received, %Package.Suback{} = suback},
-          {:dispatch, %Package.Subscribe{} = subscribe} | _other
-        ]
-      }) do
-    {:ok, {subscribe, suback}}
-  end
-end
diff --git a/test/tortoise/connection/inflight/track_test.exs b/test/tortoise/connection/inflight/track_test.exs
deleted file mode 100644
index 025ab3a8..00000000
--- a/test/tortoise/connection/inflight/track_test.exs
+++ /dev/null
@@ -1,163 +0,0 @@
-defmodule Tortoise.Connection.Inflight.TrackTest do
-  @moduledoc false
-  use ExUnit.Case
-  doctest Tortoise.Connection.Inflight.Track
-
-  alias Tortoise.Connection.Inflight.Track
-  alias Tortoise.Package
-
-  describe "incoming publish" do
-    test "progress a qos 1 receive" do
-      id = 0x0001
-      publish = %Package.Publish{qos: 1, identifier: id}
-
-      state = Track.create(:positive, publish)
-
-      assert %Track{
-               pending: [
-                 [
-                   {:dispatch, %Package.Puback{identifier: ^id}},
-                   :cleanup
-                 ]
-               ]
-             } = state
-
-      assert {next_action, resolution} = Track.next(state)
-      assert {:dispatch, %Package.Puback{identifier: ^id}} = next_action
-      assert :cleanup = resolution
-
-      assert {:ok, %Track{identifier: ^id, pending: []}} = Track.resolve(state, resolution)
-    end
-
-    test "progress a qos 2 receive" do
-      id = 0x0001
-      publish = %Package.Publish{qos: 2, identifier: id}
-
-      state = Track.create(:positive, publish)
-      assert %Track{pending: [[{:dispatch, %Package.Pubrec{}} | _] | _]} = state
-
-      {next_action, resolution} = Track.next(state)
-      assert {:dispatch, %Package.Pubrec{identifier: ^id}} = next_action
-
-      {:ok, state} = Track.resolve(state, resolution)
-      # if we send in the same resolution we should not progress
-      assert {:ok, ^state} = Track.resolve(state, resolution)
-
-      {next_action, resolution} = Track.next(state)
-      assert {:dispatch, %Package.Pubcomp{identifier: ^id}} = next_action
-      assert :cleanup = resolution
-
-      assert {:ok, %Track{identifier: ^id, pending: []}} = Track.resolve(state, resolution)
-    end
-  end
-
-  describe "outgoing publish" do
-    test "progress a qos 1 publish" do
-      id = 0x0001
-      publish = %Package.Publish{qos: 1, identifier: id}
-      caller = {self(), make_ref()}
-
-      state = Track.create({:negative, caller}, {publish, {[], nil}})
-      assert %Track{pending: [[{:dispatch, ^publish}, _] | _]} = state
-
-      {next_action, resolution} = Track.next(state)
-      assert {:dispatch, %Package.Publish{identifier: ^id}} = next_action
-      assert {:received, %Package.Puback{identifier: ^id}} = resolution
-      {:ok, state} = Track.resolve(state, resolution)
-
-      # if we send in the same resolution we should not progress
-      assert {:ok, ^state} = Track.resolve(state, resolution)
-
-      {next_action, resolution} = Track.next(state)
-      assert {:respond, ^caller} = next_action
-      assert :cleanup = resolution
-      {:ok, state} = Track.resolve(state, resolution)
-
-      # if we send in the same resolution we should not progress
-      assert {:ok, ^state} = Track.resolve(state, resolution)
-
-      assert %Track{identifier: ^id, pending: []} = state
-    end
-
-    test "progress a qos 2 publish" do
-      id = 0x0001
-      publish = %Package.Publish{qos: 2, identifier: id}
-      caller = {self(), make_ref()}
-
-      state = Track.create({:negative, caller}, {publish, {[], nil}})
-      assert %Track{pending: [[{:dispatch, ^publish}, _] | _]} = state
-
-      {next_action, resolution} = Track.next(state)
-      assert {:dispatch, %Package.Publish{identifier: ^id}} = next_action
-      assert {:received, %Package.Pubrec{identifier: ^id}} = resolution
-      {:ok, state} = Track.resolve(state, resolution)
-
-      # if we send in the same resolution we should not progress
-      assert {:ok, ^state} = Track.resolve(state, resolution)
-
-      {next_action, resolution} = Track.next(state)
-      assert {:dispatch, %Package.Pubrel{identifier: ^id}} = next_action
-      assert {:received, %Package.Pubcomp{identifier: ^id}} = resolution
-      {:ok, state} = Track.resolve(state, resolution)
-
-      # if we send in the same resolution we should not progress
-      assert {:ok, ^state} = Track.resolve(state, resolution)
-
-      {next_action, resolution} = Track.next(state)
-      assert {:respond, ^caller} = next_action
-      assert :cleanup = resolution
-      {:ok, state} = Track.resolve(state, resolution)
-
-      # if we send in the same resolution we should not progress
-      assert {:ok, ^state} = Track.resolve(state, resolution)
-
-      assert %Track{identifier: ^id, pending: []} = state
-    end
-  end
-
-  describe "subscriptions" do
-    test "progress a subscribe" do
-      id = 0x0001
-      subscribe = %Package.Subscribe{identifier: id, topics: [{"foo/bar", 0}]}
-      suback = %Package.Suback{identifier: id, acks: [ok: 0]}
-      caller = {self(), make_ref()}
-
-      state = Track.create({:negative, caller}, subscribe)
-      assert %Track{pending: [[{:dispatch, ^subscribe}, _] | _]} = state
-
-      {next_action, resolution} = Track.next(state)
-      assert {:dispatch, ^subscribe} = next_action
-      assert {:received, %Package.Suback{identifier: ^id}} = resolution
-      {:ok, state} = Track.resolve(state, {:received, suback})
-
-      {next_action, resolution} = Track.next(state)
-      assert {:respond, ^caller} = next_action
-      assert :cleanup = resolution
-      {:ok, state} = Track.resolve(state, resolution)
-
-      assert %Track{identifier: ^id, pending: []} = state
-    end
-
-    test "progress an unsubscribe" do
-      id = 0x0001
-      unsubscribe = %Package.Unsubscribe{identifier: id, topics: ["foo/bar"]}
-      unsuback = %Package.Unsuback{identifier: id}
-      caller = {self(), make_ref()}
-
-      state = Track.create({:negative, caller}, unsubscribe)
-      assert %Track{pending: [[{:dispatch, ^unsubscribe}, _] | _]} = state
-
-      {next_action, resolution} = Track.next(state)
-      assert {:dispatch, ^unsubscribe} = next_action
-      assert {:received, %Package.Unsuback{identifier: ^id}} = resolution
-      {:ok, state} = Track.resolve(state, {:received, unsuback})
-
-      {next_action, resolution} = Track.next(state)
-      assert {:respond, ^caller} = next_action
-      assert :cleanup = resolution
-      {:ok, state} = Track.resolve(state, resolution)
-
-      assert %Track{identifier: ^id, pending: []} = state
-    end
-  end
-end
diff --git a/test/tortoise/connection/inflight_test.exs b/test/tortoise/connection/inflight_test.exs
deleted file mode 100644
index f1512a57..00000000
--- a/test/tortoise/connection/inflight_test.exs
+++ /dev/null
@@ -1,305 +0,0 @@
-defmodule Tortoise.Connection.InflightTest do
-  use ExUnit.Case, async: true
-  doctest Tortoise.Connection.Inflight
-
-  alias Tortoise.Package
-  alias Tortoise.Connection.Inflight
-
-  setup context do
-    {:ok, %{client_id: context.test}}
-  end
-
-  def setup_connection(context) do
-    {:ok, client_socket, server_socket} = Tortoise.Integration.TestTCPTunnel.new()
-    connection = {Tortoise.Transport.Tcp, client_socket}
-    key = Tortoise.Registry.via_name(Tortoise.Connection, context.client_id)
-    Tortoise.Registry.put_meta(key, connection)
-
-    {:ok,
-     Map.merge(context, %{client: client_socket, server: server_socket, connection: connection})}
-  end
-
-  defp drop_connection(%{server: server} = context) do
-    :ok = :gen_tcp.close(server)
-    {:ok, Map.drop(context, [:client, :server, :connection])}
-  end
-
-  def setup_inflight(%{inflight_pid: pid} = context) when is_pid(pid) do
-    Inflight.update_connection(pid, context.connection)
-    {:ok, context}
-  end
-
-  def setup_inflight(context) do
-    {:ok, pid} = Inflight.start_link(client_id: context.client_id, parent: self())
-    setup_inflight(Map.put(context, :inflight_pid, pid))
-  end
-
-  describe "life-cycle" do
-    setup [:setup_connection]
-
-    test "start/stop", context do
-      assert {:ok, pid} = Inflight.start_link(client_id: context.client_id, parent: self())
-      assert Process.alive?(pid)
-      assert :ok = Inflight.stop(pid)
-      refute Process.alive?(pid)
-    end
-  end
-
-  describe "Publish with QoS=1" do
-    setup [:setup_connection, :setup_inflight]
-
-    test "incoming publish QoS=1", %{client_id: client_id} = context do
-      publish = %Package.Publish{identifier: 1, topic: "foo", qos: 1}
-      :ok = Inflight.track(client_id, {:incoming, publish})
-
-      puback = %Package.Puback{identifier: 1}
-      :ok = Inflight.update(client_id, {:dispatch, puback})
-      assert {:ok, data} = :gen_tcp.recv(context.server, 0, 500)
-      assert ^puback = Package.decode(data)
-    end
-
-    test "outgoing publish QoS=1", %{client_id: client_id} = context do
-      publish =
-        %Package.Publish{identifier: 1, topic: "foo", qos: 1}
-        |> Package.Meta.infer()
-
-      {:ok, ref} = Inflight.track(client_id, {:outgoing, publish})
-      assert {:ok, package} = :gen_tcp.recv(context.server, 0, 500)
-      assert ^publish = Package.decode(package)
-
-      # drop and reestablish the connection
-      {:ok, context} = drop_connection(context)
-      {:ok, context} = setup_connection(context)
-      {:ok, context} = setup_inflight(context)
-
-      # the inflight process should now re-transmit the publish
-      assert {:ok, package} = :gen_tcp.recv(context.server, 0, 500)
-      publish = %Package.Publish{publish | dup: true} |> Package.Meta.infer()
-      assert ^publish = Package.decode(package)
-
-      # simulate that we receive a puback from the server
-      Inflight.update(client_id, {:received, %Package.Puback{identifier: 1}})
-
-      # the calling process should get a result response
-      assert_receive {{Tortoise, ^client_id}, {Package.Publish, ^ref}, :ok}
-    end
-  end
-
-  describe "Publish with QoS=2" do
-    setup [:setup_connection, :setup_inflight]
-
-    test "incoming publish QoS=2", %{client_id: client_id} = context do
-      publish = %Package.Publish{identifier: 1, topic: "foo", qos: 2}
-      :ok = Inflight.track(client_id, {:incoming, publish})
-
-      pubrec = %Package.Pubrec{identifier: 1}
-      :ok = Inflight.update(client_id, {:dispatch, pubrec})
-
-      assert {:ok, data} = :gen_tcp.recv(context.server, 0, 500)
-      assert ^pubrec = Package.decode(data)
-
-      # drop and reestablish the connection
-      {:ok, context} = drop_connection(context)
-      {:ok, context} = setup_connection(context)
-      {:ok, context} = setup_inflight(context)
-
-      # now we should receive the same pubrec message
-      assert {:ok, ^data} = :gen_tcp.recv(context.server, 0, 500)
-
-      # simulate that we receive a pubrel from the server
-      Inflight.update(client_id, {:received, %Package.Pubrel{identifier: 1}})
-      # dispatch a pubcomp; we need to feed this because of user
-      # properties that can be set on the pubcomp package by the user
-      pubcomp = %Package.Pubcomp{identifier: 1}
-      Inflight.update(client_id, {:dispatch, pubcomp})
-
-      assert {:ok, encoded_pubcomp} = :gen_tcp.recv(context.server, 0, 500)
-      assert ^pubcomp = Package.decode(encoded_pubcomp)
-    end
-
-    test "outgoing publish QoS=2", %{client_id: client_id} = context do
-      publish =
-        %Package.Publish{identifier: 1, topic: "foo", qos: 2}
-        |> Package.Meta.infer()
-
-      {:ok, ref} = Inflight.track(client_id, {:outgoing, publish})
-
-      # we should transmit the publish
-      assert {:ok, package} = :gen_tcp.recv(context.server, 0, 500)
-      assert ^publish = Package.decode(package)
-      # drop and reestablish the connection
-      {:ok, context} = drop_connection(context)
-      {:ok, context} = setup_connection(context)
-      {:ok, context} = setup_inflight(context)
-      # the publish should get re-transmitted
-      publish = %Package.Publish{publish | dup: true} |> Package.Meta.infer()
-      assert {:ok, package} = :gen_tcp.recv(context.server, 0, 500)
-      assert ^publish = Package.decode(package)
-
-      # simulate that we receive a pubrel from the server
-      Inflight.update(client_id, {:received, %Package.Pubrec{identifier: 1}})
-
-      # we should send the pubrel package
-      pubrel = %Package.Pubrel{identifier: 1}
-      :ok = Inflight.update(client_id, {:dispatch, pubrel})
-
-      assert {:ok, pubrel_encoded} = :gen_tcp.recv(context.server, 0, 500)
-      assert ^pubrel = Package.decode(pubrel_encoded)
-      # drop and reestablish the connection
-      {:ok, context} = drop_connection(context)
-      {:ok, context} = setup_connection(context)
-      {:ok, context} = setup_inflight(context)
-      # re-transmit the pubrel
-      assert {:ok, ^pubrel_encoded} = :gen_tcp.recv(context.server, 0, 500)
-
-      # When we receive the pubcomp message we should respond the caller
-      Inflight.update(client_id, {:received, %Package.Pubcomp{identifier: 1}})
-      assert_receive {{Tortoise, ^client_id}, {Package.Publish, ^ref}, :ok}
-    end
-  end
-
-  describe "Subscription" do
-    setup [:setup_connection, :setup_inflight]
-
-    test "subscription", %{client_id: client_id} = context do
-      opts = [no_local: false, retain_as_published: false, retain_handling: 1]
-
-      subscribe = %Package.Subscribe{
-        identifier: 1,
-        topics: [
-          {"foo", [{:qos, 0} | opts]},
-          {"bar", [{:qos, 1} | opts]},
-          {"baz", [{:qos, 2} | opts]}
-        ]
-      }
-
-      {:ok, ref} = Inflight.track(client_id, {:outgoing, subscribe})
-
-      # send the subscribe package
-      assert {:ok, package} = :gen_tcp.recv(context.server, 0, 500)
-      assert ^subscribe = Package.decode(package)
-      # drop and reestablish the connection
-      {:ok, context} = drop_connection(context)
-      {:ok, context} = setup_connection(context)
-      {:ok, context} = setup_inflight(context)
-      # re-transmit the subscribe package
-      assert {:ok, ^package} = :gen_tcp.recv(context.server, 0, 500)
-
-      # when receiving the suback we should respond to the caller
-      suback = %Package.Suback{
-        identifier: 1,
-        acks: [{:ok, 0}, {:ok, 1}, {:ok, 2}]
-      }
-
-      Inflight.update(client_id, {:received, suback})
-
-      assert_receive {{Tortoise, ^client_id}, {Package.Subscribe, ^ref}, _}
-    end
-  end
-
-  describe "Unsubscribe" do
-    setup [:setup_connection, :setup_inflight]
-
-    test "unsubscribe", %{client_id: client_id} = context do
-      unsubscribe = %Package.Unsubscribe{
-        identifier: 1,
-        topics: ["foo", "bar", "baz"]
-      }
-
-      {:ok, ref} = Inflight.track(client_id, {:outgoing, unsubscribe})
-
-      # send the unsubscribe package
-      assert {:ok, package} = :gen_tcp.recv(context.server, 0, 500)
-      assert ^unsubscribe = Package.decode(package)
-      # drop and reestablish the connection
-      {:ok, context} = drop_connection(context)
-      {:ok, context} = setup_connection(context)
-      {:ok, context} = setup_inflight(context)
-      # re-transmit the subscribe package
-      assert {:ok, ^package} = :gen_tcp.recv(context.server, 0, 500)
-
-      # when receiving the suback we should respond to the caller
-      Inflight.update(client_id, {:received, %Package.Unsuback{identifier: 1}})
-
-      assert_receive {{Tortoise, ^client_id}, {Package.Unsubscribe, ^ref}, _}
-    end
-  end
-
-  describe "message ordering" do
-    setup [:setup_connection, :setup_inflight]
-
-    test "publish should be retransmitted in the same order", context do
-      client_id = context.client_id
-      publish1 = %Package.Publish{identifier: 250, topic: "foo", qos: 1}
-      publish2 = %Package.Publish{identifier: 500, topic: "foo", qos: 1}
-      publish3 = %Package.Publish{identifier: 100, topic: "foo", qos: 1}
-
-      {:ok, _} = Inflight.track(client_id, {:outgoing, publish1})
-      {:ok, _} = Inflight.track(client_id, {:outgoing, publish2})
-      {:ok, _} = Inflight.track(client_id, {:outgoing, publish3})
-
-      expected = Package.encode(publish1) |> IO.iodata_to_binary()
-      assert {:ok, ^expected} = :gen_tcp.recv(context.server, byte_size(expected), 500)
-      expected = Package.encode(publish2) |> IO.iodata_to_binary()
-      assert {:ok, ^expected} = :gen_tcp.recv(context.server, byte_size(expected), 500)
-      expected = Package.encode(publish3) |> IO.iodata_to_binary()
-      assert {:ok, ^expected} = :gen_tcp.recv(context.server, byte_size(expected), 500)
-
-      # drop and reestablish the connection
-      {:ok, context} = drop_connection(context)
-      {:ok, context} = setup_connection(context)
-      {:ok, context} = setup_inflight(context)
-
-      # the in flight manager should now re-transmit the publish
-      # messages in the same order they arrived
-      publish1 = %Package.Publish{publish1 | dup: true}
-      publish2 = %Package.Publish{publish2 | dup: true}
-      publish3 = %Package.Publish{publish3 | dup: true}
-
-      expected = Package.encode(publish1) |> IO.iodata_to_binary()
-      assert {:ok, ^expected} = :gen_tcp.recv(context.server, byte_size(expected), 500)
-      expected = Package.encode(publish2) |> IO.iodata_to_binary()
-      assert {:ok, ^expected} = :gen_tcp.recv(context.server, byte_size(expected), 500)
-      expected = Package.encode(publish3) |> IO.iodata_to_binary()
-      assert {:ok, ^expected} = :gen_tcp.recv(context.server, byte_size(expected), 500)
-    end
-  end
-
-  describe "resetting" do
-    setup [:setup_connection, :setup_inflight]
-
-    test "cancel outgoing inflight packages", %{client_id: client_id} do
-      publish = %Package.Publish{identifier: 1, topic: "foo", qos: 1}
-      {:ok, ref} = Inflight.track(client_id, {:outgoing, publish})
-      :ok = Inflight.reset(client_id)
-      # the calling process should get a result response
-      assert_receive {{Tortoise, ^client_id}, ^ref, {:error, :canceled}}
-    end
-  end
-
-  describe "draining" do
-    setup [:setup_connection, :setup_inflight]
-
-    test "cancel outgoing inflight packages", %{client_id: client_id} = context do
-      publish = %Package.Publish{identifier: 1, topic: "foo", qos: 1}
-      {:ok, ref} = Inflight.track(client_id, {:outgoing, publish})
-      # the publish should get dispatched
-      expected = publish |> Package.encode() |> IO.iodata_to_binary()
-      assert {:ok, ^expected} = :gen_tcp.recv(context.server, byte_size(expected), 500)
-      # start draining
-      disconnect = %Package.Disconnect{reason: :normal_disconnection}
-      :ok = Inflight.drain(client_id, disconnect)
-      # updates should have no effect at this point
-      :ok = Inflight.update(client_id, {:received, %Package.Puback{identifier: 1}})
-      # the calling process should get a result response
-      assert_receive {{Tortoise, ^client_id}, ^ref, {:error, :canceled}}
-      # Now the inflight manager should be in the draining state, new
-      # outbound messages should not get accepted
-      {:ok, ref} = Inflight.track(client_id, {:outgoing, publish})
-      assert_receive {{Tortoise, ^client_id}, ^ref, {:error, :terminating}}
-      # the remote should receive a disconnect package
-      expected = %Package.Disconnect{} |> Package.encode() |> IO.iodata_to_binary()
-      assert {:ok, ^expected} = :gen_tcp.recv(context.server, byte_size(expected), 500)
-    end
-  end
-end
diff --git a/test/tortoise/pipe_test.exs b/test/tortoise/pipe_test.exs
index 8260fe0c..5b664745 100644
--- a/test/tortoise/pipe_test.exs
+++ b/test/tortoise/pipe_test.exs
@@ -3,18 +3,17 @@ defmodule Tortoise.PipeTest do
   doctest Tortoise.Pipe
 
   alias Tortoise.{Pipe, Package}
-  alias Tortoise.Connection.Inflight
 
   setup context do
     {:ok, %{client_id: context.test}}
   end
 
-  def setup_inflight(context) do
-    opts = [client_id: context.client_id, parent: self()]
-    {:ok, inflight_pid} = Inflight.start_link(opts)
-    :ok = Inflight.update_connection(inflight_pid, context.connection)
-    {:ok, %{inflight_pid: inflight_pid}}
-  end
+  # def setup_inflight(context) do
+  #   opts = [client_id: context.client_id, parent: self()]
+  #   {:ok, inflight_pid} = Inflight.start_link(opts)
+  #   :ok = Inflight.update_connection(inflight_pid, context.connection)
+  #   {:ok, %{inflight_pid: inflight_pid}}
+  # end
 
   def setup_registry(context) do
     key = Tortoise.Registry.via_name(Tortoise.Connection, context.client_id)
@@ -128,7 +127,7 @@ defmodule Tortoise.PipeTest do
   end
 
   describe "await/1" do
-    setup [:setup_registry, :setup_connection, :setup_inflight]
+    setup [:setup_registry, :setup_connection]
 
     @tag skip: true
     test "awaiting an empty pending list should complete instantly", context do
@@ -164,14 +163,14 @@ defmodule Tortoise.PipeTest do
       # receive the QoS=1 publish so we can get the id and acknowledge it
       {:ok, package} = :gen_tcp.recv(context.server, 0, 500)
       assert %Package.Publish{identifier: id} = Package.decode(package)
-      Inflight.update(client_id, {:received, %Package.Puback{identifier: id}})
+      # Inflight.update(client_id, {:received, %Package.Puback{identifier: id}})
 
       send(child, :continue)
       # receive and acknowledge the QoS=2 publish
       {:ok, package} = :gen_tcp.recv(context.server, 0, 500)
       assert %Package.Publish{identifier: id} = Package.decode(package)
-      Inflight.update(client_id, {:received, %Package.Pubrec{identifier: id}})
-      Inflight.update(client_id, {:received, %Package.Pubcomp{identifier: id}})
+      # Inflight.update(client_id, {:received, %Package.Pubrec{identifier: id}})
+      # Inflight.update(client_id, {:received, %Package.Pubcomp{identifier: id}})
 
       # both messages should be acknowledged by now
       assert_receive {:result, result}
diff --git a/test/tortoise_test.exs b/test/tortoise_test.exs
index 051791e1..8faf5dbc 100644
--- a/test/tortoise_test.exs
+++ b/test/tortoise_test.exs
@@ -3,7 +3,6 @@ defmodule TortoiseTest do
   doctest Tortoise
 
   alias Tortoise.Package
-  alias Tortoise.Connection.Inflight
 
   setup context do
     {:ok, %{client_id: context.test, transport: Tortoise.Transport.Tcp}}
@@ -19,15 +18,8 @@ defmodule TortoiseTest do
     {:ok, %{client: client_socket, server: server_socket, connection: connection}}
   end
 
-  def setup_inflight(context) do
-    opts = [client_id: context.client_id, parent: self()]
-    {:ok, pid} = Inflight.start_link(opts)
-    :ok = Inflight.update_connection(pid, context.connection)
-    {:ok, %{inflight_pid: pid}}
-  end
-
   describe "publish/4" do
-    setup [:setup_connection, :setup_inflight]
+    setup [:setup_connection]
 
     @tag skip: true
     test "publish qos=0", context do
@@ -44,7 +36,7 @@ defmodule TortoiseTest do
     end
 
     @tag skip: true
-    test "publish qos=1 with user defined callbacks", %{client_id: client_id} = context do
+    test "publish qos=1 with user defined callbacks", context do
       parent = self()
 
       transforms = [
@@ -69,7 +61,6 @@ defmodule TortoiseTest do
       assert %Package.Publish{identifier: id, topic: "foo/bar", qos: 1, payload: nil} =
                Package.decode(data)
 
-      :ok = Inflight.update(client_id, {:received, %Package.Puback{identifier: id}})
       # check the internal transform state
       assert_receive {:callback, {Package.Publish, []}, [:init]}
       assert_receive {:callback, {Package.Puback, []}, [Package.Publish, :init]}
@@ -122,10 +113,7 @@ defmodule TortoiseTest do
                properties: [user_property: {"foo", "bar"}]
              } = Package.decode(data)
 
-      :ok = Inflight.update(context.client_id, {:received, %Package.Pubrec{identifier: id}})
-
       pubrel = %Package.Pubrel{identifier: id}
-      :ok = Inflight.update(client_id, {:dispatch, pubrel})
 
       assert {:ok, data} = :gen_tcp.recv(context.server, 0, 500)
 
@@ -136,8 +124,6 @@ defmodule TortoiseTest do
 
       assert expected_pubrel == Package.decode(data)
 
-      :ok = Inflight.update(client_id, {:received, %Package.Pubcomp{identifier: id}})
-
       assert_receive {{Tortoise, ^client_id}, {Package.Publish, ^publish_ref}, :ok}
       # check the internal state of the transform; in the test we add
       # the type of the package to the state, which we have defied as
@@ -151,7 +137,7 @@ defmodule TortoiseTest do
   end
 
   describe "publish_sync/4" do
-    setup [:setup_connection, :setup_inflight]
+    setup [:setup_connection]
 
     @tag skip: true
     test "publish qos=0", context do
@@ -162,7 +148,6 @@ defmodule TortoiseTest do
 
     @tag skip: true
     test "publish qos=1", context do
-      client_id = context.client_id
       parent = self()
 
       spawn_link(fn ->
@@ -175,7 +160,6 @@ defmodule TortoiseTest do
       assert %Package.Publish{identifier: id, topic: "foo/bar", qos: 1, payload: nil} =
                Package.decode(data)
 
-      :ok = Inflight.update(client_id, {:received, %Package.Puback{identifier: id}})
       assert_receive :done
     end
 
@@ -194,14 +178,12 @@ defmodule TortoiseTest do
       assert %Package.Publish{identifier: id, topic: "foo/bar", qos: 2, payload: nil} =
                Package.decode(data)
 
-      :ok = Inflight.update(client_id, {:received, %Package.Pubrec{identifier: id}})
       # respond with a pubrel
       pubrel = %Package.Pubrel{identifier: id}
-      :ok = Inflight.update(client_id, {:dispatch, pubrel})
+
       assert {:ok, data} = :gen_tcp.recv(context.server, 0, 500)
       assert ^pubrel = Package.decode(data)
 
-      :ok = Inflight.update(client_id, {:received, %Package.Pubcomp{identifier: id}})
       assert_receive :done
     end
   end

From 86bcda35c537c86ce5c05f069fb8e930c6dc8246 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Thu, 22 Oct 2020 19:18:18 +0100
Subject: [PATCH 217/220] Ignore code that reference a now undefined function

Still consider what to do with the "pipe" concept
---
 lib/tortoise/pipe.ex | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/lib/tortoise/pipe.ex b/lib/tortoise/pipe.ex
index 903a24f1..09e3d936 100644
--- a/lib/tortoise/pipe.ex
+++ b/lib/tortoise/pipe.ex
@@ -99,13 +99,14 @@ defmodule Tortoise.Pipe do
     end
   end
 
-  defp do_publish(%Pipe{client_id: client_id} = pipe, %Package.Publish{qos: qos} = publish)
+  defp do_publish(%Pipe{client_id: _client_id} = _pipe, %Package.Publish{qos: qos} = _publish)
        when qos in 1..2 do
-    case Inflight.track(client_id, {:outgoing, publish}) do
-      {:ok, ref} ->
-        updated_pending = [ref | pipe.pending]
-        %Pipe{pipe | pending: updated_pending}
-    end
+    # case Inflight.track(client_id, {:outgoing, publish}) do
+    #   {:ok, ref} ->
+    #     updated_pending = [ref | pipe.pending]
+    #     %Pipe{pipe | pending: updated_pending}
+    # end
+    nil
   end
 
   defp refresh(%Pipe{active: true, client_id: client_id} = pipe) do

From c743300c7cc780c547c9bab9e80e3de762f87570 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Thu, 22 Oct 2020 19:27:52 +0100
Subject: [PATCH 218/220] Add an option list to Package.encode, default empty
 list

This is in preparation for back porting protocol version 3.1.1 support
---
 lib/tortoise/encodable.ex           | 4 ++--
 lib/tortoise/package.ex             | 2 +-
 lib/tortoise/package/auth.ex        | 4 ++--
 lib/tortoise/package/connack.ex     | 2 +-
 lib/tortoise/package/connect.ex     | 6 +++---
 lib/tortoise/package/disconnect.ex  | 4 ++--
 lib/tortoise/package/pingreq.ex     | 4 +++-
 lib/tortoise/package/pingresp.ex    | 2 +-
 lib/tortoise/package/puback.ex      | 5 +++--
 lib/tortoise/package/pubcomp.ex     | 5 +++--
 lib/tortoise/package/publish.ex     | 4 ++--
 lib/tortoise/package/pubrec.ex      | 5 +++--
 lib/tortoise/package/pubrel.ex      | 5 +++--
 lib/tortoise/package/suback.ex      | 2 +-
 lib/tortoise/package/subscribe.ex   | 3 ++-
 lib/tortoise/package/unsuback.ex    | 2 +-
 lib/tortoise/package/unsubscribe.ex | 3 ++-
 17 files changed, 35 insertions(+), 27 deletions(-)

diff --git a/lib/tortoise/encodable.ex b/lib/tortoise/encodable.ex
index 09b8c6a6..02d38791 100644
--- a/lib/tortoise/encodable.ex
+++ b/lib/tortoise/encodable.ex
@@ -1,6 +1,6 @@
 defprotocol Tortoise.Encodable do
   @moduledoc false
 
-  @spec encode(t) :: iodata()
-  def encode(package)
+  @spec encode(t, Keyword.t()) :: iodata()
+  def encode(package, opts)
 end
diff --git a/lib/tortoise/package.ex b/lib/tortoise/package.ex
index 29ebd5b7..5dd3d38a 100644
--- a/lib/tortoise/package.ex
+++ b/lib/tortoise/package.ex
@@ -20,7 +20,7 @@ defmodule Tortoise.Package do
             | Package.Disconnect.t()
             | Package.Auth.t()
 
-  defdelegate encode(data), to: Tortoise.Encodable
+  defdelegate encode(data, opts \\ []), to: Tortoise.Encodable
   defdelegate decode(data), to: Tortoise.Decodable
   defdelegate generate(package), to: Tortoise.Generatable
 
diff --git a/lib/tortoise/package/auth.ex b/lib/tortoise/package/auth.ex
index b2aae0d8..2fdbfb25 100644
--- a/lib/tortoise/package/auth.ex
+++ b/lib/tortoise/package/auth.ex
@@ -42,11 +42,11 @@ defmodule Tortoise.Package.Auth do
   end
 
   defimpl Tortoise.Encodable do
-    def encode(%Package.Auth{reason: :success, properties: []} = t) do
+    def encode(%Package.Auth{reason: :success, properties: []} = t, _opts) do
       [Package.Meta.encode(t.__META__), 0]
     end
 
-    def encode(%Package.Auth{reason: reason} = t)
+    def encode(%Package.Auth{reason: reason} = t, _opts)
         when reason in [:success, :continue_authentication, :re_authenticate] do
       [
         Package.Meta.encode(t.__META__),
diff --git a/lib/tortoise/package/connack.ex b/lib/tortoise/package/connack.ex
index ffc02406..260b15e2 100644
--- a/lib/tortoise/package/connack.ex
+++ b/lib/tortoise/package/connack.ex
@@ -83,7 +83,7 @@ defmodule Tortoise.Package.Connack do
   end
 
   defimpl Tortoise.Encodable do
-    def encode(%Package.Connack{} = t) do
+    def encode(%Package.Connack{} = t, _opts) do
       [
         Package.Meta.encode(t.__META__),
         Package.variable_length_encode([
diff --git a/lib/tortoise/package/connect.ex b/lib/tortoise/package/connect.ex
index a7b21be5..90adbeca 100644
--- a/lib/tortoise/package/connect.ex
+++ b/lib/tortoise/package/connect.ex
@@ -133,7 +133,7 @@ defmodule Tortoise.Package.Connect do
   end
 
   defimpl Tortoise.Encodable do
-    def encode(%Package.Connect{client_id: client_id} = t)
+    def encode(%Package.Connect{client_id: client_id} = t, _opts)
         when is_binary(client_id) do
       [
         Package.Meta.encode(t.__META__),
@@ -147,9 +147,9 @@ defmodule Tortoise.Package.Connect do
       ]
     end
 
-    def encode(%Package.Connect{client_id: client_id} = t)
+    def encode(%Package.Connect{client_id: client_id} = t, opts)
         when is_atom(client_id) do
-      encode(%Package.Connect{t | client_id: Atom.to_string(client_id)})
+      encode(%Package.Connect{t | client_id: Atom.to_string(client_id)}, opts)
     end
 
     defp protocol_header(%{protocol: protocol, protocol_version: version}) do
diff --git a/lib/tortoise/package/disconnect.ex b/lib/tortoise/package/disconnect.ex
index 74c0699f..cbc51002 100644
--- a/lib/tortoise/package/disconnect.ex
+++ b/lib/tortoise/package/disconnect.ex
@@ -109,11 +109,11 @@ defmodule Tortoise.Package.Disconnect do
 
   # Protocols ----------------------------------------------------------
   defimpl Tortoise.Encodable do
-    def encode(%Package.Disconnect{reason: :normal_disconnection, properties: []} = t) do
+    def encode(%Package.Disconnect{reason: :normal_disconnection, properties: []} = t, _opts) do
       [Package.Meta.encode(t.__META__), 0]
     end
 
-    def encode(%Package.Disconnect{} = t) do
+    def encode(%Package.Disconnect{} = t, _opts) do
       [
         Package.Meta.encode(t.__META__),
         Package.variable_length_encode([
diff --git a/lib/tortoise/package/pingreq.ex b/lib/tortoise/package/pingreq.ex
index 8ea81d10..9500c453 100644
--- a/lib/tortoise/package/pingreq.ex
+++ b/lib/tortoise/package/pingreq.ex
@@ -17,7 +17,9 @@ defmodule Tortoise.Package.Pingreq do
 
   # Protocols ----------------------------------------------------------
   defimpl Tortoise.Encodable do
-    def encode(%Package.Pingreq{} = t) do
+    # Note: The Pingreq package is the same for both version 3.1.1 and
+    # version 5, no options apply
+    def encode(%Package.Pingreq{} = t, _opts) do
       [Package.Meta.encode(t.__META__), 0]
     end
   end
diff --git a/lib/tortoise/package/pingresp.ex b/lib/tortoise/package/pingresp.ex
index 5d602433..72c82a56 100644
--- a/lib/tortoise/package/pingresp.ex
+++ b/lib/tortoise/package/pingresp.ex
@@ -17,7 +17,7 @@ defmodule Tortoise.Package.Pingresp do
 
   # Protocols ----------------------------------------------------------
   defimpl Tortoise.Encodable do
-    def encode(%Package.Pingresp{} = t) do
+    def encode(%Package.Pingresp{} = t, _opts) do
       [Package.Meta.encode(t.__META__), 0]
     end
   end
diff --git a/lib/tortoise/package/puback.ex b/lib/tortoise/package/puback.ex
index 46be8d27..12ecd93a 100644
--- a/lib/tortoise/package/puback.ex
+++ b/lib/tortoise/package/puback.ex
@@ -71,7 +71,8 @@ defmodule Tortoise.Package.Puback do
             identifier: identifier,
             reason: :success,
             properties: []
-          } = t
+          } = t,
+          _opts
         )
         when identifier in 0x0001..0xFFFF do
       # The Reason Code and Property Length can be omitted if the
@@ -79,7 +80,7 @@ defmodule Tortoise.Package.Puback do
       [Package.Meta.encode(t.__META__), <<2, identifier::big-integer-size(16)>>]
     end
 
-    def encode(%Package.Puback{identifier: identifier} = t)
+    def encode(%Package.Puback{identifier: identifier} = t, _opts)
         when identifier in 0x0001..0xFFFF do
       [
         Package.Meta.encode(t.__META__),
diff --git a/lib/tortoise/package/pubcomp.ex b/lib/tortoise/package/pubcomp.ex
index 6795b8ee..9f598725 100644
--- a/lib/tortoise/package/pubcomp.ex
+++ b/lib/tortoise/package/pubcomp.ex
@@ -52,13 +52,14 @@ defmodule Tortoise.Package.Pubcomp do
             identifier: identifier,
             reason: :success,
             properties: []
-          } = t
+          } = t,
+          _opts
         )
         when identifier in 0x0001..0xFFFF do
       [Package.Meta.encode(t.__META__), <<2, t.identifier::big-integer-size(16)>>]
     end
 
-    def encode(%Package.Pubcomp{identifier: identifier} = t)
+    def encode(%Package.Pubcomp{identifier: identifier} = t, _opts)
         when identifier in 0x0001..0xFFFF do
       [
         Package.Meta.encode(t.__META__),
diff --git a/lib/tortoise/package/publish.ex b/lib/tortoise/package/publish.ex
index 80631ccd..5625df1c 100644
--- a/lib/tortoise/package/publish.ex
+++ b/lib/tortoise/package/publish.ex
@@ -112,7 +112,7 @@ defmodule Tortoise.Package.Publish do
 
   # Protocols ----------------------------------------------------------
   defimpl Tortoise.Encodable do
-    def encode(%Package.Publish{identifier: nil, qos: 0} = t) do
+    def encode(%Package.Publish{identifier: nil, qos: 0} = t, _opts) do
       [
         Package.Meta.encode(%{t.__META__ | flags: encode_flags(t)}),
         Package.variable_length_encode([
@@ -123,7 +123,7 @@ defmodule Tortoise.Package.Publish do
       ]
     end
 
-    def encode(%Package.Publish{identifier: identifier, qos: qos} = t)
+    def encode(%Package.Publish{identifier: identifier, qos: qos} = t, _opts)
         when identifier in 0x0001..0xFFFF and qos in 1..2 do
       [
         Package.Meta.encode(%{t.__META__ | flags: encode_flags(t)}),
diff --git a/lib/tortoise/package/pubrec.ex b/lib/tortoise/package/pubrec.ex
index b8782581..7a57e1a8 100644
--- a/lib/tortoise/package/pubrec.ex
+++ b/lib/tortoise/package/pubrec.ex
@@ -67,7 +67,8 @@ defmodule Tortoise.Package.Pubrec do
             identifier: identifier,
             reason: :success,
             properties: []
-          } = t
+          } = t,
+          _opts
         )
         when identifier in 0x0001..0xFFFF do
       # The Reason Code and Property Length can be omitted if the
@@ -75,7 +76,7 @@ defmodule Tortoise.Package.Pubrec do
       [Package.Meta.encode(t.__META__), <<2, identifier::big-integer-size(16)>>]
     end
 
-    def encode(%Package.Pubrec{identifier: identifier} = t)
+    def encode(%Package.Pubrec{identifier: identifier} = t, _opts)
         when identifier in 0x0001..0xFFFF do
       [
         Package.Meta.encode(t.__META__),
diff --git a/lib/tortoise/package/pubrel.ex b/lib/tortoise/package/pubrel.ex
index c330b4e1..488f5a3c 100644
--- a/lib/tortoise/package/pubrel.ex
+++ b/lib/tortoise/package/pubrel.ex
@@ -53,7 +53,8 @@ defmodule Tortoise.Package.Pubrel do
             identifier: identifier,
             reason: :success,
             properties: []
-          } = t
+          } = t,
+          _opts
         )
         when identifier in 0x0001..0xFFFF do
       # The Reason Code and Property Length can be omitted if the
@@ -61,7 +62,7 @@ defmodule Tortoise.Package.Pubrel do
       [Package.Meta.encode(t.__META__), <<2, identifier::big-integer-size(16)>>]
     end
 
-    def encode(%Package.Pubrel{identifier: identifier} = t)
+    def encode(%Package.Pubrel{identifier: identifier} = t, _opts)
         when identifier in 0x0001..0xFFFF do
       [
         Package.Meta.encode(t.__META__),
diff --git a/lib/tortoise/package/suback.ex b/lib/tortoise/package/suback.ex
index f84b34ee..1175dc8d 100644
--- a/lib/tortoise/package/suback.ex
+++ b/lib/tortoise/package/suback.ex
@@ -101,7 +101,7 @@ defmodule Tortoise.Package.Suback do
 
   # Protocols ----------------------------------------------------------
   defimpl Tortoise.Encodable do
-    def encode(%Package.Suback{identifier: identifier} = t)
+    def encode(%Package.Suback{identifier: identifier} = t, _opts)
         when identifier in 0x0001..0xFFFF do
       [
         Package.Meta.encode(t.__META__),
diff --git a/lib/tortoise/package/subscribe.ex b/lib/tortoise/package/subscribe.ex
index 151282c0..ef48998a 100644
--- a/lib/tortoise/package/subscribe.ex
+++ b/lib/tortoise/package/subscribe.ex
@@ -84,7 +84,8 @@ defmodule Tortoise.Package.Subscribe do
             identifier: identifier,
             # a valid subscribe package has at least one topic/opts pair
             topics: [{<<_topic_filter::binary>>, opts} | _]
-          } = t
+          } = t,
+          _opts
         )
         when identifier in 0x0001..0xFFFF and is_list(opts) do
       [
diff --git a/lib/tortoise/package/unsuback.ex b/lib/tortoise/package/unsuback.ex
index 324f903d..1737c129 100644
--- a/lib/tortoise/package/unsuback.ex
+++ b/lib/tortoise/package/unsuback.ex
@@ -78,7 +78,7 @@ defmodule Tortoise.Package.Unsuback do
 
   # Protocols ----------------------------------------------------------
   defimpl Tortoise.Encodable do
-    def encode(%Package.Unsuback{identifier: identifier} = t)
+    def encode(%Package.Unsuback{identifier: identifier} = t, _opts)
         when identifier in 0x0001..0xFFFF do
       [
         Package.Meta.encode(t.__META__),
diff --git a/lib/tortoise/package/unsubscribe.ex b/lib/tortoise/package/unsubscribe.ex
index 63222819..056afa5a 100644
--- a/lib/tortoise/package/unsubscribe.ex
+++ b/lib/tortoise/package/unsubscribe.ex
@@ -57,7 +57,8 @@ defmodule Tortoise.Package.Unsubscribe do
             identifier: identifier,
             # a valid unsubscribe package has at least one topic filter
             topics: [topic_filter | _]
-          } = t
+          } = t,
+          _opts
         )
         when identifier in 0x0001..0xFFFF and is_binary(topic_filter) do
       [

From 30fec0d578e1b9c4dca9d51816c05240190a36bd Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin.gausby@erlang-solutions.com>
Date: Thu, 22 Oct 2020 19:58:08 +0100
Subject: [PATCH 219/220] Add an option list to Package.decode, default empty
 list

This is in preparation for back porting protocol version 3.1.1 support
---
 lib/tortoise/decodable.ex           | 36 ++++++++++++++---------------
 lib/tortoise/package/auth.ex        |  6 ++---
 lib/tortoise/package/connack.ex     |  4 ++--
 lib/tortoise/package/connect.ex     |  4 ++--
 lib/tortoise/package/disconnect.ex  |  6 ++---
 lib/tortoise/package/pingreq.ex     |  4 ++--
 lib/tortoise/package/pingresp.ex    |  4 ++--
 lib/tortoise/package/puback.ex      |  6 ++---
 lib/tortoise/package/pubcomp.ex     |  6 ++---
 lib/tortoise/package/publish.ex     |  7 +++---
 lib/tortoise/package/pubrec.ex      |  6 ++---
 lib/tortoise/package/pubrel.ex      |  6 ++---
 lib/tortoise/package/suback.ex      |  4 ++--
 lib/tortoise/package/subscribe.ex   |  4 ++--
 lib/tortoise/package/unsuback.ex    |  4 ++--
 lib/tortoise/package/unsubscribe.ex |  4 ++--
 16 files changed, 56 insertions(+), 55 deletions(-)

diff --git a/lib/tortoise/decodable.ex b/lib/tortoise/decodable.ex
index f829c997..a754762e 100644
--- a/lib/tortoise/decodable.ex
+++ b/lib/tortoise/decodable.ex
@@ -1,7 +1,7 @@
 defprotocol Tortoise.Decodable do
   @moduledoc false
 
-  def decode(data)
+  def decode(data, opts \\ [])
 end
 
 defimpl Tortoise.Decodable, for: BitString do
@@ -23,27 +23,27 @@ defimpl Tortoise.Decodable, for: BitString do
     Auth
   }
 
-  def decode(<<1::4, _::4, _::binary>> = data), do: Connect.decode(data)
-  def decode(<<2::4, _::4, _::binary>> = data), do: Connack.decode(data)
-  def decode(<<3::4, _::4, _::binary>> = data), do: Publish.decode(data)
-  def decode(<<4::4, _::4, _::binary>> = data), do: Puback.decode(data)
-  def decode(<<5::4, _::4, _::binary>> = data), do: Pubrec.decode(data)
-  def decode(<<6::4, _::4, _::binary>> = data), do: Pubrel.decode(data)
-  def decode(<<7::4, _::4, _::binary>> = data), do: Pubcomp.decode(data)
-  def decode(<<8::4, _::4, _::binary>> = data), do: Subscribe.decode(data)
-  def decode(<<9::4, _::4, _::binary>> = data), do: Suback.decode(data)
-  def decode(<<10::4, _::4, _::binary>> = data), do: Unsubscribe.decode(data)
-  def decode(<<11::4, _::4, _::binary>> = data), do: Unsuback.decode(data)
-  def decode(<<12::4, _::4, _::binary>> = data), do: Pingreq.decode(data)
-  def decode(<<13::4, _::4, _::binary>> = data), do: Pingresp.decode(data)
-  def decode(<<14::4, _::4, _::binary>> = data), do: Disconnect.decode(data)
-  def decode(<<15::4, _::4, _::binary>> = data), do: Auth.decode(data)
+  def decode(<<1::4, _::4, _::binary>> = data, opts), do: Connect.decode(data, opts)
+  def decode(<<2::4, _::4, _::binary>> = data, opts), do: Connack.decode(data, opts)
+  def decode(<<3::4, _::4, _::binary>> = data, opts), do: Publish.decode(data, opts)
+  def decode(<<4::4, _::4, _::binary>> = data, opts), do: Puback.decode(data, opts)
+  def decode(<<5::4, _::4, _::binary>> = data, opts), do: Pubrec.decode(data, opts)
+  def decode(<<6::4, _::4, _::binary>> = data, opts), do: Pubrel.decode(data, opts)
+  def decode(<<7::4, _::4, _::binary>> = data, opts), do: Pubcomp.decode(data, opts)
+  def decode(<<8::4, _::4, _::binary>> = data, opts), do: Subscribe.decode(data, opts)
+  def decode(<<9::4, _::4, _::binary>> = data, opts), do: Suback.decode(data, opts)
+  def decode(<<10::4, _::4, _::binary>> = data, opts), do: Unsubscribe.decode(data, opts)
+  def decode(<<11::4, _::4, _::binary>> = data, opts), do: Unsuback.decode(data, opts)
+  def decode(<<12::4, _::4, _::binary>> = data, opts), do: Pingreq.decode(data, opts)
+  def decode(<<13::4, _::4, _::binary>> = data, opts), do: Pingresp.decode(data, opts)
+  def decode(<<14::4, _::4, _::binary>> = data, opts), do: Disconnect.decode(data, opts)
+  def decode(<<15::4, _::4, _::binary>> = data, opts), do: Auth.decode(data, opts)
 end
 
 defimpl Tortoise.Decodable, for: List do
-  def decode(data) do
+  def decode(data, opts) do
     data
     |> IO.iodata_to_binary()
-    |> Tortoise.Decodable.decode()
+    |> Tortoise.Decodable.decode(opts)
   end
 end
diff --git a/lib/tortoise/package/auth.ex b/lib/tortoise/package/auth.ex
index 2fdbfb25..2714aaad 100644
--- a/lib/tortoise/package/auth.ex
+++ b/lib/tortoise/package/auth.ex
@@ -19,12 +19,12 @@ defmodule Tortoise.Package.Auth do
             reason: nil,
             properties: []
 
-  @spec decode(binary()) :: t
-  def decode(<<@opcode::4, 0::4, 0>>) do
+  @spec decode(binary(), opts :: Keyword.t()) :: t
+  def decode(<<@opcode::4, 0::4, 0>>, _opts) do
     %__MODULE__{reason: coerce_reason_code(0x00)}
   end
 
-  def decode(<<@opcode::4, 0::4, variable_header::binary>>) do
+  def decode(<<@opcode::4, 0::4, variable_header::binary>>, _opts) do
     <<reason_code::8, properties::binary>> = Package.drop_length_prefix(variable_header)
 
     %__MODULE__{
diff --git a/lib/tortoise/package/connack.ex b/lib/tortoise/package/connack.ex
index 260b15e2..6320a430 100644
--- a/lib/tortoise/package/connack.ex
+++ b/lib/tortoise/package/connack.ex
@@ -43,8 +43,8 @@ defmodule Tortoise.Package.Connack do
             reason: :success,
             properties: []
 
-  @spec decode(binary()) :: t
-  def decode(<<@opcode::4, 0::4, variable_header::binary>>) do
+  @spec decode(binary(), opts :: Keyword.t()) :: t
+  def decode(<<@opcode::4, 0::4, variable_header::binary>>, _opts) do
     <<0::7, session_present::1, reason_code::8, properties::binary>> =
       Package.drop_length_prefix(variable_header)
 
diff --git a/lib/tortoise/package/connect.ex b/lib/tortoise/package/connect.ex
index 90adbeca..8371c544 100644
--- a/lib/tortoise/package/connect.ex
+++ b/lib/tortoise/package/connect.ex
@@ -41,8 +41,8 @@ defmodule Tortoise.Package.Connect do
             will: nil,
             properties: []
 
-  @spec decode(binary()) :: t
-  def decode(<<@opcode::4, 0::4, variable::binary>>) do
+  @spec decode(binary(), opts :: Keyword.t()) :: t
+  def decode(<<@opcode::4, 0::4, variable::binary>>, _opts) do
     <<
       4::big-integer-size(16),
       "MQTT",
diff --git a/lib/tortoise/package/disconnect.ex b/lib/tortoise/package/disconnect.ex
index cbc51002..0330b618 100644
--- a/lib/tortoise/package/disconnect.ex
+++ b/lib/tortoise/package/disconnect.ex
@@ -48,14 +48,14 @@ defmodule Tortoise.Package.Disconnect do
             reason: :normal_disconnection,
             properties: []
 
-  @spec decode(binary()) :: t
-  def decode(<<@opcode::4, 0::4, 0::8>>) do
+  @spec decode(binary(), opts :: Keyword.t()) :: t
+  def decode(<<@opcode::4, 0::4, 0::8>>, _opts) do
     # If the Remaining Length is less than 1 the value of 0x00 (Normal
     # disconnection) is used
     %__MODULE__{reason: coerce_reason_code(0x00)}
   end
 
-  def decode(<<@opcode::4, 0::4, variable_header::binary>>) do
+  def decode(<<@opcode::4, 0::4, variable_header::binary>>, _opts) do
     <<reason_code::8, properties::binary>> = drop_length_prefix(variable_header)
 
     %__MODULE__{
diff --git a/lib/tortoise/package/pingreq.ex b/lib/tortoise/package/pingreq.ex
index 9500c453..26ef0e72 100644
--- a/lib/tortoise/package/pingreq.ex
+++ b/lib/tortoise/package/pingreq.ex
@@ -10,8 +10,8 @@ defmodule Tortoise.Package.Pingreq do
           }
   defstruct __META__: %Package.Meta{opcode: @opcode, flags: 0}
 
-  @spec decode(<<_::16>>) :: t
-  def decode(<<@opcode::4, 0::4, 0>>) do
+  @spec decode(<<_::16>>, opts :: Keyword.t()) :: t
+  def decode(<<@opcode::4, 0::4, 0>>, _opts) do
     %__MODULE__{}
   end
 
diff --git a/lib/tortoise/package/pingresp.ex b/lib/tortoise/package/pingresp.ex
index 72c82a56..113716d5 100644
--- a/lib/tortoise/package/pingresp.ex
+++ b/lib/tortoise/package/pingresp.ex
@@ -10,8 +10,8 @@ defmodule Tortoise.Package.Pingresp do
           }
   defstruct __META__: %Package.Meta{opcode: @opcode, flags: 0}
 
-  @spec decode(<<_::16>>) :: t
-  def decode(<<@opcode::4, 0::4, 0>>) do
+  @spec decode(<<_::16>>, opts :: Keyword.t()) :: t
+  def decode(<<@opcode::4, 0::4, 0>>, _opts) do
     %__MODULE__{}
   end
 
diff --git a/lib/tortoise/package/puback.ex b/lib/tortoise/package/puback.ex
index 12ecd93a..7b79386e 100644
--- a/lib/tortoise/package/puback.ex
+++ b/lib/tortoise/package/puback.ex
@@ -30,8 +30,8 @@ defmodule Tortoise.Package.Puback do
             reason: :success,
             properties: []
 
-  @spec decode(binary()) :: t
-  def decode(<<@opcode::4, 0::4, 2, identifier::big-integer-size(16)>>) do
+  @spec decode(binary(), opts :: Keyword.t()) :: t
+  def decode(<<@opcode::4, 0::4, 2, identifier::big-integer-size(16)>>, _opts) do
     %__MODULE__{
       identifier: identifier,
       reason: :success,
@@ -39,7 +39,7 @@ defmodule Tortoise.Package.Puback do
     }
   end
 
-  def decode(<<@opcode::4, 0::4, variable_header::binary>>) do
+  def decode(<<@opcode::4, 0::4, variable_header::binary>>, _opts) do
     <<identifier::big-integer-size(16), reason_code::8, properties::binary>> =
       Package.drop_length_prefix(variable_header)
 
diff --git a/lib/tortoise/package/pubcomp.ex b/lib/tortoise/package/pubcomp.ex
index 9f598725..82905d88 100644
--- a/lib/tortoise/package/pubcomp.ex
+++ b/lib/tortoise/package/pubcomp.ex
@@ -21,13 +21,13 @@ defmodule Tortoise.Package.Pubcomp do
             reason: :success,
             properties: []
 
-  @spec decode(binary()) :: t
-  def decode(<<@opcode::4, 0::4, 2, identifier::big-integer-size(16)>>)
+  @spec decode(binary(), opts :: Keyword.t()) :: t
+  def decode(<<@opcode::4, 0::4, 2, identifier::big-integer-size(16)>>, _opts)
       when identifier in 0x0001..0xFFFF do
     %__MODULE__{identifier: identifier}
   end
 
-  def decode(<<@opcode::4, 0::4, variable_header::binary>>) do
+  def decode(<<@opcode::4, 0::4, variable_header::binary>>, _opts) do
     <<identifier::big-integer-size(16), reason_code::8, properties::binary>> =
       Package.drop_length_prefix(variable_header)
 
diff --git a/lib/tortoise/package/publish.ex b/lib/tortoise/package/publish.ex
index 5625df1c..4a749304 100644
--- a/lib/tortoise/package/publish.ex
+++ b/lib/tortoise/package/publish.ex
@@ -35,8 +35,8 @@ defmodule Tortoise.Package.Publish do
             retain: false,
             properties: []
 
-  @spec decode(binary()) :: t
-  def decode(<<@opcode::4, 0::1, 0::2, retain::1, length_prefixed_payload::binary>>) do
+  @spec decode(binary(), opts :: Keyword.t()) :: t
+  def decode(<<@opcode::4, 0::1, 0::2, retain::1, length_prefixed_payload::binary>>, _opts) do
     payload = drop_length_prefix(length_prefixed_payload)
     {topic, properties, payload} = decode_message(payload)
 
@@ -52,7 +52,8 @@ defmodule Tortoise.Package.Publish do
   end
 
   def decode(
-        <<@opcode::4, dup::1, qos::integer-size(2), retain::1, length_prefixed_payload::binary>>
+        <<@opcode::4, dup::1, qos::integer-size(2), retain::1, length_prefixed_payload::binary>>,
+        _opts
       ) do
     payload = drop_length_prefix(length_prefixed_payload)
     {topic, identifier, properties, payload} = decode_message_with_id(payload)
diff --git a/lib/tortoise/package/pubrec.ex b/lib/tortoise/package/pubrec.ex
index 7a57e1a8..a5c13564 100644
--- a/lib/tortoise/package/pubrec.ex
+++ b/lib/tortoise/package/pubrec.ex
@@ -29,13 +29,13 @@ defmodule Tortoise.Package.Pubrec do
             reason: :success,
             properties: []
 
-  @spec decode(binary()) :: t
-  def decode(<<@opcode::4, 0::4, 2, identifier::big-integer-size(16)>>)
+  @spec decode(binary(), opts :: Keyword.t()) :: t
+  def decode(<<@opcode::4, 0::4, 2, identifier::big-integer-size(16)>>, _opts)
       when identifier in 0x0001..0xFFFF do
     %__MODULE__{identifier: identifier, reason: :success, properties: []}
   end
 
-  def decode(<<@opcode::4, 0::4, variable_header::binary>>) do
+  def decode(<<@opcode::4, 0::4, variable_header::binary>>, _opts) do
     <<identifier::big-integer-size(16), reason_code::8, properties::binary>> =
       Package.drop_length_prefix(variable_header)
 
diff --git a/lib/tortoise/package/pubrel.ex b/lib/tortoise/package/pubrel.ex
index 488f5a3c..9cb20b3c 100644
--- a/lib/tortoise/package/pubrel.ex
+++ b/lib/tortoise/package/pubrel.ex
@@ -22,13 +22,13 @@ defmodule Tortoise.Package.Pubrel do
             reason: :success,
             properties: []
 
-  @spec decode(<<_::32>>) :: t
-  def decode(<<@opcode::4, 2::4, 2, identifier::big-integer-size(16)>>)
+  @spec decode(<<_::32>>, opts :: Keyword.t()) :: t
+  def decode(<<@opcode::4, 2::4, 2, identifier::big-integer-size(16)>>, _opts)
       when identifier in 0x0001..0xFFFF do
     %__MODULE__{identifier: identifier}
   end
 
-  def decode(<<@opcode::4, 2::4, variable_header::binary>>) do
+  def decode(<<@opcode::4, 2::4, variable_header::binary>>, _opts) do
     <<identifier::big-integer-size(16), reason_code::8, properties::binary>> =
       Package.drop_length_prefix(variable_header)
 
diff --git a/lib/tortoise/package/suback.ex b/lib/tortoise/package/suback.ex
index 1175dc8d..083276e3 100644
--- a/lib/tortoise/package/suback.ex
+++ b/lib/tortoise/package/suback.ex
@@ -33,8 +33,8 @@ defmodule Tortoise.Package.Suback do
             acks: [],
             properties: []
 
-  @spec decode(binary()) :: t
-  def decode(<<@opcode::4, 0::4, payload::binary>>) do
+  @spec decode(binary(), opts :: Keyword.t()) :: t
+  def decode(<<@opcode::4, 0::4, payload::binary>>, _opts) do
     with payload <- drop_length(payload),
          <<identifier::big-integer-size(16), rest::binary>> <- payload,
          {properties, acks} = Package.parse_variable_length(rest) do
diff --git a/lib/tortoise/package/subscribe.ex b/lib/tortoise/package/subscribe.ex
index ef48998a..54f9a82a 100644
--- a/lib/tortoise/package/subscribe.ex
+++ b/lib/tortoise/package/subscribe.ex
@@ -31,8 +31,8 @@ defmodule Tortoise.Package.Subscribe do
             topics: [],
             properties: []
 
-  @spec decode(binary()) :: t
-  def decode(<<@opcode::4, 0b0010::4, length_prefixed_payload::binary>>) do
+  @spec decode(binary(), opts :: Keyword.t()) :: t
+  def decode(<<@opcode::4, 0b0010::4, length_prefixed_payload::binary>>, _opts) do
     payload = drop_length(length_prefixed_payload)
     <<identifier::big-integer-size(16), rest::binary>> = payload
     {properties, topics} = Package.parse_variable_length(rest)
diff --git a/lib/tortoise/package/unsuback.ex b/lib/tortoise/package/unsuback.ex
index 1737c129..7abc2ae3 100644
--- a/lib/tortoise/package/unsuback.ex
+++ b/lib/tortoise/package/unsuback.ex
@@ -32,8 +32,8 @@ defmodule Tortoise.Package.Unsuback do
             results: [],
             properties: []
 
-  @spec decode(binary()) :: t | {:error, term()}
-  def decode(<<@opcode::4, 0::4, package::binary>>) do
+  @spec decode(binary(), opts :: Keyword.t()) :: t | {:error, term()}
+  def decode(<<@opcode::4, 0::4, package::binary>>, _opts) do
     with payload <- drop_length(package),
          <<identifier::big-integer-size(16), rest::binary>> <- payload,
          {properties, unsubacks} = Package.parse_variable_length(rest) do
diff --git a/lib/tortoise/package/unsubscribe.ex b/lib/tortoise/package/unsubscribe.ex
index 056afa5a..6b1a140b 100644
--- a/lib/tortoise/package/unsubscribe.ex
+++ b/lib/tortoise/package/unsubscribe.ex
@@ -20,8 +20,8 @@ defmodule Tortoise.Package.Unsubscribe do
             identifier: nil,
             properties: []
 
-  @spec decode(binary()) :: t
-  def decode(<<@opcode::4, 0b0010::4, payload::binary>>) do
+  @spec decode(binary(), opts :: Keyword.t()) :: t
+  def decode(<<@opcode::4, 0b0010::4, payload::binary>>, _opts) do
     with payload <- drop_length(payload),
          <<identifier::big-integer-size(16), rest::binary>> <- payload,
          {properties, topics} = Package.parse_variable_length(rest),

From dfb3e3e5a6e6348fbc7015a7f2ed957a4a1e6517 Mon Sep 17 00:00:00 2001
From: Martin Gausby <martin@gausby.dk>
Date: Tue, 2 Mar 2021 08:04:13 +0000
Subject: [PATCH 220/220] Simplify the nix shell

---
 .envrc    |  1 +
 shell.nix | 21 +++++----------------
 2 files changed, 6 insertions(+), 16 deletions(-)
 create mode 100644 .envrc

diff --git a/.envrc b/.envrc
new file mode 100644
index 00000000..051d09d2
--- /dev/null
+++ b/.envrc
@@ -0,0 +1 @@
+eval "$(lorri direnv)"
diff --git a/shell.nix b/shell.nix
index 2c93b324..379b1af5 100644
--- a/shell.nix
+++ b/shell.nix
@@ -1,19 +1,8 @@
-{ pkgs ? import <nixpkgs> {} }:
-
-with pkgs;
-
 let
-  inherit (lib) optional optionals;
-
-  elixir = (beam.packagesWith erlangR22).elixir;
+  pkgs = import <nixpkgs> {};
 in
-
-mkShell {
-  buildInputs = [ elixir ]
-    ++ optional stdenv.isLinux inotify-tools # For file_system on Linux.
-    ++ optionals stdenv.isDarwin (with darwin.apple_sdk.frameworks; [
-      # For file_system on macOS.
-      CoreFoundation
-      CoreServices
-    ]);
+pkgs.mkShell {
+  buildInputs = [
+    pkgs.elixir
+  ];
 }