protobuf: improve protoc compatibility

Fix a bug in _deserialize_field that causes deserialization fail in certain cases, e.g. in the case there's a nested field that is a message or an enum. The issue is that the type is optional if type_name is specified. The code falsely expects type to always be present.

Add test test_protoc_serialized_schema_deserialize to test deserialization of protoc-serialized schemas. Test a few cases that are problematic, e.g. one with an enum and another with a nested message.

karapace/protobuf/serialization.py

@@ -47,16 +47,15 @@
 
 
 def _deserialize_field(field: Any) -> FieldElement:
-    if field.type not in _TYPE_MAP:
-        raise NotImplementedError(f"Unsupported field type {field.type}")
-
     label = None
     if (field.HasField("proto3_optional") and field.proto3_optional) or Field.Label(field.label) != Field.Label.OPTIONAL:
         label = Field.Label(field.label)
     if field.HasField("type_name"):
         element_type = field.type_name
     else:
         assert field.HasField("type")
+        if field.type not in _TYPE_MAP:
+            raise NotImplementedError(f"Unsupported field type {field.type}")
         element_type = _TYPE_MAP[field.type]
     return FieldElement(DEFAULT_LOCATION, label=label, element_type=element_type, name=field.name, tag=field.number)
 

tests/schemas/protobuf.py

@@ -21,6 +21,13 @@
     + "VCE1NpbXBsZU1lc3NhZ2VQcm90b3NiBnByb3RvMw=="
 )
 
+# Created using protoc
+schema_protobuf_plain_bin_protoc = (
+    "Chl0ZXN0cy9zY2hlbWFzL3BsYWluLnByb3RvEhpjb20uY29kaW5naGFyYm91ci5wcm90b2"
+    "J1ZiJFCg1TaW1wbGVNZXNzYWdlEg8KB2NvbnRlbnQYASABKAkSEQoJZGF0ZV90aW1lGAIg"
+    "ASgJEhAKCGNvbnRlbnQyGAMgASgJQhVCE1NpbXBsZU1lc3NhZ2VQcm90b3NiBnByb3RvMw=="
+)
+
 schema_protobuf_schema_registry1 = """\
 syntax = "proto3";
 package com.codingharbour.protobuf;
@@ -118,6 +125,17 @@
     + "N5RW51bRIUChBNWV9BV0VTT01FX0ZJRUxEEABiBnByb3RvMw=="
 )
 
+# Created using protoc
+schema_protobuf_nested_message4_bin_protoc = (
+    "CiN0ZXN0cy9zY2hlbWFzL25lc3RlZF9tZXNzYWdlNC5wcm90bxIZZmFuY3kuY29tcGFueS"
+    "5pbi5wYXJ0eS52MSL5AQoOQW5vdGhlck1lc3NhZ2Ua5gEKEVdvd0FOZXN0ZWRNZXNzYWdl"
+    "GqoBCg9EZWVwbHlOZXN0ZWRNc2calgEKFUFub3RoZXJMZXZlbE9mTmVzdGluZxJ9CitpbV"
+    "90cmlja3lfaW1fcmVmZXJyaW5nX3RvX3RoZV9wcmV2aW91c19lbnVtGAEgASgOMkguZmFu"
+    "Y3kuY29tcGFueS5pbi5wYXJ0eS52MS5Bbm90aGVyTWVzc2FnZS5Xb3dBTmVzdGVkTWVzc2"
+    "FnZS5CYW1GYW5jeUVudW0iJAoMQmFtRmFuY3lFbnVtEhQKEE1ZX0FXRVNPTUVfRklFTEQQ"
+    "AGIGcHJvdG8z"
+)
+
 schema_protobuf_oneof = """\
 syntax = "proto3";
 
@@ -225,3 +243,21 @@
     + "gKEAtKBAgMEA9SA2ZvbypDCgREYXRhEhQKEERBVEFfVU5TUEVDSUZJRUQQABITCgtEQV"
     + "RBX1NFQVJDSBABGgIIARIQCgxEQVRBX0RJU1BMQVkQAg=="
 )
+
+schema_protobuf_nested_field = """\
+syntax = "proto3";
+
+message Register {
+  .Register.Metadata metadata = 1;
+  string company_number = 2;
+
+  message Metadata {
+    string id = 1;
+  }
+}"""
+
+schema_protobuf_nested_field_bin_protoc = (
+    "Cg5yZWdpc3Rlci5wcm90byJgCghSZWdpc3RlchIkCghtZXRhZGF0YRgBIAEoCzISLlJlZ2"
+    "lzdGVyLk1ldGFkYXRhEhYKDmNvbXBhbnlfbnVtYmVyGAIgASgJGhYKCE1ldGFkYXRhEgoK"
+    "AmlkGAEgASgJYgZwcm90bzM="
+)

tests/unit/test_protobuf_binary_serialization.py

@@ -9,14 +9,18 @@
     schema_protobuf_complex_bin,
     schema_protobuf_container2,
     schema_protobuf_container2_bin,
+    schema_protobuf_nested_field,
+    schema_protobuf_nested_field_bin_protoc,
     schema_protobuf_nested_message4,
     schema_protobuf_nested_message4_bin,
+    schema_protobuf_nested_message4_bin_protoc,
     schema_protobuf_oneof,
     schema_protobuf_oneof_bin,
     schema_protobuf_order_after,
     schema_protobuf_order_after_bin,
     schema_protobuf_plain,
     schema_protobuf_plain_bin,
+    schema_protobuf_plain_bin_protoc,
     schema_protobuf_references,
     schema_protobuf_references2,
     schema_protobuf_references2_bin,
@@ -69,6 +73,21 @@ def test_schema_deserialize(schema_plain, schema_serialized):
     )
 
 
+@pytest.mark.parametrize(
+    "schema_plain,schema_serialized",
+    [
+        (schema_protobuf_plain, schema_protobuf_plain_bin_protoc),
+        (schema_protobuf_nested_message4, schema_protobuf_nested_message4_bin_protoc),
+        (schema_protobuf_nested_field, schema_protobuf_nested_field_bin_protoc),
+    ],
+)
+def test_protoc_serialized_schema_deserialize(schema_plain, schema_serialized):
+    assert (
+        schema_plain.strip()
+        == ProtobufSchema("", None, None, proto_file_element=deserialize(schema_serialized)).to_schema().strip()
+    )
+
+
 @pytest.mark.parametrize(
     "schema",
     [