Merge pull request #813 from Aiven-Open/juham-improve-protoc-compatib…

…ility protobuf: improve protoc compatibility
Aiven-Open · Feb 8, 2024 · 6bdeff7 · 6bdeff7
2 parents 7794fc5 + a713162
commit 6bdeff7
Show file tree

Hide file tree

Showing 3 changed files with 60 additions and 4 deletions.
diff --git a/karapace/protobuf/serialization.py b/karapace/protobuf/serialization.py
@@ -4,6 +4,7 @@
 """
 from __future__ import annotations
 
+from karapace.errors import InvalidSchema
 from karapace.protobuf.enum_constant_element import EnumConstantElement
 from karapace.protobuf.enum_element import EnumElement
 from karapace.protobuf.field import Field
@@ -47,16 +48,16 @@
 
 
 def _deserialize_field(field: Any) -> FieldElement:
-    if field.type not in _TYPE_MAP:
-        raise NotImplementedError(f"Unsupported field type {field.type}")
-
     label = None
     if (field.HasField("proto3_optional") and field.proto3_optional) or Field.Label(field.label) != Field.Label.OPTIONAL:
         label = Field.Label(field.label)
     if field.HasField("type_name"):
         element_type = field.type_name
     else:
-        assert field.HasField("type")
+        if not field.HasField("type"):
+            raise InvalidSchema(f"field {field.name} has no type_name nor type")
+        if field.type not in _TYPE_MAP:
+            raise NotImplementedError(f"Unsupported field type {field.type}")
         element_type = _TYPE_MAP[field.type]
     return FieldElement(DEFAULT_LOCATION, label=label, element_type=element_type, name=field.name, tag=field.number)
 

diff --git a/tests/schemas/protobuf.py b/tests/schemas/protobuf.py
@@ -21,6 +21,13 @@
     + "VCE1NpbXBsZU1lc3NhZ2VQcm90b3NiBnByb3RvMw=="
 )
 
+# Created using protoc
+schema_protobuf_plain_bin_protoc = (
+    "Chl0ZXN0cy9zY2hlbWFzL3BsYWluLnByb3RvEhpjb20uY29kaW5naGFyYm91ci5wcm90b2"
+    "J1ZiJFCg1TaW1wbGVNZXNzYWdlEg8KB2NvbnRlbnQYASABKAkSEQoJZGF0ZV90aW1lGAIg"
+    "ASgJEhAKCGNvbnRlbnQyGAMgASgJQhVCE1NpbXBsZU1lc3NhZ2VQcm90b3NiBnByb3RvMw=="
+)
+
 schema_protobuf_schema_registry1 = """\
 syntax = "proto3";
 package com.codingharbour.protobuf;
@@ -118,6 +125,17 @@
     + "N5RW51bRIUChBNWV9BV0VTT01FX0ZJRUxEEABiBnByb3RvMw=="
 )
 
+# Created using protoc
+schema_protobuf_nested_message4_bin_protoc = (
+    "CiN0ZXN0cy9zY2hlbWFzL25lc3RlZF9tZXNzYWdlNC5wcm90bxIZZmFuY3kuY29tcGFueS"
+    "5pbi5wYXJ0eS52MSL5AQoOQW5vdGhlck1lc3NhZ2Ua5gEKEVdvd0FOZXN0ZWRNZXNzYWdl"
+    "GqoBCg9EZWVwbHlOZXN0ZWRNc2calgEKFUFub3RoZXJMZXZlbE9mTmVzdGluZxJ9CitpbV"
+    "90cmlja3lfaW1fcmVmZXJyaW5nX3RvX3RoZV9wcmV2aW91c19lbnVtGAEgASgOMkguZmFu"
+    "Y3kuY29tcGFueS5pbi5wYXJ0eS52MS5Bbm90aGVyTWVzc2FnZS5Xb3dBTmVzdGVkTWVzc2"
+    "FnZS5CYW1GYW5jeUVudW0iJAoMQmFtRmFuY3lFbnVtEhQKEE1ZX0FXRVNPTUVfRklFTEQQ"
+    "AGIGcHJvdG8z"
+)
+
 schema_protobuf_oneof = """\
 syntax = "proto3";
 
@@ -225,3 +243,21 @@
     + "gKEAtKBAgMEA9SA2ZvbypDCgREYXRhEhQKEERBVEFfVU5TUEVDSUZJRUQQABITCgtEQV"
     + "RBX1NFQVJDSBABGgIIARIQCgxEQVRBX0RJU1BMQVkQAg=="
 )
+
+schema_protobuf_nested_field = """\
+syntax = "proto3";
+
+message Register {
+  .Register.Metadata metadata = 1;
+  string company_number = 2;
+
+  message Metadata {
+    string id = 1;
+  }
+}"""
+
+schema_protobuf_nested_field_bin_protoc = (
+    "Cg5yZWdpc3Rlci5wcm90byJgCghSZWdpc3RlchIkCghtZXRhZGF0YRgBIAEoCzISLlJlZ2"
+    "lzdGVyLk1ldGFkYXRhEhYKDmNvbXBhbnlfbnVtYmVyGAIgASgJGhYKCE1ldGFkYXRhEgoK"
+    "AmlkGAEgASgJYgZwcm90bzM="
+)
diff --git a/tests/unit/test_protobuf_binary_serialization.py b/tests/unit/test_protobuf_binary_serialization.py
@@ -9,14 +9,18 @@
     schema_protobuf_complex_bin,
     schema_protobuf_container2,
     schema_protobuf_container2_bin,
+    schema_protobuf_nested_field,
+    schema_protobuf_nested_field_bin_protoc,
     schema_protobuf_nested_message4,
     schema_protobuf_nested_message4_bin,
+    schema_protobuf_nested_message4_bin_protoc,
     schema_protobuf_oneof,
     schema_protobuf_oneof_bin,
     schema_protobuf_order_after,
     schema_protobuf_order_after_bin,
     schema_protobuf_plain,
     schema_protobuf_plain_bin,
+    schema_protobuf_plain_bin_protoc,
     schema_protobuf_references,
     schema_protobuf_references2,
     schema_protobuf_references2_bin,
@@ -69,6 +73,21 @@ def test_schema_deserialize(schema_plain, schema_serialized):
     )
 
 
+@pytest.mark.parametrize(
+    "schema_plain,schema_serialized",
+    [
+        (schema_protobuf_plain, schema_protobuf_plain_bin_protoc),
+        (schema_protobuf_nested_message4, schema_protobuf_nested_message4_bin_protoc),
+        (schema_protobuf_nested_field, schema_protobuf_nested_field_bin_protoc),
+    ],
+)
+def test_protoc_serialized_schema_deserialize(schema_plain, schema_serialized):
+    assert (
+        schema_plain.strip()
+        == ProtobufSchema("", None, None, proto_file_element=deserialize(schema_serialized)).to_schema().strip()
+    )
+
+
 @pytest.mark.parametrize(
     "schema",
     [