diff --git a/.gitignore b/.gitignore index 142f7b8f0c..61c8731f92 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,7 @@ java/**/dependency-reduced-pom.xml **/*.html **/target .idea/ +generated/ java/dependency-reduced-pom.xml java/foo scala/.bsp @@ -38,55 +39,10 @@ bazel-fory/** **/generated java/**/generated -# IDL compiler integration test outputs -integration_tests/idl_tests/cpp/generated/ -integration_tests/idl_tests/go/addressbook*.go -integration_tests/idl_tests/go/any_example/ -integration_tests/idl_tests/go/any_example_pb/ -integration_tests/idl_tests/go/complex_fbs/ -integration_tests/idl_tests/go/complex_pb/ -integration_tests/idl_tests/go/monster/ -integration_tests/idl_tests/go/optional_types/ -integration_tests/idl_tests/go/tree/ -integration_tests/idl_tests/go/graph/ -integration_tests/idl_tests/go/collection/ -integration_tests/idl_tests/java/src/main/ -integration_tests/idl_tests/go/root/ -integration_tests/idl_tests/go/addressbook/ -integration_tests/idl_tests/java/src/main/java/collection/ -integration_tests/idl_tests/java/src/main/java/addressbook/ -integration_tests/idl_tests/java/src/main/java/complex_fbs/ -integration_tests/idl_tests/java/src/main/java/complex_pb/ -integration_tests/idl_tests/java/src/main/java/monster/ -integration_tests/idl_tests/java/src/main/java/optional_types/ -integration_tests/idl_tests/java/src/main/java/tree/ -integration_tests/idl_tests/java/src/main/java/graph/ -integration_tests/idl_tests/java/src/main/java/root/ -integration_tests/idl_tests/python/src/addressbook.py -integration_tests/idl_tests/python/src/any_example.py -integration_tests/idl_tests/python/src/any_example_pb.py -integration_tests/idl_tests/python/src/complex_fbs.py -integration_tests/idl_tests/python/src/complex_pb.py -integration_tests/idl_tests/python/src/monster.py -integration_tests/idl_tests/python/src/optional_types.py -integration_tests/idl_tests/python/src/collection.py -integration_tests/idl_tests/python/src/tree.py -integration_tests/idl_tests/python/src/graph.py -integration_tests/idl_tests/python/src/root.py -integration_tests/idl_tests/rust/src/addressbook.rs -integration_tests/idl_tests/rust/src/any_example.rs -integration_tests/idl_tests/rust/src/any_example_pb.rs -integration_tests/idl_tests/rust/src/complex_fbs.rs -integration_tests/idl_tests/rust/src/complex_pb.rs -integration_tests/idl_tests/rust/src/monster.rs -integration_tests/idl_tests/rust/src/optional_types.rs -integration_tests/idl_tests/rust/src/collection.rs -integration_tests/idl_tests/rust/src/tree.rs -integration_tests/idl_tests/rust/src/graph.rs -integration_tests/idl_tests/rust/src/root.rs javascript/**/dist/ javascript/**/node_modules/ javascript/**/build +javascript/junit.xml MODULE.bazel.lock .DS_Store **/.DS_Store diff --git a/compiler/README.md b/compiler/README.md index c530caab31..d841731d5b 100644 --- a/compiler/README.md +++ b/compiler/README.md @@ -229,8 +229,12 @@ FDL uses plain option keys without a `(fory)` prefix: ```fdl option use_record_for_java_message = true; option polymorphism = true; +option enable_auto_type_id = true; ``` +`enable_auto_type_id` defaults to `true`. Set it to `false` to keep namespace-based registration +for types that omit explicit IDs. + **Message/Enum options:** ```fdl diff --git a/compiler/extension/fory_options.proto b/compiler/extension/fory_options.proto index 3cac903e33..e77f51dc7a 100644 --- a/compiler/extension/fory_options.proto +++ b/compiler/extension/fory_options.proto @@ -28,6 +28,7 @@ // // File-level options: // option (fory).use_record_for_java_message = true; +// option (fory).enable_auto_type_id = true; // // Message options: // message MyMessage { @@ -67,6 +68,12 @@ message ForyFileOptions { // When true, type metadata is included in serialization for polymorphic // dispatch. Default: false optional bool polymorphism = 2; + + // Control auto-generation of type IDs when none are specified. + // When true (default in the compiler), IDs are generated from the package + // namespace and type name. When false, types without explicit IDs are + // registered by namespace and name. + optional bool enable_auto_type_id = 3; } extend google.protobuf.FileOptions { optional ForyFileOptions fory = 50001; } diff --git a/compiler/fory_compiler/cli.py b/compiler/fory_compiler/cli.py index 5b58ee11e4..54147a4963 100644 --- a/compiler/fory_compiler/cli.py +++ b/compiler/fory_compiler/cli.py @@ -153,6 +153,7 @@ def resolve_imports( # Create merged schema with imported types first (so they can be referenced) merged_schema = Schema( package=schema.package, + package_alias=schema.package_alias, imports=schema.imports, enums=imported_enums + schema.enums, messages=imported_messages + schema.messages, @@ -181,8 +182,34 @@ def go_package_info(schema: Schema) -> Tuple[Optional[str], str]: return None, "" +def _find_go_module_root(base_go_out: Path) -> Optional[Path]: + base_go_out = base_go_out.resolve() + for candidate in (base_go_out, *base_go_out.parents): + if (candidate / "go.mod").is_file(): + return candidate + return None + + +def _read_go_module_path(go_module_root: Path) -> Optional[str]: + module_file = go_module_root / "go.mod" + if not module_file.is_file(): + return None + try: + with module_file.open("r", encoding="utf-8") as handle: + for line in handle: + stripped = line.strip() + if stripped.startswith("module "): + return stripped.split(None, 1)[1].strip() + except OSError: + return None + return None + + def resolve_go_module_root(base_go_out: Path, schema: Schema) -> Path: """Infer the Go module root for output layout.""" + module_root = _find_go_module_root(base_go_out) + if module_root: + return module_root _, package_name = go_package_info(schema) if package_name and base_go_out.name == package_name: return base_go_out.parent @@ -195,6 +222,14 @@ def resolve_go_output_dir(base_go_out: Path, schema: Schema) -> Path: if package_name and base_go_out.name == package_name: return base_go_out if import_path: + module_path = _read_go_module_path(base_go_out) + if module_path: + module_path = module_path.rstrip("/") + if import_path.startswith(module_path): + rel_path = import_path[len(module_path) :].lstrip("/") + if rel_path: + return base_go_out / rel_path + return base_go_out last_segment = import_path.rstrip("/").split("/")[-1] if package_name and last_segment == package_name: return base_go_out / package_name @@ -501,7 +536,13 @@ def compile_file_recursive( effective_outputs = lang_output_dirs if "go" in lang_output_dirs: go_root = go_module_root or lang_output_dirs["go"] - go_out = resolve_go_output_dir(go_root, schema) + if ( + schema.get_option("go_package") is None + and lang_output_dirs["go"] != go_root + ): + go_out = lang_output_dirs["go"] + else: + go_out = resolve_go_output_dir(go_root, schema) if go_out != lang_output_dirs["go"]: effective_outputs = dict(lang_output_dirs) effective_outputs["go"] = go_out diff --git a/compiler/fory_compiler/frontend/fbs/translator.py b/compiler/fory_compiler/frontend/fbs/translator.py index ee698d51cf..c42ce0bfc3 100644 --- a/compiler/fory_compiler/frontend/fbs/translator.py +++ b/compiler/fory_compiler/frontend/fbs/translator.py @@ -77,6 +77,7 @@ def _location(self, line: int, column: int) -> SourceLocation: def translate(self) -> Schema: return Schema( package=self.schema.namespace, + package_alias=None, imports=[Import(path=inc) for inc in self.schema.includes], enums=[self._translate_enum(e) for e in self.schema.enums], unions=[self._translate_union(u) for u in self.schema.unions], diff --git a/compiler/fory_compiler/frontend/fdl/parser.py b/compiler/fory_compiler/frontend/fdl/parser.py index feb627c40f..834d4a9559 100644 --- a/compiler/fory_compiler/frontend/fdl/parser.py +++ b/compiler/fory_compiler/frontend/fdl/parser.py @@ -18,7 +18,7 @@ """Recursive descent parser for FDL.""" import warnings -from typing import List, Set +from typing import List, Set, Optional from fory_compiler.ir.ast import ( Schema, @@ -47,6 +47,7 @@ "deprecated", "use_record_for_java_message", "polymorphism", + "enable_auto_type_id", "go_nested_type_style", } @@ -92,6 +93,7 @@ KNOWN_MESSAGE_OPTIONS: Set[str] = { "id", + "alias", "evolving", "use_record_for_java", "deprecated", @@ -100,6 +102,7 @@ KNOWN_UNION_OPTIONS: Set[str] = { "id", + "alias", "deprecated", } @@ -187,6 +190,7 @@ def error(self, message: str) -> ParseError: def parse(self) -> Schema: """Parse the entire input and return a Schema.""" package = None + package_alias = None imports = [] enums = [] messages = [] @@ -197,7 +201,7 @@ def parse(self) -> Schema: if self.check(TokenType.PACKAGE): if package is not None: raise self.error("Duplicate package declaration") - package = self.parse_package() + package, package_alias = self.parse_package() elif self.check(TokenType.IMPORT): imports.append(self.parse_import()) elif self.check(TokenType.OPTION): @@ -215,6 +219,7 @@ def parse(self) -> Schema: return Schema( package=package, + package_alias=package_alias, imports=imports, enums=enums, messages=messages, @@ -233,8 +238,8 @@ def make_location(self, token: Token) -> SourceLocation: source_format=self.source_format, ) - def parse_package(self) -> str: - """Parse a package declaration: package foo.bar;""" + def parse_package(self) -> tuple[str, Optional[str]]: + """Parse a package declaration: package foo.bar [alias baz];""" self.consume(TokenType.PACKAGE) # Package name can be dotted: foo.bar.baz @@ -245,8 +250,23 @@ def parse_package(self) -> str: self.consume(TokenType.IDENT, "Expected identifier after '.'").value ) - self.consume(TokenType.SEMI, "Expected ';' after package name") - return ".".join(parts) + alias = None + if self.check(TokenType.IDENT) and self.current().value == "alias": + self.advance() # consume alias keyword + alias_parts = [ + self.consume(TokenType.IDENT, "Expected identifier after 'alias'").value + ] + while self.check(TokenType.DOT): + self.advance() + alias_parts.append( + self.consume( + TokenType.IDENT, "Expected identifier after '.' in alias" + ).value + ) + alias = ".".join(alias_parts) + + self.consume(TokenType.SEMI, "Expected ';' after package declaration") + return ".".join(parts), alias def parse_option_value(self): """Parse an option value (string, bool, int, or identifier).""" diff --git a/compiler/fory_compiler/frontend/proto/translator.py b/compiler/fory_compiler/frontend/proto/translator.py index 46f5eeddde..bf2758af4e 100644 --- a/compiler/fory_compiler/frontend/proto/translator.py +++ b/compiler/fory_compiler/frontend/proto/translator.py @@ -97,6 +97,7 @@ def _location(self, line: int, column: int) -> SourceLocation: def translate(self) -> Schema: return Schema( package=self.proto_schema.package, + package_alias=None, imports=self._translate_imports(), enums=[self._translate_enum(e) for e in self.proto_schema.enums], messages=[self._translate_message(m) for m in self.proto_schema.messages], diff --git a/compiler/fory_compiler/generators/base.py b/compiler/fory_compiler/generators/base.py index 9647af5b2d..8e67799dc7 100644 --- a/compiler/fory_compiler/generators/base.py +++ b/compiler/fory_compiler/generators/base.py @@ -181,6 +181,21 @@ def strip_enum_prefix(self, enum_name: str, value_name: str) -> str: return remainder + def format_type_id_comment(self, type_def, comment_prefix: str) -> Optional[str]: + """Format a type id comment for a message/union.""" + type_id = getattr(type_def, "type_id", None) + if type_id is None: + return None + if getattr(type_def, "id_generated", False): + source = getattr(type_def, "id_source", None) or "unknown" + return f"{comment_prefix} Type ID {type_id} is generated from {source}" + return f"{comment_prefix} Type ID {type_id} is specified manually." + + def should_register_by_id(self, type_def) -> bool: + """Return True if a type should be registered by numeric ID.""" + type_id = getattr(type_def, "type_id", None) + return type_id is not None + def get_license_header(self, comment_prefix: str = "//") -> str: """Get the Apache license header.""" lines = [ diff --git a/compiler/fory_compiler/generators/cpp.py b/compiler/fory_compiler/generators/cpp.py index a643d753ae..5c3afb4c6e 100644 --- a/compiler/fory_compiler/generators/cpp.py +++ b/compiler/fory_compiler/generators/cpp.py @@ -880,6 +880,9 @@ def generate_message_definition( lineage = parent_stack + [message] body_indent = f"{indent} " field_indent = f"{indent} " + comment = self.format_type_id_comment(message, f"{indent}//") + if comment: + lines.append(comment) lines.append(f"{indent}class {class_name} final {{") lines.append(f"{body_indent}public:") if message.fields: @@ -982,6 +985,9 @@ def generate_union_definition( ] variant_type = f"std::variant<{', '.join(case_types)}>" + comment = self.format_type_id_comment(union, f"{indent}//") + if comment: + lines.append(comment) lines.append(f"{indent}class {class_name} final {{") lines.append(f"{body_indent}public:") lines.append(f"{body_indent} enum class {case_enum} : uint32_t {{") @@ -1234,7 +1240,7 @@ def generate_union_serializer( lines.append("") lines.append(" static inline void write_type_info(WriteContext &ctx) {") lines.append( - f" auto result = ctx.write_any_typeinfo(static_cast(TypeId::TYPED_UNION), std::type_index(typeid({qualified_name})));" + f" auto result = ctx.write_any_type_info(static_cast(TypeId::TYPED_UNION), std::type_index(typeid({qualified_name})));" ) lines.append(" if (FORY_PREDICT_FALSE(!result.ok())) {") lines.append(" ctx.set_error(std::move(result).error());") @@ -1250,7 +1256,9 @@ def generate_union_serializer( lines.append(" return;") lines.append(" }") lines.append(" const TypeInfo *expected = type_info_res.value();") - lines.append(" const TypeInfo *remote = ctx.read_any_typeinfo(ctx.error());") + lines.append( + " const TypeInfo *remote = ctx.read_any_type_info(ctx.error());" + ) lines.append(" if (FORY_PREDICT_FALSE(ctx.has_error())) {") lines.append(" return;") lines.append(" }") @@ -1391,7 +1399,7 @@ def generate_union_serializer( lines.append(" return default_value();") lines.append(" }") lines.append( - " const TypeInfo *type_info = ctx.read_any_typeinfo(ctx.error());" + " const TypeInfo *type_info = ctx.read_any_type_info(ctx.error());" ) lines.append(" if (FORY_PREDICT_FALSE(ctx.has_error())) {") lines.append(" return default_value();") @@ -1858,7 +1866,7 @@ def generate_enum_registration( code_name = self.get_qualified_type_name(enum.name, parent_stack) type_name = self.get_registration_type_name(enum.name, parent_stack) - if enum.type_id is not None: + if self.should_register_by_id(enum): lines.append(f" fory.register_enum<{code_name}>({enum.type_id});") else: ns = self.package or "default" @@ -1889,7 +1897,7 @@ def generate_message_registration( ) # Register this message - if message.type_id is not None: + if self.should_register_by_id(message): lines.append(f" fory.register_struct<{code_name}>({message.type_id});") else: ns = self.package or "default" @@ -1904,7 +1912,7 @@ def generate_union_registration( code_name = self.get_qualified_type_name(union.name, parent_stack) type_name = self.get_registration_type_name(union.name, parent_stack) - if union.type_id is not None: + if self.should_register_by_id(union): lines.append(f" fory.register_union<{code_name}>({union.type_id});") else: ns = self.package or "default" diff --git a/compiler/fory_compiler/generators/go.py b/compiler/fory_compiler/generators/go.py index 34ee3410ac..812ddc4536 100644 --- a/compiler/fory_compiler/generators/go.py +++ b/compiler/fory_compiler/generators/go.py @@ -521,6 +521,9 @@ def generate_union( lines.append(")") lines.append("") + comment = self.format_type_id_comment(union, "//") + if comment: + lines.append(comment) lines.append(f"type {type_name} struct {{") lines.append(f"\tcase_ {case_type}") lines.append("\tvalue any") @@ -745,6 +748,9 @@ def generate_message( type_name = self.get_type_name(message.name, parent_stack) lineage = (parent_stack or []) + [message] + comment = self.format_type_id_comment(message, "//") + if comment: + lines.append(comment) lines.append(f"type {type_name} struct {{") # Fields @@ -1167,7 +1173,7 @@ def generate_enum_registration( code_name = self.get_type_name(enum.name, parent_stack) type_name = self.get_registration_type_name(enum.name, parent_stack) - if enum.type_id is not None: + if self.should_register_by_id(enum): lines.append( f"\tif err := f.RegisterEnum({code_name}(0), {enum.type_id}); err != nil {{" ) @@ -1210,7 +1216,7 @@ def generate_message_registration( ) # Register this message - if message.type_id is not None: + if self.should_register_by_id(message): lines.append( f"\tif err := f.RegisterStruct({code_name}{{}}, {message.type_id}); err != nil {{" ) @@ -1243,7 +1249,7 @@ def generate_union_registration( ) serializer_expr = f"fory.NewUnionSerializer({', '.join(cases)})" - if union.type_id is not None: + if self.should_register_by_id(union): lines.append( f"\tif err := f.RegisterUnion({code_name}{{}}, {union.type_id}, {serializer_expr}); err != nil {{" ) diff --git a/compiler/fory_compiler/generators/java.py b/compiler/fory_compiler/generators/java.py index 11c479e10d..a6d85b3e80 100644 --- a/compiler/fory_compiler/generators/java.py +++ b/compiler/fory_compiler/generators/java.py @@ -442,6 +442,9 @@ def generate_message_file(self, message: Message) -> GeneratedFile: lines.append("") # Class declaration + comment = self.format_type_id_comment(message, "//") + if comment: + lines.append(comment) lines.append(f"public class {message.name} {{") # Generate nested enums as static inner classes @@ -646,6 +649,9 @@ def generate_union_class( class_prefix = "public static final class" if nested else "public final class" case_enum = f"{union.name}Case" + comment = self.format_type_id_comment(union, f"{ind}//") + if comment: + lines.append(comment) lines.append(f"{ind}{class_prefix} {union.name} extends Union {{") lines.append(f"{ind} public enum {case_enum} {{") @@ -882,15 +888,15 @@ def get_union_case_type_id_expr( if isinstance(type_def, Enum): if type_def.type_id is None: return "Types.NAMED_ENUM" - return f"({type_def.type_id} << 8) | Types.ENUM" + return "Types.ENUM" if isinstance(type_def, Union): if type_def.type_id is None: return "Types.NAMED_UNION" - return f"({type_def.type_id} << 8) | Types.UNION" + return "Types.UNION" if isinstance(type_def, Message): if type_def.type_id is None: return "Types.NAMED_STRUCT" - return f"({type_def.type_id} << 8) | Types.STRUCT" + return "Types.STRUCT" return "Types.UNKNOWN" def resolve_named_type( @@ -952,6 +958,9 @@ def generate_nested_message( lineage = (parent_stack or []) + [message] # Class declaration + comment = self.format_type_id_comment(message, " " * indent + "//") + if comment: + lines.append(comment) lines.append(f"public static class {message.name} {{") # Generate nested enums @@ -1520,6 +1529,9 @@ def generate_registration_file( m for m in self.schema.messages if not self.is_imported_type(m) ] lines.append(" public static void register(Fory fory) {") + lines.append( + " org.apache.fory.resolver.TypeResolver resolver = fory.getTypeResolver();" + ) # Register enums (top-level) for enum in local_enums: @@ -1554,13 +1566,15 @@ def generate_enum_registration( class_ref = f"{parent_path}.{enum.name}" if parent_path else enum.name type_name = class_ref if parent_path else enum.name - if enum.type_id is not None: - lines.append(f" fory.register({class_ref}.class, {enum.type_id});") + if self.should_register_by_id(enum): + lines.append( + f" resolver.register({class_ref}.class, {enum.type_id}L);" + ) else: # Use FDL package for namespace (consistent across languages) ns = self.schema.package or "default" lines.append( - f' fory.register({class_ref}.class, "{ns}", "{type_name}");' + f' resolver.register({class_ref}.class, "{ns}", "{type_name}");' ) def generate_message_registration( @@ -1571,15 +1585,15 @@ def generate_message_registration( class_ref = f"{parent_path}.{message.name}" if parent_path else message.name type_name = class_ref if parent_path else message.name - if message.type_id is not None: + if self.should_register_by_id(message): lines.append( - f" fory.register({class_ref}.class, {message.type_id});" + f" resolver.register({class_ref}.class, {message.type_id}L);" ) else: # Use FDL package for namespace (consistent across languages) ns = self.schema.package or "default" lines.append( - f' fory.register({class_ref}.class, "{ns}", "{type_name}");' + f' resolver.register({class_ref}.class, "{ns}", "{type_name}");' ) # Register nested enums @@ -1604,14 +1618,14 @@ def generate_union_registration( f"new org.apache.fory.serializer.UnionSerializer(fory, {class_ref}.class)" ) - if union.type_id is not None: + if self.should_register_by_id(union): lines.append( - f" fory.registerUnion({class_ref}.class, {union.type_id}, {serializer_ref});" + f" resolver.registerUnion({class_ref}.class, {union.type_id}L, {serializer_ref});" ) else: ns = self.schema.package or "default" if parent_path: ns = f"{ns}.{parent_path}" lines.append( - f' fory.registerUnion({class_ref}.class, "{ns}", "{type_name}", {serializer_ref});' + f' resolver.registerUnion({class_ref}.class, "{ns}", "{type_name}", {serializer_ref});' ) diff --git a/compiler/fory_compiler/generators/python.py b/compiler/fory_compiler/generators/python.py index 8e70cae0f8..95fe856ca8 100644 --- a/compiler/fory_compiler/generators/python.py +++ b/compiler/fory_compiler/generators/python.py @@ -367,6 +367,9 @@ def generate_message( ind = " " * indent lineage = (parent_stack or []) + [message] + comment = self.format_type_id_comment(message, f"{ind}#") + if comment: + lines.append(comment) lines.append(f"{ind}@dataclass") lines.append(f"{ind}class {message.name}:") @@ -408,7 +411,8 @@ def generate_message( ): lines.append("") - lines.extend(self.generate_bytes_methods(message.name, indent)) + return_type = ".".join([msg.name for msg in lineage]) + lines.extend(self.generate_bytes_methods(return_type, indent)) return lines @@ -434,6 +438,9 @@ def generate_union( lines.append(f"{ind} {case_name} = {field.number}") lines.append("") + comment = self.format_type_id_comment(union, f"{ind}#") + if comment: + lines.append(comment) lines.append(f"{ind}class {union.name}(Union):") lines.append(f'{ind} __slots__ = ("_case",)') lines.append("") @@ -987,8 +994,7 @@ def generate_enum_registration( # In Python, nested class references use Outer.Inner syntax class_ref = f"{parent_path}.{enum.name}" if parent_path else enum.name type_name = class_ref if parent_path else enum.name - - if enum.type_id is not None: + if self.should_register_by_id(enum): lines.append(f" fory.register_type({class_ref}, type_id={enum.type_id})") else: ns = self.package or "default" @@ -1003,8 +1009,7 @@ def generate_message_registration( # In Python, nested class references use Outer.Inner syntax class_ref = f"{parent_path}.{message.name}" if parent_path else message.name type_name = class_ref if parent_path else message.name - - if message.type_id is not None: + if self.should_register_by_id(message): lines.append( f" fory.register_type({class_ref}, type_id={message.type_id})" ) @@ -1037,8 +1042,7 @@ def generate_union_registration( if parent_path else f"{union.name}Serializer" ) - - if union.type_id is not None: + if self.should_register_by_id(union): lines.append( f" fory.register_union({class_ref}, type_id={union.type_id}, serializer={serializer_ref}(fory))" ) diff --git a/compiler/fory_compiler/generators/rust.py b/compiler/fory_compiler/generators/rust.py index 777fce9a58..23d6fd5d3e 100644 --- a/compiler/fory_compiler/generators/rust.py +++ b/compiler/fory_compiler/generators/rust.py @@ -365,6 +365,9 @@ def generate_union( ) if self.to_pascal_case(union.name) != union.name: lines.append("#[allow(non_camel_case_types)]") + comment = self.format_type_id_comment(union, "//") + if comment: + lines.append(comment) derives = ["ForyObject", "Debug"] if not has_any: derives.extend(["Clone", "PartialEq"]) @@ -413,6 +416,9 @@ def generate_message( type_name = message.name # Derive macros + comment = self.format_type_id_comment(message, "//") + if comment: + lines.append(comment) derives = ["ForyObject", "Debug"] if not self.message_has_any(message): derives.extend(["Clone", "PartialEq", "Default"]) @@ -830,7 +836,7 @@ def generate_enum_registration( type_name = self.get_type_path(enum.name, parent_stack) reg_name = self.get_registration_type_name(enum.name, parent_stack) - if enum.type_id is not None: + if self.should_register_by_id(enum): lines.append(f" fory.register::<{type_name}>({enum.type_id})?;") else: ns = self.package or "default" @@ -866,7 +872,7 @@ def generate_message_registration( ) # Register this message - if message.type_id is not None: + if self.should_register_by_id(message): lines.append(f" fory.register::<{type_name}>({message.type_id})?;") else: ns = self.package or "default" @@ -884,7 +890,7 @@ def generate_union_registration( type_name = self.get_type_path(union.name, parent_stack) reg_name = self.get_registration_type_name(union.name, parent_stack) - if union.type_id is not None: + if self.should_register_by_id(union): lines.append(f" fory.register_union::<{type_name}>({union.type_id})?;") else: ns = self.package or "default" diff --git a/compiler/fory_compiler/ir/ast.py b/compiler/fory_compiler/ir/ast.py index 8b3c9cdf56..f84736160a 100644 --- a/compiler/fory_compiler/ir/ast.py +++ b/compiler/fory_compiler/ir/ast.py @@ -166,6 +166,8 @@ class Message: line: int = 0 column: int = 0 location: Optional[SourceLocation] = None + id_generated: bool = False + id_source: Optional[str] = None def __repr__(self) -> str: id_str = f" [id={self.type_id}]" if self.type_id is not None else "" @@ -204,6 +206,8 @@ class Enum: line: int = 0 column: int = 0 location: Optional[SourceLocation] = None + id_generated: bool = False + id_source: Optional[str] = None def __repr__(self) -> str: id_str = f" [id={self.type_id}]" if self.type_id is not None else "" @@ -222,6 +226,8 @@ class Union: line: int = 0 column: int = 0 location: Optional[SourceLocation] = None + id_generated: bool = False + id_source: Optional[str] = None def __repr__(self) -> str: id_str = f" [id={self.type_id}]" if self.type_id is not None else "" @@ -234,6 +240,7 @@ class Schema: """The root AST node representing a complete FDL file.""" package: Optional[str] + package_alias: Optional[str] = None imports: List[Import] = field(default_factory=list) enums: List[Enum] = field(default_factory=list) messages: List[Message] = field(default_factory=list) @@ -246,7 +253,11 @@ class Schema: def __repr__(self) -> str: opts = f", options={len(self.options)}" if self.options else "" - return f"Schema(package={self.package}, imports={len(self.imports)}, enums={len(self.enums)}, messages={len(self.messages)}, unions={len(self.unions)}{opts})" + alias = f", package_alias={self.package_alias}" if self.package_alias else "" + return ( + f"Schema(package={self.package}{alias}, imports={len(self.imports)}, " + f"enums={len(self.enums)}, messages={len(self.messages)}, unions={len(self.unions)}{opts})" + ) def get_option(self, name: str, default: Optional[str] = None) -> Optional[str]: """Get a file-level option value.""" diff --git a/compiler/fory_compiler/ir/type_id.py b/compiler/fory_compiler/ir/type_id.py new file mode 100644 index 0000000000..ed63a54e5a --- /dev/null +++ b/compiler/fory_compiler/ir/type_id.py @@ -0,0 +1,69 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Type ID helpers for the compiler.""" + +from __future__ import annotations + + +def murmurhash3_32(data: bytes, seed: int = 0) -> int: + """Compute MurmurHash3 x86 32-bit hash for bytes.""" + length = len(data) + nblocks = length // 4 + + h1 = seed & 0xFFFFFFFF + c1 = 0xCC9E2D51 + c2 = 0x1B873593 + + for block_index in range(nblocks): + start = block_index * 4 + k1 = int.from_bytes(data[start : start + 4], "little") + + k1 = (k1 * c1) & 0xFFFFFFFF + k1 = ((k1 << 15) | (k1 >> 17)) & 0xFFFFFFFF + k1 = (k1 * c2) & 0xFFFFFFFF + + h1 ^= k1 + h1 = ((h1 << 13) | (h1 >> 19)) & 0xFFFFFFFF + h1 = (h1 * 5 + 0xE6546B64) & 0xFFFFFFFF + + tail = data[nblocks * 4 :] + k1 = 0 + if len(tail) == 3: + k1 ^= tail[2] << 16 + if len(tail) >= 2: + k1 ^= tail[1] << 8 + if len(tail) >= 1: + k1 ^= tail[0] + k1 = (k1 * c1) & 0xFFFFFFFF + k1 = ((k1 << 15) | (k1 >> 17)) & 0xFFFFFFFF + k1 = (k1 * c2) & 0xFFFFFFFF + h1 ^= k1 + + h1 ^= length + h1 ^= h1 >> 16 + h1 = (h1 * 0x85EBCA6B) & 0xFFFFFFFF + h1 ^= h1 >> 13 + h1 = (h1 * 0xC2B2AE35) & 0xFFFFFFFF + h1 ^= h1 >> 16 + + return h1 + + +def compute_registered_type_id(full_name: str) -> int: + """Compute a registered type ID for a qualified name.""" + return murmurhash3_32(full_name.encode("utf-8")) diff --git a/compiler/fory_compiler/ir/validator.py b/compiler/fory_compiler/ir/validator.py index 0f0683147b..0a456ca5f6 100644 --- a/compiler/fory_compiler/ir/validator.py +++ b/compiler/fory_compiler/ir/validator.py @@ -34,6 +34,7 @@ SourceLocation, ) from fory_compiler.ir.types import PrimitiveKind +from fory_compiler.ir.type_id import compute_registered_type_id @dataclass @@ -60,6 +61,7 @@ def __init__(self, schema: Schema): def validate(self) -> bool: self._apply_field_defaults() + self._apply_type_id_defaults() self._check_duplicate_type_names() self._check_duplicate_type_ids() self._check_messages() @@ -71,6 +73,75 @@ def validate(self) -> bool: def _error(self, message: str, location: Optional[SourceLocation]) -> None: self.errors.append(ValidationIssue(message, location, "error")) + def _warning(self, message: str, location: Optional[SourceLocation]) -> None: + self.warnings.append(ValidationIssue(message, location, "warning")) + + def _apply_type_id_defaults(self) -> None: + enable_auto_type_id = self.schema.get_option("enable_auto_type_id", True) + if enable_auto_type_id is False: + return + used_ids = {} + for t in self.schema.get_all_types(): + if t.type_id is not None: + used_ids[t.type_id] = t + + def qualify(full_name: str) -> str: + package = self.schema.package_alias or self.schema.package + if package: + return f"{package}.{full_name}" + return full_name + + def resolve_hash_source(full_name: str, alias: Optional[str]) -> str: + if not alias: + return qualify(full_name) + if "." in alias: + return alias + package = self.schema.package_alias or self.schema.package + if package: + return f"{package}.{alias}" + return alias + + def assign_id(type_def, full_name: str) -> None: + if type_def.type_id is not None: + return + alias = type_def.options.get("alias") + source_name = resolve_hash_source(full_name, alias) + generated_id = compute_registered_type_id(source_name) + if generated_id in used_ids: + self._error( + ( + "Auto-generated type id collision for " + f"'{source_name}'. Specify an explicit [id=...] or " + 'use [alias="..."] to change the hash source.' + ), + type_def.location, + ) + return + type_def.type_id = generated_id + type_def.id_generated = True + type_def.id_source = source_name + used_ids[generated_id] = type_def + # Do not emit warnings for generated ids. + + def walk_message(message: Message, parent_path: str = "") -> None: + full_name = f"{parent_path}.{message.name}" if parent_path else message.name + assign_id(message, full_name) + for nested_enum in message.nested_enums: + nested_name = f"{full_name}.{nested_enum.name}" + assign_id(nested_enum, nested_name) + for nested_union in message.nested_unions: + nested_name = f"{full_name}.{nested_union.name}" + assign_id(nested_union, nested_name) + for nested_msg in message.nested_messages: + walk_message(nested_msg, full_name) + + for enum in self.schema.enums: + assign_id(enum, enum.name) + for union in self.schema.unions: + assign_id(union, union.name) + for message in self.schema.messages: + walk_message(message) + def _check_duplicate_type_names(self) -> None: names = {} for enum in self.schema.enums: diff --git a/compiler/fory_compiler/tests/test_auto_id.py b/compiler/fory_compiler/tests/test_auto_id.py new file mode 100644 index 0000000000..cee482c5c4 --- /dev/null +++ b/compiler/fory_compiler/tests/test_auto_id.py @@ -0,0 +1,202 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests for auto-generated type IDs.""" + +from fory_compiler.frontend.fdl.lexer import Lexer +from fory_compiler.frontend.fdl.parser import Parser +from fory_compiler.ir.type_id import compute_registered_type_id +from fory_compiler.ir.validator import SchemaValidator + + +def parse_schema(source: str): + lexer = Lexer(source) + parser = Parser(lexer.tokenize()) + return parser.parse() + + +def test_auto_id_generation_for_message_and_union(): + source = """ + package demo; + + message User { + string name = 1; + } + + union Item { + User user = 1; + string note = 2; + } + """ + schema = parse_schema(source) + validator = SchemaValidator(schema) + assert validator.validate() + + msg = schema.messages[0] + union = schema.unions[0] + + assert msg.type_id == compute_registered_type_id("demo.User") + assert msg.id_generated is True + assert msg.id_source == "demo.User" + + assert union.type_id == compute_registered_type_id("demo.Item") + assert union.id_generated is True + assert union.id_source == "demo.Item" + + +def test_alias_used_for_auto_id(): + source = """ + package demo; + + message User [alias="PersonAlias"] { + string name = 1; + } + """ + schema = parse_schema(source) + validator = SchemaValidator(schema) + assert validator.validate() + + msg = schema.messages[0] + assert msg.type_id == compute_registered_type_id("demo.PersonAlias") + assert msg.id_generated is True + assert msg.id_source == "demo.PersonAlias" + + +def test_package_alias_used_for_auto_id(): + source = """ + package demo alias alias_demo; + + message User { + string name = 1; + } + """ + schema = parse_schema(source) + validator = SchemaValidator(schema) + assert validator.validate() + + msg = schema.messages[0] + assert msg.type_id == compute_registered_type_id("alias_demo.User") + assert msg.id_generated is True + assert msg.id_source == "alias_demo.User" + + +def test_package_and_type_alias_used_for_auto_id(): + source = """ + package demo alias alias_demo; + + message User [alias="PersonAlias"] { + string name = 1; + } + """ + schema = parse_schema(source) + validator = SchemaValidator(schema) + assert validator.validate() + + msg = schema.messages[0] + assert msg.type_id == compute_registered_type_id("alias_demo.PersonAlias") + assert msg.id_generated is True + assert msg.id_source == "alias_demo.PersonAlias" + + +def test_auto_id_generation_for_enum(): + source = """ + package demo; + + enum Status { + OK = 0; + ERROR = 1; + } + """ + schema = parse_schema(source) + validator = SchemaValidator(schema) + assert validator.validate() + + enum = schema.enums[0] + assert enum.type_id == compute_registered_type_id("demo.Status") + assert enum.id_generated is True + assert enum.id_source == "demo.Status" + + +def test_auto_id_disabled_keeps_name_registration(): + source = """ + option enable_auto_type_id = false; + package demo; + + message User { + string name = 1; + } + + union Item { + User user = 1; + string note = 2; + } + + enum Status { + OK = 0; + ERROR = 1; + } + """ + schema = parse_schema(source) + validator = SchemaValidator(schema) + assert validator.validate() + + msg = schema.messages[0] + union = schema.unions[0] + enum = schema.enums[0] + + assert msg.type_id is None + assert union.type_id is None + assert enum.type_id is None + + assert msg.id_generated is False + assert union.id_generated is False + assert enum.id_generated is False + + +def test_explicit_id_not_overwritten(): + source = """ + package demo; + + message User [id=100] { + string name = 1; + } + """ + schema = parse_schema(source) + validator = SchemaValidator(schema) + assert validator.validate() + + msg = schema.messages[0] + assert msg.type_id == 100 + assert msg.id_generated is False + + +def test_auto_id_conflict_requires_explicit_resolution(): + source = """ + package demo; + + message First [alias="Shared"] { + string name = 1; + } + + message Second [alias="Shared"] { + string name = 1; + } + """ + schema = parse_schema(source) + validator = SchemaValidator(schema) + assert validator.validate() is False + assert validator.errors diff --git a/compiler/fory_compiler/tests/test_proto_frontend.py b/compiler/fory_compiler/tests/test_proto_frontend.py index 0274e6bd18..cf32816db7 100644 --- a/compiler/fory_compiler/tests/test_proto_frontend.py +++ b/compiler/fory_compiler/tests/test_proto_frontend.py @@ -69,3 +69,18 @@ def test_proto_oneof_translation(): payload_field = [f for f in event.fields if f.name == "payload"][0] assert payload_field.optional is True assert payload_field.field_type.name == "payload" + + +def test_proto_file_option_enable_auto_type_id(): + source = """ + syntax = "proto3"; + package demo; + + option (fory).enable_auto_type_id = false; + + message User { + string name = 1; + } + """ + schema = ProtoFrontend().parse(source) + assert schema.get_option("enable_auto_type_id") is False diff --git a/cpp/fory/meta/meta_string.cc b/cpp/fory/meta/meta_string.cc index 5ca868feed..a548558d47 100644 --- a/cpp/fory/meta/meta_string.cc +++ b/cpp/fory/meta/meta_string.cc @@ -282,19 +282,16 @@ MetaStringTable::read_string(Buffer &buffer, const MetaStringDecoder &decoder) { } encoding = MetaEncoding::UTF8; } else { - // Small string layout: encoding(byte) + data[len] - int8_t enc_byte_res = buffer.read_int8(error); - if (FORY_PREDICT_FALSE(!error.ok())) { - return Unexpected(std::move(error)); - } - uint8_t enc_byte = static_cast(enc_byte_res); + // Small string layout: data[len] with an encoding byte when len > 0. + // Java omits the encoding byte for empty strings. if (len == 0) { - if (enc_byte != 0) { - return Unexpected( - Error::encoding_error("Empty meta string must use UTF8 encoding")); - } encoding = MetaEncoding::UTF8; } else { + int8_t enc_byte_res = buffer.read_int8(error); + if (FORY_PREDICT_FALSE(!error.ok())) { + return Unexpected(std::move(error)); + } + uint8_t enc_byte = static_cast(enc_byte_res); FORY_TRY(enc, to_meta_encoding(enc_byte)); encoding = enc; bytes.resize(len); diff --git a/cpp/fory/meta/meta_string_test.cc b/cpp/fory/meta/meta_string_test.cc index 993c9592f1..2209c5e77a 100644 --- a/cpp/fory/meta/meta_string_test.cc +++ b/cpp/fory/meta/meta_string_test.cc @@ -404,10 +404,9 @@ TEST_F(MetaStringTest, MetaStringTableEmptyString) { MetaStringTable table; Buffer buffer; - // Empty string: len=0, encoding=UTF8 + // Empty string: len=0 (Java omits encoding byte) uint32_t header = 0 << 1; // len=0, not a reference buffer.write_var_uint32(header); - buffer.write_int8(0); // UTF8 encoding buffer.reader_index(0); auto result = table.read_string(buffer, decoder_); diff --git a/cpp/fory/serialization/any_serializer.h b/cpp/fory/serialization/any_serializer.h index 7a10dd6a10..c350709310 100644 --- a/cpp/fory/serialization/any_serializer.h +++ b/cpp/fory/serialization/any_serializer.h @@ -81,10 +81,7 @@ template <> struct Serializer { if (write_type) { uint32_t fory_type_id = type_info->type_id; - uint32_t type_id_arg = is_internal_type(fory_type_id) - ? fory_type_id - : static_cast(TypeId::UNKNOWN); - auto write_res = ctx.write_any_typeinfo(type_id_arg, concrete_type_id); + auto write_res = ctx.write_any_type_info(fory_type_id, concrete_type_id); if (FORY_PREDICT_FALSE(!write_res.ok())) { ctx.set_error(std::move(write_res).error()); return; @@ -115,7 +112,7 @@ template <> struct Serializer { return std::any(); } - const TypeInfo *type_info = ctx.read_any_typeinfo(ctx.error()); + const TypeInfo *type_info = ctx.read_any_type_info(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return std::any(); } diff --git a/cpp/fory/serialization/array_serializer.h b/cpp/fory/serialization/array_serializer.h index 54713e70e0..388fa8de50 100644 --- a/cpp/fory/serialization/array_serializer.h +++ b/cpp/fory/serialization/array_serializer.h @@ -61,11 +61,11 @@ struct Serializer< }(); static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - uint32_t actual = ctx.read_var_uint32(ctx.error()); + uint32_t actual = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } @@ -80,7 +80,7 @@ struct Serializer< bool has_generics = false) { write_not_null_ref_flag(ctx, ref_mode); if (write_type) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } write_data_generic(arr, ctx, has_generics); } @@ -125,7 +125,7 @@ struct Serializer< return std::array(); } if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return std::array(); } @@ -188,11 +188,11 @@ template struct Serializer> { static constexpr TypeId type_id = TypeId::BOOL_ARRAY; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - uint32_t actual = ctx.read_var_uint32(ctx.error()); + uint32_t actual = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } @@ -207,7 +207,7 @@ template struct Serializer> { bool has_generics = false) { write_not_null_ref_flag(ctx, ref_mode); if (write_type) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } write_data_generic(arr, ctx, has_generics); } @@ -243,7 +243,7 @@ template struct Serializer> { return std::array(); } if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return std::array(); } diff --git a/cpp/fory/serialization/basic_serializer.h b/cpp/fory/serialization/basic_serializer.h index 8ce28f9189..7d72990dab 100644 --- a/cpp/fory/serialization/basic_serializer.h +++ b/cpp/fory/serialization/basic_serializer.h @@ -43,11 +43,11 @@ template <> struct Serializer { static constexpr TypeId type_id = TypeId::BOOL; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - uint32_t actual = ctx.read_var_uint32(ctx.error()); + uint32_t actual = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } @@ -61,7 +61,7 @@ template <> struct Serializer { bool write_type, bool = false) { write_not_null_ref_flag(ctx, ref_mode); if (write_type) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } write_data(value, ctx); } @@ -80,7 +80,7 @@ template <> struct Serializer { return false; } if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return false; } @@ -114,11 +114,11 @@ template <> struct Serializer { static constexpr TypeId type_id = TypeId::INT8; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - uint32_t actual = ctx.read_var_uint32(ctx.error()); + uint32_t actual = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } @@ -132,7 +132,7 @@ template <> struct Serializer { bool write_type, bool = false) { write_not_null_ref_flag(ctx, ref_mode); if (write_type) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } write_data(value, ctx); } @@ -152,7 +152,7 @@ template <> struct Serializer { return 0; } if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return 0; } @@ -184,11 +184,11 @@ template <> struct Serializer { static constexpr TypeId type_id = TypeId::INT16; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - uint32_t actual = ctx.read_var_uint32(ctx.error()); + uint32_t actual = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } @@ -202,7 +202,7 @@ template <> struct Serializer { bool write_type, bool = false) { write_not_null_ref_flag(ctx, ref_mode); if (write_type) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } write_data(value, ctx); } @@ -223,7 +223,7 @@ template <> struct Serializer { return 0; } if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return 0; } @@ -255,11 +255,11 @@ template <> struct Serializer { static constexpr TypeId type_id = TypeId::VARINT32; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - uint32_t actual = ctx.read_var_uint32(ctx.error()); + uint32_t actual = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } @@ -273,7 +273,7 @@ template <> struct Serializer { bool write_type, bool = false) { write_not_null_ref_flag(ctx, ref_mode); if (write_type) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } write_data(value, ctx); } @@ -294,7 +294,7 @@ template <> struct Serializer { return 0; } if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return 0; } @@ -326,11 +326,11 @@ template <> struct Serializer { static constexpr TypeId type_id = TypeId::VARINT64; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - uint32_t actual = ctx.read_var_uint32(ctx.error()); + uint32_t actual = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } @@ -344,7 +344,7 @@ template <> struct Serializer { bool write_type, bool = false) { write_not_null_ref_flag(ctx, ref_mode); if (write_type) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } write_data(value, ctx); } @@ -365,7 +365,7 @@ template <> struct Serializer { return 0; } if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return 0; } @@ -397,11 +397,11 @@ template <> struct Serializer { static constexpr TypeId type_id = TypeId::FLOAT32; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - uint32_t actual = ctx.read_var_uint32(ctx.error()); + uint32_t actual = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } @@ -415,7 +415,7 @@ template <> struct Serializer { bool write_type, bool = false) { write_not_null_ref_flag(ctx, ref_mode); if (write_type) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } write_data(value, ctx); } @@ -434,7 +434,7 @@ template <> struct Serializer { return 0.0f; } if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return 0.0f; } @@ -466,11 +466,11 @@ template <> struct Serializer { static constexpr TypeId type_id = TypeId::FLOAT64; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - uint32_t actual = ctx.read_var_uint32(ctx.error()); + uint32_t actual = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } @@ -484,7 +484,7 @@ template <> struct Serializer { bool write_type, bool = false) { write_not_null_ref_flag(ctx, ref_mode); if (write_type) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } write_data(value, ctx); } @@ -504,7 +504,7 @@ template <> struct Serializer { return 0.0; } if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return 0.0; } @@ -540,11 +540,11 @@ template <> struct Serializer { static constexpr TypeId type_id = TypeId::CHAR; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - uint32_t actual = ctx.read_var_uint32(ctx.error()); + uint32_t actual = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } @@ -558,7 +558,7 @@ template <> struct Serializer { bool write_type, bool = false) { write_not_null_ref_flag(ctx, ref_mode); if (write_type) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } write_data(value, ctx); } @@ -577,7 +577,7 @@ template <> struct Serializer { return '\0'; } if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return '\0'; } @@ -609,11 +609,11 @@ template <> struct Serializer { static constexpr TypeId type_id = TypeId::CHAR16; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - uint32_t actual = ctx.read_var_uint32(ctx.error()); + uint32_t actual = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } @@ -627,7 +627,7 @@ template <> struct Serializer { bool write_type, bool = false) { write_not_null_ref_flag(ctx, ref_mode); if (write_type) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } write_data(value, ctx); } @@ -648,7 +648,7 @@ template <> struct Serializer { return u'\0'; } if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return u'\0'; } @@ -683,11 +683,11 @@ template <> struct Serializer { static constexpr TypeId type_id = TypeId::CHAR32; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - uint32_t actual = ctx.read_var_uint32(ctx.error()); + uint32_t actual = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } @@ -701,7 +701,7 @@ template <> struct Serializer { bool write_type, bool = false) { write_not_null_ref_flag(ctx, ref_mode); if (write_type) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } write_data(value, ctx); } @@ -722,7 +722,7 @@ template <> struct Serializer { return U'\0'; } if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return U'\0'; } diff --git a/cpp/fory/serialization/collection_serializer.h b/cpp/fory/serialization/collection_serializer.h index 2a793a2ac2..a780d1c0c1 100644 --- a/cpp/fory/serialization/collection_serializer.h +++ b/cpp/fory/serialization/collection_serializer.h @@ -284,8 +284,17 @@ inline void write_collection_data_slow(const Container &coll, WriteContext &ctx, if (is_same_type && !(bitmap & COLL_DECL_ELEMENT_TYPE)) { if constexpr (elem_is_polymorphic) { // write concrete type info for polymorphic elements - ctx.write_any_typeinfo(static_cast(TypeId::UNKNOWN), - first_type); + auto type_info_res = ctx.type_resolver().get_type_info(first_type); + if (FORY_PREDICT_FALSE(!type_info_res.ok())) { + ctx.set_error(std::move(type_info_res).error()); + return; + } + auto write_res = + ctx.write_any_type_info(type_info_res.value()->type_id, first_type); + if (FORY_PREDICT_FALSE(!write_res.ok())) { + ctx.set_error(std::move(write_res).error()); + return; + } } else { Serializer::write_type_info(ctx); } @@ -407,7 +416,7 @@ inline Container read_collection_data_slow(ReadContext &ctx, uint32_t length) { // Read element type info if IS_SAME_TYPE && !IS_DECL_ELEMENT_TYPE const TypeInfo *elem_type_info = nullptr; if (is_same_type && !is_decl_type) { - elem_type_info = ctx.read_any_typeinfo(ctx.error()); + elem_type_info = ctx.read_any_type_info(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return result; } @@ -520,11 +529,11 @@ struct Serializer< }(); static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - uint32_t actual = ctx.read_var_uint32(ctx.error()); + uint32_t actual = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } @@ -539,7 +548,7 @@ struct Serializer< bool has_generics = false) { write_not_null_ref_flag(ctx, ref_mode); if (write_type) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } write_data(vec, ctx); } @@ -578,7 +587,7 @@ struct Serializer< } if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return std::vector(); } @@ -628,11 +637,11 @@ struct Serializer< static constexpr TypeId type_id = TypeId::LIST; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - uint32_t actual = ctx.read_var_uint32(ctx.error()); + uint32_t actual = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } @@ -652,12 +661,11 @@ struct Serializer< // Optional type info for polymorphic containers if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return std::vector(); } - uint32_t low = type_id_read & 0xffu; - if (low != static_cast(type_id)) { + if (type_id_read != static_cast(type_id)) { ctx.set_error( Error::type_mismatch(type_id_read, static_cast(type_id))); return std::vector(); @@ -694,7 +702,7 @@ struct Serializer< // Read element type info if IS_SAME_TYPE is set but IS_DECL_ELEMENT_TYPE // is not. if (is_same_type && !is_decl_type) { - const TypeInfo *elem_type_info = ctx.read_any_typeinfo(ctx.error()); + const TypeInfo *elem_type_info = ctx.read_any_type_info(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return std::vector(); } @@ -763,7 +771,7 @@ struct Serializer< // write type info if requested if (write_type) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } write_data_generic(vec, ctx, has_generics); @@ -818,11 +826,11 @@ template struct Serializer> { static constexpr TypeId type_id = TypeId::BOOL_ARRAY; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - uint32_t actual = ctx.read_var_uint32(ctx.error()); + uint32_t actual = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } @@ -837,7 +845,7 @@ template struct Serializer> { bool has_generics = false) { write_not_null_ref_flag(ctx, ref_mode); if (write_type) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } write_data(vec, ctx); } @@ -871,7 +879,7 @@ template struct Serializer> { } if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return std::vector(); } @@ -920,11 +928,11 @@ template struct Serializer> { static constexpr TypeId type_id = TypeId::LIST; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - uint32_t actual = ctx.read_var_uint32(ctx.error()); + uint32_t actual = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } @@ -944,12 +952,11 @@ template struct Serializer> { // Optional type info for polymorphic containers if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return std::list(); } - uint32_t low = type_id_read & 0xffu; - if (low != static_cast(type_id)) { + if (type_id_read != static_cast(type_id)) { ctx.set_error( Error::type_mismatch(type_id_read, static_cast(type_id))); return std::list(); @@ -986,7 +993,7 @@ template struct Serializer> { // Read element type info if IS_SAME_TYPE is set but IS_DECL_ELEMENT_TYPE // is not. if (is_same_type && !is_decl_type) { - const TypeInfo *elem_type_info = ctx.read_any_typeinfo(ctx.error()); + const TypeInfo *elem_type_info = ctx.read_any_type_info(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return std::list(); } @@ -1054,7 +1061,7 @@ template struct Serializer> { // write type info if requested if (write_type) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } write_data_generic(lst, ctx, has_generics); @@ -1111,11 +1118,11 @@ template struct Serializer> { static constexpr TypeId type_id = TypeId::LIST; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - uint32_t actual = ctx.read_var_uint32(ctx.error()); + uint32_t actual = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } @@ -1135,12 +1142,11 @@ template struct Serializer> { // Optional type info for polymorphic containers if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return std::deque(); } - uint32_t low = type_id_read & 0xffu; - if (low != static_cast(type_id)) { + if (type_id_read != static_cast(type_id)) { ctx.set_error( Error::type_mismatch(type_id_read, static_cast(type_id))); return std::deque(); @@ -1177,7 +1183,7 @@ template struct Serializer> { // Read element type info if IS_SAME_TYPE is set but IS_DECL_ELEMENT_TYPE // is not. if (is_same_type && !is_decl_type) { - const TypeInfo *elem_type_info = ctx.read_any_typeinfo(ctx.error()); + const TypeInfo *elem_type_info = ctx.read_any_type_info(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return std::deque(); } @@ -1245,7 +1251,7 @@ template struct Serializer> { // write type info if requested if (write_type) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } write_data_generic(deq, ctx, has_generics); @@ -1303,11 +1309,11 @@ struct Serializer> { static constexpr TypeId type_id = TypeId::LIST; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - uint32_t actual = ctx.read_var_uint32(ctx.error()); + uint32_t actual = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } @@ -1327,12 +1333,11 @@ struct Serializer> { // Optional type info for polymorphic containers if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return std::forward_list(); } - uint32_t low = type_id_read & 0xffu; - if (low != static_cast(type_id)) { + if (type_id_read != static_cast(type_id)) { ctx.set_error( Error::type_mismatch(type_id_read, static_cast(type_id))); return std::forward_list(); @@ -1374,7 +1379,7 @@ struct Serializer> { // Read element type info if IS_SAME_TYPE is set but IS_DECL_ELEMENT_TYPE // is not. if (is_same_type && !is_decl_type) { - const TypeInfo *elem_type_info = ctx.read_any_typeinfo(ctx.error()); + const TypeInfo *elem_type_info = ctx.read_any_type_info(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return std::forward_list(); } @@ -1444,7 +1449,7 @@ struct Serializer> { // write type info if requested if (write_type) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } write_data_generic(lst, ctx, has_generics); @@ -1629,8 +1634,17 @@ struct Serializer> { // write element type info if IS_SAME_TYPE && !IS_DECL_ELEMENT_TYPE if (is_same_type && !(bitmap & COLL_DECL_ELEMENT_TYPE)) { if constexpr (elem_is_polymorphic) { - ctx.write_any_typeinfo(static_cast(TypeId::UNKNOWN), - first_type); + auto type_info_res = ctx.type_resolver().get_type_info(first_type); + if (FORY_PREDICT_FALSE(!type_info_res.ok())) { + ctx.set_error(std::move(type_info_res).error()); + return; + } + auto write_res = ctx.write_any_type_info( + type_info_res.value()->type_id, first_type); + if (FORY_PREDICT_FALSE(!write_res.ok())) { + ctx.set_error(std::move(write_res).error()); + return; + } } else { Serializer::write_type_info(ctx); } @@ -1726,11 +1740,11 @@ struct Serializer> { static constexpr TypeId type_id = TypeId::SET; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - uint32_t actual = ctx.read_var_uint32(ctx.error()); + uint32_t actual = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } @@ -1746,7 +1760,7 @@ struct Serializer> { write_not_null_ref_flag(ctx, ref_mode); if (write_type) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } write_data_generic(set, ctx, has_generics); @@ -1781,7 +1795,7 @@ struct Serializer> { // Read type info if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return std::set(); } @@ -1820,7 +1834,7 @@ struct Serializer> { bool is_same_type = (bitmap & COLL_IS_SAME_TYPE) != 0; if (is_same_type && !is_decl_type) { - const TypeInfo *elem_type_info = ctx.read_any_typeinfo(ctx.error()); + const TypeInfo *elem_type_info = ctx.read_any_type_info(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return std::set(); } @@ -1898,11 +1912,11 @@ struct Serializer> { static constexpr TypeId type_id = TypeId::SET; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - uint32_t actual = ctx.read_var_uint32(ctx.error()); + uint32_t actual = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } @@ -1918,7 +1932,7 @@ struct Serializer> { write_not_null_ref_flag(ctx, ref_mode); if (write_type) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } write_data_generic(set, ctx, has_generics); @@ -1954,7 +1968,7 @@ struct Serializer> { // Read type info if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return std::unordered_set(); } @@ -1995,7 +2009,7 @@ struct Serializer> { bool is_same_type = (bitmap & COLL_IS_SAME_TYPE) != 0; if (is_same_type && !is_decl_type) { - const TypeInfo *elem_type_info = ctx.read_any_typeinfo(ctx.error()); + const TypeInfo *elem_type_info = ctx.read_any_type_info(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return std::unordered_set(); } diff --git a/cpp/fory/serialization/collection_serializer_test.cc b/cpp/fory/serialization/collection_serializer_test.cc index d664510792..0394ff2566 100644 --- a/cpp/fory/serialization/collection_serializer_test.cc +++ b/cpp/fory/serialization/collection_serializer_test.cc @@ -74,6 +74,7 @@ Fory create_fory() { void register_types(Fory &fory) { fory.register_struct(100); fory.register_struct(101); + fory.register_struct("test", "Animal"); fory.register_struct("test", "Dog"); fory.register_struct("test", "Cat"); } diff --git a/cpp/fory/serialization/context.cc b/cpp/fory/serialization/context.cc index 90d6824fcc..5cc78b7a28 100644 --- a/cpp/fory/serialization/context.cc +++ b/cpp/fory/serialization/context.cc @@ -101,7 +101,7 @@ static void write_encoded_meta_string(Buffer &buffer, if (encoded_len > k_small_string_threshold) { // For large strings, write pre-computed hash buffer.write_int64(encoded.hash); - } else { + } else if (encoded_len > 0) { // For small strings, write encoding byte buffer.write_int8(static_cast(encoded.encoding)); } @@ -112,14 +112,16 @@ static void write_encoded_meta_string(Buffer &buffer, } Result -WriteContext::write_enum_typeinfo(const std::type_index &type) { +WriteContext::write_enum_type_info(const std::type_index &type) { FORY_TRY(type_info, type_resolver_->get_type_info(type)); uint32_t type_id = type_info->type_id; - uint32_t type_id_low = type_id & 0xff; - - buffer_.write_var_uint32(type_id); - - if (type_id_low == static_cast(TypeId::NAMED_ENUM)) { + buffer_.write_uint8(static_cast(type_id)); + if (type_id == static_cast(TypeId::ENUM)) { + if (type_info->user_type_id == kInvalidUserTypeId) { + return Unexpected(Error::type_error("User type id is required for enum")); + } + buffer_.write_var_uint32(type_info->user_type_id); + } else if (type_id == static_cast(TypeId::NAMED_ENUM)) { if (config_->compatible) { // write type meta inline using streaming protocol FORY_RETURN_NOT_OK(write_type_meta(type)); @@ -140,17 +142,20 @@ WriteContext::write_enum_typeinfo(const std::type_index &type) { } Result -WriteContext::write_enum_typeinfo(const TypeInfo *type_info) { +WriteContext::write_enum_type_info(const TypeInfo *type_info) { if (!type_info) { return Unexpected(Error::type_error("Enum type not registered")); } uint32_t type_id = type_info->type_id; - uint32_t type_id_low = type_id & 0xff; - - buffer_.write_var_uint32(type_id); - if (type_id_low == static_cast(TypeId::NAMED_ENUM)) { + buffer_.write_uint8(static_cast(type_id)); + if (type_id == static_cast(TypeId::ENUM)) { + if (type_info->user_type_id == kInvalidUserTypeId) { + return Unexpected(Error::type_error("User type id is required for enum")); + } + buffer_.write_var_uint32(type_info->user_type_id); + } else if (type_id == static_cast(TypeId::NAMED_ENUM)) { if (config_->compatible) { // write type meta inline using streaming protocol write_type_meta(type_info); @@ -171,11 +176,12 @@ WriteContext::write_enum_typeinfo(const TypeInfo *type_info) { } Result -WriteContext::write_any_typeinfo(uint32_t fory_type_id, - const std::type_index &concrete_type_id) { +WriteContext::write_any_type_info(uint32_t fory_type_id, + const std::type_index &concrete_type_id) { // Check if it's an internal type if (is_internal_type(fory_type_id)) { - buffer_.write_var_uint32(fory_type_id); + // write type_id + buffer_.write_uint8(static_cast(fory_type_id)); FORY_TRY(type_info, type_resolver_->get_type_info_by_id(fory_type_id)); return type_info; } @@ -185,21 +191,25 @@ WriteContext::write_any_typeinfo(uint32_t fory_type_id, uint32_t type_id = type_info->type_id; // write type_id - buffer_.write_var_uint32(type_id); + buffer_.write_uint8(static_cast(type_id)); // Handle different type categories based on low byte - uint32_t type_id_low = type_id & 0xff; - switch (type_id_low) { - case static_cast(TypeId::NAMED_COMPATIBLE_STRUCT): - case static_cast(TypeId::COMPATIBLE_STRUCT): { + switch (static_cast(type_id)) { + case TypeId::ENUM: + case TypeId::STRUCT: + case TypeId::EXT: + case TypeId::TYPED_UNION: + buffer_.write_var_uint32(type_info->user_type_id); + break; + case TypeId::COMPATIBLE_STRUCT: + case TypeId::NAMED_COMPATIBLE_STRUCT: // write type meta inline using streaming protocol FORY_RETURN_NOT_OK(write_type_meta(concrete_type_id)); break; - } - case static_cast(TypeId::NAMED_ENUM): - case static_cast(TypeId::NAMED_EXT): - case static_cast(TypeId::NAMED_STRUCT): - case static_cast(TypeId::NAMED_UNION): { + case TypeId::NAMED_ENUM: + case TypeId::NAMED_EXT: + case TypeId::NAMED_STRUCT: + case TypeId::NAMED_UNION: if (config_->compatible) { // write type meta inline using streaming protocol FORY_RETURN_NOT_OK(write_type_meta(concrete_type_id)); @@ -214,7 +224,6 @@ WriteContext::write_any_typeinfo(uint32_t fory_type_id, } } break; - } default: // For other types, just writing type_id is sufficient break; @@ -223,6 +232,52 @@ WriteContext::write_any_typeinfo(uint32_t fory_type_id, return type_info; } +Result +WriteContext::write_any_type_info(const TypeInfo *type_info) { + if (FORY_PREDICT_FALSE(type_info == nullptr)) { + return Unexpected(Error::invalid("TypeInfo is null")); + } + uint32_t type_id = type_info->type_id; + buffer_.write_uint8(static_cast(type_id)); + + switch (static_cast(type_id)) { + case TypeId::ENUM: + case TypeId::STRUCT: + case TypeId::EXT: + case TypeId::TYPED_UNION: + buffer_.write_var_uint32(type_info->user_type_id); + break; + case TypeId::COMPATIBLE_STRUCT: + case TypeId::NAMED_COMPATIBLE_STRUCT: + // write type meta inline using streaming protocol + write_type_meta(type_info); + break; + case TypeId::NAMED_ENUM: + case TypeId::NAMED_EXT: + case TypeId::NAMED_STRUCT: + case TypeId::NAMED_UNION: + if (config_->compatible) { + // write type meta inline using streaming protocol + write_type_meta(type_info); + } else { + // write pre-encoded namespace and type_name + if (type_info->encoded_namespace && type_info->encoded_type_name) { + write_encoded_meta_string(buffer_, *type_info->encoded_namespace); + write_encoded_meta_string(buffer_, *type_info->encoded_type_name); + } else { + return Unexpected( + Error::invalid("Encoded meta strings not initialized for type")); + } + } + break; + default: + // For other types, just writing type_id is sufficient + break; + } + + return Result(); +} + Result WriteContext::write_struct_type_info(const std::type_index &type_id) { // get type info with single lookup @@ -230,18 +285,20 @@ WriteContext::write_struct_type_info(const std::type_index &type_id) { uint32_t fory_type_id = type_info->type_id; // write type_id - buffer_.write_var_uint32(fory_type_id); - - // Handle different struct type categories based on low byte - uint32_t type_id_low = fory_type_id & 0xff; - switch (type_id_low) { - case static_cast(TypeId::NAMED_COMPATIBLE_STRUCT): - case static_cast(TypeId::COMPATIBLE_STRUCT): { + buffer_.write_uint8(static_cast(fory_type_id)); + switch (static_cast(fory_type_id)) { + case TypeId::ENUM: + case TypeId::STRUCT: + case TypeId::EXT: + case TypeId::TYPED_UNION: + buffer_.write_var_uint32(type_info->user_type_id); + break; + case TypeId::COMPATIBLE_STRUCT: + case TypeId::NAMED_COMPATIBLE_STRUCT: // write type meta inline using streaming protocol FORY_RETURN_NOT_OK(write_type_meta(type_id)); break; - } - case static_cast(TypeId::NAMED_STRUCT): { + case TypeId::NAMED_STRUCT: if (config_->compatible) { // write type meta inline using streaming protocol FORY_RETURN_NOT_OK(write_type_meta(type_id)); @@ -256,7 +313,6 @@ WriteContext::write_struct_type_info(const std::type_index &type_id) { } } break; - } default: // STRUCT type - just writing type_id is sufficient break; @@ -270,18 +326,20 @@ WriteContext::write_struct_type_info(const TypeInfo *type_info) { uint32_t fory_type_id = type_info->type_id; // write type_id - buffer_.write_var_uint32(fory_type_id); - - // Handle different struct type categories based on low byte - uint32_t type_id_low = fory_type_id & 0xff; - switch (type_id_low) { - case static_cast(TypeId::NAMED_COMPATIBLE_STRUCT): - case static_cast(TypeId::COMPATIBLE_STRUCT): { + buffer_.write_uint8(static_cast(fory_type_id)); + switch (static_cast(fory_type_id)) { + case TypeId::ENUM: + case TypeId::STRUCT: + case TypeId::EXT: + case TypeId::TYPED_UNION: + buffer_.write_var_uint32(type_info->user_type_id); + break; + case TypeId::COMPATIBLE_STRUCT: + case TypeId::NAMED_COMPATIBLE_STRUCT: // write type meta inline using streaming protocol write_type_meta(type_info); break; - } - case static_cast(TypeId::NAMED_STRUCT): { + case TypeId::NAMED_STRUCT: if (config_->compatible) { // write type meta inline using streaming protocol write_type_meta(type_info); @@ -296,7 +354,6 @@ WriteContext::write_struct_type_info(const TypeInfo *type_info) { } } break; - } default: // STRUCT type - just writing type_id is sufficient break; @@ -349,14 +406,31 @@ ReadContext::read_enum_type_info(const std::type_index &type, Result ReadContext::read_enum_type_info(uint32_t base_type_id) { - FORY_TRY(type_info, read_any_typeinfo()); - uint32_t type_id_low = type_info->type_id & 0xff; - // Accept both ENUM and NAMED_ENUM as compatible types - if (type_id_low != static_cast(TypeId::ENUM) && - type_id_low != static_cast(TypeId::NAMED_ENUM)) { - return Unexpected(Error::type_mismatch(type_info->type_id, base_type_id)); + Error error; + uint32_t type_id = buffer_->read_uint8(error); + if (type_id == static_cast(TypeId::ENUM)) { + uint32_t user_type_id = buffer_->read_var_uint32(error); + if (FORY_PREDICT_FALSE(!error.ok())) { + return Unexpected(std::move(error)); + } + FORY_TRY(type_info, + type_resolver_->get_user_type_info_by_id(type_id, user_type_id)); + return type_info; + } else if (type_id == static_cast(TypeId::NAMED_ENUM)) { + if (config_->compatible) { + // Read type meta inline using streaming protocol + return read_type_meta(); + } + FORY_TRY(namespace_str, + meta_string_table_.read_string(*buffer_, k_namespace_decoder)); + FORY_TRY(type_name, + meta_string_table_.read_string(*buffer_, k_type_name_decoder)); + FORY_TRY(type_info, + type_resolver_->get_type_info_by_name(namespace_str, type_name)); + return type_info; } - return type_info; + + return Unexpected(Error::type_mismatch(type_id, base_type_id)); } // Maximum number of parsed type defs to cache (avoid OOM from malicious input) @@ -407,9 +481,17 @@ Result ReadContext::read_type_meta() { local_type_info = result.value(); } } else { - auto result = type_resolver_->get_type_info_by_id(parsed_meta->type_id); - if (result.ok()) { - local_type_info = result.value(); + if (parsed_meta->user_type_id != kInvalidUserTypeId) { + auto result = type_resolver_->get_user_type_info_by_id( + parsed_meta->type_id, parsed_meta->user_type_id); + if (result.ok()) { + local_type_info = result.value(); + } + } else { + auto result = type_resolver_->get_type_info_by_id(parsed_meta->type_id); + if (result.ok()) { + local_type_info = result.value(); + } } } @@ -423,6 +505,7 @@ Result ReadContext::read_type_meta() { parsed_meta->field_infos); } type_info->type_id = local_type_info->type_id; + type_info->user_type_id = local_type_info->user_type_id; type_info->type_meta = std::move(parsed_meta); type_info->type_def = local_type_info->type_def; // CRITICAL: copy the harness from the registered type_info @@ -434,6 +517,7 @@ Result ReadContext::read_type_meta() { } else { // No local type - create stub TypeInfo with parsed meta type_info->type_id = parsed_meta->type_id; + type_info->user_type_id = parsed_meta->user_type_id; type_info->type_meta = std::move(parsed_meta); } @@ -462,25 +546,33 @@ ReadContext::get_type_info_by_index(size_t index) const { return reading_type_infos_[index]; } -Result ReadContext::read_any_typeinfo() { +Result ReadContext::read_any_type_info() { Error error; - uint32_t type_id = buffer_->read_var_uint32(error); + uint32_t type_id = buffer_->read_uint8(error); if (FORY_PREDICT_FALSE(!error.ok())) { return Unexpected(std::move(error)); } - uint32_t type_id_low = type_id & 0xff; - - // Use streaming protocol for type meta - switch (type_id_low) { - case static_cast(TypeId::NAMED_COMPATIBLE_STRUCT): - case static_cast(TypeId::COMPATIBLE_STRUCT): { + switch (static_cast(type_id)) { + case TypeId::ENUM: + case TypeId::STRUCT: + case TypeId::EXT: + case TypeId::TYPED_UNION: { + uint32_t user_type_id = buffer_->read_var_uint32(error); + if (FORY_PREDICT_FALSE(!error.ok())) { + return Unexpected(std::move(error)); + } + FORY_TRY(type_info, + type_resolver_->get_user_type_info_by_id(type_id, user_type_id)); + return type_info; + } + case TypeId::COMPATIBLE_STRUCT: + case TypeId::NAMED_COMPATIBLE_STRUCT: // Read type meta inline using streaming protocol return read_type_meta(); - } - case static_cast(TypeId::NAMED_ENUM): - case static_cast(TypeId::NAMED_EXT): - case static_cast(TypeId::NAMED_STRUCT): - case static_cast(TypeId::NAMED_UNION): { + case TypeId::NAMED_ENUM: + case TypeId::NAMED_EXT: + case TypeId::NAMED_STRUCT: + case TypeId::NAMED_UNION: { if (config_->compatible) { // Read type meta inline using streaming protocol return read_type_meta(); @@ -501,8 +593,8 @@ Result ReadContext::read_any_typeinfo() { } } -const TypeInfo *ReadContext::read_any_typeinfo(Error &error) { - auto result = read_any_typeinfo(); +const TypeInfo *ReadContext::read_any_type_info(Error &error) { + auto result = read_any_type_info(); if (!result.ok()) { error = std::move(result).error(); return nullptr; diff --git a/cpp/fory/serialization/context.h b/cpp/fory/serialization/context.h index 2b0e0b0f84..be11dd2dcd 100644 --- a/cpp/fory/serialization/context.h +++ b/cpp/fory/serialization/context.h @@ -23,6 +23,7 @@ #include "fory/serialization/config.h" #include "fory/serialization/ref_resolver.h" #include "fory/serialization/type_info.h" +#include "fory/type/type.h" #include "fory/util/buffer.h" #include "fory/util/error.h" #include "fory/util/result.h" @@ -254,8 +255,12 @@ class WriteContext { /// @param concrete_type_id The runtime type_index for concrete type /// @return TypeInfo for the written type, or error Result - write_any_typeinfo(uint32_t fory_type_id, - const std::type_index &concrete_type_id); + write_any_type_info(uint32_t fory_type_id, + const std::type_index &concrete_type_id); + + /// write type information using an existing TypeInfo. + /// Avoids extra type lookup when the TypeInfo is already known. + Result write_any_type_info(const TypeInfo *type_info); /// Fast path for writing struct type info - does a single type lookup /// and handles all struct type categories (STRUCT, NAMED_STRUCT, @@ -277,8 +282,22 @@ class WriteContext { /// Only for STRUCT types (not NAMED_STRUCT, COMPATIBLE_STRUCT, etc.) /// /// @param type_id The pre-computed Fory type_id - inline void write_struct_type_id_direct(uint32_t type_id) { - buffer_.write_var_uint32(type_id); + inline void write_struct_type_id_direct(uint32_t type_id, + uint32_t user_type_id) { + buffer_.write_uint8(static_cast(type_id)); + switch (static_cast(type_id)) { + case TypeId::ENUM: + case TypeId::STRUCT: + case TypeId::COMPATIBLE_STRUCT: + case TypeId::EXT: + case TypeId::TYPED_UNION: + FORY_CHECK(user_type_id != kInvalidUserTypeId) + << "User type id is required for struct type"; + buffer_.write_var_uint32(user_type_id); + break; + default: + break; + } } /// get the type_id for a type. Used to cache type_id for fast writes. @@ -286,16 +305,16 @@ class WriteContext { uint32_t get_type_id_for_cache(const std::type_index &type_idx); /// write type info for a registered enum type. - /// Looks up the type info and delegates to write_any_typeinfo. - Result write_enum_typeinfo(const std::type_index &type); + /// Looks up the type info and delegates to write_any_type_info. + Result write_enum_type_info(const std::type_index &type); /// write type info for a registered enum type using TypeInfo pointer. /// Avoids type_index creation and lookup overhead. - Result write_enum_typeinfo(const TypeInfo *type_info); + Result write_enum_type_info(const TypeInfo *type_info); /// write type info for a registered enum type using compile-time type lookup. /// Faster than the std::type_index version - uses type_index(). - template Result write_enum_typeinfo(); + template Result write_enum_type_info(); /// reset context for reuse. void reset(); @@ -557,7 +576,7 @@ class ReadContext { Result get_type_info_by_index(size_t index) const; /// Read type information dynamically from buffer based on type ID. - /// This mirrors Rust's read_any_typeinfo implementation. + /// This mirrors Rust's read_any_type_info implementation. /// /// Handles different type categories: /// - COMPATIBLE_STRUCT/NAMED_COMPATIBLE_STRUCT: read meta_index @@ -566,13 +585,13 @@ class ReadContext { /// - Other types: look up by type_id /// /// @return const pointer to TypeInfo for the read type, or error - Result read_any_typeinfo(); + Result read_any_type_info(); /// Read type information dynamically from buffer based on type ID. /// Same as above but uses context's error state instead of returning Result. /// @param error Output parameter to receive any error /// @return const pointer to TypeInfo for the read type, or nullptr on error - const TypeInfo *read_any_typeinfo(Error &error); + const TypeInfo *read_any_type_info(Error &error); /// reset context for reuse. void reset(); diff --git a/cpp/fory/serialization/enum_serializer.h b/cpp/fory/serialization/enum_serializer.h index 961ecd189b..ce125c412c 100644 --- a/cpp/fory/serialization/enum_serializer.h +++ b/cpp/fory/serialization/enum_serializer.h @@ -48,11 +48,11 @@ struct Serializer>> { static inline void write_type_info(WriteContext &ctx) { // Use compile-time type lookup for faster enum type info writing - ctx.write_enum_typeinfo(); + ctx.write_enum_type_info(); } static inline void read_type_info(ReadContext &ctx) { - const TypeInfo *type_info = ctx.read_any_typeinfo(ctx.error()); + const TypeInfo *type_info = ctx.read_any_type_info(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } diff --git a/cpp/fory/serialization/map_serializer.h b/cpp/fory/serialization/map_serializer.h index b6ad5a5eb0..5bd9bea51b 100644 --- a/cpp/fory/serialization/map_serializer.h +++ b/cpp/fory/serialization/map_serializer.h @@ -61,7 +61,7 @@ template inline void read_type_info(ReadContext &ctx) { /// Read polymorphic type info from buffer. Returns nullptr on error. inline const TypeInfo *read_polymorphic_type_info(ReadContext &ctx) { - return ctx.read_any_typeinfo(ctx.error()); + return ctx.read_any_type_info(ctx.error()); } // ============================================================================ @@ -218,8 +218,8 @@ inline void write_map_data_slow(const MapType &map, WriteContext &ctx, bool need_write_header = true; // Track current chunk's types for polymorphic handling - uint32_t current_key_type_id = 0; - uint32_t current_val_type_id = 0; + const TypeInfo *current_key_type_info = nullptr; + const TypeInfo *current_val_type_info = nullptr; for (const auto &[key, value] : map) { // Check if key or value is null (for nullable types: optional, shared_ptr, @@ -274,8 +274,14 @@ inline void write_map_data_slow(const MapType &map, WriteContext &ctx, "Polymorphic key shared_ptr must not point to void")); return; } - auto res = ctx.write_any_typeinfo( - static_cast(TypeId::UNKNOWN), concrete_type_id); + auto type_info_res = + ctx.type_resolver().get_type_info(concrete_type_id); + if (!type_info_res.ok()) { + ctx.set_error(std::move(type_info_res).error()); + return; + } + auto res = ctx.write_any_type_info(type_info_res.value()->type_id, + concrete_type_id); if (!res.ok()) { ctx.set_error(std::move(res).error()); return; @@ -321,8 +327,14 @@ inline void write_map_data_slow(const MapType &map, WriteContext &ctx, "Polymorphic value shared_ptr must not point to void")); return; } - auto res = ctx.write_any_typeinfo( - static_cast(TypeId::UNKNOWN), concrete_type_id); + auto type_info_res = + ctx.type_resolver().get_type_info(concrete_type_id); + if (!type_info_res.ok()) { + ctx.set_error(std::move(type_info_res).error()); + return; + } + auto res = ctx.write_any_type_info(type_info_res.value()->type_id, + concrete_type_id); if (!res.ok()) { ctx.set_error(std::move(res).error()); return; @@ -343,8 +355,8 @@ inline void write_map_data_slow(const MapType &map, WriteContext &ctx, } // get type IDs for polymorphic types - uint32_t key_type_id = 0; - uint32_t val_type_id = 0; + const TypeInfo *key_type_info = nullptr; + const TypeInfo *val_type_info = nullptr; if constexpr (key_is_polymorphic) { auto concrete_type_id = get_concrete_type_id(key); auto key_type_info_res = @@ -353,7 +365,7 @@ inline void write_map_data_slow(const MapType &map, WriteContext &ctx, ctx.set_error(std::move(key_type_info_res).error()); return; } - key_type_id = key_type_info_res.value()->type_id; + key_type_info = key_type_info_res.value(); } if constexpr (val_is_polymorphic) { auto concrete_type_id = get_concrete_type_id(value); @@ -363,14 +375,19 @@ inline void write_map_data_slow(const MapType &map, WriteContext &ctx, ctx.set_error(std::move(val_type_info_res).error()); return; } - val_type_id = val_type_info_res.value()->type_id; + val_type_info = val_type_info_res.value(); } // Check if we need to start a new chunk due to type changes bool types_changed = false; if constexpr (key_is_polymorphic || val_is_polymorphic) { - types_changed = (key_type_id != current_key_type_id) || - (val_type_id != current_val_type_id); + if constexpr (key_is_polymorphic) { + types_changed = key_type_info != current_key_type_info; + } + if constexpr (val_is_polymorphic) { + types_changed = + types_changed || (val_type_info != current_val_type_info); + } } if (need_write_header || types_changed) { @@ -412,9 +429,17 @@ inline void write_map_data_slow(const MapType &map, WriteContext &ctx, if (!is_key_declared || key_is_polymorphic) { if constexpr (key_is_polymorphic) { auto concrete_type_id = get_concrete_type_id(key); - // Use UNKNOWN for polymorphic shared_ptr - auto res = ctx.write_any_typeinfo( - static_cast(TypeId::UNKNOWN), concrete_type_id); + if (FORY_PREDICT_FALSE(key_type_info == nullptr)) { + auto type_info_res = + ctx.type_resolver().get_type_info(concrete_type_id); + if (!type_info_res.ok()) { + ctx.set_error(std::move(type_info_res).error()); + return; + } + key_type_info = type_info_res.value(); + } + auto res = + ctx.write_any_type_info(key_type_info->type_id, concrete_type_id); if (!res.ok()) { ctx.set_error(std::move(res).error()); return; @@ -427,9 +452,17 @@ inline void write_map_data_slow(const MapType &map, WriteContext &ctx, if (!is_val_declared || val_is_polymorphic) { if constexpr (val_is_polymorphic) { auto concrete_type_id = get_concrete_type_id(value); - // Use UNKNOWN for polymorphic shared_ptr - auto res = ctx.write_any_typeinfo( - static_cast(TypeId::UNKNOWN), concrete_type_id); + if (FORY_PREDICT_FALSE(val_type_info == nullptr)) { + auto type_info_res = + ctx.type_resolver().get_type_info(concrete_type_id); + if (!type_info_res.ok()) { + ctx.set_error(std::move(type_info_res).error()); + return; + } + val_type_info = type_info_res.value(); + } + auto res = + ctx.write_any_type_info(val_type_info->type_id, concrete_type_id); if (!res.ok()) { ctx.set_error(std::move(res).error()); return; @@ -440,8 +473,8 @@ inline void write_map_data_slow(const MapType &map, WriteContext &ctx, } need_write_header = false; - current_key_type_id = key_type_id; - current_val_type_id = val_type_id; + current_key_type_info = key_type_info; + current_val_type_info = val_type_info; } // write key-value pair @@ -482,8 +515,8 @@ inline void write_map_data_slow(const MapType &map, WriteContext &ctx, write_chunk_size(ctx, header_offset, pair_counter); pair_counter = 0; need_write_header = true; - current_key_type_id = 0; - current_val_type_id = 0; + current_key_type_info = nullptr; + current_val_type_info = nullptr; } } @@ -894,11 +927,11 @@ struct Serializer> { using MapType = std::map; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - const TypeInfo *type_info = ctx.read_any_typeinfo(ctx.error()); + const TypeInfo *type_info = ctx.read_any_type_info(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } @@ -916,7 +949,7 @@ struct Serializer> { write_not_null_ref_flag(ctx, ref_mode); if (write_type) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } write_data_generic(map, ctx, has_generics); @@ -958,7 +991,7 @@ struct Serializer> { } if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return MapType{}; } @@ -1023,7 +1056,7 @@ struct Serializer> { write_not_null_ref_flag(ctx, ref_mode); if (write_type) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } constexpr bool is_fast_path = @@ -1073,7 +1106,7 @@ struct Serializer> { } if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return MapType{}; } diff --git a/cpp/fory/serialization/serialization_test.cc b/cpp/fory/serialization/serialization_test.cc index 47fe4b312d..a089b0b2a1 100644 --- a/cpp/fory/serialization/serialization_test.cc +++ b/cpp/fory/serialization/serialization_test.cc @@ -234,16 +234,15 @@ TEST(SerializationTest, EnumSerializesOrdinalValue) { << "Serialization failed: " << bytes_result.error().to_string(); std::vector bytes = bytes_result.value(); - // Xlang spec: enums are serialized as varuint32, not fixed int32_t - // With registration, type_id = (1 << 8) + ENUM = 279, which takes 2 bytes as - // varuint32 Expected: 2 (header) + 1 (ref flag) + 2 (type id as varuint) + 1 - // (ordinal as varuint32) = 6 bytes - ASSERT_GE(bytes.size(), 2 + 1 + 2 + 1); + // Xlang spec: enums are serialized as varuint32, not fixed int32_t. + // With registration, we write type_id (ENUM) + user_type_id (1), both as + // varuint32. Expected: 2 (header) + 1 (ref flag) + 1 (type id) + + // 1 (user type id) + 1 (ordinal) = 6 bytes + ASSERT_GE(bytes.size(), 2 + 1 + 1 + 1 + 1); size_t offset = 2; EXPECT_EQ(bytes[offset], static_cast(NOT_NULL_VALUE_FLAG)); - // Type ID 279 = (1 << 8) + ENUM encoded as varuint32: 0x97, 0x02 - EXPECT_EQ(bytes[offset + 1], 0x97); - EXPECT_EQ(bytes[offset + 2], 0x02); + EXPECT_EQ(bytes[offset + 1], static_cast(TypeId::ENUM)); + EXPECT_EQ(bytes[offset + 2], 1); // Ordinal 2 encoded as varuint32 is just 1 byte with value 2 EXPECT_EQ(bytes[offset + 3], 2); } @@ -257,13 +256,12 @@ TEST(SerializationTest, OldEnumSerializesOrdinalValue) { << "Serialization failed: " << bytes_result.error().to_string(); std::vector bytes = bytes_result.value(); - // With registration, type_id = (1 << 8) + ENUM = 279, which takes 2 bytes - ASSERT_GE(bytes.size(), 2 + 1 + 2 + 1); + // With registration, type_id + user_type_id take 2 bytes + ASSERT_GE(bytes.size(), 2 + 1 + 1 + 1 + 1); size_t offset = 2; EXPECT_EQ(bytes[offset], static_cast(NOT_NULL_VALUE_FLAG)); - // Type ID 279 encoded as varuint32: 0x97, 0x02 - EXPECT_EQ(bytes[offset + 1], 0x97); - EXPECT_EQ(bytes[offset + 2], 0x02); + EXPECT_EQ(bytes[offset + 1], static_cast(TypeId::ENUM)); + EXPECT_EQ(bytes[offset + 2], 1); // Ordinal 2 encoded as varuint32 is just 1 byte with value 2 EXPECT_EQ(bytes[offset + 3], 2); } @@ -277,13 +275,12 @@ TEST(SerializationTest, EnumOrdinalMappingHandlesNonZeroStart) { << "Serialization failed: " << bytes_result.error().to_string(); std::vector bytes = bytes_result.value(); - // With registration, type_id = (1 << 8) + ENUM = 279, which takes 2 bytes - ASSERT_GE(bytes.size(), 2 + 1 + 2 + 1); + // With registration, type_id + user_type_id take 2 bytes + ASSERT_GE(bytes.size(), 2 + 1 + 1 + 1 + 1); size_t offset = 2; EXPECT_EQ(bytes[offset], static_cast(NOT_NULL_VALUE_FLAG)); - // Type ID 279 encoded as varuint32: 0x97, 0x02 - EXPECT_EQ(bytes[offset + 1], 0x97); - EXPECT_EQ(bytes[offset + 2], 0x02); + EXPECT_EQ(bytes[offset + 1], static_cast(TypeId::ENUM)); + EXPECT_EQ(bytes[offset + 2], 1); // Ordinal 0 encoded as varuint32 is just 1 byte with value 0 EXPECT_EQ(bytes[offset + 3], 0); @@ -303,8 +300,8 @@ TEST(SerializationTest, EnumOrdinalMappingRejectsInvalidOrdinal) { std::vector bytes = bytes_result.value(); size_t offset = 2; - // With registration, type_id takes 2 bytes, ordinal is at offset + 3 - // Replace the valid ordinal with an invalid one (99 as varuint32) + // With registration, type_id + user_type_id take 2 bytes, ordinal is at + // offset + 3 Replace the valid ordinal with an invalid one (99 as varuint32) bytes[offset + 3] = 99; auto decode = fory.deserialize(bytes.data(), bytes.size()); @@ -320,13 +317,12 @@ TEST(SerializationTest, OldEnumOrdinalMappingHandlesNonZeroStart) { << "Serialization failed: " << bytes_result.error().to_string(); std::vector bytes = bytes_result.value(); - // With registration, type_id = (1 << 8) + ENUM = 279, which takes 2 bytes - ASSERT_GE(bytes.size(), 2 + 1 + 2 + 1); + // With registration, type_id + user_type_id take 2 bytes + ASSERT_GE(bytes.size(), 2 + 1 + 1 + 1 + 1); size_t offset = 2; EXPECT_EQ(bytes[offset], static_cast(NOT_NULL_VALUE_FLAG)); - // Type ID 279 encoded as varuint32: 0x97, 0x02 - EXPECT_EQ(bytes[offset + 1], 0x97); - EXPECT_EQ(bytes[offset + 2], 0x02); + EXPECT_EQ(bytes[offset + 1], static_cast(TypeId::ENUM)); + EXPECT_EQ(bytes[offset + 2], 1); // Ordinal 0 encoded as varuint32 is just 1 byte with value 0 EXPECT_EQ(bytes[offset + 3], 0); diff --git a/cpp/fory/serialization/serializer.h b/cpp/fory/serialization/serializer.h index e323011bc5..173576976c 100644 --- a/cpp/fory/serialization/serializer.h +++ b/cpp/fory/serialization/serializer.h @@ -192,15 +192,14 @@ FORY_ALWAYS_INLINE bool read_null_only_flag(ReadContext &ctx, inline bool type_id_matches(uint32_t actual, uint32_t expected) { if (actual == expected) return true; - uint32_t low_actual = actual & 0xffu; // For structs, allow STRUCT/COMPATIBLE_STRUCT/NAMED_*/etc. if (expected == static_cast(TypeId::STRUCT)) { - return low_actual == static_cast(TypeId::STRUCT) || - low_actual == static_cast(TypeId::COMPATIBLE_STRUCT) || - low_actual == static_cast(TypeId::NAMED_STRUCT) || - low_actual == static_cast(TypeId::NAMED_COMPATIBLE_STRUCT); + return actual == static_cast(TypeId::STRUCT) || + actual == static_cast(TypeId::COMPATIBLE_STRUCT) || + actual == static_cast(TypeId::NAMED_STRUCT) || + actual == static_cast(TypeId::NAMED_COMPATIBLE_STRUCT); } - return low_actual == expected; + return actual == expected; } // ============================================================================ diff --git a/cpp/fory/serialization/skip.cc b/cpp/fory/serialization/skip.cc index 5b75d67688..1941b7dd6e 100644 --- a/cpp/fory/serialization/skip.cc +++ b/cpp/fory/serialization/skip.cc @@ -213,13 +213,29 @@ void skip_struct(ReadContext &ctx, const FieldType &) { } // Read remote type_id - uint32_t remote_type_id = ctx.read_var_uint32(ctx.error()); + uint32_t remote_type_id = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } - uint32_t type_id_low = remote_type_id & 0xff; - TypeId remote_tid = static_cast(type_id_low); + uint32_t user_type_id = 0; + switch (static_cast(remote_type_id)) { + case TypeId::ENUM: + case TypeId::STRUCT: + case TypeId::EXT: + case TypeId::TYPED_UNION: + user_type_id = ctx.read_var_uint32(ctx.error()); + if (FORY_PREDICT_FALSE(ctx.has_error())) { + return; + } + break; + default: + break; + } + TypeId remote_tid = static_cast(remote_type_id); + bool has_user_type_id = + remote_tid == TypeId::ENUM || remote_tid == TypeId::STRUCT || + remote_tid == TypeId::EXT || remote_tid == TypeId::TYPED_UNION; const TypeInfo *type_info = nullptr; @@ -236,7 +252,10 @@ void skip_struct(ReadContext &ctx, const FieldType &) { } else { // Plain STRUCT: look up by type_id, read struct_version if enabled auto type_info_res = - ctx.type_resolver().get_type_info_by_id(remote_type_id); + has_user_type_id + ? ctx.type_resolver().get_user_type_info_by_id(remote_type_id, + user_type_id) + : ctx.type_resolver().get_type_info_by_id(remote_type_id); if (FORY_PREDICT_FALSE(!type_info_res.ok())) { ctx.set_error(std::move(type_info_res).error()); return; @@ -282,13 +301,27 @@ void skip_ext(ReadContext &ctx, const FieldType &) { } // Read remote type_id from buffer - Java always writes type_id for ext - uint32_t remote_type_id = ctx.read_var_uint32(ctx.error()); + uint32_t remote_type_id = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } - uint32_t low = remote_type_id & 0xffu; - TypeId remote_tid = static_cast(low); + uint32_t user_type_id = 0; + switch (static_cast(remote_type_id)) { + case TypeId::ENUM: + case TypeId::STRUCT: + case TypeId::COMPATIBLE_STRUCT: + case TypeId::EXT: + case TypeId::TYPED_UNION: + user_type_id = ctx.read_var_uint32(ctx.error()); + if (FORY_PREDICT_FALSE(ctx.has_error())) { + return; + } + break; + default: + break; + } + TypeId remote_tid = static_cast(remote_type_id); const TypeInfo *type_info = nullptr; @@ -303,8 +336,8 @@ void skip_ext(ReadContext &ctx, const FieldType &) { type_info = type_info_res.value(); } else { // ID-based ext: look up by remote type_id we just read - auto type_info_res = - ctx.type_resolver().get_type_info_by_id(remote_type_id); + auto type_info_res = ctx.type_resolver().get_user_type_info_by_id( + remote_type_id, user_type_id); if (FORY_PREDICT_FALSE(!type_info_res.ok())) { ctx.set_error(std::move(type_info_res).error()); return; @@ -353,7 +386,7 @@ void skip_unknown(ReadContext &ctx) { // UNKNOWN type means the actual type info is written inline. // We need to read the type info and then skip based on the actual type. // This is used for polymorphic fields like List. - const TypeInfo *type_info = ctx.read_any_typeinfo(ctx.error()); + const TypeInfo *type_info = ctx.read_any_type_info(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } @@ -363,8 +396,7 @@ void skip_unknown(ReadContext &ctx) { } // Check the actual type and skip accordingly - uint32_t low = type_info->type_id & 0xffu; - TypeId actual_tid = static_cast(low); + TypeId actual_tid = static_cast(type_info->type_id); switch (actual_tid) { case TypeId::STRUCT: @@ -425,7 +457,7 @@ void skip_union(ReadContext &ctx) { } // Read and skip the alternative's type info - const TypeInfo *type_info = ctx.read_any_typeinfo(ctx.error()); + const TypeInfo *type_info = ctx.read_any_type_info(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } @@ -455,14 +487,8 @@ void skip_field_value(ReadContext &ctx, const FieldType &field_type, } } - // skip based on low 8 bits of the type ID. - // - // xlang user types encode the user type id in the high bits and the - // logical TypeId in the low 8 bits (see Java XtypeResolver and Rust - // TypeId conventions). For skipping we only care about the logical - // category (STRUCT/ENUM/EXT/etc.), so mask off the user id portion. - uint32_t low = field_type.type_id & 0xffu; - TypeId tid = static_cast(low); + // skip based on the logical TypeId (already in 0..255) + TypeId tid = static_cast(field_type.type_id); switch (tid) { case TypeId::BOOL: diff --git a/cpp/fory/serialization/smart_ptr_serializer_test.cc b/cpp/fory/serialization/smart_ptr_serializer_test.cc index 8c4b03c29d..1c8b105f62 100644 --- a/cpp/fory/serialization/smart_ptr_serializer_test.cc +++ b/cpp/fory/serialization/smart_ptr_serializer_test.cc @@ -216,6 +216,7 @@ struct PolymorphicUniqueHolder { TEST(SmartPtrSerializerTest, PolymorphicSharedPtrDerived1) { auto fory = create_serializer(true); fory.register_struct(200); + fory.register_struct("test", "Base"); auto register_result = fory.register_struct("test", "Derived1"); ASSERT_TRUE(register_result.ok()) << "Failed to register Derived1: " << register_result.error().to_string(); @@ -244,6 +245,7 @@ TEST(SmartPtrSerializerTest, PolymorphicSharedPtrDerived1) { TEST(SmartPtrSerializerTest, PolymorphicSharedPtrDerived2) { auto fory = create_serializer(true); fory.register_struct(200); + fory.register_struct("test", "Base"); auto register_result = fory.register_struct("test", "Derived2"); ASSERT_TRUE(register_result.ok()) << "Failed to register Derived2: " << register_result.error().to_string(); @@ -272,6 +274,7 @@ TEST(SmartPtrSerializerTest, PolymorphicSharedPtrDerived2) { TEST(SmartPtrSerializerTest, PolymorphicUniquePtrDerived1) { auto fory = create_serializer(true); fory.register_struct(201); + fory.register_struct("test", "Base"); auto register_result = fory.register_struct("test", "Derived1"); ASSERT_TRUE(register_result.ok()) << "Failed to register Derived1: " << register_result.error().to_string(); @@ -300,6 +303,7 @@ TEST(SmartPtrSerializerTest, PolymorphicUniquePtrDerived1) { TEST(SmartPtrSerializerTest, PolymorphicUniquePtrDerived2) { auto fory = create_serializer(true); fory.register_struct(201); + fory.register_struct("test", "Base"); auto register_result = fory.register_struct("test", "Derived2"); ASSERT_TRUE(register_result.ok()) << "Failed to register Derived2: " << register_result.error().to_string(); diff --git a/cpp/fory/serialization/smart_ptr_serializers.h b/cpp/fory/serialization/smart_ptr_serializers.h index 0c30040c38..2ffe0ca98d 100644 --- a/cpp/fory/serialization/smart_ptr_serializers.h +++ b/cpp/fory/serialization/smart_ptr_serializers.h @@ -270,7 +270,7 @@ template struct Serializer> { static inline void write_type_info(WriteContext &ctx) { if constexpr (std::is_polymorphic_v) { // For polymorphic types, type info must be written dynamically - ctx.write_var_uint32(static_cast(TypeId::UNKNOWN)); + ctx.write_uint8(static_cast(TypeId::UNKNOWN)); } else { Serializer::write_type_info(ctx); } @@ -279,7 +279,7 @@ template struct Serializer> { static inline void read_type_info(ReadContext &ctx) { if constexpr (std::is_polymorphic_v) { // For polymorphic types, type info is read dynamically - ctx.read_any_typeinfo(ctx.error()); + ctx.read_any_type_info(ctx.error()); // Type checking is done at value read time } else { Serializer::read_type_info(ctx); @@ -309,8 +309,8 @@ template struct Serializer> { } const TypeInfo *type_info = type_info_res.value(); if (write_type) { - auto write_res = ctx.write_any_typeinfo( - static_cast(TypeId::UNKNOWN), concrete_type_id); + auto write_res = + ctx.write_any_type_info(type_info->type_id, concrete_type_id); if (!write_res.ok()) { ctx.set_error(std::move(write_res).error()); return; @@ -354,8 +354,8 @@ template struct Serializer> { // write type info if requested if (write_type) { - auto write_res = ctx.write_any_typeinfo( - static_cast(TypeId::UNKNOWN), concrete_type_id); + auto write_res = + ctx.write_any_type_info(type_info->type_id, concrete_type_id); if (!write_res.ok()) { ctx.set_error(std::move(write_res).error()); return; @@ -429,7 +429,7 @@ template struct Serializer> { if constexpr (is_polymorphic) { if (read_type) { // Polymorphic path: read type info from stream to get concrete type - const TypeInfo *type_info = ctx.read_any_typeinfo(ctx.error()); + const TypeInfo *type_info = ctx.read_any_type_info(ctx.error()); if (ctx.has_error()) { return nullptr; } @@ -519,7 +519,7 @@ template struct Serializer> { DynDepthGuard dyn_depth_guard(ctx); // Read type info from stream to get the concrete type - const TypeInfo *type_info = ctx.read_any_typeinfo(ctx.error()); + const TypeInfo *type_info = ctx.read_any_type_info(ctx.error()); if (ctx.has_error()) { return nullptr; } @@ -775,8 +775,8 @@ template struct Serializer> { } const TypeInfo *type_info = type_info_res.value(); if (write_type) { - auto write_res = ctx.write_any_typeinfo( - static_cast(TypeId::UNKNOWN), concrete_type_id); + auto write_res = + ctx.write_any_type_info(type_info->type_id, concrete_type_id); if (!write_res.ok()) { ctx.set_error(std::move(write_res).error()); return; @@ -809,8 +809,8 @@ template struct Serializer> { } const TypeInfo *type_info = type_info_res.value(); if (write_type) { - auto write_res = ctx.write_any_typeinfo( - static_cast(TypeId::UNKNOWN), concrete_type_id); + auto write_res = + ctx.write_any_type_info(type_info->type_id, concrete_type_id); if (!write_res.ok()) { ctx.set_error(std::move(write_res).error()); return; @@ -879,7 +879,7 @@ template struct Serializer> { if constexpr (is_polymorphic) { if (read_type) { // Polymorphic path: read type info from stream to get concrete type - const TypeInfo *type_info = ctx.read_any_typeinfo(ctx.error()); + const TypeInfo *type_info = ctx.read_any_type_info(ctx.error()); if (ctx.has_error()) { return nullptr; } @@ -939,7 +939,7 @@ template struct Serializer> { DynDepthGuard dyn_depth_guard(ctx); // Read type info from stream to get the concrete type - const TypeInfo *type_info = ctx.read_any_typeinfo(ctx.error()); + const TypeInfo *type_info = ctx.read_any_type_info(ctx.error()); if (ctx.has_error()) { return nullptr; } diff --git a/cpp/fory/serialization/string_serializer.h b/cpp/fory/serialization/string_serializer.h index 6dc9bc931a..139564b93a 100644 --- a/cpp/fory/serialization/string_serializer.h +++ b/cpp/fory/serialization/string_serializer.h @@ -236,11 +236,11 @@ template <> struct Serializer { static constexpr TypeId type_id = TypeId::STRING; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - uint32_t actual = ctx.read_var_uint32(ctx.error()); + uint32_t actual = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } @@ -254,7 +254,7 @@ template <> struct Serializer { RefMode ref_mode, bool write_type, bool = false) { write_not_null_ref_flag(ctx, ref_mode); if (write_type) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } write_data(value, ctx); } @@ -275,7 +275,7 @@ template <> struct Serializer { return std::string(); } if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return std::string(); } @@ -313,14 +313,14 @@ template <> struct Serializer { static constexpr TypeId type_id = TypeId::STRING; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void write(std::string_view value, WriteContext &ctx, RefMode ref_mode, bool write_type, bool = false) { write_not_null_ref_flag(ctx, ref_mode); if (write_type) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } write_data(value, ctx); } @@ -344,11 +344,11 @@ template <> struct Serializer { static constexpr TypeId type_id = TypeId::STRING; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - uint32_t actual = ctx.read_var_uint32(ctx.error()); + uint32_t actual = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } @@ -362,7 +362,7 @@ template <> struct Serializer { RefMode ref_mode, bool write_type, bool = false) { write_not_null_ref_flag(ctx, ref_mode); if (write_type) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } write_data(value, ctx); } @@ -384,7 +384,7 @@ template <> struct Serializer { return std::u16string(); } if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return std::u16string(); } diff --git a/cpp/fory/serialization/struct_serializer.h b/cpp/fory/serialization/struct_serializer.h index 67471c493c..7c7e5ae351 100644 --- a/cpp/fory/serialization/struct_serializer.h +++ b/cpp/fory/serialization/struct_serializer.h @@ -2812,7 +2812,7 @@ struct Serializer>> { /// Read and validate type info. /// This consumes the type_id and meta index from the buffer. static void read_type_info(ReadContext &ctx) { - const TypeInfo *type_info = ctx.read_any_typeinfo(ctx.error()); + const TypeInfo *type_info = ctx.read_any_type_info(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } @@ -2845,10 +2845,9 @@ struct Serializer>> { uint32_t tid = type_info->type_id; // Fast path: check if this is a simple STRUCT type (no meta needed) - uint32_t type_id_low = tid & 0xff; - if (type_id_low == static_cast(TypeId::STRUCT)) { + if (tid == static_cast(TypeId::STRUCT)) { // Simple STRUCT - just write the type_id directly - ctx.write_struct_type_id_direct(tid); + ctx.write_struct_type_id_direct(tid, type_info->user_type_id); } else { // Complex type (NAMED_STRUCT, COMPATIBLE_STRUCT, etc.) - use TypeInfo* ctx.write_struct_type_info(type_info); @@ -2945,16 +2944,30 @@ struct Serializer>> { // In compatible mode: always use remote TypeMeta for schema evolution if (read_type) { // Read type_id - uint32_t remote_type_id = ctx.read_var_uint32(ctx.error()); + uint32_t remote_type_id = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return T{}; } - uint8_t remote_type_id_low = remote_type_id & 0xff; + uint32_t remote_user_type_id = 0; + switch (static_cast(remote_type_id)) { + case TypeId::ENUM: + case TypeId::STRUCT: + case TypeId::EXT: + case TypeId::TYPED_UNION: + remote_user_type_id = ctx.read_var_uint32(ctx.error()); + if (FORY_PREDICT_FALSE(ctx.has_error())) { + return T{}; + } + break; + default: + break; + } const bool remote_has_meta = - remote_type_id_low == + remote_type_id == static_cast(TypeId::COMPATIBLE_STRUCT) || - remote_type_id_low == + remote_type_id == static_cast(TypeId::NAMED_COMPATIBLE_STRUCT); + (void)remote_user_type_id; if (remote_has_meta) { // Read TypeMeta inline using streaming protocol auto remote_type_info_res = ctx.read_type_meta(); @@ -2991,27 +3004,57 @@ struct Serializer>> { // FAST PATH: For simple numeric type IDs (not named types), we can // just read the varint and compare directly without hash lookup. - // Named types have type_id_low in ranges that require metadata - // parsing. - uint8_t expected_type_id_low = expected_type_id & 0xff; - if (expected_type_id_low != - static_cast(TypeId::NAMED_ENUM) && - expected_type_id_low != static_cast(TypeId::NAMED_EXT) && - expected_type_id_low != - static_cast(TypeId::NAMED_STRUCT)) { + // Named types require metadata parsing. + if (expected_type_id != static_cast(TypeId::NAMED_ENUM) && + expected_type_id != static_cast(TypeId::NAMED_EXT) && + expected_type_id != static_cast(TypeId::NAMED_STRUCT) && + expected_type_id != static_cast(TypeId::NAMED_UNION) && + expected_type_id != + static_cast(TypeId::NAMED_COMPATIBLE_STRUCT) && + expected_type_id != + static_cast(TypeId::COMPATIBLE_STRUCT)) { // Simple type ID - just read and compare varint directly - uint32_t remote_type_id = ctx.read_var_uint32(ctx.error()); + uint32_t remote_type_id = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return T{}; } + uint32_t remote_user_type_id = 0; + switch (static_cast(remote_type_id)) { + case TypeId::ENUM: + case TypeId::STRUCT: + case TypeId::EXT: + case TypeId::TYPED_UNION: + remote_user_type_id = ctx.read_var_uint32(ctx.error()); + if (FORY_PREDICT_FALSE(ctx.has_error())) { + return T{}; + } + break; + default: + break; + } if (remote_type_id != expected_type_id) { ctx.set_error( Error::type_mismatch(remote_type_id, expected_type_id)); return T{}; } + switch (static_cast(expected_type_id)) { + case TypeId::ENUM: + case TypeId::STRUCT: + case TypeId::EXT: + case TypeId::TYPED_UNION: + if (type_info->user_type_id == kInvalidUserTypeId || + remote_user_type_id != type_info->user_type_id) { + ctx.set_error(Error::type_mismatch(remote_user_type_id, + type_info->user_type_id)); + return T{}; + } + break; + default: + break; + } } else { // Named type - need to parse full type info - const TypeInfo *remote_info = ctx.read_any_typeinfo(ctx.error()); + const TypeInfo *remote_info = ctx.read_any_type_info(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return T{}; } diff --git a/cpp/fory/serialization/struct_test.cc b/cpp/fory/serialization/struct_test.cc index 207aab796a..8d7cc21da5 100644 --- a/cpp/fory/serialization/struct_test.cc +++ b/cpp/fory/serialization/struct_test.cc @@ -410,6 +410,8 @@ inline void register_all_test_types(Fory &fory) { fory.register_struct(type_id++); fory.register_struct(type_id++); fory.register_struct(type_id++); + fory.register_enum(type_id++); + fory.register_enum(type_id++); } template void test_roundtrip(const T &original) { diff --git a/cpp/fory/serialization/temporal_serializers.h b/cpp/fory/serialization/temporal_serializers.h index fefb9841e8..046599ae8f 100644 --- a/cpp/fory/serialization/temporal_serializers.h +++ b/cpp/fory/serialization/temporal_serializers.h @@ -60,11 +60,11 @@ template <> struct Serializer { static constexpr TypeId type_id = TypeId::DURATION; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - uint32_t actual = ctx.read_var_uint32(ctx.error()); + uint32_t actual = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } @@ -79,7 +79,7 @@ template <> struct Serializer { bool has_generics = false) { write_not_null_ref_flag(ctx, ref_mode); if (write_type) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } write_data(duration, ctx); } @@ -101,7 +101,7 @@ template <> struct Serializer { return Duration(0); } if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return Duration(0); } @@ -137,11 +137,11 @@ template <> struct Serializer { static constexpr TypeId type_id = TypeId::TIMESTAMP; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - uint32_t actual = ctx.read_var_uint32(ctx.error()); + uint32_t actual = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } @@ -156,7 +156,7 @@ template <> struct Serializer { bool has_generics = false) { write_not_null_ref_flag(ctx, ref_mode); if (write_type) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } write_data(timestamp, ctx); } @@ -187,7 +187,7 @@ template <> struct Serializer { return Timestamp(std::chrono::nanoseconds(0)); } if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return Timestamp(std::chrono::nanoseconds(0)); } @@ -227,11 +227,11 @@ template <> struct Serializer { static constexpr TypeId type_id = TypeId::DATE; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - uint32_t actual = ctx.read_var_uint32(ctx.error()); + uint32_t actual = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } @@ -246,7 +246,7 @@ template <> struct Serializer { bool has_generics = false) { write_not_null_ref_flag(ctx, ref_mode); if (write_type) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } write_data(date, ctx); } @@ -266,7 +266,7 @@ template <> struct Serializer { return Date(); } if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return Date(); } diff --git a/cpp/fory/serialization/tuple_serializer.h b/cpp/fory/serialization/tuple_serializer.h index 2122455177..c817c13ec1 100644 --- a/cpp/fory/serialization/tuple_serializer.h +++ b/cpp/fory/serialization/tuple_serializer.h @@ -173,7 +173,7 @@ inline Tuple read_tuple_elements_heterogeneous(ReadContext &ctx, // skip any extra elements beyond tuple size while (index < length && !ctx.has_error()) { // skip value - read type and skip data - ctx.read_any_typeinfo(ctx.error()); + ctx.read_any_type_info(ctx.error()); ++index; } @@ -220,11 +220,11 @@ template <> struct Serializer> { static constexpr TypeId type_id = TypeId::LIST; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - const TypeInfo *type_info = ctx.read_any_typeinfo(ctx.error()); + const TypeInfo *type_info = ctx.read_any_type_info(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } @@ -239,7 +239,7 @@ template <> struct Serializer> { bool has_generics = false) { write_not_null_ref_flag(ctx, ref_mode); if (write_type) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } write_data(std::tuple<>(), ctx); } @@ -261,7 +261,7 @@ template <> struct Serializer> { } if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return std::tuple<>(); } @@ -296,11 +296,11 @@ template struct Serializer> { using IndexSeq = std::index_sequence_for; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - const TypeInfo *type_info = ctx.read_any_typeinfo(ctx.error()); + const TypeInfo *type_info = ctx.read_any_type_info(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } @@ -315,7 +315,7 @@ template struct Serializer> { bool has_generics = false) { write_not_null_ref_flag(ctx, ref_mode); if (write_type) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } write_data_generic(tuple, ctx, has_generics); } @@ -357,7 +357,7 @@ template struct Serializer> { } if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return TupleType{}; } @@ -397,7 +397,7 @@ template struct Serializer> { if (is_same_type) { // Read element type info once - ctx.read_any_typeinfo(ctx.error()); + ctx.read_any_type_info(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return TupleType{}; } diff --git a/cpp/fory/serialization/type_info.h b/cpp/fory/serialization/type_info.h index dc6565df8c..1379727da5 100644 --- a/cpp/fory/serialization/type_info.h +++ b/cpp/fory/serialization/type_info.h @@ -34,6 +34,8 @@ namespace fory { namespace serialization { +constexpr uint32_t kInvalidUserTypeId = 0xFFFFFFFFu; + // Forward declarations - these allow Harness function pointers to compile // without including the full headers class WriteContext; @@ -111,10 +113,12 @@ struct CachedMetaString { }; /// TypeInfo holds metadata about a type for serialization purposes. -/// This is used by read_any_typeinfo() and write_any_typeinfo() to track +/// This is used by read_any_type_info() and write_any_type_info() to track /// type information across language boundaries (xlang serialization). struct TypeInfo { uint32_t type_id = 0; + // Stored as unsigned; 0xffffffff means "unset". + uint32_t user_type_id = kInvalidUserTypeId; std::string namespace_name; std::string type_name; bool register_by_name = false; diff --git a/cpp/fory/serialization/type_resolver.cc b/cpp/fory/serialization/type_resolver.cc index 2e0f28c202..dbfa69623a 100644 --- a/cpp/fory/serialization/type_resolver.cc +++ b/cpp/fory/serialization/type_resolver.cc @@ -54,8 +54,10 @@ Result FieldType::write_to(Buffer &buffer, bool write_flag, if (nullable_val) { header |= 2; } + buffer.write_var_uint32(header); + } else { + buffer.write_uint8(static_cast(header)); } - buffer.write_var_uint32(header); // write generics for list/set/map if (type_id == static_cast(TypeId::LIST) || @@ -82,7 +84,8 @@ Result FieldType::read_from(Buffer &buffer, bool read_flag, bool nullable_val, bool ref_tracking_val) { Error error; - uint32_t header = buffer.read_var_uint32(error); + uint32_t header = + read_flag ? buffer.read_var_uint32(error) : buffer.read_uint8(error); if (FORY_PREDICT_FALSE(!error.ok())) { return Unexpected(std::move(error)); } @@ -102,6 +105,7 @@ Result FieldType::read_from(Buffer &buffer, bool read_flag, } FieldType ft(tid, null, ref_track); + ft.user_type_id = kInvalidUserTypeId; // Read generics for list/set/map if (tid == static_cast(TypeId::LIST) || @@ -130,8 +134,34 @@ Result, Error> FieldInfo::to_bytes() const { // ref_tracking:1bit | const bool use_tag_id = field_id >= 0; uint8_t encoding_idx = use_tag_id ? 3 : 0; // TAG_ID or UTF8 + std::vector encoded_name; + if (!use_tag_id) { + static const MetaStringEncoder k_field_name_encoder('$', '_'); + static const std::vector k_field_name_encoding_list = { + MetaEncoding::UTF8, MetaEncoding::ALL_TO_LOWER_SPECIAL, + MetaEncoding::LOWER_UPPER_DIGIT_SPECIAL}; + + FORY_TRY(encoded, k_field_name_encoder.encode(field_name, + k_field_name_encoding_list)); + switch (encoded.encoding) { + case MetaEncoding::UTF8: + encoding_idx = 0; + break; + case MetaEncoding::ALL_TO_LOWER_SPECIAL: + encoding_idx = 1; + break; + case MetaEncoding::LOWER_UPPER_DIGIT_SPECIAL: + encoding_idx = 2; + break; + default: + return Unexpected(Error::encoding_error( + "Unsupported field name encoding: " + + std::to_string(static_cast(encoded.encoding)))); + } + encoded_name = std::move(encoded.bytes); + } size_t name_size = - use_tag_id ? static_cast(field_id) + 1 : field_name.size(); + use_tag_id ? static_cast(field_id) + 1 : encoded_name.size(); uint8_t header = (std::min(FIELD_NAME_SIZE_THRESHOLD, name_size - 1) << 2) & 0x3C; @@ -154,8 +184,7 @@ Result, Error> FieldInfo::to_bytes() const { // write field name only when tag ID is not used. if (!use_tag_id) { - buffer.write_bytes(reinterpret_cast(field_name.data()), - field_name.size()); + buffer.write_bytes(encoded_name.data(), encoded_name.size()); } // CRITICAL FIX: Use writer_index() not size() to get actual bytes written! @@ -214,8 +243,16 @@ Result FieldInfo::from_bytes(Buffer &buffer) { } static const MetaStringDecoder k_field_name_decoder('$', '_'); - - FORY_TRY(encoding, to_meta_encoding(encoding_idx)); + static const MetaEncoding k_field_name_encodings[] = { + MetaEncoding::UTF8, MetaEncoding::ALL_TO_LOWER_SPECIAL, + MetaEncoding::LOWER_UPPER_DIGIT_SPECIAL}; + if (encoding_idx >= + sizeof(k_field_name_encodings) / sizeof(k_field_name_encodings[0])) { + return Unexpected( + Error::encoding_error("Invalid field name encoding index: " + + std::to_string(static_cast(encoding_idx)))); + } + MetaEncoding encoding = k_field_name_encodings[encoding_idx]; FORY_TRY(decoded_name, k_field_name_decoder.decode( name_bytes.data(), name_bytes.size(), encoding)); @@ -239,10 +276,40 @@ static const MetaEncoding k_type_name_encodings[] = { MetaEncoding::LOWER_UPPER_DIGIT_SPECIAL, MetaEncoding::FIRST_TO_LOWER_SPECIAL}; -inline Result write_meta_name(Buffer &buffer, - const std::string &name) { - const uint8_t encoding_idx = 0; // UTF8 in both encoding tables - const size_t len = name.size(); +static const std::vector k_namespace_encoding_list = { + MetaEncoding::UTF8, MetaEncoding::ALL_TO_LOWER_SPECIAL, + MetaEncoding::LOWER_UPPER_DIGIT_SPECIAL}; + +static const std::vector k_type_name_encoding_list = { + MetaEncoding::UTF8, MetaEncoding::ALL_TO_LOWER_SPECIAL, + MetaEncoding::LOWER_UPPER_DIGIT_SPECIAL, + MetaEncoding::FIRST_TO_LOWER_SPECIAL}; + +static const MetaStringEncoder k_namespace_encoder('.', '_'); +static const MetaStringEncoder k_type_name_encoder('$', '_'); + +inline Result encoding_to_index(MetaEncoding encoding, + const MetaEncoding *encodings, + size_t enc_count) { + for (size_t i = 0; i < enc_count; ++i) { + if (encodings[i] == encoding) { + return static_cast(i); + } + } + return Unexpected( + Error::encoding_error("Unsupported meta string encoding: " + + std::to_string(static_cast(encoding)))); +} + +inline Result +write_meta_name(Buffer &buffer, const std::string &name, + const MetaStringEncoder &encoder, + const std::vector &allowed_encodings, + const MetaEncoding *encodings, size_t enc_count) { + FORY_TRY(encoded, encoder.encode(name, allowed_encodings)); + FORY_TRY(encoding_idx, + encoding_to_index(encoded.encoding, encodings, enc_count)); + const size_t len = encoded.bytes.size(); if (len >= BIG_NAME_THRESHOLD) { uint8_t header = @@ -254,9 +321,8 @@ inline Result write_meta_name(Buffer &buffer, buffer.write_uint8(header); } - if (!name.empty()) { - buffer.write_bytes(reinterpret_cast(name.data()), - static_cast(name.size())); + if (len > 0) { + buffer.write_bytes(encoded.bytes.data(), static_cast(len)); } return Result(); } @@ -304,6 +370,7 @@ read_meta_name(Buffer &buffer, const MetaStringDecoder &decoder, TypeMeta TypeMeta::from_fields(uint32_t tid, const std::string &ns, const std::string &name, bool by_name, + uint32_t user_type_id, std::vector fields) { for (const auto &field : fields) { FORY_CHECK(!field.field_name.empty()) @@ -311,6 +378,7 @@ TypeMeta TypeMeta::from_fields(uint32_t tid, const std::string &ns, } TypeMeta meta; meta.type_id = tid; + meta.user_type_id = user_type_id; meta.namespace_str = ns; meta.type_name = name; meta.register_by_name = by_name; @@ -338,10 +406,21 @@ Result, Error> TypeMeta::to_bytes() const { // write namespace and type name (if registered by name) using the // same compact meta string format as Rust/Java. if (register_by_name) { - FORY_RETURN_NOT_OK(write_meta_name(layer_buffer, namespace_str)); - FORY_RETURN_NOT_OK(write_meta_name(layer_buffer, type_name)); + FORY_RETURN_NOT_OK(write_meta_name( + layer_buffer, namespace_str, k_namespace_encoder, + k_namespace_encoding_list, k_namespace_encodings, + sizeof(k_namespace_encodings) / sizeof(k_namespace_encodings[0]))); + FORY_RETURN_NOT_OK(write_meta_name( + layer_buffer, type_name, k_type_name_encoder, k_type_name_encoding_list, + k_type_name_encodings, + sizeof(k_type_name_encodings) / sizeof(k_type_name_encodings[0]))); } else { - layer_buffer.write_var_uint32(type_id); + layer_buffer.write_uint8(static_cast(type_id)); + if (user_type_id == kInvalidUserTypeId) { + return Unexpected( + Error::type_error("User type id is required for this type")); + } + layer_buffer.write_var_uint32(user_type_id); } // write field infos @@ -422,6 +501,7 @@ TypeMeta::from_bytes(Buffer &buffer, const TypeMeta *local_type_info) { // Read type ID or namespace/type name uint32_t type_id = 0; + uint32_t user_type_id = kInvalidUserTypeId; std::string namespace_str; std::string type_name; @@ -441,11 +521,16 @@ TypeMeta::from_bytes(Buffer &buffer, const TypeMeta *local_type_info) { sizeof(k_type_name_encodings[0]))); type_name = std::move(tn); } else { - uint32_t tid = buffer.read_var_uint32(error); + uint32_t tid = buffer.read_uint8(error); if (FORY_PREDICT_FALSE(!error.ok())) { return Unexpected(std::move(error)); } type_id = tid; + uint32_t uid = buffer.read_var_uint32(error); + if (FORY_PREDICT_FALSE(!error.ok())) { + return Unexpected(std::move(error)); + } + user_type_id = uid; } // Read field infos @@ -476,6 +561,7 @@ TypeMeta::from_bytes(Buffer &buffer, const TypeMeta *local_type_info) { auto meta = std::make_unique(); meta->hash = meta_hash; meta->type_id = type_id; + meta->user_type_id = user_type_id; meta->namespace_str = std::move(namespace_str); meta->type_name = std::move(type_name); meta->register_by_name = register_by_name; @@ -521,6 +607,7 @@ TypeMeta::from_bytes_with_header(Buffer &buffer, int64_t header) { // Read type ID or namespace/type name uint32_t type_id = 0; + uint32_t user_type_id = kInvalidUserTypeId; std::string namespace_str; std::string type_name; @@ -540,11 +627,27 @@ TypeMeta::from_bytes_with_header(Buffer &buffer, int64_t header) { sizeof(k_type_name_encodings[0]))); type_name = std::move(tn); } else { - uint32_t tid = buffer.read_var_uint32(error); + uint32_t tid = buffer.read_uint8(error); if (FORY_PREDICT_FALSE(!error.ok())) { return Unexpected(std::move(error)); } type_id = tid; + switch (static_cast(type_id)) { + case TypeId::ENUM: + case TypeId::STRUCT: + case TypeId::COMPATIBLE_STRUCT: + case TypeId::EXT: + case TypeId::TYPED_UNION: { + uint32_t uid = buffer.read_var_uint32(error); + if (FORY_PREDICT_FALSE(!error.ok())) { + return Unexpected(std::move(error)); + } + user_type_id = uid; + break; + } + default: + break; + } } // Read field infos @@ -569,6 +672,7 @@ TypeMeta::from_bytes_with_header(Buffer &buffer, int64_t header) { auto meta = std::make_unique(); meta->hash = meta_hash; meta->type_id = type_id; + meta->user_type_id = user_type_id; meta->namespace_str = std::move(namespace_str); meta->type_name = std::move(type_name); meta->register_by_name = register_by_name; @@ -661,6 +765,8 @@ bool is_compress(uint32_t type_id) { type_id == static_cast(TypeId::TAGGED_UINT64); } +bool field_type_needs_user_type_id(uint32_t) { return false; } + std::string field_sort_key(const FieldInfo &field) { if (field.field_id >= 0) { return std::to_string(field.field_id); @@ -841,8 +947,7 @@ void TypeMeta::assign_field_ids(const TypeMeta *local_type, // NAMED_ENUM, EXT vs NAMED_EXT, and their ID-based variants) are // treated as equal for schema-evolution matching. auto normalize_type_id = [](uint32_t tid) { - uint32_t low = tid & 0xffu; - switch (static_cast(low)) { + switch (static_cast(tid)) { case TypeId::STRUCT: case TypeId::COMPATIBLE_STRUCT: case TypeId::NAMED_STRUCT: @@ -874,6 +979,14 @@ void TypeMeta::assign_field_ids(const TypeMeta *local_type, if (normalize_type_id(a.type_id) != normalize_type_id(b.type_id)) { return false; } + if (field_type_needs_user_type_id(a.type_id) || + field_type_needs_user_type_id(b.type_id)) { + if ((a.user_type_id != kInvalidUserTypeId || + b.user_type_id != kInvalidUserTypeId) && + a.user_type_id != b.user_type_id) { + return false; + } + } if (a.generics.size() != b.generics.size()) { return false; } @@ -1117,6 +1230,7 @@ int32_t TypeMeta::compute_struct_version(const TypeMeta &meta) { std::unique_ptr TypeInfo::deep_clone() const { auto cloned = std::make_unique(); cloned->type_id = type_id; + cloned->user_type_id = user_type_id; cloned->namespace_name = namespace_name; cloned->type_name = type_name; cloned->register_by_name = register_by_name; @@ -1233,6 +1347,9 @@ TypeResolver::build_final_type_resolver() { for (const auto &[key, old_ptr] : type_info_by_id_) { final_resolver->type_info_by_id_.put(key, ptr_map[old_ptr]); } + for (const auto &[key, old_ptr] : user_type_info_by_id_) { + final_resolver->user_type_info_by_id_.put(key, ptr_map[old_ptr]); + } for (const auto &[key, old_ptr] : type_info_by_name_) { final_resolver->type_info_by_name_[key] = ptr_map[old_ptr]; } @@ -1248,37 +1365,27 @@ TypeResolver::build_final_type_resolver() { for (const auto &[rust_type_id, partial_ptr] : final_resolver->partial_type_infos_) { // Call the harness's sorted_field_infos function to get complete field info - auto fields_result = - partial_ptr->harness.sorted_field_infos_fn(*final_resolver); - if (!fields_result.ok()) { - return Unexpected(fields_result.error()); - } - auto sorted_fields = std::move(fields_result).value(); + FORY_TRY(sorted_fields, + partial_ptr->harness.sorted_field_infos_fn(*final_resolver)); // Build complete TypeMeta TypeMeta meta = TypeMeta::from_fields( partial_ptr->type_id, partial_ptr->namespace_name, partial_ptr->type_name, partial_ptr->register_by_name, - std::move(sorted_fields)); + partial_ptr->user_type_id, std::move(sorted_fields)); // Serialize TypeMeta to bytes - auto type_def_result = meta.to_bytes(); - if (!type_def_result.ok()) { - return Unexpected(type_def_result.error()); - } + FORY_TRY(type_def, meta.to_bytes()); // Update the TypeInfo in place - partial_ptr->type_def = std::move(type_def_result).value(); + partial_ptr->type_def = std::move(type_def); // Parse the serialized TypeMeta back to create unique_ptr Buffer buffer(partial_ptr->type_def.data(), static_cast(partial_ptr->type_def.size()), false); buffer.writer_index(static_cast(partial_ptr->type_def.size())); - auto parsed_meta_result = TypeMeta::from_bytes(buffer, nullptr); - if (!parsed_meta_result.ok()) { - return Unexpected(parsed_meta_result.error()); - } - partial_ptr->type_meta = std::move(parsed_meta_result).value(); + FORY_TRY(parsed_meta, TypeMeta::from_bytes(buffer, nullptr)); + partial_ptr->type_meta = std::move(parsed_meta); } // Clear partial_type_infos in the final resolver since they're all completed @@ -1315,6 +1422,9 @@ std::unique_ptr TypeResolver::clone() const { for (const auto &[key, old_ptr] : type_info_by_id_) { cloned->type_info_by_id_.put(key, ptr_map[old_ptr]); } + for (const auto &[key, old_ptr] : user_type_info_by_id_) { + cloned->user_type_info_by_id_.put(key, ptr_map[old_ptr]); + } for (const auto &[key, old_ptr] : type_info_by_name_) { cloned->type_info_by_name_[key] = ptr_map[old_ptr]; } @@ -1329,8 +1439,10 @@ std::unique_ptr TypeResolver::clone() const { void TypeResolver::register_builtin_types() { // Register internal type IDs without harnesses (deserialization is static) - // These are needed so read_any_typeinfo can find them by type_id + // These are needed so read_any_type_info can find them by type_id auto register_type_id_only = [this](TypeId type_id) { + FORY_CHECK(static_cast(type_id) < 256) + << "Internal type id overflow: " << static_cast(type_id); auto info = std::make_unique(); info->type_id = static_cast(type_id); info->register_by_name = false; diff --git a/cpp/fory/serialization/type_resolver.h b/cpp/fory/serialization/type_resolver.h index 44574e66e2..2a8e61d871 100644 --- a/cpp/fory/serialization/type_resolver.h +++ b/cpp/fory/serialization/type_resolver.h @@ -70,6 +70,10 @@ class TypeResolver; class WriteContext; class ReadContext; +template +Result +get_type_info_with_resolver(TypeResolver &resolver); + // ============================================================================ // TypeIndex primary template (fallback) // Specializations for primitives and containers are in serializer_traits.h @@ -101,19 +105,23 @@ template struct TypeIndex { class FieldType { public: uint32_t type_id; + // Stored as unsigned; 0xffffffff means "unset". + uint32_t user_type_id; bool nullable; bool ref_tracking; RefMode ref_mode; // Precomputed from nullable and ref_tracking std::vector generics; FieldType() - : type_id(0), nullable(false), ref_tracking(false), - ref_mode(RefMode::None) {} + : type_id(0), user_type_id(kInvalidUserTypeId), nullable(false), + ref_tracking(false), ref_mode(RefMode::None) {} FieldType(uint32_t tid, bool null, bool ref_track = false, - std::vector gens = {}) - : type_id(tid), nullable(null), ref_tracking(ref_track), - ref_mode(make_ref_mode(null, ref_track)), generics(std::move(gens)) {} + std::vector gens = {}, + uint32_t user_tid = kInvalidUserTypeId) + : type_id(tid), user_type_id(user_tid), nullable(null), + ref_tracking(ref_track), ref_mode(make_ref_mode(null, ref_track)), + generics(std::move(gens)) {} /// write field type to buffer /// @param buffer Target buffer @@ -133,6 +141,7 @@ class FieldType { bool operator==(const FieldType &other) const { return type_id == other.type_id && nullable == other.nullable && + user_type_id == other.user_type_id && ref_tracking == other.ref_tracking && generics == other.generics; } @@ -177,18 +186,23 @@ constexpr size_t MAX_PARSED_NUM_TYPE_DEFS = 8192; /// Used for schema evolution to compare remote and local type schemas class TypeMeta { public: - int64_t hash; // Type hash for fast comparison - uint32_t type_id; // Type ID (for non-named registration) - std::string namespace_str; // Namespace (for named registration) - std::string type_name; // Type name (for named registration) - bool register_by_name; // Whether registered by name + int64_t hash; // Type hash for fast comparison + uint32_t type_id; // Type ID (for non-named registration) + // Stored as unsigned; 0xffffffff means "unset". + uint32_t user_type_id; // User type ID (for non-named registration) + std::string namespace_str; // Namespace (for named registration) + std::string type_name; // Type name (for named registration) + bool register_by_name; // Whether registered by name std::vector field_infos; // Field information - TypeMeta() : hash(0), type_id(0), register_by_name(false) {} + TypeMeta() + : hash(0), type_id(0), user_type_id(kInvalidUserTypeId), + register_by_name(false) {} /// Create TypeMeta from field information static TypeMeta from_fields(uint32_t tid, const std::string &ns, const std::string &name, bool by_name, + uint32_t user_type_id, std::vector fields); /// write type meta to buffer (for serialization) @@ -329,9 +343,14 @@ template struct FieldTypeBuilder>>> { using Inner = typename decay_t::element_type; static FieldType build(bool) { - FieldType inner = FieldTypeBuilder::build(true); - inner.nullable = true; - return inner; + if constexpr (std::is_polymorphic_v) { + FieldType ft(to_type_id(TypeId::UNKNOWN), true); + return ft; + } else { + FieldType inner = FieldTypeBuilder::build(true); + inner.nullable = true; + return inner; + } } }; @@ -339,9 +358,14 @@ template struct FieldTypeBuilder>>> { using Inner = typename decay_t::element_type; static FieldType build(bool) { - FieldType inner = FieldTypeBuilder::build(true); - inner.nullable = true; - return inner; + if constexpr (std::is_polymorphic_v) { + FieldType ft(to_type_id(TypeId::UNKNOWN), true); + return ft; + } else { + FieldType inner = FieldTypeBuilder::build(true); + inner.nullable = true; + return inner; + } } }; @@ -451,9 +475,9 @@ struct FieldTypeBuilder>>> { if constexpr (tuple_size > 0) { add_element_types(ft, std::make_index_sequence{}); } else { - // Empty tuple: use STRUCT as stub element type for schema encoding + // Empty tuple: use UNKNOWN as stub element type for schema encoding ft.generics.push_back( - FieldType(static_cast(TypeId::STRUCT), false)); + FieldType(static_cast(TypeId::UNKNOWN), false)); } return ft; } @@ -487,6 +511,144 @@ struct FieldTypeBuilder< } }; +inline bool field_type_needs_user_type_id(uint32_t) { return false; } + +template +Result build_field_type_with_resolver(TypeResolver &resolver, + bool nullable); + +template +Result add_tuple_element_types(TypeResolver &resolver, + FieldType &ft) { + if constexpr (Index < std::tuple_size_v) { + using Elem = std::tuple_element_t; + FORY_TRY(elem_ft, build_field_type_with_resolver(resolver, false)); + ft.generics.push_back(std::move(elem_ft)); + return add_tuple_element_types(resolver, ft); + } else { + return Result(); + } +} + +template +Result build_field_type_with_resolver(TypeResolver &resolver, + bool nullable) { + using Decayed = decay_t; + if constexpr (is_optional_v) { + using Inner = typename Decayed::value_type; + FORY_TRY(inner, build_field_type_with_resolver(resolver, true)); + inner.nullable = true; + return inner; + } else if constexpr (is_shared_ptr_v) { + using Inner = typename Decayed::element_type; + if constexpr (std::is_polymorphic_v) { + return FieldType(to_type_id(TypeId::UNKNOWN), true); + } else { + FORY_TRY(inner, build_field_type_with_resolver(resolver, true)); + inner.nullable = true; + return inner; + } + } else if constexpr (::fory::detail::is_shared_weak_v) { + using Inner = nullable_element_t; + if constexpr (std::is_polymorphic_v) { + return FieldType(to_type_id(TypeId::UNKNOWN), true); + } else { + FORY_TRY(inner, build_field_type_with_resolver(resolver, true)); + inner.nullable = true; + return inner; + } + } else if constexpr (is_unique_ptr_v) { + using Inner = typename Decayed::element_type; + if constexpr (std::is_polymorphic_v) { + return FieldType(to_type_id(TypeId::UNKNOWN), true); + } else { + FORY_TRY(inner, build_field_type_with_resolver(resolver, true)); + inner.nullable = true; + return inner; + } + } else if constexpr (is_vector_v) { + using Vec = Decayed; + using Element = element_type_t; + constexpr TypeId vec_type_id = Serializer::type_id; + if constexpr (vec_type_id == TypeId::LIST) { + FORY_TRY(elem, build_field_type_with_resolver(resolver, false)); + FieldType ft(to_type_id(vec_type_id), nullable); + ft.generics.push_back(std::move(elem)); + return ft; + } else { + return FieldType(to_type_id(vec_type_id), nullable); + } + } else if constexpr (is_list_v) { + using Element = typename Decayed::value_type; + FORY_TRY(elem, build_field_type_with_resolver(resolver, false)); + FieldType ft(to_type_id(TypeId::LIST), nullable); + ft.generics.push_back(std::move(elem)); + return ft; + } else if constexpr (is_deque_v) { + using Element = typename Decayed::value_type; + FORY_TRY(elem, build_field_type_with_resolver(resolver, false)); + FieldType ft(to_type_id(TypeId::LIST), nullable); + ft.generics.push_back(std::move(elem)); + return ft; + } else if constexpr (is_forward_list_v) { + using Element = typename Decayed::value_type; + FORY_TRY(elem, build_field_type_with_resolver(resolver, false)); + FieldType ft(to_type_id(TypeId::LIST), nullable); + ft.generics.push_back(std::move(elem)); + return ft; + } else if constexpr (is_set_like_v) { + using Set = Decayed; + using Element = element_type_t; + FORY_TRY(elem, build_field_type_with_resolver(resolver, false)); + FieldType ft(to_type_id(Serializer::type_id), nullable); + ft.generics.push_back(std::move(elem)); + return ft; + } else if constexpr (is_map_like_v) { + using Map = Decayed; + using Key = key_type_t; + using Value = mapped_type_t; + FORY_TRY(key_ft, build_field_type_with_resolver(resolver, false)); + FORY_TRY(val_ft, build_field_type_with_resolver(resolver, false)); + FieldType ft(to_type_id(Serializer::type_id), nullable); + ft.generics.push_back(std::move(key_ft)); + ft.generics.push_back(std::move(val_ft)); + return ft; + } else if constexpr (is_string_view_v) { + using StringT = std::basic_string; + return build_field_type_with_resolver(resolver, nullable); + } else if constexpr (is_tuple_v) { + constexpr size_t tuple_size = std::tuple_size_v; + FieldType ft(to_type_id(Serializer::type_id), nullable); + if constexpr (tuple_size > 0) { + FORY_RETURN_IF_ERROR((add_tuple_element_types(resolver, ft))); + } else { + ft.generics.push_back( + FieldType(static_cast(TypeId::UNKNOWN), false)); + } + return ft; + } else { + FieldType ft = FieldTypeBuilder::build(nullable); + if (is_user_type(static_cast(ft.type_id))) { + FORY_TRY(info, get_type_info_with_resolver(resolver)); + uint32_t resolved_type_id = info->type_id; + switch (static_cast(resolved_type_id)) { + case TypeId::NAMED_ENUM: + resolved_type_id = static_cast(TypeId::ENUM); + break; + case TypeId::TYPED_UNION: + case TypeId::NAMED_UNION: + resolved_type_id = static_cast(TypeId::UNION); + break; + default: + break; + } + ft.type_id = resolved_type_id; + } + return ft; + } +} + // Helper template functions to compute is_nullable and track_ref at compile // time. These replace constexpr lambdas which have issues on MSVC. template struct FieldInfoBuilder { static FieldInfo build() { - const auto meta = fory_field_info(T{}); - const auto field_names = decltype(meta)::Names; - const auto &field_ptrs = decltype(meta)::ptrs_ref(); + using MetaDesc = decltype(fory_field_info(std::declval())); + const auto field_names = MetaDesc::Names; + const auto &field_ptrs = MetaDesc::ptrs_ref(); // Convert camel_case field name to snake_case for cross-language // compatibility @@ -692,6 +854,79 @@ template struct FieldInfoBuilder { info.field_id = field_id; return info; } + + static Result build_with_resolver(TypeResolver &resolver) { + using MetaDesc = decltype(fory_field_info(std::declval())); + const auto field_names = MetaDesc::Names; + const auto &field_ptrs = MetaDesc::ptrs_ref(); + + // Convert camel_case field name to snake_case for cross-language + // compatibility + std::string_view original_name = field_names[Index]; + constexpr size_t max_snake_len = 128; // Reasonable max for field names + auto [snake_buffer, snake_len] = + ::fory::to_snake_case(original_name); + std::string field_name(snake_buffer.data(), snake_len); + + const auto field_ptr = std::get(field_ptrs); + using RawFieldType = + typename meta::RemoveMemberPointerCVRefT; + using ActualFieldType = + std::remove_cv_t>; + // unwrap fory::field<> to get the underlying type for FieldTypeBuilder + using UnwrappedFieldType = fory::unwrap_field_t; + + // get nullable and track_ref from field tags (FORY_FIELD_TAGS or + // fory::field<>) + constexpr bool is_nullable = + compute_is_nullable(); + constexpr bool track_ref = compute_track_ref(); + constexpr int16_t field_id = compute_field_id(); + + FORY_TRY(field_type, build_field_type_with_resolver( + resolver, false)); + + // Override type_id for unsigned types based on encoding from + // FORY_FIELD_CONFIG + using InnerType = unwrap_optional_inner_t; + constexpr uint32_t unsigned_tid = + compute_unsigned_type_id(); + if constexpr (unsigned_tid != 0 && is_unsigned_integer_v) { + field_type.type_id = unsigned_tid; + } + + // Override type_id for signed types based on encoding from + // FORY_FIELD_CONFIG + constexpr uint32_t signed_tid = + compute_signed_type_id(); + if constexpr (signed_tid != 0) { + field_type.type_id = signed_tid; + } + + if constexpr (::fory::detail::has_field_config_v) { + constexpr int16_t override_id = + ::fory::detail::GetFieldConfigEntry::type_id_override; + if constexpr (override_id >= 0) { + field_type.type_id = static_cast(override_id); + } + } + + // Override nullable and ref_tracking from field-level metadata + field_type.nullable = is_nullable; + field_type.ref_tracking = track_ref; + field_type.ref_mode = make_ref_mode(is_nullable, track_ref); +#ifdef FORY_DEBUG + // DEBUG: Print field info for debugging fingerprint mismatch + std::cerr << "[xlang][debug] FieldInfoBuilder T=" << typeid(T).name() + << " Index=" << Index << " field=" << field_name + << " type_id=" << field_type.type_id << " has_tags=" + << ::fory::detail::has_field_tags_v << " is_nullable=" + << is_nullable << " track_ref=" << track_ref << std::endl; +#endif + FieldInfo info(std::move(field_name), std::move(field_type)); + info.field_id = field_id; + return info; + } }; template @@ -702,6 +937,35 @@ std::vector build_field_infos(std::index_sequence) { return fields; } +template +Result append_field_infos(TypeResolver &, std::vector &, + std::index_sequence<>) { + return Result(); +} + +template +Result append_field_infos(TypeResolver &resolver, + std::vector &fields, + std::index_sequence) { + FORY_TRY(info, (FieldInfoBuilder::build_with_resolver(resolver))); + fields.push_back(std::move(info)); + return append_field_infos(resolver, fields, + std::index_sequence{}); +} + +template +Result, Error> +build_field_infos_with_resolver(TypeResolver &resolver, + std::index_sequence) { + std::vector fields; + fields.reserve(sizeof...(Indices)); + auto append_fields = [&](auto index_sequence) -> Result { + return append_field_infos(resolver, fields, index_sequence); + }; + FORY_RETURN_IF_ERROR(append_fields(std::index_sequence{})); + return fields; +} + } // namespace detail // ============================================================================ @@ -733,10 +997,15 @@ class TypeResolver { template TypeMeta clone_struct_meta(); template const std::vector &sorted_indices(); - /// get type info by type ID (for non-namespaced types) + /// get type info by type ID (for internal types) /// @return const pointer to TypeInfo if found, error otherwise Result get_type_info_by_id(uint32_t type_id) const; + /// get type info by type ID and user type ID (for user-registered types) + /// @return const pointer to TypeInfo if found, error otherwise + Result + get_user_type_info_by_id(uint32_t type_id, uint32_t user_type_id) const; + /// get type info by namespace and type name (for namespaced types) /// @return const pointer to TypeInfo if found, error otherwise Result @@ -807,26 +1076,32 @@ class TypeResolver { template static Result, Error> - build_struct_type_info(uint32_t type_id, std::string ns, - std::string type_name, bool register_by_name); + build_struct_type_info(uint32_t type_id, uint32_t user_type_id, + std::string ns, std::string type_name, + bool register_by_name); template static Result, Error> - build_enum_type_info(uint32_t type_id, std::string ns, std::string type_name, - bool register_by_name); + build_enum_type_info(uint32_t type_id, uint32_t user_type_id, std::string ns, + std::string type_name, bool register_by_name); template static Result, Error> - build_ext_type_info(uint32_t type_id, std::string ns, std::string type_name, - bool register_by_name); + build_ext_type_info(uint32_t type_id, uint32_t user_type_id, std::string ns, + std::string type_name, bool register_by_name); template static Result, Error> - build_union_type_info(uint32_t type_id, std::string ns, std::string type_name, - bool register_by_name); + build_union_type_info(uint32_t type_id, uint32_t user_type_id, std::string ns, + std::string type_name, bool register_by_name); template static Harness make_struct_harness(); + template static Harness make_struct_harness_impl(std::true_type); + + template + static Harness make_struct_harness_impl(std::false_type); + template static Harness make_serializer_harness(); template @@ -838,6 +1113,10 @@ class TypeResolver { static void *harness_read_adapter(ReadContext &ctx, RefMode ref_mode, bool read_type_info); + template + static void *harness_read_adapter_abstract(ReadContext &ctx, RefMode ref_mode, + bool read_type_info); + template static void harness_write_data_adapter(const void *value, WriteContext &ctx, bool has_generics); @@ -845,6 +1124,9 @@ class TypeResolver { template static void *harness_read_data_adapter(ReadContext &ctx); + template + static void *harness_read_data_adapter_abstract(ReadContext &ctx); + template static Result, Error> harness_struct_sorted_fields(TypeResolver &resolver); @@ -857,8 +1139,13 @@ class TypeResolver { static void *harness_read_compatible_adapter(ReadContext &ctx, const TypeInfo *ti); + template + static void *harness_read_compatible_adapter_abstract(ReadContext &ctx, + const TypeInfo *ti); + static std::string make_name_key(const std::string &ns, const std::string &name); + static uint64_t make_user_type_key(uint32_t type_id, uint32_t user_type_id); /// Register a TypeInfo, taking ownership and storing in primary storage. /// Returns pointer to the stored TypeInfo (owned by TypeResolver). @@ -889,6 +1176,7 @@ class TypeResolver { // FlatIntMap is optimized for integer keys with minimal overhead util::U64PtrMap type_info_by_ctid_{256}; util::U32PtrMap type_info_by_id_{256}; + util::U64PtrMap user_type_info_by_id_{256}; absl::flat_hash_map type_info_by_name_; util::U64PtrMap partial_type_infos_{256}; @@ -957,6 +1245,12 @@ template inline std::any any_read_adapter(ReadContext &ctx) { return std::any(std::make_shared(std::move(value))); } +template +inline std::any any_read_adapter_abstract(ReadContext &ctx) { + ctx.set_error(Error::type_error("Cannot deserialize abstract type")); + return std::any(); +} + } // namespace detail template const TypeMeta &TypeResolver::struct_meta() { @@ -996,6 +1290,12 @@ Result TypeResolver::get_type_info() const { return Unexpected(Error::type_error("Type not registered")); } +template +Result +get_type_info_with_resolver(TypeResolver &resolver) { + return resolver.get_type_info(); +} + template Result TypeResolver::register_any_type() { check_registration_thread(); constexpr uint32_t static_type_id = @@ -1024,21 +1324,21 @@ template Result TypeResolver::register_any_type() { template Result TypeResolver::register_by_id(uint32_t type_id) { check_registration_thread(); - if (type_id == 0) { - return Unexpected( - Error::invalid("type_id must be non-zero for register_by_id")); + if (type_id == kInvalidUserTypeId) { + return Unexpected(Error::invalid( + "type_id must be in range [0, 0xfffffffe] for register_by_id")); } constexpr uint64_t ctid = type_index(); if constexpr (is_fory_serializable_v) { - // encode type_id: shift left by 8 bits and add type category in low byte uint32_t actual_type_id = - compatible_ - ? (type_id << 8) + static_cast(TypeId::COMPATIBLE_STRUCT) - : (type_id << 8) + static_cast(TypeId::STRUCT); + compatible_ ? static_cast(TypeId::COMPATIBLE_STRUCT) + : static_cast(TypeId::STRUCT); + uint32_t user_type_id = type_id; - FORY_TRY(info, build_struct_type_info(actual_type_id, "", "", false)); + FORY_TRY(info, build_struct_type_info(actual_type_id, user_type_id, "", + "", false)); if (!info->harness.valid()) { return Unexpected( Error::invalid("Harness for registered type is incomplete")); @@ -1051,10 +1351,11 @@ Result TypeResolver::register_by_id(uint32_t type_id) { register_type_internal_runtime(std::type_index(typeid(T)), stored_ptr); return Result(); } else if constexpr (std::is_enum_v) { - uint32_t actual_type_id = - (type_id << 8) + static_cast(TypeId::ENUM); + uint32_t actual_type_id = static_cast(TypeId::ENUM); + uint32_t user_type_id = type_id; - FORY_TRY(info, build_enum_type_info(actual_type_id, "", "", false)); + FORY_TRY(info, build_enum_type_info(actual_type_id, user_type_id, "", "", + false)); if (!info->harness.valid()) { return Unexpected( Error::invalid("Harness for registered enum type is incomplete")); @@ -1089,8 +1390,8 @@ TypeResolver::register_by_name(const std::string &ns, compatible_ ? static_cast(TypeId::NAMED_COMPATIBLE_STRUCT) : static_cast(TypeId::NAMED_STRUCT); - FORY_TRY(info, - build_struct_type_info(actual_type_id, ns, type_name, true)); + FORY_TRY(info, build_struct_type_info(actual_type_id, kInvalidUserTypeId, + ns, type_name, true)); if (!info->harness.valid()) { return Unexpected( Error::invalid("Harness for registered type is incomplete")); @@ -1102,8 +1403,8 @@ TypeResolver::register_by_name(const std::string &ns, return Result(); } else if constexpr (std::is_enum_v) { uint32_t actual_type_id = static_cast(TypeId::NAMED_ENUM); - FORY_TRY(info, - build_enum_type_info(actual_type_id, ns, type_name, true)); + FORY_TRY(info, build_enum_type_info(actual_type_id, kInvalidUserTypeId, + ns, type_name, true)); if (!info->harness.valid()) { return Unexpected( Error::invalid("Harness for registered enum type is incomplete")); @@ -1124,17 +1425,18 @@ TypeResolver::register_by_name(const std::string &ns, template Result TypeResolver::register_ext_type_by_id(uint32_t type_id) { check_registration_thread(); - if (type_id == 0) { - return Unexpected( - Error::invalid("type_id must be non-zero for register_ext_type_by_id")); + if (type_id == kInvalidUserTypeId) { + return Unexpected(Error::invalid("type_id must be in range [0, 0xfffffffe] " + "for register_ext_type_by_id")); } constexpr uint64_t ctid = type_index(); - // encode type_id: shift left by 8 bits and add type category in low byte - uint32_t actual_type_id = (type_id << 8) + static_cast(TypeId::EXT); + uint32_t actual_type_id = static_cast(TypeId::EXT); + uint32_t user_type_id = type_id; - FORY_TRY(info, build_ext_type_info(actual_type_id, "", "", false)); + FORY_TRY(info, + build_ext_type_info(actual_type_id, user_type_id, "", "", false)); FORY_TRY(stored_ptr, register_type_internal(ctid, std::move(info))); partial_type_infos_.put(ctid, stored_ptr); @@ -1155,7 +1457,8 @@ TypeResolver::register_ext_type_by_name(const std::string &ns, constexpr uint64_t ctid = type_index(); uint32_t actual_type_id = static_cast(TypeId::NAMED_EXT); - FORY_TRY(info, build_ext_type_info(actual_type_id, ns, type_name, true)); + FORY_TRY(info, build_ext_type_info(actual_type_id, kInvalidUserTypeId, ns, + type_name, true)); FORY_TRY(stored_ptr, register_type_internal(ctid, std::move(info))); partial_type_infos_.put(ctid, stored_ptr); @@ -1166,16 +1469,17 @@ TypeResolver::register_ext_type_by_name(const std::string &ns, template Result TypeResolver::register_union_by_id(uint32_t type_id) { check_registration_thread(); - if (type_id == 0) { - return Unexpected( - Error::invalid("type_id must be non-zero for register_union_by_id")); + if (type_id == kInvalidUserTypeId) { + return Unexpected(Error::invalid( + "type_id must be in range [0, 0xfffffffe] for register_union_by_id")); } constexpr uint64_t ctid = type_index(); - uint32_t actual_type_id = - (type_id << 8) + static_cast(TypeId::TYPED_UNION); + uint32_t actual_type_id = static_cast(TypeId::TYPED_UNION); + uint32_t user_type_id = type_id; - FORY_TRY(info, build_union_type_info(actual_type_id, "", "", false)); + FORY_TRY(info, build_union_type_info(actual_type_id, user_type_id, "", "", + false)); FORY_TRY(stored_ptr, register_type_internal(ctid, std::move(info))); partial_type_infos_.put(ctid, stored_ptr); @@ -1196,7 +1500,8 @@ TypeResolver::register_union_by_name(const std::string &ns, constexpr uint64_t ctid = type_index(); uint32_t actual_type_id = static_cast(TypeId::NAMED_UNION); - FORY_TRY(info, build_union_type_info(actual_type_id, ns, type_name, true)); + FORY_TRY(info, build_union_type_info(actual_type_id, kInvalidUserTypeId, + ns, type_name, true)); FORY_TRY(stored_ptr, register_type_internal(ctid, std::move(info))); partial_type_infos_.put(ctid, stored_ptr); @@ -1206,8 +1511,8 @@ TypeResolver::register_union_by_name(const std::string &ns, template Result, Error> -TypeResolver::build_struct_type_info(uint32_t type_id, std::string ns, - std::string type_name, +TypeResolver::build_struct_type_info(uint32_t type_id, uint32_t user_type_id, + std::string ns, std::string type_name, bool register_by_name) { static_assert(is_fory_serializable_v, "build_struct_type_info requires FORY_STRUCT types"); @@ -1218,17 +1523,18 @@ TypeResolver::build_struct_type_info(uint32_t type_id, std::string ns, auto entry = std::make_unique(); entry->type_id = type_id; + entry->user_type_id = user_type_id; entry->namespace_name = std::move(ns); entry->register_by_name = register_by_name; entry->is_external = false; - const auto meta_desc = fory_field_info(T{}); - constexpr size_t field_count = decltype(meta_desc)::Size; - const auto field_names = decltype(meta_desc)::Names; + using MetaDesc = decltype(fory_field_info(std::declval())); + constexpr size_t field_count = MetaDesc::Size; + const auto field_names = MetaDesc::Names; std::string resolved_name = type_name; if (resolved_name.empty()) { - resolved_name = std::string(decltype(meta_desc)::Name); + resolved_name = std::string(MetaDesc::Name); } if (register_by_name && resolved_name.empty()) { return Unexpected(Error::invalid( @@ -1261,18 +1567,6 @@ TypeResolver::build_struct_type_info(uint32_t type_id, std::string ns, entry->sorted_indices.push_back(it->second); } - TypeMeta meta = - TypeMeta::from_fields(type_id, entry->namespace_name, entry->type_name, - register_by_name, std::move(sorted_fields)); - - FORY_TRY(type_def, meta.to_bytes()); - entry->type_def = std::move(type_def); - - Buffer buffer(entry->type_def.data(), - static_cast(entry->type_def.size()), false); - buffer.writer_index(static_cast(entry->type_def.size())); - FORY_TRY(parsed_meta, TypeMeta::from_bytes(buffer, nullptr)); - entry->type_meta = std::move(parsed_meta); entry->harness = make_struct_harness(); // Pre-encode namespace and type_name for efficient writing @@ -1286,8 +1580,8 @@ TypeResolver::build_struct_type_info(uint32_t type_id, std::string ns, template Result, Error> -TypeResolver::build_enum_type_info(uint32_t type_id, std::string ns, - std::string type_name, +TypeResolver::build_enum_type_info(uint32_t type_id, uint32_t user_type_id, + std::string ns, std::string type_name, bool register_by_name) { static_assert(std::is_enum_v, "build_enum_type_info requires enum types"); @@ -1298,6 +1592,7 @@ TypeResolver::build_enum_type_info(uint32_t type_id, std::string ns, auto entry = std::make_unique(); entry->type_id = type_id; + entry->user_type_id = user_type_id; entry->namespace_name = std::move(ns); entry->register_by_name = register_by_name; entry->is_external = false; @@ -1329,11 +1624,12 @@ TypeResolver::build_enum_type_info(uint32_t type_id, std::string ns, template Result, Error> -TypeResolver::build_ext_type_info(uint32_t type_id, std::string ns, - std::string type_name, +TypeResolver::build_ext_type_info(uint32_t type_id, uint32_t user_type_id, + std::string ns, std::string type_name, bool register_by_name) { auto entry = std::make_unique(); entry->type_id = type_id; + entry->user_type_id = user_type_id; entry->namespace_name = ns; entry->type_name = type_name; entry->register_by_name = register_by_name; @@ -1354,9 +1650,9 @@ TypeResolver::build_ext_type_info(uint32_t type_id, std::string ns, // Create TypeMeta for extension type (with empty fields) and generate // type_def. Extension types need type_def for meta sharing in compatible // mode. - TypeMeta meta = - TypeMeta::from_fields(type_id, entry->namespace_name, entry->type_name, - register_by_name, std::vector{}); + TypeMeta meta = TypeMeta::from_fields( + type_id, entry->namespace_name, entry->type_name, register_by_name, + entry->user_type_id, std::vector{}); FORY_TRY(type_def, meta.to_bytes()); entry->type_def = std::move(type_def); @@ -1365,11 +1661,12 @@ TypeResolver::build_ext_type_info(uint32_t type_id, std::string ns, template Result, Error> -TypeResolver::build_union_type_info(uint32_t type_id, std::string ns, - std::string type_name, +TypeResolver::build_union_type_info(uint32_t type_id, uint32_t user_type_id, + std::string ns, std::string type_name, bool register_by_name) { auto entry = std::make_unique(); entry->type_id = type_id; + entry->user_type_id = user_type_id; entry->namespace_name = std::move(ns); if (!type_name.empty()) { entry->type_name = std::move(type_name); @@ -1392,6 +1689,27 @@ TypeResolver::build_union_type_info(uint32_t type_id, std::string ns, } template Harness TypeResolver::make_struct_harness() { + constexpr bool needs_abstract_harness = + std::is_abstract_v || !std::is_default_constructible_v; + return make_struct_harness_impl( + std::bool_constant{}); +} + +template +Harness TypeResolver::make_struct_harness_impl(std::true_type) { + Harness harness(&TypeResolver::harness_write_adapter, + &TypeResolver::harness_read_adapter_abstract, + &TypeResolver::harness_write_data_adapter, + &TypeResolver::harness_read_data_adapter_abstract, + &TypeResolver::harness_struct_sorted_fields, + &TypeResolver::harness_read_compatible_adapter_abstract); + harness.any_write_fn = &detail::any_write_adapter; + harness.any_read_fn = &detail::any_read_adapter_abstract; + return harness; +} + +template +Harness TypeResolver::make_struct_harness_impl(std::false_type) { Harness harness(&TypeResolver::harness_write_adapter, &TypeResolver::harness_read_adapter, &TypeResolver::harness_write_data_adapter, @@ -1433,6 +1751,13 @@ void *TypeResolver::harness_read_adapter(ReadContext &ctx, RefMode ref_mode, return new T(std::move(value)); } +template +void *TypeResolver::harness_read_adapter_abstract(ReadContext &ctx, RefMode, + bool) { + ctx.set_error(Error::type_error("Cannot deserialize abstract type")); + return nullptr; +} + template void TypeResolver::harness_write_data_adapter(const void *value, WriteContext &ctx, @@ -1450,6 +1775,12 @@ void *TypeResolver::harness_read_data_adapter(ReadContext &ctx) { return new T(std::move(value)); } +template +void *TypeResolver::harness_read_data_adapter_abstract(ReadContext &ctx) { + ctx.set_error(Error::type_error("Cannot deserialize abstract type")); + return nullptr; +} + template void *TypeResolver::harness_read_compatible_adapter(ReadContext &ctx, const TypeInfo *ti) { @@ -1460,15 +1791,22 @@ void *TypeResolver::harness_read_compatible_adapter(ReadContext &ctx, return new T(std::move(value)); } +template +void *TypeResolver::harness_read_compatible_adapter_abstract(ReadContext &ctx, + const TypeInfo *) { + ctx.set_error(Error::type_error("Cannot deserialize abstract type")); + return nullptr; +} + template Result, Error> -TypeResolver::harness_struct_sorted_fields(TypeResolver &) { +TypeResolver::harness_struct_sorted_fields(TypeResolver &resolver) { static_assert(is_fory_serializable_v, "harness_struct_sorted_fields requires FORY_STRUCT types"); - const auto meta_desc = fory_field_info(T{}); - constexpr size_t field_count = decltype(meta_desc)::Size; - auto fields = - detail::build_field_infos(std::make_index_sequence{}); + using MetaDesc = decltype(fory_field_info(std::declval())); + constexpr size_t field_count = MetaDesc::Size; + FORY_TRY(fields, detail::build_field_infos_with_resolver( + resolver, std::make_index_sequence{})); auto sorted = TypeMeta::sort_field_infos(std::move(fields)); return sorted; } @@ -1489,6 +1827,12 @@ inline std::string TypeResolver::make_name_key(const std::string &ns, return key; } +inline uint64_t TypeResolver::make_user_type_key(uint32_t type_id, + uint32_t user_type_id) { + return (static_cast(type_id) << 32) | + static_cast(user_type_id); +} + inline Result TypeResolver::register_type_internal(uint64_t ctid, std::unique_ptr info) { @@ -1503,7 +1847,7 @@ TypeResolver::register_type_internal(uint64_t ctid, type_info_by_ctid_.put(ctid, raw_ptr); - if (raw_ptr->type_id != 0 && !raw_ptr->register_by_name) { + if (::fory::is_internal_type(raw_ptr->type_id)) { TypeInfo *existing = type_info_by_id_.get_or_default(raw_ptr->type_id, nullptr); if (existing != nullptr && existing != raw_ptr) { @@ -1511,6 +1855,16 @@ TypeResolver::register_type_internal(uint64_t ctid, std::to_string(raw_ptr->type_id))); } type_info_by_id_.put(raw_ptr->type_id, raw_ptr); + } else if (!raw_ptr->register_by_name && + raw_ptr->user_type_id != kInvalidUserTypeId) { + uint64_t key = make_user_type_key(raw_ptr->type_id, raw_ptr->user_type_id); + TypeInfo *existing = user_type_info_by_id_.get_or_default(key, nullptr); + if (existing != nullptr && existing != raw_ptr) { + return Unexpected(Error::invalid( + "Type id already registered: " + std::to_string(raw_ptr->type_id) + + "/" + std::to_string(raw_ptr->user_type_id))); + } + user_type_info_by_id_.put(key, raw_ptr); } if (raw_ptr->register_by_name) { @@ -1544,6 +1898,19 @@ TypeResolver::get_type_info_by_id(uint32_t type_id) const { std::to_string(type_id))); } +inline Result +TypeResolver::get_user_type_info_by_id(uint32_t type_id, + uint32_t user_type_id) const { + uint64_t key = make_user_type_key(type_id, user_type_id); + TypeInfo *info = user_type_info_by_id_.get_or_default(key, nullptr); + if (info != nullptr) { + return info; + } + return Unexpected(Error::type_error( + "TypeInfo not found for type_id: " + std::to_string(type_id) + + ", user_type_id: " + std::to_string(user_type_id))); +} + inline Result TypeResolver::get_type_info_by_name(const std::string &ns, const std::string &type_name) const { @@ -1561,9 +1928,9 @@ TypeResolver::get_type_info_by_name(const std::string &ns, // TypeResolver to be complete) // ============================================================================ -template Result WriteContext::write_enum_typeinfo() { +template Result WriteContext::write_enum_type_info() { FORY_TRY(type_info, type_resolver_->get_type_info()); - return write_enum_typeinfo(type_info); + return write_enum_type_info(type_info); } } // namespace serialization diff --git a/cpp/fory/serialization/union_serializer.h b/cpp/fory/serialization/union_serializer.h index 6592b3bd22..47ab55fae2 100644 --- a/cpp/fory/serialization/union_serializer.h +++ b/cpp/fory/serialization/union_serializer.h @@ -421,8 +421,12 @@ struct Serializer>> { static constexpr TypeId type_id = TypeId::UNION; static inline void write_type_info(WriteContext &ctx) { - auto result = ctx.write_any_typeinfo( - static_cast(TypeId::TYPED_UNION), std::type_index(typeid(T))); + auto type_info_res = ctx.type_resolver().template get_type_info(); + if (FORY_PREDICT_FALSE(!type_info_res.ok())) { + ctx.set_error(std::move(type_info_res).error()); + return; + } + auto result = ctx.write_any_type_info(type_info_res.value()); if (FORY_PREDICT_FALSE(!result.ok())) { ctx.set_error(std::move(result).error()); } @@ -435,7 +439,7 @@ struct Serializer>> { return; } const TypeInfo *expected = type_info_res.value(); - const TypeInfo *remote = ctx.read_any_typeinfo(ctx.error()); + const TypeInfo *remote = ctx.read_any_type_info(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } @@ -488,7 +492,7 @@ struct Serializer>> { nullable)) { return; } - ctx.write_var_uint32(field_type_id); + ctx.write_uint8(static_cast(field_type_id)); detail::write_union_value_data(value, ctx, field_type_id); return; } @@ -573,7 +577,7 @@ struct Serializer>> { detail::default_union_case_value()); return; } - uint32_t actual_type_id = ctx.read_var_uint32(ctx.error()); + uint32_t actual_type_id = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { result = default_value(); return; @@ -620,7 +624,7 @@ struct Serializer>> { Error::invalid_data("Unknown reference flag in union value")); return default_value(); } - const TypeInfo *type_info = ctx.read_any_typeinfo(ctx.error()); + const TypeInfo *type_info = ctx.read_any_type_info(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return default_value(); } diff --git a/cpp/fory/serialization/unsigned_serializer.h b/cpp/fory/serialization/unsigned_serializer.h index b6961b55e6..e53ff250f2 100644 --- a/cpp/fory/serialization/unsigned_serializer.h +++ b/cpp/fory/serialization/unsigned_serializer.h @@ -42,11 +42,11 @@ template <> struct Serializer { static constexpr TypeId type_id = TypeId::UINT8; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - uint32_t actual = ctx.read_var_uint32(ctx.error()); + uint32_t actual = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(actual != static_cast(type_id))) { ctx.set_error( Error::type_mismatch(actual, static_cast(type_id))); @@ -78,7 +78,7 @@ template <> struct Serializer { return 0; } if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(type_id_read != static_cast(type_id))) { ctx.set_error( Error::type_mismatch(type_id_read, static_cast(type_id))); @@ -106,11 +106,11 @@ template <> struct Serializer { static constexpr TypeId type_id = TypeId::UINT16; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - uint32_t actual = ctx.read_var_uint32(ctx.error()); + uint32_t actual = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(actual != static_cast(type_id))) { ctx.set_error( Error::type_mismatch(actual, static_cast(type_id))); @@ -142,7 +142,7 @@ template <> struct Serializer { return 0; } if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(type_id_read != static_cast(type_id))) { ctx.set_error( Error::type_mismatch(type_id_read, static_cast(type_id))); @@ -170,11 +170,11 @@ template <> struct Serializer { static constexpr TypeId type_id = TypeId::VAR_UINT32; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - uint32_t actual = ctx.read_var_uint32(ctx.error()); + uint32_t actual = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(actual != static_cast(type_id))) { ctx.set_error( Error::type_mismatch(actual, static_cast(type_id))); @@ -206,7 +206,7 @@ template <> struct Serializer { return 0; } if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(type_id_read != static_cast(type_id))) { ctx.set_error( Error::type_mismatch(type_id_read, static_cast(type_id))); @@ -234,11 +234,11 @@ template <> struct Serializer { static constexpr TypeId type_id = TypeId::VAR_UINT64; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - uint32_t actual = ctx.read_var_uint32(ctx.error()); + uint32_t actual = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(actual != static_cast(type_id))) { ctx.set_error( Error::type_mismatch(actual, static_cast(type_id))); @@ -270,7 +270,7 @@ template <> struct Serializer { return 0; } if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(type_id_read != static_cast(type_id))) { ctx.set_error( Error::type_mismatch(type_id_read, static_cast(type_id))); @@ -303,11 +303,11 @@ template struct Serializer> { static constexpr TypeId type_id = TypeId::INT8_ARRAY; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - uint32_t actual = ctx.read_var_uint32(ctx.error()); + uint32_t actual = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE( !type_id_matches(actual, static_cast(type_id)))) { ctx.set_error( @@ -351,7 +351,7 @@ template struct Serializer> { return std::array(); } if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(type_id_read != static_cast(type_id))) { ctx.set_error( Error::type_mismatch(type_id_read, static_cast(type_id))); @@ -387,11 +387,11 @@ template struct Serializer> { static constexpr TypeId type_id = TypeId::UINT16_ARRAY; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - uint32_t actual = ctx.read_var_uint32(ctx.error()); + uint32_t actual = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE( !type_id_matches(actual, static_cast(type_id)))) { ctx.set_error( @@ -435,7 +435,7 @@ template struct Serializer> { return std::array(); } if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(type_id_read != static_cast(type_id))) { ctx.set_error( Error::type_mismatch(type_id_read, static_cast(type_id))); @@ -471,11 +471,11 @@ template struct Serializer> { static constexpr TypeId type_id = TypeId::UINT32_ARRAY; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - uint32_t actual = ctx.read_var_uint32(ctx.error()); + uint32_t actual = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE( !type_id_matches(actual, static_cast(type_id)))) { ctx.set_error( @@ -519,7 +519,7 @@ template struct Serializer> { return std::array(); } if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(type_id_read != static_cast(type_id))) { ctx.set_error( Error::type_mismatch(type_id_read, static_cast(type_id))); @@ -555,11 +555,11 @@ template struct Serializer> { static constexpr TypeId type_id = TypeId::UINT64_ARRAY; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - uint32_t actual = ctx.read_var_uint32(ctx.error()); + uint32_t actual = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE( !type_id_matches(actual, static_cast(type_id)))) { ctx.set_error( @@ -603,7 +603,7 @@ template struct Serializer> { return std::array(); } if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(type_id_read != static_cast(type_id))) { ctx.set_error( Error::type_mismatch(type_id_read, static_cast(type_id))); @@ -644,11 +644,11 @@ template <> struct Serializer> { static constexpr TypeId type_id = TypeId::BINARY; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - uint32_t actual = ctx.read_var_uint32(ctx.error()); + uint32_t actual = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE( !type_id_matches(actual, static_cast(type_id)))) { ctx.set_error( @@ -661,7 +661,7 @@ template <> struct Serializer> { bool has_generics = false) { write_not_null_ref_flag(ctx, ref_mode); if (write_type) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } write_data_generic(vec, ctx, has_generics); } @@ -692,7 +692,7 @@ template <> struct Serializer> { return std::vector(); } if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(type_id_read != static_cast(type_id))) { ctx.set_error( Error::type_mismatch(type_id_read, static_cast(type_id))); @@ -730,11 +730,11 @@ template <> struct Serializer> { static constexpr TypeId type_id = TypeId::UINT16_ARRAY; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - uint32_t actual = ctx.read_var_uint32(ctx.error()); + uint32_t actual = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE( !type_id_matches(actual, static_cast(type_id)))) { ctx.set_error( @@ -784,7 +784,7 @@ template <> struct Serializer> { return std::vector(); } if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(type_id_read != static_cast(type_id))) { ctx.set_error( Error::type_mismatch(type_id_read, static_cast(type_id))); @@ -832,11 +832,11 @@ template <> struct Serializer> { static constexpr TypeId type_id = TypeId::UINT32_ARRAY; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - uint32_t actual = ctx.read_var_uint32(ctx.error()); + uint32_t actual = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE( !type_id_matches(actual, static_cast(type_id)))) { ctx.set_error( @@ -886,7 +886,7 @@ template <> struct Serializer> { return std::vector(); } if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(type_id_read != static_cast(type_id))) { ctx.set_error( Error::type_mismatch(type_id_read, static_cast(type_id))); @@ -934,11 +934,11 @@ template <> struct Serializer> { static constexpr TypeId type_id = TypeId::UINT64_ARRAY; static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { - uint32_t actual = ctx.read_var_uint32(ctx.error()); + uint32_t actual = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE( !type_id_matches(actual, static_cast(type_id)))) { ctx.set_error( @@ -988,7 +988,7 @@ template <> struct Serializer> { return std::vector(); } if (read_type) { - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(type_id_read != static_cast(type_id))) { ctx.set_error( Error::type_mismatch(type_id_read, static_cast(type_id))); diff --git a/cpp/fory/serialization/variant_serializer.h b/cpp/fory/serialization/variant_serializer.h index e028e66e28..041c273add 100644 --- a/cpp/fory/serialization/variant_serializer.h +++ b/cpp/fory/serialization/variant_serializer.h @@ -41,12 +41,12 @@ template <> struct Serializer { TypeId::NONE; // Use NONE for empty/not-applicable type static inline void write_type_info(WriteContext &ctx) { - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { // Read and validate type_id for monostate - ctx.read_var_uint32(ctx.error()); + ctx.read_uint8(ctx.error()); // Accept any type since monostate is just a marker with no data } @@ -175,12 +175,12 @@ template struct Serializer> { static inline void write_type_info(WriteContext &ctx) { // write UNION type_id to indicate variant/sum type - ctx.write_var_uint32(static_cast(type_id)); + ctx.write_uint8(static_cast(type_id)); } static inline void read_type_info(ReadContext &ctx) { // Read and validate UNION type_id - uint32_t type_id_read = ctx.read_var_uint32(ctx.error()); + uint32_t type_id_read = ctx.read_uint8(ctx.error()); if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } diff --git a/cpp/fory/serialization/xlang_test_main.cc b/cpp/fory/serialization/xlang_test_main.cc index fde2ba032f..cf8e087f17 100644 --- a/cpp/fory/serialization/xlang_test_main.cc +++ b/cpp/fory/serialization/xlang_test_main.cc @@ -762,9 +762,14 @@ template <> struct Serializer { // Delegate dynamic typeinfo to WriteContext so that user type // ids and named registrations are encoded consistently with // other ext types. - auto result = - ctx.write_any_typeinfo(static_cast(TypeId::UNKNOWN), - std::type_index(typeid(MyExt))); + auto type_info_res = + ctx.type_resolver().get_type_info(std::type_index(typeid(MyExt))); + if (!type_info_res.ok()) { + ctx.set_error(std::move(type_info_res).error()); + return; + } + auto result = ctx.write_any_type_info(type_info_res.value()->type_id, + std::type_index(typeid(MyExt))); if (!result.ok()) { ctx.set_error(std::move(result).error()); return; @@ -790,7 +795,7 @@ template <> struct Serializer { } if (read_type) { // Validate dynamic type info and consume any named metadata. - const TypeInfo *type_info = ctx.read_any_typeinfo(ctx.error()); + const TypeInfo *type_info = ctx.read_any_type_info(ctx.error()); if (ctx.has_error()) { return MyExt{}; } diff --git a/cpp/fory/type/type.h b/cpp/fory/type/type.h index fbfe1c8bc5..75964282e5 100644 --- a/cpp/fory/type/type.h +++ b/cpp/fory/type/type.h @@ -22,7 +22,7 @@ #include // For fixed-width integer types namespace fory { -enum class TypeId : int32_t { +enum class TypeId : uint8_t { // Unknown/polymorphic type marker. UNKNOWN = 0, // a boolean value (true or false). @@ -145,8 +145,34 @@ enum class TypeId : int32_t { BOUND = 67 }; -inline bool is_user_type(int32_t type_id) { - switch (static_cast(type_id)) { +enum class TypeRegistrationKind : int32_t { + INTERNAL = 0, + BY_ID = 1, + BY_NAME = 2 +}; + +inline constexpr TypeRegistrationKind +get_type_registration_kind(TypeId type_id) { + switch (type_id) { + case TypeId::ENUM: + case TypeId::STRUCT: + case TypeId::COMPATIBLE_STRUCT: + case TypeId::EXT: + case TypeId::TYPED_UNION: + return TypeRegistrationKind::BY_ID; + case TypeId::NAMED_ENUM: + case TypeId::NAMED_STRUCT: + case TypeId::NAMED_COMPATIBLE_STRUCT: + case TypeId::NAMED_EXT: + case TypeId::NAMED_UNION: + return TypeRegistrationKind::BY_NAME; + default: + return TypeRegistrationKind::INTERNAL; + } +} + +inline bool is_user_type(TypeId type_id) { + switch (type_id) { case TypeId::ENUM: case TypeId::NAMED_ENUM: case TypeId::STRUCT: @@ -163,8 +189,8 @@ inline bool is_user_type(int32_t type_id) { } } -inline bool is_namespaced_type(int32_t type_id) { - switch (static_cast(type_id)) { +inline bool is_namespaced_type(TypeId type_id) { + switch (type_id) { case TypeId::NAMED_ENUM: case TypeId::NAMED_STRUCT: case TypeId::NAMED_COMPATIBLE_STRUCT: @@ -176,8 +202,8 @@ inline bool is_namespaced_type(int32_t type_id) { } } -inline bool is_type_share_meta(int32_t type_id) { - switch (static_cast(type_id)) { +inline bool is_type_share_meta(TypeId type_id) { + switch (type_id) { case TypeId::NAMED_ENUM: case TypeId::NAMED_STRUCT: case TypeId::NAMED_EXT: @@ -215,17 +241,6 @@ inline constexpr bool is_internal_type(uint32_t type_id) { return false; } // Internal types are all types that are NOT user types or UNKNOWN - uint32_t tid_low = type_id & 0xff; - return tid_low != static_cast(TypeId::ENUM) && - tid_low != static_cast(TypeId::NAMED_ENUM) && - tid_low != static_cast(TypeId::STRUCT) && - tid_low != static_cast(TypeId::COMPATIBLE_STRUCT) && - tid_low != static_cast(TypeId::NAMED_STRUCT) && - tid_low != static_cast(TypeId::NAMED_COMPATIBLE_STRUCT) && - tid_low != static_cast(TypeId::EXT) && - tid_low != static_cast(TypeId::NAMED_EXT) && - tid_low != static_cast(TypeId::TYPED_UNION) && - tid_low != static_cast(TypeId::NAMED_UNION) && - tid_low != static_cast(TypeId::UNKNOWN); + return !is_user_type(static_cast(type_id)); } } // namespace fory diff --git a/cpp/fory/util/flat_int_map_test.cc b/cpp/fory/util/flat_int_map_test.cc index 8459f1f4b6..d43c5d45de 100644 --- a/cpp/fory/util/flat_int_map_test.cc +++ b/cpp/fory/util/flat_int_map_test.cc @@ -485,15 +485,14 @@ TEST_F(FlatIntMapTest, U32PtrMap_ConstFind) { EXPECT_EQ(entry->key, 1u); } -// Test for type_id lookups (simulates type_info_by_id_ usage) +// Test for type_id lookups (simulates internal type_info_by_id_ usage) TEST_F(FlatIntMapTest, U32PtrMap_TypeIdLookups) { U32PtrMap map(256); - // Simulate type ID values used in TypeResolver + // Simulate internal type ID values used in TypeResolver std::vector type_ids; for (uint32_t i = 1; i <= 50; ++i) { - // Simulate encoded type_id: (user_id << 8) + TypeId - uint32_t type_id = (i << 8) + 100; // 100 = some TypeId enum value + uint32_t type_id = i; type_ids.push_back(type_id); map.put(type_id, reinterpret_cast(static_cast(i))); } diff --git a/dart/packages/fory/lib/src/const/obj_type.dart b/dart/packages/fory/lib/src/const/obj_type.dart index 42e7f4d9b5..d91c252628 100644 --- a/dart/packages/fory/lib/src/const/obj_type.dart +++ b/dart/packages/fory/lib/src/const/obj_type.dart @@ -21,6 +21,9 @@ /// TODO: foryJava manages writeRef for these types more precisely, such as control for time types and strings, not just basic types. Here we only control basic types. library; +// 0xffffffff means "unset" for user type id. +const int kInvalidUserTypeId = 0xFFFFFFFF; + enum ObjType { /// Unknown/polymorphic type marker. For example, a field is a parent class/non-specific class, /// which cannot be analyzed during static code generation. @@ -211,6 +214,13 @@ enum ObjType { || this == NAMED_COMPATIBLE_STRUCT; } + bool needsUserTypeId() { + return this == ENUM + || this == STRUCT + || this == COMPATIBLE_STRUCT + || this == EXT; + } + bool isTimeType() { return this == TIMESTAMP || this == DATE diff --git a/dart/packages/fory/lib/src/exception/registration_exception.dart b/dart/packages/fory/lib/src/exception/registration_exception.dart index b9abe0b3c2..d41d077950 100644 --- a/dart/packages/fory/lib/src/exception/registration_exception.dart +++ b/dart/packages/fory/lib/src/exception/registration_exception.dart @@ -33,7 +33,7 @@ class UnregisteredTagException extends ForyException { } class UnregisteredTypeException extends ForyException { - final Type _type; + final Object _type; UnregisteredTypeException(this._type); @@ -80,4 +80,4 @@ class DuplicatedTypeRegistrationException extends ForyException { buf.write('\nBut you try to register another tag: '); buf.writeln(_newTag); } -} \ No newline at end of file +} diff --git a/dart/packages/fory/lib/src/fory_context.dart b/dart/packages/fory/lib/src/fory_context.dart index 2fa23254ad..0cc38e438b 100644 --- a/dart/packages/fory/lib/src/fory_context.dart +++ b/dart/packages/fory/lib/src/fory_context.dart @@ -20,6 +20,7 @@ import 'dart:collection'; import 'package:fory/src/config/fory_config.dart'; import 'package:fory/src/exception/registration_exception.dart' show DuplicatedTagRegistrationException, DuplicatedTypeRegistrationException; +import 'package:fory/src/collection/long_long_key.dart'; import 'package:fory/src/meta/type_info.dart'; import 'package:fory/src/serializer/serializer.dart'; import 'package:fory/src/serializer/serializer_pool.dart'; @@ -41,6 +42,7 @@ class ForyContext { final ForyConfig conf; final Map tag2TypeInfo; final Map type2TypeInfo; + final Map userTypeId2TypeInfo; late final List objTypeId2TypeInfo; late final Serializer abstractListSer; @@ -48,7 +50,8 @@ class ForyContext { ForyContext(this.conf) : tag2TypeInfo = HashMap(), - type2TypeInfo = HashMap(); + type2TypeInfo = HashMap(), + userTypeId2TypeInfo = HashMap(); void initForDefaultTypes() { type2TypeInfo.addEntries(_defaultTypeInfos); @@ -75,5 +78,8 @@ class ForyContext { } tag2TypeInfo[typeInfo.tag!] = typeInfo; type2TypeInfo[typeInfo.dartType] = typeInfo; + if (typeInfo.objType.needsUserTypeId() && typeInfo.userTypeId != kInvalidUserTypeId) { + userTypeId2TypeInfo[LongLongKey(typeInfo.objType.id, typeInfo.userTypeId)] = typeInfo; + } } } diff --git a/dart/packages/fory/lib/src/meta/type_info.dart b/dart/packages/fory/lib/src/meta/type_info.dart index d858890f7d..7286626a0a 100644 --- a/dart/packages/fory/lib/src/meta/type_info.dart +++ b/dart/packages/fory/lib/src/meta/type_info.dart @@ -27,6 +27,8 @@ class TypeInfo { final String? tag; final MetaStringBytes? typeNameBytes; final MetaStringBytes? nsBytes; + // Stored as unsigned 32-bit; -1 (0xffffffff) means "unset". + final int userTypeId; late Serializer ser; TypeInfo( @@ -35,6 +37,7 @@ class TypeInfo { this.tag, this.typeNameBytes, this.nsBytes, + {this.userTypeId = kInvalidUserTypeId} ); TypeInfo.fromInnerType( @@ -43,5 +46,6 @@ class TypeInfo { this.ser, ) : tag = null, typeNameBytes = null, - nsBytes = null; -} \ No newline at end of file + nsBytes = null, + userTypeId = kInvalidUserTypeId; +} diff --git a/dart/packages/fory/lib/src/resolver/impl/meta_string_resolver_impl.dart b/dart/packages/fory/lib/src/resolver/impl/meta_string_resolver_impl.dart index c57256c7e5..5b6a7bcc2f 100644 --- a/dart/packages/fory/lib/src/resolver/impl/meta_string_resolver_impl.dart +++ b/dart/packages/fory/lib/src/resolver/impl/meta_string_resolver_impl.dart @@ -39,6 +39,8 @@ class MetaStringResolverImpl extends MetaStringResolver { final Map _longLong2MSB = HashMap(); final List _readId2MSB = []; final Map _metaString2MSB = HashMap(); + // Empty meta string uses utf8 encoding id 0 and hashCode 256 (see MetaStringBytes.of). + final MetaStringBytes _emptyMetaStringBytes = MetaStringBytes(Uint8List(0), 256); MetaStringResolverImpl(); @@ -88,6 +90,9 @@ class MetaStringResolverImpl extends MetaStringResolver { /// [v1] and [v2] are the two parts of the long long key /// [encoding] is the encoding of the string bytes MetaStringBytes _readSmallStringBytes(ByteReader reader, int len) { + if (len == 0) { + return _emptyMetaStringBytes; + } late final int v1; int v2 = 0; int encoding = reader.readInt8(); diff --git a/dart/packages/fory/lib/src/resolver/impl/meta_string_writing_resolver_impl.dart b/dart/packages/fory/lib/src/resolver/impl/meta_string_writing_resolver_impl.dart index 717022d82f..b1a98b8e07 100644 --- a/dart/packages/fory/lib/src/resolver/impl/meta_string_writing_resolver_impl.dart +++ b/dart/packages/fory/lib/src/resolver/impl/meta_string_writing_resolver_impl.dart @@ -41,9 +41,9 @@ final class MetaStringWritingResolverImpl extends MetaStringWritingResolver{ bw.writeVarUint32Small7(bytesLen << 1); if (bytesLen > smallStringThreshold){ bw.writeInt64(msb.hashCode); - }else { + }else if (bytesLen != 0) { bw.writeInt8(msb.encoding.id); } bw.writeBytes(msb.bytes); } -} \ No newline at end of file +} diff --git a/dart/packages/fory/lib/src/resolver/impl/xtype_resolver_impl.dart b/dart/packages/fory/lib/src/resolver/impl/xtype_resolver_impl.dart index 099b2ca107..d231e2d118 100644 --- a/dart/packages/fory/lib/src/resolver/impl/xtype_resolver_impl.dart +++ b/dart/packages/fory/lib/src/resolver/impl/xtype_resolver_impl.dart @@ -153,9 +153,19 @@ final class XtypeResolverImpl extends XtypeResolver { @override TypeInfo readTypeInfo(ByteReader br) { - int xtypeId = br.readVarUint32Small14(); + int xtypeId = br.readUint8(); ObjType xtype = ObjType.fromId(xtypeId)!; switch(xtype){ + case ObjType.ENUM: + case ObjType.STRUCT: + case ObjType.COMPATIBLE_STRUCT: + case ObjType.EXT: + int userTypeId = br.readVarUint32(); + TypeInfo? idTypeInfo = _ctx.userTypeId2TypeInfo[LongLongKey(xtypeId, userTypeId)]; + if (idTypeInfo != null) { + return idTypeInfo; + } + throw UnregisteredTypeException(xtype); case ObjType.NAMED_ENUM: case ObjType.NAMED_STRUCT: case ObjType.NAMED_COMPATIBLE_STRUCT: @@ -207,8 +217,14 @@ final class XtypeResolverImpl extends XtypeResolver { if (typeInfo == null){ throw UnregisteredTypeException(dartType); } - bw.writeVarUint32Small7(typeInfo.objType.id); + bw.writeUint8(typeInfo.objType.id); switch(typeInfo.objType){ + case ObjType.ENUM: + case ObjType.STRUCT: + case ObjType.COMPATIBLE_STRUCT: + case ObjType.EXT: + bw.writeVarUint32(typeInfo.userTypeId); + break; case ObjType.NAMED_ENUM: case ObjType.NAMED_STRUCT: case ObjType.NAMED_COMPATIBLE_STRUCT: diff --git a/docs/compiler/fdl-syntax.md b/docs/compiler/fdl-syntax.md index 2a3c1e1151..082111ec5f 100644 --- a/docs/compiler/fdl-syntax.md +++ b/docs/compiler/fdl-syntax.md @@ -67,12 +67,19 @@ The package declaration defines the namespace for all types in the file. package com.example.models; ``` +You can optionally specify a package alias used for auto-generated type IDs: + +```protobuf +package com.example.models alias models_v1; +``` + **Rules:** - Optional but recommended - Must appear before any type definitions - Only one package declaration per file - Used for namespace-based type registration +- Package alias is used for auto-ID hashing **Language Mapping:** @@ -262,6 +269,7 @@ FDL supports protobuf-style extension options for Fory-specific configuration: ```protobuf option (fory).use_record_for_java_message = true; option (fory).polymorphism = true; +option (fory).enable_auto_type_id = true; ``` **Available File Options:** @@ -270,6 +278,7 @@ option (fory).polymorphism = true; | ----------------------------- | ------ | ------------------------------------------------------------ | | `use_record_for_java_message` | bool | Generate Java records instead of classes | | `polymorphism` | bool | Enable polymorphism for all types | +| `enable_auto_type_id` | bool | Auto-generate numeric type IDs when omitted (default: true) | | `go_nested_type_style` | string | Go nested type naming: `underscore` (default) or `camelcase` | See the [Fory Extension Options](#fory-extension-options) section for complete documentation of message, enum, and field options. @@ -426,7 +435,7 @@ enum Status { } ``` -### With Type ID +### With Explicit Type ID ```protobuf enum Status [id=100] { @@ -508,7 +517,7 @@ enum_value := IDENTIFIER '=' INTEGER ';' - Enum names must be unique within the file - Enum values must have explicit integer assignments - Value integers must be unique within the enum (no aliases) -- Type ID (`[id=100]`) is optional but recommended for cross-language use +- Type ID (`[id=100]`) is optional for enums but recommended for cross-language use **Example with All Features:** @@ -538,7 +547,7 @@ message Person { } ``` -### With Type ID +### With Explicit Type ID ```protobuf message Person [id=101] { @@ -547,6 +556,34 @@ message Person [id=101] { } ``` +### Without Explicit Type ID + +```protobuf +message Person { // Auto-generated when enable_auto_type_id = true + string name = 1; + int32 age = 2; +} +``` + +### Type Registration + +FDL uses numeric type IDs for message, union, and enum registration. By default, +if you omit `id`, the compiler auto-generates one using +`MurmurHash3(utf8(package.type_name))` (32-bit). If a package/type name alias is +specified, the alias is used instead. When `enable_auto_type_id = false`, types +without explicit IDs are registered by namespace and name instead of receiving +generated IDs. + +```protobuf +message User [id=100] { ... } // Registered with ID 100 +message Config { ... } // ID auto-generated when enable_auto_type_id = true +``` + +Namespace-based registration is still available when calling runtime APIs +directly. IDL-generated code uses explicit IDs when provided. If an auto-generated ID +conflicts, the compiler raises an error and asks you to specify an explicit `id` or an +`alias` to change the hash source. + ### Reserved Fields Reserve field numbers or names to prevent reuse after removing fields: @@ -582,6 +619,15 @@ message_body := (option_stmt | reserved_stmt | nested_type | field_def)* nested_type := enum_def | message_def ``` +**Rules:** + +- Message and union type IDs are required for ID-based registration. If omitted and + `enable_auto_type_id = true` (default), they are auto-generated (see [Type IDs](#type-ids)); + if `enable_auto_type_id = false`, they are registered by namespace and name. +- Numeric type IDs (manual or auto-generated) must be globally unique (including nested types). + If an auto-generated ID conflicts, the compiler raises an error and asks for an explicit `id` + or an `alias` to change the hash source. + ## Nested Types Messages can contain nested message and enum definitions. This is useful for defining types that are closely related to their parent message. @@ -668,7 +714,9 @@ message OtherMessage { - Nested type names must be unique within their parent message - Nested types can have their own type IDs -- Type IDs must be globally unique (including nested types) +- Numeric type IDs must be globally unique (including nested types); if an auto-generated ID + conflicts, the compiler raises an error and asks for an explicit `id` or an `alias` + (auto-generation happens only when `enable_auto_type_id = true`) - Within a message, you can reference nested types by simple name - From outside, use the qualified name (Parent.Child) @@ -700,7 +748,11 @@ message Person [id=100] { - Cases cannot be `optional`, `repeated`, or `ref` - Union cases do not support field options - Case types can be primitives, enums, messages, or other named types -- Union type IDs (`[id=...]`) are optional but recommended for cross-language use +- Union type IDs (`[id=...]`) are required for ID-based registration. If omitted and + `enable_auto_type_id = true`, the compiler auto-generates one using + `MurmurHash3(utf8(package.type_name))` (32-bit); otherwise, unions are registered + by namespace and name. +- Use `[alias="..."]` to change the hash source without renaming the union **Grammar:** @@ -933,36 +985,57 @@ message Example { ## Type IDs -Type IDs enable efficient cross-language serialization: +Type IDs enable efficient cross-language serialization and are used for +messages, unions, and enums. When `enable_auto_type_id = true` (default) and +`id` is omitted, the compiler auto-generates one using +`MurmurHash3(utf8(package.type_name))` (32-bit) and annotates it in generated +code. When `enable_auto_type_id = false`, types without explicit IDs are +registered by namespace and name instead. Collisions are detected at +compile-time across the current file and all imports; when a collision occurs, +the compiler raises an error and asks for an explicit `id` or an `alias`. ```protobuf enum Color [id=100] { ... } message User [id=101] { ... } -message Order [id=102] { ... } +union Event [id=102] { ... } ``` -### With Type ID (Recommended) +Enum type IDs remain optional; if omitted they are auto-generated using the same +hash when `enable_auto_type_id = true`. + +### With Explicit Type ID ```protobuf message User [id=101] { ... } message User [id=101, deprecated=true] { ... } // Multiple options ``` -- Serialized as compact integer -- Fast lookup during deserialization -- Must be globally unique across all types -- Recommended for production use - -### Without Type ID +### Without Explicit Type ID ```protobuf -message Config { ... } +message Config { ... } // Auto-generated when enable_auto_type_id = true ``` -- Registered using namespace + name -- More flexible for development -- Slightly larger serialized size -- Uses package as namespace: `"package.Config"` +You can set `[alias="..."]` to change the hash source without renaming the type. + +### Pay-as-you-go principle + +Type ID Specification + +- IDs: Messages, unions, and enums use numeric IDs; if omitted and + `enable_auto_type_id = true`, the compiler auto-generates one. +- Auto-generation: If no ID is provided, fory generates one using + MurmurHash3(utf8(package.type_name)) (32-bit). If a package alias is specified, + the alias is used instead of the package name; if a type alias is specified, + the alias is used instead of the type name. +- Space Efficiency: + - Manual IDs (0-127): Encoded as 1 byte (Varint). Ideal for high-frequency messages. + - Generated IDs: Usually large integers, taking 4-5 bytes in the wire format (varuint32). +- Conflict Resolution: While the collision probability is extremely low, conflicts are detected + at compile-time. The compiler raises an error and asks you to specify an explicit `id` or use + the `alias` option to change the hash source. + +Explicit is better than implicit, but automation is better than toil. ### ID Assignment Strategy @@ -1048,7 +1121,7 @@ message Order [id=204] { optional timestamp shipped_at = 9; } -// Config without type ID (uses namespace registration) +// Config without explicit type ID (auto-generated when enable_auto_type_id = true) message ShopConfig { string store_name = 1; string currency = 2; @@ -1066,12 +1139,13 @@ FDL supports protobuf-style extension options for Fory-specific configuration. T ```protobuf option (fory).use_record_for_java_message = true; option (fory).polymorphism = true; +option (fory).enable_auto_type_id = true; ``` -| Option | Type | Description | -| ----------------------------- | ---- | ---------------------------------------- | -| `use_record_for_java_message` | bool | Generate Java records instead of classes | -| `polymorphism` | bool | Enable polymorphism for all types | +| Option | Type | Description | +| ----------------------------- | ---- | ----------------------------------------------------------- | +| `use_record_for_java_message` | bool | Generate Java records instead of classes | +| `enable_auto_type_id` | bool | Auto-generate numeric type IDs when omitted (default: true) | ### Message-Level Fory Options @@ -1086,16 +1160,31 @@ message MyMessage { } ``` -| Option | Type | Description | -| --------------------- | ------ | ----------------------------------------------------------------------------------- | -| `id` | int | Type ID for serialization (sets type_id) | -| `evolving` | bool | Schema evolution support (default: true). When false, schema is fixed like a struct | -| `use_record_for_java` | bool | Generate Java record for this message | -| `deprecated` | bool | Mark this message as deprecated | -| `namespace` | string | Custom namespace for type registration | +| Option | Type | Description | +| --------------------- | ------ | ------------------------------------------------------------------------------------ | +| `id` | int | Type ID for serialization (auto-generated if omitted and enable_auto_type_id = true) | +| `alias` | string | Alternate name used as hash source for auto-generated IDs | +| `evolving` | bool | Schema evolution support (default: true). When false, schema is fixed like a struct | +| `use_record_for_java` | bool | Generate Java record for this message | +| `deprecated` | bool | Mark this message as deprecated | +| `namespace` | string | Custom namespace for type registration | **Note:** `option (fory).id = 100` is equivalent to the inline syntax `message MyMessage [id=100]`. +### Union-Level Fory Options + +```protobuf +union MyUnion [id=100, alias="MyUnionAlias"] { + string text = 1; +} +``` + +| Option | Type | Description | +| ------------ | ------ | ------------------------------------------------------------------------------------ | +| `id` | int | Type ID for serialization (auto-generated if omitted and enable_auto_type_id = true) | +| `alias` | string | Alternate name used as hash source for auto-generated IDs | +| `deprecated` | bool | Mark this union as deprecated | + ### Enum-Level Fory Options ```protobuf @@ -1209,7 +1298,7 @@ message ForyFieldOptions { ``` file := [package_decl] file_option* import_decl* type_def* -package_decl := 'package' package_name ';' +package_decl := 'package' package_name ['alias' package_name] ';' package_name := IDENTIFIER ('.' IDENTIFIER)* file_option := 'option' option_name '=' option_value ';' diff --git a/docs/compiler/generated-code.md b/docs/compiler/generated-code.md index 92708c9bdc..7abed81103 100644 --- a/docs/compiler/generated-code.md +++ b/docs/compiler/generated-code.md @@ -832,7 +832,8 @@ int main() { ## Name-Based Registration -When types don't have explicit type IDs, they use namespace-based registration: +When types don't have explicit type IDs and `enable_auto_type_id = false`, they use +namespace-based registration: ### FDL diff --git a/docs/compiler/index.md b/docs/compiler/index.md index 694419cc0d..fd62dd62d2 100644 --- a/docs/compiler/index.md +++ b/docs/compiler/index.md @@ -28,25 +28,40 @@ FDL provides a simple, intuitive syntax for defining cross-language data structu ```protobuf package example; -enum Status [id=100] { +enum Status { PENDING = 0; ACTIVE = 1; COMPLETED = 2; } -message User [id=101] { +message User { string name = 1; int32 age = 2; optional string email = 3; repeated string tags = 4; } -message Order [id=102] { +message Order { ref User customer = 1; repeated Item items = 2; Status status = 3; map metadata = 4; } + +message Dog [id=104] { + string name = 1; + int32 bark_volume = 2; +} + +message Cat [id=105] { + string name = 1; + int32 lives = 2; +} + +union Animal [id=106] { + Dog dog = 1; + Cat cat = 2; +} ``` ## Why FDL? @@ -84,8 +99,7 @@ Generated code uses native language constructs: ### 1. Install the Compiler ```bash -cd compiler -pip install -e . +pip install fory-compiler ``` ### 2. Write Your Schema @@ -95,7 +109,7 @@ Create `example.fdl`: ```protobuf package example; -message Person [id=100] { +message Person { string name = 1; int32 age = 2; optional string email = 3; @@ -117,26 +131,20 @@ foryc example.fdl --lang java,python --output ./generated **Java:** ```java -Fory fory = Fory.builder().withLanguage(Language.XLANG).build(); -ExampleForyRegistration.register(fory); - Person person = new Person(); person.setName("Alice"); person.setAge(30); -byte[] data = fory.serialize(person); +byte[] data = person.toBytes(); ``` **Python:** ```python import pyfory -from example import Person, register_example_types - -fory = pyfory.Fory() -register_example_types(fory) +from example import Person person = Person(name="Alice", age=30) -data = fory.serialize(person) +data = bytes(person) # or `person.to_bytes()` ``` ## Documentation @@ -152,22 +160,6 @@ data = fory.serialize(person) ## Key Concepts -### Type Registration - -FDL supports two registration modes: - -**Numeric Type IDs** - Fast and compact: - -```protobuf -message User [id=100] { ... } // Registered with ID 100 -``` - -**Namespace-based** - Flexible and readable: - -```protobuf -message Config { ... } // Registered as "package.Config" -``` - ### Field Modifiers - **`optional`**: Field can be null/None diff --git a/docs/guide/cpp/custom-serializers.md b/docs/guide/cpp/custom-serializers.md index 4a1a1ece30..8441537152 100644 --- a/docs/guide/cpp/custom-serializers.md +++ b/docs/guide/cpp/custom-serializers.md @@ -58,7 +58,7 @@ struct Serializer { (void)has_generics; write_not_null_ref_flag(ctx, ref_mode); if (write_type) { - auto result = ctx.write_any_typeinfo( + auto result = ctx.write_any_type_info( static_cast(TypeId::UNKNOWN), std::type_index(typeid(MyExt))); if (!result.ok()) { @@ -88,7 +88,7 @@ struct Serializer { return MyExt{}; } if (read_type) { - const TypeInfo *type_info = ctx.read_any_typeinfo(ctx.error()); + const TypeInfo *type_info = ctx.read_any_type_info(ctx.error()); if (ctx.has_error()) { return MyExt{}; } @@ -190,7 +190,7 @@ struct Serializer { (void)has_generics; write_not_null_ref_flag(ctx, ref_mode); if (write_type) { - auto result = ctx.write_any_typeinfo( + auto result = ctx.write_any_type_info( static_cast(TypeId::UNKNOWN), std::type_index(typeid(CustomType))); if (!result.ok()) { @@ -220,7 +220,7 @@ struct Serializer { return CustomType{}; } if (read_type) { - const TypeInfo *type_info = ctx.read_any_typeinfo(ctx.error()); + const TypeInfo *type_info = ctx.read_any_type_info(ctx.error()); if (ctx.has_error()) { return CustomType{}; } diff --git a/go/fory/array.go b/go/fory/array.go index d52463f916..9b8b9c17d3 100644 --- a/go/fory/array.go +++ b/go/fory/array.go @@ -30,7 +30,7 @@ func writeArrayRefAndType(ctx *WriteContext, refMode RefMode, writeType bool, va ctx.Buffer().WriteInt8(NotNullValueFlag) } if writeType { - ctx.Buffer().WriteVarUint32Small7(uint32(typeId)) + ctx.Buffer().WriteUint8(uint8(typeId)) } return false } @@ -55,12 +55,13 @@ func readArrayRefAndType(ctx *ReadContext, refMode RefMode, readType bool, value } } if readType { - typeID := buf.ReadVarUint32Small7(err) + typeID := uint32(buf.ReadUint8(err)) if ctx.HasError() { return false } - if IsNamespacedType(TypeId(typeID)) { - ctx.TypeResolver().readTypeInfoWithTypeID(buf, typeID, err) + if typeID != uint32(LIST) { + ctx.SetError(DeserializationErrorf("array type mismatch: expected LIST (%d), got %d", LIST, typeID)) + return false } } return false @@ -82,7 +83,7 @@ func (s arraySerializer) WriteData(ctx *WriteContext, value reflect.Value) { } func (s arraySerializer) Write(ctx *WriteContext, refMode RefMode, writeType bool, hasGenerics bool, value reflect.Value) { - writeArrayRefAndType(ctx, refMode, writeType, value, -LIST) + writeArrayRefAndType(ctx, refMode, writeType, value, LIST) if ctx.HasError() { return } @@ -175,10 +176,10 @@ func (s *arrayConcreteValueSerializer) WriteData(ctx *WriteContext, value reflec if elemTypeInfo != nil { internalTypeID = elemTypeInfo.TypeID } - if IsNamespacedType(TypeId(internalTypeID)) { + if elemTypeInfo != nil { ctx.TypeResolver().WriteTypeInfo(buf, elemTypeInfo, ctx.Err()) } else { - buf.WriteVarUint32Small7(uint32(internalTypeID)) + buf.WriteUint8(uint8(internalTypeID)) } // Write elements @@ -216,7 +217,7 @@ func (s *arrayConcreteValueSerializer) WriteData(ctx *WriteContext, value reflec } func (s *arrayConcreteValueSerializer) Write(ctx *WriteContext, refMode RefMode, writeType bool, hasGenerics bool, value reflect.Value) { - writeArrayRefAndType(ctx, refMode, writeType, value, -LIST) + writeArrayRefAndType(ctx, refMode, writeType, value, LIST) if ctx.HasError() { return } @@ -239,12 +240,7 @@ func (s *arrayConcreteValueSerializer) ReadData(ctx *ReadContext, value reflect. // Read element type info if present if (collectFlag & CollectionIsSameType) != 0 { if (collectFlag & CollectionIsDeclElementType) == 0 { - typeID := buf.ReadVarUint32(err) - if ctx.HasError() { - return - } - // Read additional metadata for namespaced types - ctx.TypeResolver().readTypeInfoWithTypeID(buf, typeID, err) + ctx.TypeResolver().ReadTypeInfo(buf, err) } } diff --git a/go/fory/enum.go b/go/fory/enum.go index 99e5181c82..1a1e63c617 100644 --- a/go/fory/enum.go +++ b/go/fory/enum.go @@ -85,7 +85,7 @@ func (s *enumSerializer) Read(ctx *ReadContext, refMode RefMode, readType bool, } } if readType { - _ = ctx.buffer.ReadVarUint32Small7(err) + _ = ctx.buffer.ReadUint8(err) } if ctx.HasError() { return diff --git a/go/fory/field_info.go b/go/fory/field_info.go index 33466bc0fb..c10766a847 100644 --- a/go/fory/field_info.go +++ b/go/fory/field_info.go @@ -260,13 +260,11 @@ func fieldHasNonPrimitiveSerializer(field *FieldInfo) bool { if field.Serializer == nil { return false } - // ENUM (numeric ID), NAMED_ENUM (namespace/typename), NAMED_STRUCT, NAMED_COMPATIBLE_STRUCT, NAMED_EXT + // ENUM (numeric ID), NAMED_STRUCT, NAMED_COMPATIBLE_STRUCT, NAMED_EXT // all require special serialization and should not use the primitive fast path // Note: ENUM uses unsigned VarUint32Small7 for ordinals, not signed zigzag varint - // Use internal type ID (low 8 bits) since registered types have composite TypeIds like (userID << 8) | internalID - internalTypeId := TypeId(field.Meta.TypeId & 0xFF) - switch internalTypeId { - case ENUM, NAMED_ENUM, NAMED_STRUCT, NAMED_COMPATIBLE_STRUCT, NAMED_EXT: + switch TypeId(field.Meta.TypeId) { + case ENUM, NAMED_STRUCT, NAMED_COMPATIBLE_STRUCT, NAMED_EXT: return true default: return false @@ -278,8 +276,8 @@ func isEnumField(field *FieldInfo) bool { if field.Serializer == nil { return false } - internalTypeId := field.Meta.TypeId & 0xFF - return internalTypeId == ENUM || internalTypeId == NAMED_ENUM + internalTypeId := field.Meta.TypeId + return internalTypeId == ENUM } // getFieldCategory returns the category for sorting remainingFields: @@ -292,17 +290,17 @@ func getFieldCategory(field *FieldInfo) int { if isNullableFixedSizePrimitive(field.DispatchId) || isNullableVarintPrimitive(field.DispatchId) { return 0 } - internalId := int16(field.Meta.TypeId & 0xFF) - if internalId == UNKNOWN { + typeId := field.Meta.TypeId + if typeId == UNKNOWN { return 4 } - if isUserDefinedType(internalId) { + if isUserDefinedType(typeId) { return 4 } - if internalId == LIST || internalId == SET { + if typeId == LIST || typeId == SET { return 2 } - if internalId == MAP { + if typeId == MAP { return 3 } // Internal build-in types: sorted by typeId, then sort key (matches Java build-in group) @@ -518,10 +516,7 @@ func isStructFieldType(ft FieldType) bool { } typeId := ft.TypeId() // Check base type IDs that need type info (struct and ext, NOT enum) - // Always check the internal type ID (low byte) to handle composite type IDs - // which may be negative when stored as int32 (e.g., -2288 = (short)128784) - internalTypeId := TypeId(typeId & 0xFF) - switch internalTypeId { + switch TypeId(typeId) { case STRUCT, NAMED_STRUCT, COMPATIBLE_STRUCT, NAMED_COMPATIBLE_STRUCT, EXT, NAMED_EXT: return true @@ -613,7 +608,7 @@ func ComputeStructFingerprint(fields []FieldFingerprintInfo) string { // Field sorting helpers type triple struct { - typeID int16 + typeID TypeId serializer Serializer name string nullable bool @@ -711,7 +706,7 @@ func sortFields( // Java sorts by: compressed (varint) types last, then by size (largest first), then by type ID (descending) // Fixed types: BOOL, INT8, UINT8, INT16, UINT16, INT32, UINT32, INT64, UINT64, FLOAT32, FLOAT64 // Varint types: VARINT32, VARINT64, VAR_UINT32, VAR_UINT64, TAGGED_INT64, TAGGED_UINT64 - isVarintTypeId := func(typeID int16) bool { + isVarintTypeId := func(typeID TypeId) bool { return typeID == VARINT32 || typeID == VARINT64 || typeID == VAR_UINT32 || typeID == VAR_UINT64 || typeID == TAGGED_INT64 || typeID == TAGGED_UINT64 diff --git a/go/fory/fory.go b/go/fory/fory.go index da9d680657..fea76a3857 100644 --- a/go/fory/fory.go +++ b/go/fory/fory.go @@ -189,12 +189,22 @@ func NewFory(opts ...Option) *Fory { return New(opts...) } +func validateUserTypeID(typeID uint32) error { + if typeID > maxUserTypeID { + return fmt.Errorf("typeID must be in range [0, 0xfffffffe], got %d", typeID) + } + return nil +} + // RegisterStruct registers a struct type with a numeric ID for cross-language serialization. // This is compatible with Java's fory.register(Class, int) method. // type_ can be either a reflect.Type or an instance of the type -// typeID should be the user type ID in the range 0-8192 (the internal type ID will be added automatically) +// typeID should be the user type ID in the range 0-0xfffffffe (0xffffffff is reserved for "unset"). // Note: For enum types, use RegisterEnum instead. func (f *Fory) RegisterStruct(type_ any, typeID uint32) error { + if err := validateUserTypeID(typeID); err != nil { + return err + } var t reflect.Type if rt, ok := type_.(reflect.Type); ok { t = rt @@ -220,20 +230,20 @@ func (f *Fory) RegisterStruct(type_ any, typeID uint32) error { internalTypeID = STRUCT } - // Calculate full type ID: (userID << 8) | internalTypeID - fullTypeID := (typeID << 8) | uint32(internalTypeID) - - return f.typeResolver.RegisterStruct(t, fullTypeID) + return f.typeResolver.RegisterStruct(t, internalTypeID, typeID) } // RegisterUnion registers a union type with a numeric ID for cross-language serialization. // type_ can be either a reflect.Type or an instance of the union type. -// typeID should be the user type ID in the range 0-8192 (the internal type ID will be added automatically). +// typeID should be the user type ID in the range 0-0xfffffffe (0xffffffff is reserved for "unset"). // serializer must implement union payload encoding/decoding. func (f *Fory) RegisterUnion(type_ any, typeID uint32, serializer Serializer) error { if serializer == nil { return fmt.Errorf("RegisterUnion requires a non-nil serializer") } + if err := validateUserTypeID(typeID); err != nil { + return err + } var t reflect.Type if rt, ok := type_.(reflect.Type); ok { t = rt @@ -246,8 +256,7 @@ func (f *Fory) RegisterUnion(type_ any, typeID uint32, serializer Serializer) er if t.Kind() != reflect.Struct { return fmt.Errorf("RegisterUnion only supports struct types; got: %v", t.Kind()) } - fullTypeID := (typeID << 8) | uint32(TYPED_UNION) - return f.typeResolver.RegisterUnion(t, fullTypeID, serializer) + return f.typeResolver.RegisterUnion(t, typeID, serializer) } // RegisterNamedUnion registers a union type with a namespace + type name for cross-language serialization. @@ -309,8 +318,11 @@ func (f *Fory) RegisterNamedStruct(type_ any, typeName string) error { // In Go, enums are typically defined as int-based types (e.g., type Color int32). // This method creates an enum serializer that writes/reads the enum value as VarUint32Small7. // type_ can be either a reflect.Type or an instance of the enum type -// typeID should be the user type ID in the range 0-8192 (the internal type ID will be added automatically) +// typeID should be the user type ID in the range 0-0xfffffffe (0xffffffff is reserved for "unset"). func (f *Fory) RegisterEnum(type_ any, typeID uint32) error { + if err := validateUserTypeID(typeID); err != nil { + return err + } var t reflect.Type if rt, ok := type_.(reflect.Type); ok { t = rt @@ -330,10 +342,7 @@ func (f *Fory) RegisterEnum(type_ any, typeID uint32) error { return fmt.Errorf("RegisterEnum only supports numeric types (Go enums); got: %v", t.Kind()) } - // Calculate full type ID: (userID << 8) | ENUM - fullTypeID := (typeID << 8) | uint32(ENUM) - - return f.typeResolver.RegisterEnum(t, fullTypeID) + return f.typeResolver.RegisterEnum(t, typeID) } // RegisterNamedEnum registers an enum type with a name for cross-language serialization. @@ -372,8 +381,11 @@ func (f *Fory) RegisterNamedEnum(type_ any, typeName string) error { // RegisterExtension registers a type as an extension type with a numeric ID. // Extension types use a custom serializer provided by the user. -// typeID should be the user type ID in the range 0-8192. +// typeID should be the user type ID in the range 0-0xfffffffe (0xffffffff is reserved for "unset"). func (f *Fory) RegisterExtension(type_ any, typeID uint32, serializer ExtensionSerializer) error { + if err := validateUserTypeID(typeID); err != nil { + return err + } var t reflect.Type if rt, ok := type_.(reflect.Type); ok { t = rt @@ -992,48 +1004,48 @@ func Deserialize[T any](f *Fory, data []byte, target *T) error { err := f.readCtx.Err() switch t := any(target).(type) { case *bool: - _ = buf.ReadInt8(err) // null flag - _ = buf.ReadVarUint32Small7(err) // type ID + _ = buf.ReadInt8(err) // null flag + _ = buf.ReadUint8(err) // type ID *t = buf.ReadBool(err) return f.readCtx.CheckError() case *int8: _ = buf.ReadInt8(err) - _ = buf.ReadVarUint32Small7(err) + _ = buf.ReadUint8(err) *t = buf.ReadInt8(err) return f.readCtx.CheckError() case *int16: _ = buf.ReadInt8(err) - _ = buf.ReadVarUint32Small7(err) + _ = buf.ReadUint8(err) *t = buf.ReadInt16(err) return f.readCtx.CheckError() case *int32: _ = buf.ReadInt8(err) - _ = buf.ReadVarUint32Small7(err) + _ = buf.ReadUint8(err) *t = buf.ReadVarint32(err) return f.readCtx.CheckError() case *int64: _ = buf.ReadInt8(err) - _ = buf.ReadVarUint32Small7(err) + _ = buf.ReadUint8(err) *t = buf.ReadVarint64(err) return f.readCtx.CheckError() case *int: _ = buf.ReadInt8(err) - _ = buf.ReadVarUint32Small7(err) + _ = buf.ReadUint8(err) *t = int(buf.ReadVarint64(err)) return f.readCtx.CheckError() case *float32: _ = buf.ReadInt8(err) - _ = buf.ReadVarUint32Small7(err) + _ = buf.ReadUint8(err) *t = buf.ReadFloat32(err) return f.readCtx.CheckError() case *float64: _ = buf.ReadInt8(err) - _ = buf.ReadVarUint32Small7(err) + _ = buf.ReadUint8(err) *t = buf.ReadFloat64(err) return f.readCtx.CheckError() case *string: - _ = buf.ReadInt8(err) // null flag - _ = buf.ReadVarUint32Small7(err) // type ID + _ = buf.ReadInt8(err) // null flag + _ = buf.ReadUint8(err) // type ID *t = f.readCtx.ReadString() return f.readCtx.CheckError() case *[]byte: diff --git a/go/fory/map.go b/go/fory/map.go index ddc38c2b1d..f2489601f3 100644 --- a/go/fory/map.go +++ b/go/fory/map.go @@ -583,7 +583,7 @@ func writeMapRefAndType(ctx *WriteContext, refMode RefMode, writeType bool, valu ctx.buffer.WriteInt8(NotNullValueFlag) } if writeType { - ctx.buffer.WriteVarUint32Small7(uint32(MAP)) + ctx.buffer.WriteUint8(uint8(MAP)) } return false } @@ -614,7 +614,7 @@ func readMapRefAndType(ctx *ReadContext, refMode RefMode, readType bool, value r } } if readType { - buf.ReadVarUint32Small7(ctxErr) + buf.ReadUint8(ctxErr) } return false } diff --git a/go/fory/map_primitive.go b/go/fory/map_primitive.go index edd9e415a2..21a4bd7b5d 100644 --- a/go/fory/map_primitive.go +++ b/go/fory/map_primitive.go @@ -50,8 +50,8 @@ func writeMapStringString(buf *ByteBuffer, m map[string]string, hasGenerics bool // Write type info for generic reader compatibility buf.WriteUint8(0) buf.WriteUint8(uint8(chunkSize)) - buf.WriteVarUint32Small7(uint32(STRING)) // key type - buf.WriteVarUint32Small7(uint32(STRING)) // value type + buf.WriteUint8(uint8(STRING)) // key type + buf.WriteUint8(uint8(STRING)) // value type } // WriteData chunk entries @@ -92,7 +92,7 @@ func readMapStringString(buf *ByteBuffer, err *Error) map[string]string { // Null key with non-null value valueDeclared := (chunkHeader & VALUE_DECL_TYPE) != 0 if !valueDeclared { - buf.ReadVarUint32Small7(err) // skip value type + buf.ReadUint8(err) // skip value type } v := readString(buf, err) result[""] = v // empty string as null key @@ -102,7 +102,7 @@ func readMapStringString(buf *ByteBuffer, err *Error) map[string]string { // Non-null key with null value keyDeclared := (chunkHeader & KEY_DECL_TYPE) != 0 if !keyDeclared { - buf.ReadVarUint32Small7(err) // skip key type + buf.ReadUint8(err) // skip key type } k := readString(buf, err) result[k] = "" // empty string as null value @@ -115,10 +115,10 @@ func readMapStringString(buf *ByteBuffer, err *Error) map[string]string { // Read type info if not DECL_TYPE if (chunkHeader & KEY_DECL_TYPE) == 0 { - buf.ReadVarUint32Small7(err) // skip key type + buf.ReadUint8(err) // skip key type } if (chunkHeader & VALUE_DECL_TYPE) == 0 { - buf.ReadVarUint32Small7(err) // skip value type + buf.ReadUint8(err) // skip value type } // ReadData chunk entries @@ -154,8 +154,8 @@ func writeMapStringInt64(buf *ByteBuffer, m map[string]int64, hasGenerics bool) } else { buf.WriteUint8(0) buf.WriteUint8(uint8(chunkSize)) - buf.WriteVarUint32Small7(uint32(STRING)) // key type - buf.WriteVarUint32Small7(uint32(VARINT64)) // value type + buf.WriteUint8(uint8(STRING)) // key type + buf.WriteUint8(uint8(VARINT64)) // value type } count := 0 @@ -191,10 +191,10 @@ func readMapStringInt64(buf *ByteBuffer, err *Error) map[string]int64 { chunkSize := int(buf.ReadUint8(err)) if (chunkHeader & KEY_DECL_TYPE) == 0 { - buf.ReadVarUint32Small7(err) + buf.ReadUint8(err) } if (chunkHeader & VALUE_DECL_TYPE) == 0 { - buf.ReadVarUint32Small7(err) + buf.ReadUint8(err) } for i := 0; i < chunkSize && size > 0; i++ { k := readString(buf, err) @@ -228,8 +228,8 @@ func writeMapStringInt32(buf *ByteBuffer, m map[string]int32, hasGenerics bool) } else { buf.WriteUint8(0) buf.WriteUint8(uint8(chunkSize)) - buf.WriteVarUint32Small7(uint32(STRING)) // key type - buf.WriteVarUint32Small7(uint32(VARINT32)) // value type + buf.WriteUint8(uint8(STRING)) // key type + buf.WriteUint8(uint8(VARINT32)) // value type } count := 0 @@ -265,10 +265,10 @@ func readMapStringInt32(buf *ByteBuffer, err *Error) map[string]int32 { chunkSize := int(buf.ReadUint8(err)) if (chunkHeader & KEY_DECL_TYPE) == 0 { - buf.ReadVarUint32Small7(err) + buf.ReadUint8(err) } if (chunkHeader & VALUE_DECL_TYPE) == 0 { - buf.ReadVarUint32Small7(err) + buf.ReadUint8(err) } for i := 0; i < chunkSize && size > 0; i++ { k := readString(buf, err) @@ -302,8 +302,8 @@ func writeMapStringInt(buf *ByteBuffer, m map[string]int, hasGenerics bool) { } else { buf.WriteUint8(0) buf.WriteUint8(uint8(chunkSize)) - buf.WriteVarUint32Small7(uint32(STRING)) // key type - buf.WriteVarUint32Small7(uint32(VARINT64)) // value type (int serialized as varint64) + buf.WriteUint8(uint8(STRING)) // key type + buf.WriteUint8(uint8(VARINT64)) // value type (int serialized as varint64) } count := 0 @@ -339,10 +339,10 @@ func readMapStringInt(buf *ByteBuffer, err *Error) map[string]int { chunkSize := int(buf.ReadUint8(err)) if (chunkHeader & KEY_DECL_TYPE) == 0 { - buf.ReadVarUint32Small7(err) + buf.ReadUint8(err) } if (chunkHeader & VALUE_DECL_TYPE) == 0 { - buf.ReadVarUint32Small7(err) + buf.ReadUint8(err) } for i := 0; i < chunkSize && size > 0; i++ { k := readString(buf, err) @@ -376,8 +376,8 @@ func writeMapStringFloat64(buf *ByteBuffer, m map[string]float64, hasGenerics bo } else { buf.WriteUint8(0) buf.WriteUint8(uint8(chunkSize)) - buf.WriteVarUint32Small7(uint32(STRING)) // key type - buf.WriteVarUint32Small7(uint32(FLOAT64)) // value type + buf.WriteUint8(uint8(STRING)) // key type + buf.WriteUint8(uint8(FLOAT64)) // value type } count := 0 @@ -413,10 +413,10 @@ func readMapStringFloat64(buf *ByteBuffer, err *Error) map[string]float64 { chunkSize := int(buf.ReadUint8(err)) if (chunkHeader & KEY_DECL_TYPE) == 0 { - buf.ReadVarUint32Small7(err) + buf.ReadUint8(err) } if (chunkHeader & VALUE_DECL_TYPE) == 0 { - buf.ReadVarUint32Small7(err) + buf.ReadUint8(err) } for i := 0; i < chunkSize && size > 0; i++ { k := readString(buf, err) @@ -450,8 +450,8 @@ func writeMapStringBool(buf *ByteBuffer, m map[string]bool, hasGenerics bool) { } else { buf.WriteUint8(0) buf.WriteUint8(uint8(chunkSize)) - buf.WriteVarUint32Small7(uint32(STRING)) // key type - buf.WriteVarUint32Small7(uint32(BOOL)) // value type + buf.WriteUint8(uint8(STRING)) // key type + buf.WriteUint8(uint8(BOOL)) // value type } count := 0 @@ -491,10 +491,10 @@ func readMapStringBool(buf *ByteBuffer, err *Error) map[string]bool { keyDeclType := (chunkHeader & KEY_DECL_TYPE) != 0 valDeclType := (chunkHeader & VALUE_DECL_TYPE) != 0 if !keyDeclType { - buf.ReadVarUint32Small7(err) // skip key type info + buf.ReadUint8(err) // skip key type info } if !valDeclType { - buf.ReadVarUint32Small7(err) // skip value type info + buf.ReadUint8(err) // skip value type info } for i := 0; i < chunkSize && size > 0; i++ { @@ -529,8 +529,8 @@ func writeMapInt32Int32(buf *ByteBuffer, m map[int32]int32, hasGenerics bool) { } else { buf.WriteUint8(0) buf.WriteUint8(uint8(chunkSize)) - buf.WriteVarUint32Small7(uint32(VARINT32)) // key type - buf.WriteVarUint32Small7(uint32(VARINT32)) // value type + buf.WriteUint8(uint8(VARINT32)) // key type + buf.WriteUint8(uint8(VARINT32)) // value type } count := 0 @@ -566,10 +566,10 @@ func readMapInt32Int32(buf *ByteBuffer, err *Error) map[int32]int32 { chunkSize := int(buf.ReadUint8(err)) if (chunkHeader & KEY_DECL_TYPE) == 0 { - buf.ReadVarUint32Small7(err) + buf.ReadUint8(err) } if (chunkHeader & VALUE_DECL_TYPE) == 0 { - buf.ReadVarUint32Small7(err) + buf.ReadUint8(err) } for i := 0; i < chunkSize && size > 0; i++ { k := buf.ReadVarint32(err) @@ -603,8 +603,8 @@ func writeMapInt64Int64(buf *ByteBuffer, m map[int64]int64, hasGenerics bool) { } else { buf.WriteUint8(0) buf.WriteUint8(uint8(chunkSize)) - buf.WriteVarUint32Small7(uint32(VARINT64)) // key type - buf.WriteVarUint32Small7(uint32(VARINT64)) // value type + buf.WriteUint8(uint8(VARINT64)) // key type + buf.WriteUint8(uint8(VARINT64)) // value type } count := 0 @@ -640,10 +640,10 @@ func readMapInt64Int64(buf *ByteBuffer, err *Error) map[int64]int64 { chunkSize := int(buf.ReadUint8(err)) if (chunkHeader & KEY_DECL_TYPE) == 0 { - buf.ReadVarUint32Small7(err) + buf.ReadUint8(err) } if (chunkHeader & VALUE_DECL_TYPE) == 0 { - buf.ReadVarUint32Small7(err) + buf.ReadUint8(err) } for i := 0; i < chunkSize && size > 0; i++ { k := buf.ReadVarint64(err) @@ -677,8 +677,8 @@ func writeMapIntInt(buf *ByteBuffer, m map[int]int, hasGenerics bool) { } else { buf.WriteUint8(0) buf.WriteUint8(uint8(chunkSize)) - buf.WriteVarUint32Small7(uint32(VARINT64)) // key type (int serialized as varint64) - buf.WriteVarUint32Small7(uint32(VARINT64)) // value type + buf.WriteUint8(uint8(VARINT64)) // key type (int serialized as varint64) + buf.WriteUint8(uint8(VARINT64)) // value type } count := 0 @@ -714,10 +714,10 @@ func readMapIntInt(buf *ByteBuffer, err *Error) map[int]int { chunkSize := int(buf.ReadUint8(err)) if (chunkHeader & KEY_DECL_TYPE) == 0 { - buf.ReadVarUint32Small7(err) + buf.ReadUint8(err) } if (chunkHeader & VALUE_DECL_TYPE) == 0 { - buf.ReadVarUint32Small7(err) + buf.ReadUint8(err) } for i := 0; i < chunkSize && size > 0; i++ { k := buf.ReadVarint64(err) diff --git a/go/fory/meta_string_resolver.go b/go/fory/meta_string_resolver.go index f0df9b8038..a20651647e 100644 --- a/go/fory/meta_string_resolver.go +++ b/go/fory/meta_string_resolver.go @@ -70,6 +70,8 @@ type MetaStringResolver struct { metaStrToMetaStrBytes map[*meta.MetaString]*MetaStringBytes // Conversion cache } +var emptyMetaStringBytes = NewMetaStringBytes([]byte{}, 256) + func NewMetaStringResolver() *MetaStringResolver { return &MetaStringResolver{ hashToMetaStrBytes: make(map[int64]*MetaStringBytes), @@ -92,7 +94,9 @@ func (r *MetaStringResolver) WriteMetaStringBytes(buf *ByteBuffer, m *MetaString // Small strings store encoding in header if m.Length <= SmallStringThreshold { - buf.WriteByte(byte(m.Encoding)) + if m.Length != 0 { + buf.WriteByte(byte(m.Encoding)) + } } else { // Large strings include full hash binErr := binary.Write(buf, binary.LittleEndian, m.Hashcode) @@ -135,6 +139,10 @@ func (r *MetaStringResolver) ReadMetaStringBytes(buf *ByteBuffer, ctxErr *Error) // Small string optimization if length <= SmallStringThreshold { + if length == 0 { + r.dynamicIDToEnumString = append(r.dynamicIDToEnumString, emptyMetaStringBytes) + return emptyMetaStringBytes, nil + } // ReadData encoding and data encByte := buf.ReadByte(ctxErr) encoding = Encoding(encByte) @@ -208,6 +216,10 @@ func (r *MetaStringResolver) GetMetaStrBytes(metastr *meta.MetaString) *MetaStri data := metastr.GetEncodedBytes() length := len(data) + if length == 0 { + r.metaStrToMetaStrBytes[metastr] = emptyMetaStringBytes + return emptyMetaStringBytes + } if length <= SmallStringThreshold { // Small string: use direct bytes as hash components var v1, v2 int64 @@ -236,7 +248,10 @@ func ComputeMetaStringHash(data []byte, encoding meta.Encoding) int64 { length := len(data) var hashcode int64 - if length <= SmallStringThreshold { + if length == 0 { + hashcode = 256 + hashcode |= int64(encoding) + } else if length <= SmallStringThreshold { // Small string: use direct bytes as hash components var v1, v2 int64 if length <= 8 { diff --git a/go/fory/optional_serializer.go b/go/fory/optional_serializer.go index c85fc76c67..35480da6a6 100644 --- a/go/fory/optional_serializer.go +++ b/go/fory/optional_serializer.go @@ -287,12 +287,12 @@ func (s *optionalSerializer) Read(ctx *ReadContext, refMode RefMode, readType bo // No null flag. } if readType { - typeID := buf.ReadVarUint32Small7(ctx.Err()) + typeID := uint32(buf.ReadUint8(ctx.Err())) if ctx.HasError() { return } - internalTypeID := TypeId(typeID & 0xFF) - if IsNamespacedType(TypeId(typeID)) || internalTypeID == COMPATIBLE_STRUCT || internalTypeID == STRUCT { + internalTypeID := TypeId(typeID) + if IsNamespacedType(internalTypeID) || internalTypeID == COMPATIBLE_STRUCT || internalTypeID == STRUCT { typeInfo := ctx.TypeResolver().readTypeInfoWithTypeID(buf, typeID, ctx.Err()) if structSer, ok := typeInfo.Serializer.(*structSerializer); ok && len(structSer.fieldDefs) > 0 { valueField := s.valueField(value) diff --git a/go/fory/pointer.go b/go/fory/pointer.go index 63bdd0ebc4..50a9809b0b 100644 --- a/go/fory/pointer.go +++ b/go/fory/pointer.go @@ -142,13 +142,13 @@ func (s *ptrToValueSerializer) Read(ctx *ReadContext, refMode RefMode, readType } if readType { // Read type info - in compatible mode this contains the serializer with fieldDefs - typeID := buf.ReadVarUint32Small7(ctxErr) + typeID := uint32(buf.ReadUint8(ctxErr)) if ctx.HasError() { return } - internalTypeID := TypeId(typeID & 0xFF) + internalTypeID := TypeId(typeID) // Check if this is a struct type that needs type meta reading - if IsNamespacedType(TypeId(typeID)) || internalTypeID == COMPATIBLE_STRUCT || internalTypeID == STRUCT { + if IsNamespacedType(internalTypeID) || internalTypeID == COMPATIBLE_STRUCT || internalTypeID == STRUCT { typeInfo := ctx.TypeResolver().readTypeInfoWithTypeID(buf, typeID, ctxErr) // Use the serializer from TypeInfo which has the remote field definitions if structSer, ok := typeInfo.Serializer.(*structSerializer); ok && len(structSer.fieldDefs) > 0 { diff --git a/go/fory/primitive.go b/go/fory/primitive.go index 3057d0dfae..2c316d549e 100644 --- a/go/fory/primitive.go +++ b/go/fory/primitive.go @@ -38,7 +38,7 @@ func (s boolSerializer) Write(ctx *WriteContext, refMode RefMode, writeType bool ctx.buffer.WriteInt8(NotNullValueFlag) } if writeType { - ctx.buffer.WriteVarUint32Small7(uint32(BOOL)) + ctx.buffer.WriteUint8(uint8(BOOL)) } s.WriteData(ctx, value) } @@ -57,7 +57,7 @@ func (s boolSerializer) Read(ctx *ReadContext, refMode RefMode, readType bool, h } } if readType { - _ = ctx.buffer.ReadVarUint32Small7(err) + _ = ctx.buffer.ReadUint8(err) } if ctx.HasError() { return @@ -85,7 +85,7 @@ func (s int8Serializer) Write(ctx *WriteContext, refMode RefMode, writeType bool ctx.buffer.WriteInt8(NotNullValueFlag) } if writeType { - ctx.buffer.WriteVarUint32Small7(uint32(INT8)) + ctx.buffer.WriteUint8(uint8(INT8)) } s.WriteData(ctx, value) } @@ -104,7 +104,7 @@ func (s int8Serializer) Read(ctx *ReadContext, refMode RefMode, readType bool, h } } if readType { - _ = ctx.buffer.ReadVarUint32Small7(err) + _ = ctx.buffer.ReadUint8(err) } if ctx.HasError() { return @@ -131,7 +131,7 @@ func (s byteSerializer) Write(ctx *WriteContext, refMode RefMode, writeType bool ctx.buffer.WriteInt8(NotNullValueFlag) } if writeType { - ctx.buffer.WriteVarUint32Small7(uint32(UINT8)) + ctx.buffer.WriteUint8(uint8(UINT8)) } s.WriteData(ctx, value) } @@ -150,7 +150,7 @@ func (s byteSerializer) Read(ctx *ReadContext, refMode RefMode, readType bool, h } } if readType { - _ = ctx.buffer.ReadVarUint32Small7(err) + _ = ctx.buffer.ReadUint8(err) } if ctx.HasError() { return @@ -177,7 +177,7 @@ func (s uint16Serializer) Write(ctx *WriteContext, refMode RefMode, writeType bo ctx.buffer.WriteInt8(NotNullValueFlag) } if writeType { - ctx.buffer.WriteVarUint32Small7(uint32(UINT16)) + ctx.buffer.WriteUint8(uint8(UINT16)) } s.WriteData(ctx, value) } @@ -196,7 +196,7 @@ func (s uint16Serializer) Read(ctx *ReadContext, refMode RefMode, readType bool, } } if readType { - _ = ctx.buffer.ReadVarUint32Small7(err) + _ = ctx.buffer.ReadUint8(err) } if ctx.HasError() { return @@ -223,7 +223,7 @@ func (s uint32Serializer) Write(ctx *WriteContext, refMode RefMode, writeType bo ctx.buffer.WriteInt8(NotNullValueFlag) } if writeType { - ctx.buffer.WriteVarUint32Small7(uint32(VAR_UINT32)) + ctx.buffer.WriteUint8(uint8(VAR_UINT32)) } s.WriteData(ctx, value) } @@ -242,7 +242,7 @@ func (s uint32Serializer) Read(ctx *ReadContext, refMode RefMode, readType bool, } } if readType { - _ = ctx.buffer.ReadVarUint32Small7(err) + _ = ctx.buffer.ReadUint8(err) } if ctx.HasError() { return @@ -269,7 +269,7 @@ func (s uint64Serializer) Write(ctx *WriteContext, refMode RefMode, writeType bo ctx.buffer.WriteInt8(NotNullValueFlag) } if writeType { - ctx.buffer.WriteVarUint32Small7(uint32(VAR_UINT64)) + ctx.buffer.WriteUint8(uint8(VAR_UINT64)) } s.WriteData(ctx, value) } @@ -288,7 +288,7 @@ func (s uint64Serializer) Read(ctx *ReadContext, refMode RefMode, readType bool, } } if readType { - _ = ctx.buffer.ReadVarUint32Small7(err) + _ = ctx.buffer.ReadUint8(err) } if ctx.HasError() { return @@ -314,7 +314,7 @@ func (s uintSerializer) Write(ctx *WriteContext, refMode RefMode, writeType bool ctx.buffer.WriteInt8(NotNullValueFlag) } if writeType { - ctx.buffer.WriteVarUint32Small7(uint32(VAR_UINT64)) + ctx.buffer.WriteUint8(uint8(VAR_UINT64)) } s.WriteData(ctx, value) } @@ -332,7 +332,7 @@ func (s uintSerializer) Read(ctx *ReadContext, refMode RefMode, readType bool, h } } if readType { - _ = ctx.buffer.ReadVarUint32Small7(err) + _ = ctx.buffer.ReadUint8(err) } if ctx.HasError() { return @@ -358,7 +358,7 @@ func (s int16Serializer) Write(ctx *WriteContext, refMode RefMode, writeType boo ctx.buffer.WriteInt8(NotNullValueFlag) } if writeType { - ctx.buffer.WriteVarUint32Small7(uint32(INT16)) + ctx.buffer.WriteUint8(uint8(INT16)) } s.WriteData(ctx, value) } @@ -376,7 +376,7 @@ func (s int16Serializer) Read(ctx *ReadContext, refMode RefMode, readType bool, } } if readType { - _ = ctx.buffer.ReadVarUint32Small7(err) + _ = ctx.buffer.ReadUint8(err) } if ctx.HasError() { return @@ -402,7 +402,7 @@ func (s int32Serializer) Write(ctx *WriteContext, refMode RefMode, writeType boo ctx.buffer.WriteInt8(NotNullValueFlag) } if writeType { - ctx.buffer.WriteVarUint32Small7(uint32(INT32)) + ctx.buffer.WriteUint8(uint8(INT32)) } s.WriteData(ctx, value) } @@ -420,7 +420,7 @@ func (s int32Serializer) Read(ctx *ReadContext, refMode RefMode, readType bool, } } if readType { - _ = ctx.buffer.ReadVarUint32Small7(err) + _ = ctx.buffer.ReadUint8(err) } if ctx.HasError() { return @@ -446,7 +446,7 @@ func (s int64Serializer) Write(ctx *WriteContext, refMode RefMode, writeType boo ctx.buffer.WriteInt8(NotNullValueFlag) } if writeType { - ctx.buffer.WriteVarUint32Small7(uint32(INT64)) + ctx.buffer.WriteUint8(uint8(INT64)) } s.WriteData(ctx, value) } @@ -464,7 +464,7 @@ func (s int64Serializer) Read(ctx *ReadContext, refMode RefMode, readType bool, } } if readType { - _ = ctx.buffer.ReadVarUint32Small7(err) + _ = ctx.buffer.ReadUint8(err) } if ctx.HasError() { return @@ -488,7 +488,7 @@ func (s intSerializer) Write(ctx *WriteContext, refMode RefMode, writeType bool, ctx.buffer.WriteInt8(NotNullValueFlag) } if writeType { - ctx.buffer.WriteVarUint32Small7(uint32(INT64)) + ctx.buffer.WriteUint8(uint8(INT64)) } s.WriteData(ctx, value) } @@ -506,7 +506,7 @@ func (s intSerializer) Read(ctx *ReadContext, refMode RefMode, readType bool, ha } } if readType { - _ = ctx.buffer.ReadVarUint32Small7(err) + _ = ctx.buffer.ReadUint8(err) } if ctx.HasError() { return @@ -532,7 +532,7 @@ func (s float32Serializer) Write(ctx *WriteContext, refMode RefMode, writeType b ctx.buffer.WriteInt8(NotNullValueFlag) } if writeType { - ctx.buffer.WriteVarUint32Small7(uint32(FLOAT32)) + ctx.buffer.WriteUint8(uint8(FLOAT32)) } s.WriteData(ctx, value) } @@ -550,7 +550,7 @@ func (s float32Serializer) Read(ctx *ReadContext, refMode RefMode, readType bool } } if readType { - _ = ctx.buffer.ReadVarUint32Small7(err) + _ = ctx.buffer.ReadUint8(err) } if ctx.HasError() { return @@ -576,7 +576,7 @@ func (s float64Serializer) Write(ctx *WriteContext, refMode RefMode, writeType b ctx.buffer.WriteInt8(NotNullValueFlag) } if writeType { - ctx.buffer.WriteVarUint32Small7(uint32(FLOAT64)) + ctx.buffer.WriteUint8(uint8(FLOAT64)) } s.WriteData(ctx, value) } @@ -594,7 +594,7 @@ func (s float64Serializer) Read(ctx *ReadContext, refMode RefMode, readType bool } } if readType { - _ = ctx.buffer.ReadVarUint32Small7(err) + _ = ctx.buffer.ReadUint8(err) } if ctx.HasError() { return @@ -628,7 +628,7 @@ func (s float16Serializer) Write(ctx *WriteContext, refMode RefMode, writeType b ctx.buffer.WriteInt8(NotNullValueFlag) } if writeType { - ctx.buffer.WriteVarUint32Small7(uint32(FLOAT16)) + ctx.buffer.WriteUint8(uint8(FLOAT16)) } s.WriteData(ctx, value) } @@ -652,7 +652,7 @@ func (s float16Serializer) Read(ctx *ReadContext, refMode RefMode, readType bool } } if readType { - _ = ctx.buffer.ReadVarUint32Small7(err) + _ = ctx.buffer.ReadUint8(err) } if ctx.HasError() { return diff --git a/go/fory/reader.go b/go/fory/reader.go index e8fd1a8639..e7a1df1710 100644 --- a/go/fory/reader.go +++ b/go/fory/reader.go @@ -178,8 +178,7 @@ func (c *ReadContext) ReadBinary() []byte { } func (c *ReadContext) ReadTypeId() TypeId { - // Use VarUint32Small7 encoding to match Java's xlang serialization - return TypeId(c.buffer.ReadVarUint32Small7(c.Err())) + return TypeId(c.buffer.ReadUint8(c.Err())) } func (c *ReadContext) getTypeInfoByType(type_ reflect.Type) *TypeInfo { @@ -264,7 +263,7 @@ func (c *ReadContext) ReadBoolSlice(refMode RefMode, readType bool) []bool { } } if readType { - _ = c.buffer.ReadVarUint32Small7(err) + _ = c.buffer.ReadUint8(err) } return ReadBoolSlice(c.buffer, err) } @@ -278,7 +277,7 @@ func (c *ReadContext) ReadInt8Slice(refMode RefMode, readType bool) []int8 { } } if readType { - _ = c.buffer.ReadVarUint32Small7(err) + _ = c.buffer.ReadUint8(err) } return ReadInt8Slice(c.buffer, err) } @@ -292,7 +291,7 @@ func (c *ReadContext) ReadInt16Slice(refMode RefMode, readType bool) []int16 { } } if readType { - _ = c.buffer.ReadVarUint32Small7(err) + _ = c.buffer.ReadUint8(err) } return ReadInt16Slice(c.buffer, err) } @@ -306,7 +305,7 @@ func (c *ReadContext) ReadInt32Slice(refMode RefMode, readType bool) []int32 { } } if readType { - _ = c.buffer.ReadVarUint32Small7(err) + _ = c.buffer.ReadUint8(err) } return ReadInt32Slice(c.buffer, err) } @@ -320,7 +319,7 @@ func (c *ReadContext) ReadInt64Slice(refMode RefMode, readType bool) []int64 { } } if readType { - _ = c.buffer.ReadVarUint32Small7(err) + _ = c.buffer.ReadUint8(err) } return ReadInt64Slice(c.buffer, err) } @@ -334,7 +333,7 @@ func (c *ReadContext) ReadUint16Slice(refMode RefMode, readType bool) []uint16 { } } if readType { - _ = c.buffer.ReadVarUint32Small7(err) + _ = c.buffer.ReadUint8(err) } return ReadUint16Slice(c.buffer, err) } @@ -348,7 +347,7 @@ func (c *ReadContext) ReadUint32Slice(refMode RefMode, readType bool) []uint32 { } } if readType { - _ = c.buffer.ReadVarUint32Small7(err) + _ = c.buffer.ReadUint8(err) } return ReadUint32Slice(c.buffer, err) } @@ -362,7 +361,7 @@ func (c *ReadContext) ReadUint64Slice(refMode RefMode, readType bool) []uint64 { } } if readType { - _ = c.buffer.ReadVarUint32Small7(err) + _ = c.buffer.ReadUint8(err) } return ReadUint64Slice(c.buffer, err) } @@ -376,7 +375,7 @@ func (c *ReadContext) ReadIntSlice(refMode RefMode, readType bool) []int { } } if readType { - _ = c.buffer.ReadVarUint32Small7(err) + _ = c.buffer.ReadUint8(err) } return ReadIntSlice(c.buffer, err) } @@ -390,7 +389,7 @@ func (c *ReadContext) ReadUintSlice(refMode RefMode, readType bool) []uint { } } if readType { - _ = c.buffer.ReadVarUint32Small7(err) + _ = c.buffer.ReadUint8(err) } return ReadUintSlice(c.buffer, err) } @@ -404,7 +403,7 @@ func (c *ReadContext) ReadFloat32Slice(refMode RefMode, readType bool) []float32 } } if readType { - _ = c.buffer.ReadVarUint32Small7(err) + _ = c.buffer.ReadUint8(err) } return ReadFloat32Slice(c.buffer, err) } @@ -418,7 +417,7 @@ func (c *ReadContext) ReadFloat64Slice(refMode RefMode, readType bool) []float64 } } if readType { - _ = c.buffer.ReadVarUint32Small7(err) + _ = c.buffer.ReadUint8(err) } return ReadFloat64Slice(c.buffer, err) } @@ -432,7 +431,7 @@ func (c *ReadContext) ReadByteSlice(refMode RefMode, readType bool) []byte { } } if readType { - _ = c.buffer.ReadVarUint32Small7(err) + _ = c.buffer.ReadUint8(err) } size := c.buffer.ReadLength(err) return c.buffer.ReadBinary(size, err) @@ -447,7 +446,7 @@ func (c *ReadContext) ReadStringSlice(refMode RefMode, readType bool) []string { } } if readType { - _ = c.buffer.ReadVarUint32Small7(err) + _ = c.buffer.ReadUint8(err) } return ReadStringSlice(c.buffer, err) } @@ -461,7 +460,7 @@ func (c *ReadContext) ReadStringStringMap(refMode RefMode, readType bool) map[st } } if readType { - _ = c.buffer.ReadVarUint32Small7(err) + _ = c.buffer.ReadUint8(err) } return readMapStringString(c.buffer, err) } @@ -475,7 +474,7 @@ func (c *ReadContext) ReadStringInt64Map(refMode RefMode, readType bool) map[str } } if readType { - _ = c.buffer.ReadVarUint32Small7(err) + _ = c.buffer.ReadUint8(err) } return readMapStringInt64(c.buffer, err) } @@ -489,7 +488,7 @@ func (c *ReadContext) ReadStringInt32Map(refMode RefMode, readType bool) map[str } } if readType { - _ = c.buffer.ReadVarUint32Small7(err) + _ = c.buffer.ReadUint8(err) } return readMapStringInt32(c.buffer, err) } @@ -503,7 +502,7 @@ func (c *ReadContext) ReadStringIntMap(refMode RefMode, readType bool) map[strin } } if readType { - _ = c.buffer.ReadVarUint32Small7(err) + _ = c.buffer.ReadUint8(err) } return readMapStringInt(c.buffer, err) } @@ -517,7 +516,7 @@ func (c *ReadContext) ReadStringFloat64Map(refMode RefMode, readType bool) map[s } } if readType { - _ = c.buffer.ReadVarUint32Small7(err) + _ = c.buffer.ReadUint8(err) } return readMapStringFloat64(c.buffer, err) } @@ -531,7 +530,7 @@ func (c *ReadContext) ReadStringBoolMap(refMode RefMode, readType bool) map[stri } } if readType { - _ = c.buffer.ReadVarUint32Small7(err) + _ = c.buffer.ReadUint8(err) } return readMapStringBool(c.buffer, err) } @@ -545,7 +544,7 @@ func (c *ReadContext) ReadInt32Int32Map(refMode RefMode, readType bool) map[int3 } } if readType { - _ = c.buffer.ReadVarUint32Small7(err) + _ = c.buffer.ReadUint8(err) } return readMapInt32Int32(c.buffer, err) } @@ -559,7 +558,7 @@ func (c *ReadContext) ReadInt64Int64Map(refMode RefMode, readType bool) map[int6 } } if readType { - _ = c.buffer.ReadVarUint32Small7(err) + _ = c.buffer.ReadUint8(err) } return readMapInt64Int64(c.buffer, err) } @@ -573,7 +572,7 @@ func (c *ReadContext) ReadIntIntMap(refMode RefMode, readType bool) map[int]int } } if readType { - _ = c.buffer.ReadVarUint32Small7(err) + _ = c.buffer.ReadUint8(err) } return readMapIntInt(c.buffer, err) } @@ -678,7 +677,7 @@ func (c *ReadContext) ReadValue(value reflect.Value, refMode RefMode, readType b // Create a new instance var newValue reflect.Value var valueToSet reflect.Value - internalTypeID := TypeId(typeInfo.TypeID & 0xFF) + internalTypeID := TypeId(typeInfo.TypeID) // For named struct types, create a pointer type to support circular references. // In Java/xlang serialization, objects are always by reference, so when deserializing @@ -879,7 +878,7 @@ func (c *ReadContext) ReadArrayValue(target reflect.Value, refMode RefMode, read // Read type ID if requested (will be slice type in stream) if readType { - c.buffer.ReadVarUint32Small7(c.Err()) + c.buffer.ReadUint8(c.Err()) } // Get slice serializer to read the data diff --git a/go/fory/set.go b/go/fory/set.go index fa08b8698c..2105b3e9df 100644 --- a/go/fory/set.go +++ b/go/fory/set.go @@ -120,13 +120,7 @@ func (s setSerializer) Write(ctx *WriteContext, refMode RefMode, writeType bool, } } if writeType { - // For polymorphic set elements, need to write full type info - typeInfo, err := ctx.TypeResolver().getTypeInfo(value, true) - if err != nil { - ctx.SetError(FromError(err)) - return - } - ctx.TypeResolver().WriteTypeInfo(ctx.buffer, typeInfo, ctx.Err()) + ctx.buffer.WriteUint8(uint8(SET)) } s.writeDataWithGenerics(ctx, value, hasGenerics) } @@ -202,7 +196,7 @@ func (s setSerializer) writeHeader(ctx *WriteContext, buf *ByteBuffer, keys []re // 1. All elements have same type (IS_SAME_TYPE is set) // 2. Element type is NOT declared from schema (IS_DECL_ELEMENT_TYPE is NOT set) if hasSameType && !hasGenerics && elemTypeInfo != nil { - buf.WriteVarUint32Small7(uint32(elemTypeInfo.TypeID)) + ctx.TypeResolver().WriteTypeInfo(buf, elemTypeInfo, ctx.Err()) } return byte(collectFlag), elemTypeInfo @@ -269,7 +263,7 @@ func (s setSerializer) writeDifferentTypes(ctx *WriteContext, buf *ByteBuffer, k ctx.SetError(FromError(err)) return } - buf.WriteVarUint32Small7(uint32(typeInfo.TypeID)) + ctx.TypeResolver().WriteTypeInfo(buf, typeInfo, ctx.Err()) if !refWritten { typeInfo.Serializer.WriteData(ctx, key) if ctx.HasError() { @@ -279,14 +273,14 @@ func (s setSerializer) writeDifferentTypes(ctx *WriteContext, buf *ByteBuffer, k } else if hasNull { // No ref tracking but may have nulls - write NotNullValueFlag before type + data buf.WriteInt8(NotNullValueFlag) - buf.WriteVarUint32Small7(uint32(typeInfo.TypeID)) + ctx.TypeResolver().WriteTypeInfo(buf, typeInfo, ctx.Err()) typeInfo.Serializer.WriteData(ctx, key) if ctx.HasError() { return } } else { // No ref tracking and no nulls - write type + data directly - buf.WriteVarUint32Small7(uint32(typeInfo.TypeID)) + ctx.TypeResolver().WriteTypeInfo(buf, typeInfo, ctx.Err()) typeInfo.Serializer.WriteData(ctx, key) if ctx.HasError() { return @@ -482,10 +476,14 @@ func (s setSerializer) Read(ctx *ReadContext, refMode RefMode, readType bool, ha } } if readType { - // ReadData and discard type info for sets - typeID := uint32(buf.ReadVarUint32Small7(ctxErr)) - if IsNamespacedType(TypeId(typeID)) { - ctx.TypeResolver().readTypeInfoWithTypeID(buf, typeID, ctxErr) + // Read and discard type ID for sets + typeID := uint32(buf.ReadUint8(ctxErr)) + if ctx.HasError() { + return + } + if typeID != uint32(SET) { + ctx.SetError(DeserializationErrorf("set type mismatch: expected SET (%d), got %d", SET, typeID)) + return } } s.ReadData(ctx, value) diff --git a/go/fory/skip.go b/go/fory/skip.go index 2eea25d4c4..b0b36c5af7 100644 --- a/go/fory/skip.go +++ b/go/fory/skip.go @@ -51,18 +51,17 @@ func SkipFieldValueWithTypeFlag(ctx *ReadContext, fieldDef FieldDef, readRefFlag } // Read type info (typeID + meta_index) - wroteTypeID := ctx.buffer.ReadVarUint32Small7(err) - internalID := wroteTypeID & 0xff + wroteTypeID := uint32(ctx.buffer.ReadUint8(err)) + internalID := TypeId(wroteTypeID) // Check if it's an EXT type first - EXT types don't have meta info like structs - if internalID == uint32(EXT) { - // EXT types with numeric ID - try to find the registered serializer - serializer := ctx.TypeResolver().getSerializerByTypeID(wroteTypeID) - if serializer != nil { + if internalID == EXT { + typeInfo := ctx.TypeResolver().readTypeInfoWithTypeID(ctx.buffer, wroteTypeID, err) + if typeInfo != nil && typeInfo.Serializer != nil { // Use the serializer to read and discard the value var dummy any dummyVal := reflect.ValueOf(&dummy).Elem() - serializer.Read(ctx, RefModeNone, false, false, dummyVal) + typeInfo.Serializer.Read(ctx, RefModeNone, false, false, dummyVal) return } // If no serializer is registered, we can't skip this type @@ -71,7 +70,7 @@ func SkipFieldValueWithTypeFlag(ctx *ReadContext, fieldDef FieldDef, readRefFlag } // Check if it's a NAMED_EXT type - need to read type info to find serializer - if internalID == uint32(NAMED_EXT) { + if internalID == NAMED_EXT { typeInfo := ctx.TypeResolver().readTypeInfoWithTypeID(ctx.buffer, wroteTypeID, err) if typeInfo.Serializer != nil { // Use the serializer to read and discard the value @@ -85,15 +84,15 @@ func SkipFieldValueWithTypeFlag(ctx *ReadContext, fieldDef FieldDef, readRefFlag } // Check if it's a struct type - need to read type info and skip struct data - if internalID == uint32(COMPATIBLE_STRUCT) || internalID == uint32(STRUCT) || - internalID == uint32(NAMED_STRUCT) || internalID == uint32(NAMED_COMPATIBLE_STRUCT) { + if internalID == COMPATIBLE_STRUCT || internalID == STRUCT || + internalID == NAMED_STRUCT || internalID == NAMED_COMPATIBLE_STRUCT { typeInfo := ctx.TypeResolver().readTypeInfoWithTypeID(ctx.buffer, wroteTypeID, err) // Now skip the struct data using the typeInfo from the written type skipStruct(ctx, typeInfo) return } - if IsNamespacedType(TypeId(wroteTypeID)) { + if IsNamespacedType(internalID) { typeInfo := ctx.TypeResolver().readTypeInfoWithTypeID(ctx.buffer, wroteTypeID, err) // Now skip the struct data using the typeInfo from the written type skipStruct(ctx, typeInfo) @@ -133,17 +132,17 @@ func SkipAnyValue(ctx *ReadContext, readRefFlag bool) { } // ReadData type_id first - typeID := ctx.buffer.ReadVarUint32Small7(err) + typeID := uint32(ctx.buffer.ReadUint8(err)) if ctx.HasError() { return } - internalID := typeID & 0xff + internalID := TypeId(typeID) // For struct-like types, also read meta_index to get type_info var fieldDef FieldDef var typeInfo *TypeInfo - switch TypeId(internalID) { + switch internalID { case LIST, SET: fieldDef = FieldDef{ fieldType: NewCollectionFieldType(TypeId(typeID), NewSimpleFieldType(UNKNOWN)), @@ -168,7 +167,7 @@ func SkipAnyValue(ctx *ReadContext, readRefFlag bool) { fieldType: NewSimpleFieldType(TypeId(typeID)), nullable: true, } - case COMPATIBLE_STRUCT, NAMED_COMPATIBLE_STRUCT, STRUCT, NAMED_STRUCT: + case COMPATIBLE_STRUCT, NAMED_COMPATIBLE_STRUCT, STRUCT, NAMED_STRUCT, EXT, TYPED_UNION: // Read type info using the shared meta reader when enabled. typeInfo = ctx.TypeResolver().readTypeInfoWithTypeID(ctx.buffer, typeID, err) if ctx.HasError() { @@ -179,6 +178,13 @@ func SkipAnyValue(ctx *ReadContext, readRefFlag bool) { nullable: true, } default: + if internalID == ENUM || internalID == STRUCT || + internalID == EXT || internalID == TYPED_UNION { + ctx.buffer.ReadVarUint32(err) + if ctx.HasError() { + return + } + } fieldDef = FieldDef{ fieldType: NewSimpleFieldType(TypeId(typeID)), nullable: true, @@ -195,7 +201,7 @@ func SkipAnyValue(ctx *ReadContext, readRefFlag bool) { func readTypeInfoForSkip(ctx *ReadContext, fieldTypeId TypeId) *TypeInfo { err := ctx.Err() // Read the actual typeID from buffer (Java writes typeID for struct fields) - typeID := ctx.buffer.ReadVarUint32Small7(err) + typeID := uint32(ctx.buffer.ReadUint8(err)) if ctx.HasError() { return nil } @@ -226,7 +232,7 @@ func skipCollection(ctx *ReadContext, fieldDef FieldDef) { var elemTypeInfo *TypeInfo if isSameType && !isDeclared { // ReadData element type info - first read the typeID from buffer - typeID := ctx.buffer.ReadVarUint32Small7(err) + typeID := uint32(ctx.buffer.ReadUint8(err)) if ctx.HasError() { return } @@ -326,7 +332,7 @@ func skipMap(ctx *ReadContext, fieldDef FieldDef) { var valueDef FieldDef var valueTypeInfo *TypeInfo if !valueDeclared { - typeID := ctx.buffer.ReadVarUint32Small7(bufErr) + typeID := uint32(ctx.buffer.ReadUint8(bufErr)) if ctx.HasError() { return } @@ -358,7 +364,7 @@ func skipMap(ctx *ReadContext, fieldDef FieldDef) { var keyDef FieldDef var keyTypeInfo *TypeInfo if !keyDeclared { - typeID := ctx.buffer.ReadVarUint32Small7(bufErr) + typeID := uint32(ctx.buffer.ReadUint8(bufErr)) if ctx.HasError() { return } @@ -396,7 +402,7 @@ func skipMap(ctx *ReadContext, fieldDef FieldDef) { var keyDef, valueDef FieldDef var keyTypeInfo, valueTypeInfo *TypeInfo if !keyDeclared { - typeID := ctx.buffer.ReadVarUint32Small7(bufErr) + typeID := uint32(ctx.buffer.ReadUint8(bufErr)) if ctx.HasError() { return } @@ -410,7 +416,7 @@ func skipMap(ctx *ReadContext, fieldDef FieldDef) { } if !valueDeclared { - typeID := ctx.buffer.ReadVarUint32Small7(bufErr) + typeID := uint32(ctx.buffer.ReadUint8(bufErr)) if ctx.HasError() { return } @@ -518,43 +524,37 @@ func skipValue(ctx *ReadContext, fieldDef FieldDef, readRefFlag bool, isField bo typeIDNum := uint32(fieldDef.fieldType.TypeId()) - // Check if it's a user-defined type (high bits set, meaning type_id > 255) - if typeIDNum > 255 { - internalID := typeIDNum & 0xff - // Handle struct-like types - if internalID == uint32(COMPATIBLE_STRUCT) || internalID == uint32(STRUCT) || - internalID == uint32(NAMED_STRUCT) || internalID == uint32(NAMED_COMPATIBLE_STRUCT) || - internalID == uint32(UNKNOWN) { - // If type_info is provided (from SkipAnyValue), use skipStruct directly - if typeInfo != nil { - skipStruct(ctx, typeInfo) - return - } - // Otherwise we need to read type info - ti := ctx.TypeResolver().readTypeInfoWithTypeID(ctx.buffer, typeIDNum, err) - skipStruct(ctx, ti) - return - } else if internalID == uint32(ENUM) || internalID == uint32(NAMED_ENUM) { - _ = ctx.buffer.ReadVarUint32(err) - return - } else if internalID == uint32(EXT) || internalID == uint32(NAMED_EXT) { - // EXT types use custom serializers - try to find the registered serializer - serializer := ctx.TypeResolver().getSerializerByTypeID(typeIDNum) - if serializer != nil { - // Use the serializer to read and discard the value - // Create a dummy value to read into - var dummy any - dummyVal := reflect.ValueOf(&dummy).Elem() - serializer.Read(ctx, RefModeNone, false, false, dummyVal) - return - } - // If no serializer is registered, we can't skip this type - ctx.SetError(DeserializationErrorf("cannot skip EXT type %d: no serializer registered", typeIDNum)) + internalID := TypeId(typeIDNum) + // Handle struct-like types + if internalID == COMPATIBLE_STRUCT || internalID == STRUCT || + internalID == NAMED_STRUCT || internalID == NAMED_COMPATIBLE_STRUCT || + internalID == UNKNOWN { + // If type_info is provided (from SkipAnyValue), use skipStruct directly + if typeInfo != nil { + skipStruct(ctx, typeInfo) return - } else { - ctx.SetError(DeserializationErrorf("unknown type id: %d (internal_id: %d)", typeIDNum, internalID)) + } + // Otherwise we need to read type info + ti := ctx.TypeResolver().readTypeInfoWithTypeID(ctx.buffer, typeIDNum, err) + skipStruct(ctx, ti) + return + } + if internalID == ENUM { + // Enum values are encoded as ordinal only (VarUint32Small7) for xlang. + _ = ctx.buffer.ReadUint8(err) + return + } + if internalID == EXT || internalID == NAMED_EXT || internalID == TYPED_UNION || internalID == NAMED_UNION { + typeInfo := ctx.TypeResolver().readTypeInfoWithTypeID(ctx.buffer, typeIDNum, err) + if typeInfo != nil && typeInfo.Serializer != nil { + // Use the serializer to read and discard the value + var dummy any + dummyVal := reflect.ValueOf(&dummy).Elem() + typeInfo.Serializer.Read(ctx, RefModeNone, false, false, dummyVal) return } + ctx.SetError(DeserializationErrorf("cannot skip type %d: no serializer registered", typeIDNum)) + return } // Match on built-in types @@ -569,9 +569,9 @@ func skipValue(ctx *ReadContext, fieldDef FieldDef, readRefFlag bool, isField bo case INT16: _ = ctx.buffer.ReadInt16(err) case INT32: - _ = ctx.buffer.ReadVarUint32Small7(err) + _ = ctx.buffer.ReadUint8(err) case VARINT32: - _ = ctx.buffer.ReadVarUint32Small7(err) + _ = ctx.buffer.ReadUint8(err) case INT64, VARINT64, TAGGED_INT64: _ = ctx.buffer.ReadVarint64(err) @@ -631,7 +631,7 @@ func skipValue(ctx *ReadContext, fieldDef FieldDef, readRefFlag bool, isField bo // Date/Time types case DATE: - _ = ctx.buffer.ReadVarUint32Small7(err) + _ = ctx.buffer.ReadUint8(err) case TIMESTAMP: _ = ctx.buffer.ReadInt64(err) _ = ctx.buffer.ReadUint32(err) @@ -666,7 +666,7 @@ func skipValue(ctx *ReadContext, fieldDef FieldDef, readRefFlag bool, isField bo skipStruct(ctx, ti) // Enum types - case ENUM, NAMED_ENUM: + case ENUM: _ = ctx.buffer.ReadVarUint32(err) // Unsigned integer types diff --git a/go/fory/slice.go b/go/fory/slice.go index 72fba1fd29..bd3a9aa7ee 100644 --- a/go/fory/slice.go +++ b/go/fory/slice.go @@ -56,7 +56,7 @@ func writeSliceRefAndType(ctx *WriteContext, refMode RefMode, writeType bool, va ctx.Buffer().WriteInt8(NotNullValueFlag) } if writeType { - ctx.Buffer().WriteVarUint32Small7(uint32(typeId)) + ctx.Buffer().WriteUint8(uint8(typeId)) } return false } @@ -89,7 +89,7 @@ func readSliceRefAndType(ctx *ReadContext, refMode RefMode, readType bool, value } var typeId uint32 if readType { - typeId = buf.ReadVarUint32Small7(ctxErr) + typeId = uint32(buf.ReadUint8(ctxErr)) } return false, typeId } @@ -281,9 +281,7 @@ func (s *sliceSerializer) ReadData(ctx *ReadContext, value reflect.Value) { // We must consume these bytes for protocol compliance if (collectFlag & CollectionIsSameType) != 0 { if (collectFlag & CollectionIsDeclElementType) == 0 { - typeID := buf.ReadVarUint32Small7(ctxErr) - // ReadData additional metadata for namespaced types - ctx.TypeResolver().readTypeInfoWithTypeID(buf, typeID, ctxErr) + ctx.TypeResolver().ReadTypeInfo(buf, ctxErr) } } diff --git a/go/fory/slice_primitive.go b/go/fory/slice_primitive.go index 0b6c7dc05b..200d144c63 100644 --- a/go/fory/slice_primitive.go +++ b/go/fory/slice_primitive.go @@ -601,7 +601,7 @@ func (s stringSliceSerializer) writeDataWithGenerics(ctx *WriteContext, value re } else { // When element type is not known, write CollectionIsSameType and element type info buf.WriteInt8(int8(CollectionIsSameType)) - buf.WriteVarUint32Small7(uint32(STRING)) + buf.WriteUint8(uint8(STRING)) } // Write elements directly (no ref flag for strings) @@ -653,7 +653,7 @@ func (s stringSliceSerializer) ReadData(ctx *ReadContext, value reflect.Value) { // Read element type info if present (when CollectionIsSameType but not CollectionIsDeclElementType) if (collectFlag&CollectionIsSameType) != 0 && (collectFlag&CollectionIsDeclElementType) == 0 { - _ = buf.ReadVarUint32Small7(ctxErr) // Read and discard type ID (we know it's STRING) + _ = buf.ReadUint8(ctxErr) // Read and discard type ID (we know it's STRING) } result := make([]string, length) @@ -1296,7 +1296,7 @@ func WriteStringSlice(buf *ByteBuffer, value []string, hasGenerics bool) { buf.WriteInt8(int8(CollectionDeclSameType)) } else { buf.WriteInt8(int8(CollectionIsSameType)) - buf.WriteVarUint32Small7(uint32(STRING)) + buf.WriteUint8(uint8(STRING)) } for i := 0; i < length; i++ { writeString(buf, value[i]) @@ -1314,7 +1314,7 @@ func ReadStringSlice(buf *ByteBuffer, err *Error) []string { } collectFlag := buf.ReadInt8(err) if (collectFlag&CollectionIsSameType) != 0 && (collectFlag&CollectionIsDeclElementType) == 0 { - _ = buf.ReadVarUint32Small7(err) // Read and discard element type ID + _ = buf.ReadUint8(err) // Read and discard element type ID } result := make([]string, length) trackRefs := (collectFlag & CollectionTrackingRef) != 0 diff --git a/go/fory/string.go b/go/fory/string.go index be87d91d03..cde49328bb 100644 --- a/go/fory/string.go +++ b/go/fory/string.go @@ -117,7 +117,7 @@ func (s stringSerializer) Write(ctx *WriteContext, refMode RefMode, writeType bo ctx.buffer.WriteInt8(NotNullValueFlag) } if writeType { - ctx.buffer.WriteVarUint32Small7(uint32(STRING)) + ctx.buffer.WriteUint8(uint8(STRING)) } s.WriteData(ctx, value) } @@ -142,7 +142,7 @@ func (s stringSerializer) Read(ctx *ReadContext, refMode RefMode, readType bool, } } if readType { - _ = ctx.buffer.ReadVarUint32Small7(err) + _ = ctx.buffer.ReadUint8(err) } if ctx.HasError() { return @@ -166,7 +166,7 @@ func (s ptrToStringSerializer) Write(ctx *WriteContext, refMode RefMode, writeTy ctx.buffer.WriteInt8(NotNullValueFlag) } if writeType { - ctx.buffer.WriteVarUint32Small7(uint32(STRING)) + ctx.buffer.WriteUint8(uint8(STRING)) } s.WriteData(ctx, value) } @@ -184,7 +184,7 @@ func (s ptrToStringSerializer) Read(ctx *ReadContext, refMode RefMode, readType } } if readType { - _ = ctx.buffer.ReadVarUint32Small7(err) + _ = ctx.buffer.ReadUint8(err) } if ctx.HasError() { return diff --git a/go/fory/struct.go b/go/fory/struct.go index 53b81f0a50..18dec4e847 100644 --- a/go/fory/struct.go +++ b/go/fory/struct.go @@ -45,6 +45,7 @@ type structSerializer struct { type_ reflect.Type structHash int32 typeID uint32 + userTypeID uint32 // Pre-sorted and categorized fields (embedded for cache locality) fieldGroup FieldGroup @@ -72,9 +73,10 @@ func newStructSerializerFromTypeDef(type_ reflect.Type, name string, fieldDefs [ name = type_.Name() } return &structSerializer{ - type_: type_, - name: name, - fieldDefs: fieldDefs, + type_: type_, + name: name, + userTypeID: invalidUserTypeID, + fieldDefs: fieldDefs, } } @@ -86,8 +88,9 @@ func newStructSerializer(type_ reflect.Type, name string) *structSerializer { name = type_.Name() } return &structSerializer{ - type_: type_, - name: name, + type_: type_, + name: name, + userTypeID: invalidUserTypeID, } } @@ -135,8 +138,7 @@ func computeLocalNullable(typeResolver *TypeResolver, field reflect.StructField, fieldType = optionalInfo.valueType } typeId := typeResolver.getTypeIdByType(fieldType) - internalId := TypeId(typeId & 0xFF) - isEnum := internalId == ENUM || internalId == NAMED_ENUM + isEnum := typeId == ENUM var nullableFlag bool if typeResolver.fory.config.IsXlang { nullableFlag = isOptional || field.Type.Kind() == reflect.Ptr @@ -376,8 +378,7 @@ func (s *structSerializer) initFields(typeResolver *TypeResolver) error { // - In native mode: Go's natural semantics apply - slice/map/interface can be nil, // so they are nullable by default. // Can be overridden by explicit fory tag `fory:"nullable"`. - internalId := fieldTypeId & 0xFF - isEnum := internalId == ENUM || internalId == NAMED_ENUM + isEnum := fieldTypeId == ENUM // Determine nullable based on mode // In xlang mode: only pointer types are nullable by default (per xlang spec) @@ -688,9 +689,8 @@ func (s *structSerializer) initFieldsFromTypeDef(typeResolver *TypeResolver) err isPolymorphicField := def.fieldType.TypeId() == UNKNOWN defTypeId := def.fieldType.TypeId() // Check if field is an enum - either by type ID or by serializer type - // The type ID may be a composite value with namespace bits, so check the low 8 bits - internalDefTypeId := defTypeId & 0xFF - isEnumField := internalDefTypeId == NAMED_ENUM || internalDefTypeId == ENUM + internalDefTypeId := defTypeId + isEnumField := internalDefTypeId == ENUM if !isEnumField && fieldSerializer != nil { _, isEnumField = fieldSerializer.(*enumSerializer) } @@ -700,8 +700,8 @@ func (s *structSerializer) initFieldsFromTypeDef(typeResolver *TypeResolver) err shouldRead = true fieldType = localType } else if typeLookupFailed && isEnumField { - // For enum fields with failed TypeDef lookup (NAMED_ENUM stores by namespace/typename, not typeId), - // check if local field is a numeric type (Go enums are int-based) + // For enum fields with failed TypeDef lookup, check if local field is a numeric type + // (Go enums are int-based) // Also handle pointer enum fields (*EnumType) localKind := localType.Kind() elemKind := localKind @@ -735,7 +735,7 @@ func (s *structSerializer) initFieldsFromTypeDef(typeResolver *TypeResolver) err shouldRead = true fieldType = localType } - } else if typeLookupFailed && isPrimitiveType(int16(internalDefTypeId)) { + } else if typeLookupFailed && isPrimitiveType(TypeId(internalDefTypeId)) { baseLocal := localType if optInfo, ok := getOptionalInfo(baseLocal); ok { baseLocal = optInfo.valueType @@ -747,7 +747,7 @@ func (s *structSerializer) initFieldsFromTypeDef(typeResolver *TypeResolver) err shouldRead = true fieldType = localType } - } else if typeLookupFailed && isPrimitiveArrayType(int16(internalDefTypeId)) { + } else if typeLookupFailed && isPrimitiveArrayType(TypeId(internalDefTypeId)) { // Primitive arrays/slices use array type IDs but may not be registered in typeIDToTypeInfo. // Allow reading using the local slice/array type when the type IDs match. localTypeId := typeIdFromKind(localType) @@ -1020,12 +1020,12 @@ func (s *structSerializer) initFieldsFromTypeDef(typeResolver *TypeResolver) err s.typeDefDiffers = true break } - remoteTypeId := TypeId(s.fieldDefs[i].fieldType.TypeId() & 0xFF) + remoteTypeId := TypeId(s.fieldDefs[i].fieldType.TypeId()) localTypeId := typeResolver.getTypeIdByType(field.Meta.Type) if localTypeId == 0 { localTypeId = typeIdFromKind(field.Meta.Type) } - localTypeId = TypeId(localTypeId & 0xFF) + localTypeId = TypeId(localTypeId) if !typeIdEqualForDiff(remoteTypeId, localTypeId) { if DebugOutputEnabled && s.type_ != nil { fmt.Printf("[Go][fory-debug] [%s] typeDefDiffers: type ID mismatch idx=%d name=%q tagID=%d remote=%d local=%d\n", @@ -1083,8 +1083,7 @@ func (s *structSerializer) computeHash() int32 { } } // Unions use UNION type ID in fingerprints, regardless of typed/named variants. - internalId := typeId & 0xFF - if internalId == TYPED_UNION || internalId == NAMED_UNION || internalId == UNION { + if typeId == TYPED_UNION || typeId == NAMED_UNION || typeId == UNION { typeId = UNION } // For user-defined types (struct, ext types), use UNKNOWN in fingerprint @@ -1125,7 +1124,7 @@ func (s *structSerializer) computeHash() int32 { typeId = LIST } } else if fieldTypeForHash.Kind() == reflect.Slice { - if !isPrimitiveArrayType(int16(typeId)) && typeId != BINARY { + if !isPrimitiveArrayType(TypeId(typeId)) && typeId != BINARY { typeId = LIST } } else if fieldTypeForHash.Kind() == reflect.Map { @@ -2385,43 +2384,6 @@ func (s *structSerializer) Read(ctx *ReadContext, refMode RefMode, readType bool } } if readType { - if !ctx.Compatible() && s.type_ != nil { - typeID := buf.ReadVarUint32Small7(ctxErr) - if ctxErr.HasError() { - return - } - if s.typeID != 0 && typeID == s.typeID && !IsNamespacedType(TypeId(typeID)) { - s.ReadData(ctx, value) - return - } - if IsNamespacedType(TypeId(typeID)) { - // Expected type is known: skip namespace/type meta and read data directly. - ctx.TypeResolver().metaStringResolver.ReadMetaStringBytes(buf, ctxErr) - ctx.TypeResolver().metaStringResolver.ReadMetaStringBytes(buf, ctxErr) - if ctxErr.HasError() { - return - } - s.ReadData(ctx, value) - return - } - internalTypeID := TypeId(typeID & 0xFF) - if internalTypeID == COMPATIBLE_STRUCT || internalTypeID == STRUCT { - typeInfo := ctx.TypeResolver().readTypeInfoWithTypeID(buf, typeID, ctxErr) - if ctxErr.HasError() { - return - } - if structSer, ok := typeInfo.Serializer.(*structSerializer); ok && len(structSer.fieldDefs) > 0 { - structSer.ReadData(ctx, value) - return - } - if s.typeID != 0 && typeID == s.typeID { - s.ReadData(ctx, value) - return - } - } - ctx.SetError(DeserializationError("unexpected type id for struct")) - return - } if s.type_ != nil { serializer := ctx.TypeResolver().ReadTypeInfoForType(buf, s.type_, ctxErr) if ctxErr.HasError() { @@ -2438,11 +2400,12 @@ func (s *structSerializer) Read(ctx *ReadContext, refMode RefMode, readType bool s.ReadData(ctx, value) return } - // Fallback: read type info based on typeID when expected type is unknown - typeID := buf.ReadVarUint32Small7(ctxErr) - internalTypeID := TypeId(typeID & 0xFF) - if IsNamespacedType(TypeId(typeID)) || internalTypeID == COMPATIBLE_STRUCT || internalTypeID == STRUCT { - typeInfo := ctx.TypeResolver().readTypeInfoWithTypeID(buf, typeID, ctxErr) + // Fallback: read type info when expected type is unknown + typeInfo := ctx.TypeResolver().ReadTypeInfo(buf, ctxErr) + if ctxErr.HasError() { + return + } + if typeInfo != nil { if structSer, ok := typeInfo.Serializer.(*structSerializer); ok && len(structSer.fieldDefs) > 0 { structSer.ReadData(ctx, value) return diff --git a/go/fory/time.go b/go/fory/time.go index 000108a4ea..3eb2086464 100644 --- a/go/fory/time.go +++ b/go/fory/time.go @@ -38,7 +38,7 @@ func (s dateSerializer) Write(ctx *WriteContext, refMode RefMode, writeType bool ctx.buffer.WriteInt8(NotNullValueFlag) } if writeType { - ctx.buffer.WriteVarUint32Small7(uint32(DATE)) + ctx.buffer.WriteUint8(uint8(DATE)) } s.WriteData(ctx, value) } @@ -58,7 +58,7 @@ func (s dateSerializer) Read(ctx *ReadContext, refMode RefMode, readType bool, h } } if readType { - _ = ctx.buffer.ReadVarUint32Small7(err) + _ = ctx.buffer.ReadUint8(err) } if ctx.HasError() { return @@ -92,7 +92,7 @@ func (s timeSerializer) Write(ctx *WriteContext, refMode RefMode, writeType bool ctx.buffer.WriteInt8(NotNullValueFlag) } if writeType { - ctx.buffer.WriteVarUint32Small7(uint32(TIMESTAMP)) + ctx.buffer.WriteUint8(uint8(TIMESTAMP)) } s.WriteData(ctx, value) } @@ -112,7 +112,7 @@ func (s timeSerializer) Read(ctx *ReadContext, refMode RefMode, readType bool, h } } if readType { - _ = ctx.buffer.ReadVarUint32Small7(err) + _ = ctx.buffer.ReadUint8(err) } if ctx.HasError() { return diff --git a/go/fory/type_def.go b/go/fory/type_def.go index 6fa0a65098..9a0d416218 100644 --- a/go/fory/type_def.go +++ b/go/fory/type_def.go @@ -44,7 +44,9 @@ typeDef are layout as following: - next variable bytes: field definitions (see below) */ type TypeDef struct { - typeId uint32 // Full composite type ID (userId << 8 | internalTypeId) + typeId uint32 + // User type ID is stored as unsigned uint32; 0xffffffff means unset. + userTypeId uint32 nsName *MetaStringBytes typeName *MetaStringBytes compressed bool @@ -54,9 +56,10 @@ type TypeDef struct { type_ reflect.Type } -func NewTypeDef(typeId uint32, nsName, typeName *MetaStringBytes, registerByName, compressed bool, fieldDefs []FieldDef) *TypeDef { +func NewTypeDef(typeId uint32, userTypeId uint32, nsName, typeName *MetaStringBytes, registerByName, compressed bool, fieldDefs []FieldDef) *TypeDef { return &TypeDef{ typeId: typeId, + userTypeId: userTypeId, nsName: nsName, typeName: typeName, compressed: compressed, @@ -93,8 +96,8 @@ func (td *TypeDef) String() string { for i, fd := range td.fieldDefs { fieldStrs[i] = fd.String() } - return fmt.Sprintf("TypeDef{typeId=%d, ns=%s, type=%s, registerByName=%v, compressed=%v, fields=[%s]}", - td.typeId, nsStr, typeStr, td.registerByName, td.compressed, strings.Join(fieldStrs, ", ")) + return fmt.Sprintf("TypeDef{typeId=%d, userTypeId=%d, ns=%s, type=%s, registerByName=%v, compressed=%v, fields=[%s]}", + td.typeId, td.userTypeId, nsStr, typeStr, td.registerByName, td.compressed, strings.Join(fieldStrs, ", ")) } // ComputeDiff computes the diff between this (decoded/remote) TypeDef and a local TypeDef. @@ -237,6 +240,7 @@ func (td *TypeDef) buildTypeInfoWithResolver(resolver *TypeResolver) (TypeInfo, } else { // Known struct type - use structSerializer with fieldDefs structSer := newStructSerializerFromTypeDef(type_, "", td.fieldDefs) + structSer.userTypeID = td.userTypeId // Eagerly initialize the struct serializer with pre-computed field metadata if resolver != nil { if err := structSer.initialize(resolver); err != nil { @@ -250,6 +254,7 @@ func (td *TypeDef) buildTypeInfoWithResolver(resolver *TypeResolver) (TypeInfo, info := TypeInfo{ Type: type_, TypeID: td.typeId, + UserTypeID: td.userTypeId, Serializer: serializer, PkgPathBytes: td.nsName, NameBytes: td.typeName, @@ -353,9 +358,9 @@ func buildTypeDef(fory *Fory, value reflect.Value) (*TypeDef, error) { if err != nil { return nil, fmt.Errorf("failed to get type info for value %v: %w", value, err) } - typeId := uint32(infoPtr.TypeID) // Use full uint32 type ID - registerByName := IsNamespacedType(TypeId(typeId & 0xFF)) - typeDef := NewTypeDef(typeId, infoPtr.PkgPathBytes, infoPtr.NameBytes, registerByName, false, fieldDefs) + typeId := uint32(infoPtr.TypeID) + registerByName := IsNamespacedType(TypeId(typeId)) + typeDef := NewTypeDef(typeId, infoPtr.UserTypeID, infoPtr.PkgPathBytes, infoPtr.NameBytes, registerByName, false, fieldDefs) // encoding the typeDef, and save the encoded bytes encoded, err := encodingTypeDef(fory.typeResolver, typeDef) @@ -527,8 +532,7 @@ func buildFieldDefs(fory *Fory, value reflect.Value) ([]FieldDef, error) { // so they are nullable by default. // Can be overridden by explicit fory tag `fory:"nullable"` typeId := ft.TypeId() - internalId := TypeId(typeId & 0xFF) - isEnumField := internalId == ENUM || internalId == NAMED_ENUM + isEnumField := typeId == ENUM // Determine nullable based on mode // In xlang mode: only pointer types are nullable by default (per xlang spec) // In native mode: Go's natural semantics - all nil-able types are nullable @@ -627,6 +631,7 @@ func buildFieldDefs(fory *Fory, value reflect.Value) ([]FieldDef, error) { // FieldType interface represents different field types, including object, collection, and map types type FieldType interface { TypeId() TypeId + UserTypeId() uint32 String() string write(*ByteBuffer) writeWithFlags(*ByteBuffer, bool, bool) // writeWithFlags writes typeId with nullable/trackingRef flags @@ -636,15 +641,17 @@ type FieldType interface { // BaseFieldType provides common functionality for field types type BaseFieldType struct { - typeId TypeId + typeId TypeId + userTypeId uint32 } -func (b *BaseFieldType) TypeId() TypeId { return b.typeId } +func (b *BaseFieldType) TypeId() TypeId { return b.typeId } +func (b *BaseFieldType) UserTypeId() uint32 { return b.userTypeId } func (b *BaseFieldType) String() string { return fmt.Sprintf("FieldType{typeId=%d}", b.typeId) } func (b *BaseFieldType) write(buffer *ByteBuffer) { - buffer.WriteVarUint32Small7(uint32(b.typeId)) + buffer.WriteUint8(uint8(b.typeId)) } // writeWithFlags writes the typeId with nullable and trackingRef flags packed into the value. @@ -677,6 +684,9 @@ func getFieldTypeSerializerWithResolver(resolver *TypeResolver, ft FieldType) (S } func (b *BaseFieldType) getTypeInfo(fory *Fory) (TypeInfo, error) { + if isUserDefinedType(b.typeId) { + return TypeInfo{}, nil + } info, err := fory.typeResolver.getTypeInfoById(uint32(b.typeId)) if err != nil { return TypeInfo{}, err @@ -688,6 +698,9 @@ func (b *BaseFieldType) getTypeInfo(fory *Fory) (TypeInfo, error) { } func (b *BaseFieldType) getTypeInfoWithResolver(resolver *TypeResolver) (TypeInfo, error) { + if isUserDefinedType(b.typeId) { + return TypeInfo{}, nil + } info, err := resolver.getTypeInfoById(uint32(b.typeId)) if err != nil { return TypeInfo{}, err @@ -701,9 +714,8 @@ func (b *BaseFieldType) getTypeInfoWithResolver(resolver *TypeResolver) (TypeInf // readFieldType reads field type info from the buffer according to the TypeId // This is called for top-level field types where flags are NOT embedded in the type ID func readFieldType(buffer *ByteBuffer, err *Error) (FieldType, error) { - typeId := buffer.ReadVarUint32Small7(err) - // Use internal type ID (low byte) for switch, but store the full typeId - internalTypeId := TypeId(typeId & 0xFF) + typeId := buffer.ReadUint8(err) + internalTypeId := TypeId(typeId) switch internalTypeId { case LIST, SET: @@ -725,9 +737,11 @@ func readFieldType(buffer *ByteBuffer, err *Error) (FieldType, error) { } return NewMapFieldType(TypeId(typeId), keyType, valueType), nil case UNKNOWN, EXT, STRUCT, NAMED_STRUCT, COMPATIBLE_STRUCT, NAMED_COMPATIBLE_STRUCT: - return NewDynamicFieldType(TypeId(typeId)), nil + ft := NewDynamicFieldType(TypeId(typeId)) + return ft, nil } - return NewSimpleFieldType(TypeId(typeId)), nil + ft := NewSimpleFieldType(TypeId(typeId)) + return ft, nil } // readFieldTypeWithFlags reads field type info where flags are embedded in the type ID @@ -738,8 +752,7 @@ func readFieldTypeWithFlags(buffer *ByteBuffer, err *Error) (FieldType, error) { // trackingRef := (rawValue & 0b1) != 0 // Not used currently // nullable := (rawValue & 0b10) != 0 // Not used currently typeId := rawValue >> 2 - // Use internal type ID (low byte) for switch, but store the full typeId - internalTypeId := TypeId(typeId & 0xFF) + internalTypeId := TypeId(typeId) switch internalTypeId { case LIST, SET: @@ -759,9 +772,11 @@ func readFieldTypeWithFlags(buffer *ByteBuffer, err *Error) (FieldType, error) { } return NewMapFieldType(TypeId(typeId), keyType, valueType), nil case UNKNOWN, EXT, STRUCT, NAMED_STRUCT, COMPATIBLE_STRUCT, NAMED_COMPATIBLE_STRUCT: - return NewDynamicFieldType(TypeId(typeId)), nil + ft := NewDynamicFieldType(TypeId(typeId)) + return ft, nil } - return NewSimpleFieldType(TypeId(typeId)), nil + ft := NewSimpleFieldType(TypeId(typeId)) + return ft, nil } // CollectionFieldType represents collection types like List, Slice @@ -772,7 +787,7 @@ type CollectionFieldType struct { func NewCollectionFieldType(typeId TypeId, elementType FieldType) *CollectionFieldType { return &CollectionFieldType{ - BaseFieldType: BaseFieldType{typeId: typeId}, + BaseFieldType: BaseFieldType{typeId: typeId, userTypeId: invalidUserTypeID}, elementType: elementType, } } @@ -855,7 +870,7 @@ type MapFieldType struct { func NewMapFieldType(typeId TypeId, keyType, valueType FieldType) *MapFieldType { return &MapFieldType{ - BaseFieldType: BaseFieldType{typeId: typeId}, + BaseFieldType: BaseFieldType{typeId: typeId, userTypeId: invalidUserTypeID}, keyType: keyType, valueType: valueType, } @@ -950,7 +965,8 @@ type SimpleFieldType struct { func NewSimpleFieldType(typeId TypeId) *SimpleFieldType { return &SimpleFieldType{ BaseFieldType: BaseFieldType{ - typeId: typeId & 0xff, + typeId: typeId, + userTypeId: invalidUserTypeID, }, } } @@ -967,7 +983,8 @@ type DynamicFieldType struct { func NewDynamicFieldType(typeId TypeId) *DynamicFieldType { return &DynamicFieldType{ BaseFieldType: BaseFieldType{ - typeId: typeId, + typeId: typeId, + userTypeId: invalidUserTypeID, }, } } @@ -983,7 +1000,6 @@ func (d *DynamicFieldType) getTypeInfo(fory *Fory) (TypeInfo, error) { func (d *DynamicFieldType) getTypeInfoWithResolver(resolver *TypeResolver) (TypeInfo, error) { // Try to resolve the actual type from the resolver - // The typeId might be a composite ID (customId << 8 + baseType) typeId := d.typeId // First try direct lookup @@ -992,19 +1008,6 @@ func (d *DynamicFieldType) getTypeInfoWithResolver(resolver *TypeResolver) (Type return *info, nil } - // If direct lookup fails and it's a composite ID, extract the custom ID - baseType := typeId & 0xFF - if baseType == NAMED_STRUCT || baseType == NAMED_COMPATIBLE_STRUCT || baseType == COMPATIBLE_STRUCT { - customId := typeId >> 8 - if customId > 0 { - // Try looking up by the custom ID - info, err = resolver.getTypeInfoById(uint32(customId)) - if err == nil && info != nil { - return *info, nil - } - } - } - // Fallback to any for unknown types return TypeInfo{Type: reflect.TypeOf((*any)(nil)).Elem(), Serializer: nil}, nil } @@ -1107,17 +1110,25 @@ func buildFieldType(fory *Fory, fieldValue reflect.Value) (FieldType, error) { return nil, err } typeId = TypeId(typeInfo.TypeID) + if typeId == NAMED_ENUM { + typeId = ENUM + } + if typeId == NAMED_UNION || typeId == TYPED_UNION { + typeId = UNION + } if isUserDefinedType(typeId) { - internalTypeId := TypeId(typeId & 0xFF) - switch internalTypeId { - case UNION, TYPED_UNION, NAMED_UNION, ENUM, NAMED_ENUM: - return NewSimpleFieldType(typeId), nil + switch typeId { + case UNION, ENUM: + ft := NewSimpleFieldType(typeId) + return ft, nil } - return NewDynamicFieldType(typeId), nil + ft := NewDynamicFieldType(typeId) + return ft, nil } - return NewSimpleFieldType(typeId), nil + ft := NewSimpleFieldType(typeId) + return ft, nil } const ( @@ -1128,13 +1139,13 @@ const ( // Field name encoding flags (2 bits in header) const ( - FieldNameEncodingUTF8 = 0 // UTF-8 encoding + FieldNameEncodingUTF8 = 0 // UTF_8 encoding FieldNameEncodingAllToLowerSpecial = 1 // ALL_TO_LOWER_SPECIAL encoding FieldNameEncodingLowerUpperDigit = 2 // LOWER_UPPER_DIGIT_SPECIAL encoding FieldNameEncodingTagID = 3 // Use tag ID instead of field name ) -// Encoding `UTF8/ALL_TO_LOWER_SPECIAL/LOWER_UPPER_DIGIT_SPECIAL` for fieldName +// Encoding `UTF_8/ALL_TO_LOWER_SPECIAL/LOWER_UPPER_DIGIT_SPECIAL` for fieldName // Note: TAG_ID (0b11) is a special encoding that uses tag ID instead of field name var fieldNameEncodings = []meta.Encoding{ meta.UTF_8, @@ -1173,7 +1184,7 @@ func writeSimpleName(buffer *ByteBuffer, metaBytes *MetaStringBytes, encoder *me encoding := metaBytes.Encoding // Get encoding flags (0-2) - Java uses 2 bits for package encoding: - // 0=UTF8, 1=ALL_TO_LOWER_SPECIAL, 2=LOWER_UPPER_DIGIT_SPECIAL + // 0=UTF_8, 1=ALL_TO_LOWER_SPECIAL, 2=LOWER_UPPER_DIGIT_SPECIAL var encodingFlags byte switch encoding { case meta.UTF_8: @@ -1216,7 +1227,7 @@ func writeSimpleTypeName(buffer *ByteBuffer, metaBytes *MetaStringBytes, encoder encoding := metaBytes.Encoding // Get encoding flags (0-3) - Java uses 2 bits for typename encoding: - // 0=UTF8, 1=ALL_TO_LOWER_SPECIAL, 2=LOWER_UPPER_DIGIT_SPECIAL, 3=FIRST_TO_LOWER_SPECIAL + // 0=UTF_8, 1=ALL_TO_LOWER_SPECIAL, 2=LOWER_UPPER_DIGIT_SPECIAL, 3=FIRST_TO_LOWER_SPECIAL var encodingFlags byte switch encoding { case meta.UTF_8: @@ -1264,9 +1275,11 @@ func encodingTypeDef(typeResolver *TypeResolver, typeDef *TypeDef) ([]byte, erro return nil, fmt.Errorf("failed to write typename: %w", err) } } else { - // Java uses WriteVarUint32 for type ID (unsigned varint) - // typeDef.typeId is already int32, no need for conversion - buffer.WriteVarUint32(uint32(typeDef.typeId)) + buffer.WriteUint8(uint8(typeDef.typeId)) + if typeDef.userTypeId == invalidUserTypeID { + return nil, fmt.Errorf("missing user type ID for typeID %d", typeDef.typeId) + } + buffer.WriteVarUint32(typeDef.userTypeId) } if err := writeFieldDefs(typeResolver, buffer, typeDef.fieldDefs); err != nil { @@ -1461,6 +1474,7 @@ func decodeTypeDef(fory *Fory, buffer *ByteBuffer, header int64) (*TypeDef, erro // ReadData name or type ID according to the registerByName flag var typeId uint32 + userTypeId := invalidUserTypeID var nsBytes, nameBytes *MetaStringBytes var type_ reflect.Type if registeredByName { @@ -1475,7 +1489,7 @@ func decodeTypeDef(fory *Fory, buffer *ByteBuffer, header int64) (*TypeDef, erro nsSize = int(metaBuffer.ReadVarUint32Small7(&metaErr)) + BIG_NAME_THRESHOLD } - // Java pkg encoding: 0=UTF8, 1=ALL_TO_LOWER_SPECIAL, 2=LOWER_UPPER_DIGIT_SPECIAL + // Java pkg encoding: 0=UTF_8, 1=ALL_TO_LOWER_SPECIAL, 2=LOWER_UPPER_DIGIT_SPECIAL var nsEncoding meta.Encoding switch nsEncodingFlags { case 0: @@ -1501,7 +1515,7 @@ func decodeTypeDef(fory *Fory, buffer *ByteBuffer, header int64) (*TypeDef, erro typeSize = int(metaBuffer.ReadVarUint32Small7(&metaErr)) + BIG_NAME_THRESHOLD } - // Java typename encoding: 0=UTF8, 1=ALL_TO_LOWER_SPECIAL, 2=LOWER_UPPER_DIGIT_SPECIAL, 3=FIRST_TO_LOWER_SPECIAL + // Java typename encoding: 0=UTF_8, 1=ALL_TO_LOWER_SPECIAL, 2=LOWER_UPPER_DIGIT_SPECIAL, 3=FIRST_TO_LOWER_SPECIAL var typeEncoding meta.Encoding switch typeEncodingFlags { case 0: @@ -1558,6 +1572,7 @@ func decodeTypeDef(fory *Fory, buffer *ByteBuffer, header int64) (*TypeDef, erro type_ = type_.Elem() } typeId = uint32(info.TypeID) + userTypeId = info.UserTypeID } else { // Type not registered - use NAMED_STRUCT as default typeId // The type_ will remain nil and will be set from field definitions later @@ -1565,14 +1580,15 @@ func decodeTypeDef(fory *Fory, buffer *ByteBuffer, header int64) (*TypeDef, erro type_ = nil } } else { - // Java uses WriteVarUint32 for type ID in TypeDef - // The type ID is a composite: (userID << 8) | internalTypeID - typeId = metaBuffer.ReadVarUint32(&metaErr) - // Try to get the type from registry using the full type ID - if info, exists := fory.typeResolver.typeIDToTypeInfo[typeId]; exists { + typeId = uint32(metaBuffer.ReadUint8(&metaErr)) + userTypeId = metaBuffer.ReadVarUint32(&metaErr) + if info, exists := fory.typeResolver.userTypeIdToTypeInfo[userTypeId]; exists { type_ = info.Type - } else { - //Type not registered - will be built from field definitions + } else if info, exists := fory.typeResolver.typeIDToTypeInfo[typeId]; exists { + type_ = info.Type + } + if type_ == nil { + // Type not registered - will be built from field definitions type_ = nil } } @@ -1592,7 +1608,7 @@ func decodeTypeDef(fory *Fory, buffer *ByteBuffer, header int64) (*TypeDef, erro encoded := buildTypeDefEncoded(globalHeader, metaSizeBits, extraMetaSize, encodedMeta) // Create TypeDef - typeDef := NewTypeDef(typeId, nsBytes, nameBytes, registeredByName, isCompressed, fieldInfos) + typeDef := NewTypeDef(typeId, userTypeId, nsBytes, nameBytes, registeredByName, isCompressed, fieldInfos) typeDef.encoded = encoded typeDef.type_ = type_ diff --git a/go/fory/type_def_test.go b/go/fory/type_def_test.go index aa336ce1c1..df31e60d2a 100644 --- a/go/fory/type_def_test.go +++ b/go/fory/type_def_test.go @@ -151,6 +151,9 @@ func checkFieldDef(t *testing.T, original, decoded FieldDef) { func checkFieldTypeRecursively(t *testing.T, original, decoded FieldType, path string) { // Check TypeId assert.Equal(t, original.TypeId(), decoded.TypeId(), "FieldType TypeId mismatch at path: %s", path) + if isUserTypeRegisteredById(original.TypeId()) || isUserTypeRegisteredById(decoded.TypeId()) { + assert.Equal(t, original.UserTypeId(), decoded.UserTypeId(), "FieldType UserTypeId mismatch at path: %s", path) + } // Check type consistency based on the actual type switch originalType := original.(type) { diff --git a/go/fory/type_resolver.go b/go/fory/type_resolver.go index 971fb56278..5fd1ca9b8d 100644 --- a/go/fory/type_resolver.go +++ b/go/fory/type_resolver.go @@ -50,6 +50,10 @@ const ( useStringValue = 0 useStringId = 1 SMALL_STRING_THRESHOLD = 16 + // 0xffffffff is reserved for "unset". + maxUserTypeID uint32 = 0xfffffffe + invalidUserTypeID uint32 = 0xffffffff + internalTypeIDLimit = 0xFF ) var ( @@ -131,12 +135,14 @@ type TypeInfo struct { NameBytes *MetaStringBytes IsDynamic bool TypeID uint32 - DispatchId DispatchId - Serializer Serializer - NeedWriteDef bool - NeedWriteRef bool // Whether this type needs reference tracking - hashValue uint64 - TypeDef *TypeDef + // User type ID is stored as unsigned uint32; 0xffffffff means unset. + UserTypeID uint32 + DispatchId DispatchId + Serializer Serializer + NeedWriteDef bool + NeedWriteRef bool // Whether this type needs reference tracking + hashValue uint64 + TypeDef *TypeDef } type ( namedTypeKey [2]string @@ -153,7 +159,7 @@ type TypeResolver struct { typeToTypeInfo map[reflect.Type]string typeToTypeTag map[reflect.Type]string typeInfoToType map[string]reflect.Type - typeIdToType map[int16]reflect.Type + typeIdToType map[TypeId]reflect.Type dynamicStringToId map[string]int16 dynamicIdToString map[int16]string dynamicStringId int16 @@ -168,11 +174,11 @@ type TypeResolver struct { metaStrToStr map[string]string metaStrToClass map[string]reflect.Type hashToMetaString map[uint64]string - hashToClassInfo map[uint64]*TypeInfo // Type tracking dynamicWrittenMetaStr []string typeIDToTypeInfo map[uint32]*TypeInfo + userTypeIdToTypeInfo map[uint32]*TypeInfo typeIDCounter uint32 dynamicWriteStringID uint32 @@ -202,7 +208,7 @@ func newTypeResolver(fory *Fory) *TypeResolver { r := &TypeResolver{ typeTagToSerializers: map[string]Serializer{}, typeToSerializers: map[reflect.Type]Serializer{}, - typeIdToType: map[int16]reflect.Type{}, + typeIdToType: map[TypeId]reflect.Type{}, typeToTypeInfo: map[reflect.Type]string{}, typeInfoToType: map[string]reflect.Type{}, dynamicStringToId: map[string]int16{}, @@ -216,10 +222,10 @@ func newTypeResolver(fory *Fory) *TypeResolver { metaStrToStr: make(map[string]string), metaStrToClass: make(map[string]reflect.Type), hashToMetaString: make(map[uint64]string), - hashToClassInfo: make(map[uint64]*TypeInfo), dynamicWrittenMetaStr: make([]string, 0), typeIDToTypeInfo: make(map[uint32]*TypeInfo), + userTypeIdToTypeInfo: make(map[uint32]*TypeInfo), typeIDCounter: 300, dynamicWriteStringID: 0, @@ -284,12 +290,12 @@ func newTypeResolver(fory *Fory) *TypeResolver { // 3. Register complete type information (critical for proper serialization) // Codegen serializers are for named structs - _, err := r.registerType(type_, uint32(NAMED_STRUCT), pkgPath, typeName, codegenSerializer, false) + _, err := r.registerType(type_, uint32(NAMED_STRUCT), invalidUserTypeID, pkgPath, typeName, codegenSerializer, false) if err != nil { panic(fmt.Errorf("failed to register codegen type %s: %v", typeTag, err)) } // 4. Register pointer type information - _, err = r.registerType(ptrType, uint32(NAMED_STRUCT), pkgPath, typeName, ptrCodegenSer, false) + _, err = r.registerType(ptrType, uint32(NAMED_STRUCT), invalidUserTypeID, pkgPath, typeName, ptrCodegenSer, false) if err != nil { panic(fmt.Errorf("failed to register codegen pointer type %s: %v", typeTag, err)) } @@ -436,7 +442,7 @@ func (r *TypeResolver) initialize() { {genericSetType, SET, setSerializer{}}, } for _, elem := range serializers { - _, err := r.registerType(elem.Type, uint32(elem.TypeId), "", "", elem.Serializer, true) + _, err := r.registerType(elem.Type, uint32(elem.TypeId), invalidUserTypeID, "", "", elem.Serializer, true) if err != nil { panic(fmt.Errorf("init type error: %v", err)) } @@ -479,7 +485,7 @@ func (r *TypeResolver) registerSerializer(type_ reflect.Type, typeId TypeId, s S // Collection types (LIST, SET, MAP) can have multiple Go types mapping to them // Primitive array types can also have multiple Go types (e.g., []int and []int64 both map to INT64_ARRAY on 64-bit systems) // Also skip if type ID already registered (e.g., string and *string both map to STRING) - if !IsNamespacedType(typeId) && !isCollectionType(int16(typeId)) && !isPrimitiveArrayType(int16(typeId)) { + if !IsNamespacedType(typeId) && !isCollectionType(typeId) && !isPrimitiveArrayType(typeId) { if typeId > NotSupportCrossLanguage { if _, ok := r.typeIdToType[typeId]; !ok { r.typeIdToType[typeId] = type_ @@ -489,16 +495,50 @@ func (r *TypeResolver) registerSerializer(type_ reflect.Type, typeId TypeId, s S return nil } -// RegisterStruct registers a type with a numeric type ID for cross-language serialization. -// This is used when the full type ID (user_id << 8 | internal_id) is already calculated. -func (r *TypeResolver) RegisterStruct(type_ reflect.Type, fullTypeID uint32) error { +func validateOptionalFields(type_ reflect.Type) error { + if type_ == nil { + return nil + } + if type_.Kind() == reflect.Ptr { + type_ = type_.Elem() + } + if type_.Kind() != reflect.Struct { + return nil + } + for i := 0; i < type_.NumField(); i++ { + field := type_.Field(i) + if field.PkgPath != "" { + continue + } + foryTag := parseForyTag(field) + if foryTag.Ignore { + continue + } + optionalInfo, isOptional := getOptionalInfo(field.Type) + if isOptional { + if err := validateOptionalValueType(optionalInfo.valueType); err != nil { + return fmt.Errorf("field %s: %w", field.Name, err) + } + } + } + return nil +} + +// RegisterStruct registers a type with a numeric user type ID for cross-language serialization. +func (r *TypeResolver) RegisterStruct(type_ reflect.Type, typeID TypeId, userTypeID uint32) error { // Check if already registered - if info, ok := r.typeIDToTypeInfo[fullTypeID]; ok { - return fmt.Errorf("type %s with id %d has been registered", info.Type, fullTypeID) + if info, ok := r.userTypeIdToTypeInfo[userTypeID]; ok { + if info.Type == type_ { + return nil + } + return fmt.Errorf("type %s with id %d has been registered", info.Type, userTypeID) } switch type_.Kind() { case reflect.Struct: + if err := validateOptionalFields(type_); err != nil { + return err + } // For struct types, check if serializer already registered if prev, ok := r.typeToSerializers[type_]; ok { return fmt.Errorf("type %s already has a serializer %s registered", type_, prev) @@ -508,10 +548,6 @@ func (r *TypeResolver) RegisterStruct(type_ reflect.Type, fullTypeID uint32) err tag := type_.Name() serializer := newStructSerializer(type_, tag) r.typeToSerializers[type_] = serializer - if err := serializer.initialize(r); err != nil { - delete(r.typeToSerializers, type_) - return err - } r.typeToTypeInfo[type_] = "@" + tag r.typeInfoToType["@"+tag] = type_ @@ -529,13 +565,13 @@ func (r *TypeResolver) RegisterStruct(type_ reflect.Type, fullTypeID uint32) err r.typeInfoToType["*@"+tag] = ptrType // Register value type with fullTypeID - _, err := r.registerType(type_, fullTypeID, "", "", serializer, false) + _, err := r.registerType(type_, uint32(typeID), userTypeID, "", "", serializer, false) if err != nil { return fmt.Errorf("failed to register type by ID: %w", err) } // Register pointer type with same fullTypeID (Java treats value and pointer types the same) - _, err = r.registerType(ptrType, fullTypeID, "", "", ptrSerializer, false) + _, err = r.registerType(ptrType, uint32(typeID), userTypeID, "", "", ptrSerializer, false) if err != nil { return fmt.Errorf("failed to register pointer type by ID: %w", err) } @@ -547,17 +583,13 @@ func (r *TypeResolver) RegisterStruct(type_ reflect.Type, fullTypeID uint32) err return nil } -// RegisterUnion registers a union type with a numeric type ID for cross-language serialization. -// The fullTypeID should already be calculated as (user_id << 8) | TYPED_UNION. -func (r *TypeResolver) RegisterUnion(type_ reflect.Type, fullTypeID uint32, serializer Serializer) error { +// RegisterUnion registers a union type with a numeric user type ID for cross-language serialization. +func (r *TypeResolver) RegisterUnion(type_ reflect.Type, userTypeID uint32, serializer Serializer) error { if serializer == nil { return fmt.Errorf("RegisterUnion requires a non-nil serializer") } - if info, ok := r.typeIDToTypeInfo[fullTypeID]; ok { - return fmt.Errorf("type %s with id %d has been registered", info.Type, fullTypeID) - } - if TypeId(fullTypeID&0xFF) != TYPED_UNION { - return fmt.Errorf("RegisterUnion requires internal type ID TYPED_UNION, got %d", fullTypeID&0xFF) + if info, ok := r.userTypeIdToTypeInfo[userTypeID]; ok { + return fmt.Errorf("type %s with id %d has been registered", info.Type, userTypeID) } if type_.Kind() != reflect.Struct { return fmt.Errorf("RegisterUnion only supports struct types; got: %v", type_.Kind()) @@ -578,22 +610,22 @@ func (r *TypeResolver) RegisterUnion(type_ reflect.Type, fullTypeID uint32, seri r.typeToTypeInfo[ptrType] = "*@" + tag r.typeInfoToType["*@"+tag] = ptrType - _, err := r.registerType(type_, fullTypeID, "", "", serializer, false) + _, err := r.registerType(type_, uint32(TYPED_UNION), userTypeID, "", "", serializer, false) if err != nil { return fmt.Errorf("failed to register union by ID: %w", err) } - _, err = r.registerType(ptrType, fullTypeID, "", "", ptrSerializer, false) + _, err = r.registerType(ptrType, uint32(TYPED_UNION), userTypeID, "", "", ptrSerializer, false) if err != nil { return fmt.Errorf("failed to register pointer union by ID: %w", err) } return nil } -// RegisterEnum registers an enum type (numeric type in Go) with a full type ID -func (r *TypeResolver) RegisterEnum(type_ reflect.Type, fullTypeID uint32) error { +// RegisterEnum registers an enum type (numeric type in Go) with a user type ID. +func (r *TypeResolver) RegisterEnum(type_ reflect.Type, userTypeID uint32) error { // Check if already registered - if info, ok := r.typeIDToTypeInfo[fullTypeID]; ok { - return fmt.Errorf("type %s with id %d has been registered", info.Type, fullTypeID) + if info, ok := r.userTypeIdToTypeInfo[userTypeID]; ok { + return fmt.Errorf("type %s with id %d has been registered", info.Type, userTypeID) } // Verify it's a numeric type @@ -606,7 +638,7 @@ func (r *TypeResolver) RegisterEnum(type_ reflect.Type, fullTypeID uint32) error } // Create enum serializer - serializer := &enumSerializer{type_: type_, typeID: fullTypeID} + serializer := &enumSerializer{type_: type_, typeID: uint32(ENUM)} tag := type_.Name() r.typeToSerializers[type_] = serializer @@ -616,13 +648,14 @@ func (r *TypeResolver) RegisterEnum(type_ reflect.Type, fullTypeID uint32) error // Create TypeInfo with serializer typeInfo := &TypeInfo{ Type: type_, - TypeID: fullTypeID, + TypeID: uint32(ENUM), + UserTypeID: userTypeID, Serializer: serializer, IsDynamic: isDynamicType(type_), DispatchId: GetDispatchId(type_), hashValue: calcTypeHash(type_), } - r.typeIDToTypeInfo[fullTypeID] = typeInfo + r.userTypeIdToTypeInfo[userTypeID] = typeInfo r.typesInfo[type_] = typeInfo return nil @@ -670,7 +703,7 @@ func (r *TypeResolver) RegisterNamedEnum(type_ reflect.Type, namespace, typeName r.typeInfoToType["@"+tag] = type_ // Register the type - _, err := r.registerType(type_, typeId, namespace, typeName, serializer, false) + _, err := r.registerType(type_, typeId, invalidUserTypeID, namespace, typeName, serializer, false) if err != nil { return fmt.Errorf("failed to register enum by name: %w", err) } @@ -700,6 +733,9 @@ func (r *TypeResolver) RegisterNamedStruct( if typeName == "" && namespace != "" { return fmt.Errorf("typeName cannot be empty if namespace is provided") } + if typeId > 0 && typeId > maxUserTypeID { + return fmt.Errorf("typeId must be in range [0, 0xfffffffe], got %d", typeId) + } var tag string if namespace == "" { tag = typeName @@ -721,30 +757,33 @@ func (r *TypeResolver) RegisterNamedStruct( r.typeTagToSerializers[tag] = ptrSerializer r.typeToTypeInfo[ptrType] = "*@" + tag r.typeInfoToType["*@"+tag] = ptrType + var internalTypeID TypeId + userTypeID := invalidUserTypeID if typeId == 0 { if r.metaShareEnabled() { - typeId = (typeId << 8) | NAMED_COMPATIBLE_STRUCT + internalTypeID = NAMED_COMPATIBLE_STRUCT } else { - typeId = (typeId << 8) | NAMED_STRUCT + internalTypeID = NAMED_STRUCT } } else { if r.metaShareEnabled() { - typeId = (typeId << 8) | COMPATIBLE_STRUCT + internalTypeID = COMPATIBLE_STRUCT } else { - typeId = (typeId << 8) | STRUCT + internalTypeID = STRUCT } + userTypeID = typeId } if registerById { - if info, ok := r.typeIDToTypeInfo[typeId]; ok { + if info, ok := r.userTypeIdToTypeInfo[userTypeID]; ok { return fmt.Errorf("type %s with id %d has been registered", info.Type, typeId) } } // For named structs, directly register both their value and pointer types with same typeId - _, err := r.registerType(type_, typeId, namespace, typeName, nil, false) + _, err := r.registerType(type_, uint32(internalTypeID), userTypeID, namespace, typeName, nil, false) if err != nil { return fmt.Errorf("failed to register named structs: %w", err) } - _, err = r.registerType(ptrType, typeId, namespace, typeName, nil, false) + _, err = r.registerType(ptrType, uint32(internalTypeID), userTypeID, namespace, typeName, nil, false) if err != nil { return fmt.Errorf("failed to register named structs: %w", err) } @@ -795,11 +834,11 @@ func (r *TypeResolver) RegisterNamedUnion( r.typeInfoToType["*@"+tag] = ptrType typeId := uint32(NAMED_UNION) - _, err := r.registerType(type_, typeId, namespace, typeName, serializer, false) + _, err := r.registerType(type_, typeId, invalidUserTypeID, namespace, typeName, serializer, false) if err != nil { return fmt.Errorf("failed to register union by name: %w", err) } - _, err = r.registerType(ptrType, typeId, namespace, typeName, ptrSerializer, false) + _, err = r.registerType(ptrType, typeId, invalidUserTypeID, namespace, typeName, ptrSerializer, false) if err != nil { return fmt.Errorf("failed to register pointer union by name: %w", err) } @@ -855,16 +894,15 @@ func (r *TypeResolver) RegisterNamedExtension( r.typeToTypeInfo[ptrType] = "*@" + tag r.typeInfoToType["*@"+tag] = ptrType - // Use NAMED_STRUCT type ID to match Java's behavior - // (Java uses NAMED_STRUCT when register() + registerSerializer() are used) - typeId := uint32(NAMED_STRUCT) + // Use NAMED_EXT type ID for extension types + typeId := uint32(NAMED_EXT) // Register both value and pointer types - _, err := r.registerType(type_, typeId, namespace, typeName, nil, false) + _, err := r.registerType(type_, typeId, invalidUserTypeID, namespace, typeName, nil, false) if err != nil { return fmt.Errorf("failed to register extension type: %w", err) } - _, err = r.registerType(ptrType, typeId, namespace, typeName, nil, false) + _, err = r.registerType(ptrType, typeId, invalidUserTypeID, namespace, typeName, nil, false) if err != nil { return fmt.Errorf("failed to register extension type: %w", err) } @@ -877,6 +915,9 @@ func (r *TypeResolver) RegisterExtension( userTypeID uint32, userSerializer ExtensionSerializer, ) error { + if userTypeID > maxUserTypeID { + return fmt.Errorf("typeID must be in range [0, 0xfffffffe], got %d", userTypeID) + } if userSerializer == nil { return fmt.Errorf("serializer cannot be nil for extension type %s", type_) } @@ -892,16 +933,14 @@ func (r *TypeResolver) RegisterExtension( ptrSerializer := &ptrToValueSerializer{valueSerializer: serializer} r.typeToSerializers[ptrType] = ptrSerializer - // Use EXT type ID to match Java's behavior for extension types - fullTypeID := (userTypeID << 8) | uint32(EXT) - // Register type info for both value and pointer types typeInfo := &TypeInfo{ Type: type_, - TypeID: fullTypeID, + TypeID: uint32(EXT), + UserTypeID: userTypeID, Serializer: serializer, } - r.typeIDToTypeInfo[fullTypeID] = typeInfo + r.userTypeIdToTypeInfo[userTypeID] = typeInfo r.typesInfo[type_] = typeInfo r.typesInfo[ptrType] = typeInfo @@ -928,11 +967,11 @@ func (r *TypeResolver) getTypeIdByType(type_ reflect.Type) TypeId { type_ = info.valueType } if info, ok := r.typesInfo[type_]; ok { - return TypeId(info.TypeID & 0xFF) // Extract base type ID + return TypeId(info.TypeID) } if type_ != nil && type_.Kind() == reflect.Ptr { if info, ok := r.typesInfo[type_.Elem()]; ok { - return TypeId(info.TypeID & 0xFF) + return TypeId(info.TypeID) } } return 0 @@ -949,7 +988,7 @@ func (r *TypeResolver) getSerializerByTypeTag(typeTag string) (Serializer, error // getSerializerByTypeID returns the serializer for a given type ID, or nil if not found. func (r *TypeResolver) getSerializerByTypeID(typeID uint32) Serializer { // First try to get the type from typeIdToType - if t, ok := r.typeIdToType[int16(typeID)]; ok { + if t, ok := r.typeIdToType[TypeId(typeID)]; ok { if serializer, ok := r.typeToSerializers[t]; ok { return serializer } @@ -1048,6 +1087,7 @@ func (r *TypeResolver) getTypeInfo(value reflect.Value, create bool) (*TypeInfo, NameBytes: elemInfo.NameBytes, IsDynamic: elemInfo.IsDynamic, TypeID: elemInfo.TypeID, + UserTypeID: elemInfo.UserTypeID, DispatchId: elemInfo.DispatchId, Serializer: ptrSerializer, NeedWriteDef: elemInfo.NeedWriteDef, @@ -1225,6 +1265,7 @@ func (r *TypeResolver) getTypeInfo(value reflect.Value, create bool) (*TypeInfo, return r.registerType( type_, typeID, + invalidUserTypeID, pkgPath, typeName, nil, // serializer will be created during registration @@ -1243,6 +1284,7 @@ func isMultiDimensionaSlice(v reflect.Value) bool { func (r *TypeResolver) registerType( type_ reflect.Type, typeID uint32, + userTypeID uint32, namespace string, typeName string, serializer Serializer, @@ -1255,8 +1297,11 @@ func (r *TypeResolver) registerType( if typeName == "" && namespace != "" { panic("namespace provided without typeName") } + if internal && typeID > internalTypeIDLimit { + panic(fmt.Sprintf("internal type id overflow: %d", typeID)) + } if internal && serializer != nil { - if err := r.registerSerializer(type_, TypeId(typeID&0xFF), serializer); err != nil { + if err := r.registerSerializer(type_, TypeId(typeID), serializer); err != nil { panic(fmt.Errorf("impossible error: %s", err)) } } @@ -1298,6 +1343,7 @@ func (r *TypeResolver) registerType( typeInfo := &TypeInfo{ Type: type_, TypeID: typeID, + UserTypeID: userTypeID, Serializer: serializer, PkgPathBytes: nsBytes, // Encoded namespace bytes NameBytes: typeBytes, // Encoded type name bytes @@ -1308,6 +1354,7 @@ func (r *TypeResolver) registerType( } if structSer, ok := serializer.(*structSerializer); ok { structSer.typeID = typeID + structSer.userTypeID = userTypeID } // Update resolver caches: r.typesInfo[type_] = typeInfo // Cache by type string @@ -1326,7 +1373,17 @@ func (r *TypeResolver) registerType( } // Cache by type ID (for cross-language support) - if r.isXlang && !IsNamespacedType(TypeId(typeID)) { + if (TypeId(typeID) == ENUM || TypeId(typeID) == STRUCT || + TypeId(typeID) == COMPATIBLE_STRUCT || TypeId(typeID) == EXT || + TypeId(typeID) == TYPED_UNION) && + userTypeID != invalidUserTypeID { + if _, ok := r.userTypeIdToTypeInfo[userTypeID]; !ok { + r.userTypeIdToTypeInfo[userTypeID] = typeInfo + } + } else if TypeId(typeID) != ENUM && TypeId(typeID) != STRUCT && + TypeId(typeID) != COMPATIBLE_STRUCT && TypeId(typeID) != EXT && + TypeId(typeID) != TYPED_UNION && + !IsNamespacedType(TypeId(typeID)) { /* This function is required to maintain the typeID registry: all types are registered at startup, and we keep this table updated. @@ -1367,14 +1424,15 @@ func (r *TypeResolver) WriteTypeInfo(buffer *ByteBuffer, typeInfo *TypeInfo, err if typeInfo == nil { return } - // Extract the internal type ID (lower 8 bits) typeID := typeInfo.TypeID - internalTypeID := TypeId(typeID & 0xFF) - // WriteData the type ID to buffer using VarUint32Small7 encoding (matches Java) - buffer.WriteVarUint32Small7(typeID) + buffer.WriteUint8(uint8(typeID)) - // Handle type meta based on internal type ID (matching Java XtypeResolver.writeClassInfo) - switch internalTypeID { + // Handle type meta based on internal type ID (matching Java XtypeResolver.writeTypeInfo) + switch TypeId(typeID) { + case ENUM, STRUCT, EXT, TYPED_UNION: + buffer.WriteVarUint32(typeInfo.UserTypeID) + case COMPATIBLE_STRUCT, NAMED_COMPATIBLE_STRUCT: + r.writeSharedTypeMeta(buffer, typeInfo, err) case NAMED_ENUM, NAMED_STRUCT, NAMED_EXT, NAMED_UNION: if r.metaShareEnabled() { r.writeSharedTypeMeta(buffer, typeInfo, err) @@ -1384,11 +1442,8 @@ func (r *TypeResolver) WriteTypeInfo(buffer *ByteBuffer, typeInfo *TypeInfo, err r.metaStringResolver.WriteMetaStringBytes(buffer, typeInfo.PkgPathBytes, err) // WriteData type name metadata r.metaStringResolver.WriteMetaStringBytes(buffer, typeInfo.NameBytes, err) - case NAMED_COMPATIBLE_STRUCT, COMPATIBLE_STRUCT: - // Meta share must be enabled for compatible mode - if r.metaShareEnabled() { - r.writeSharedTypeMeta(buffer, typeInfo, err) - } + default: + break } } @@ -1959,15 +2014,55 @@ func (r *TypeResolver) readTypeByReadTag(buffer *ByteBuffer, err *Error) reflect return nil } +func (r *TypeResolver) resolveTypeInfoByMetaBytes(nsBytes, typeBytes *MetaStringBytes, + compositeKey nsTypeKey, typeID uint32, err *Error) *TypeInfo { + if typeInfo, exists := r.nsTypeToTypeInfo[compositeKey]; exists { + return typeInfo + } + + ns, decErr := r.namespaceDecoder.Decode(nsBytes.Data, nsBytes.Encoding) + if decErr != nil { + err.SetError(fmt.Errorf("namespace decode failed: %w", decErr)) + return nil + } + + typeName, decErr := r.typeNameDecoder.Decode(typeBytes.Data, typeBytes.Encoding) + if decErr != nil { + err.SetError(fmt.Errorf("typename decode failed: %w", decErr)) + return nil + } + + nameKey := [2]string{ns, typeName} + if typeInfo, exists := r.namedTypeToTypeInfo[nameKey]; exists { + r.nsTypeToTypeInfo[compositeKey] = typeInfo + return typeInfo + } + + fullName := typeName + if ns != "" { + fullName = ns + "." + typeName + } + err.SetError(fmt.Errorf("unregistered type: %s (typeID: %d)", fullName, typeID)) + return nil +} + // ReadTypeInfo reads type info from buffer and returns it. // This is exported for use by generated code. func (r *TypeResolver) ReadTypeInfo(buffer *ByteBuffer, err *Error) *TypeInfo { - // ReadData variable-length type ID using VarUint32Small7 encoding (matches Java) - typeID := buffer.ReadVarUint32Small7(err) - internalTypeID := TypeId(typeID & 0xFF) + typeID := uint32(buffer.ReadUint8(err)) + internalTypeID := TypeId(typeID) - // Handle type meta based on internal type ID (matching Java XtypeResolver.readClassInfo) switch internalTypeID { + case ENUM, STRUCT, EXT, TYPED_UNION: + userTypeID := buffer.ReadVarUint32(err) + if err.HasError() { + return nil + } + if typeInfo, exists := r.userTypeIdToTypeInfo[userTypeID]; exists { + return typeInfo + } + case COMPATIBLE_STRUCT, NAMED_COMPATIBLE_STRUCT: + return r.readSharedTypeMeta(buffer, err) case NAMED_ENUM, NAMED_STRUCT, NAMED_EXT, NAMED_UNION: if r.metaShareEnabled() { return r.readSharedTypeMeta(buffer, err) @@ -1982,41 +2077,9 @@ func (r *TypeResolver) ReadTypeInfo(buffer *ByteBuffer, err *Error) *TypeInfo { compositeKey := nsTypeKey{nsBytes.Hashcode, typeBytes.Hashcode} // For pointer and value types, use the negative ID system // to obtain the correct TypeInfo for subsequent deserialization - if typeInfo, exists := r.nsTypeToTypeInfo[compositeKey]; exists { - return typeInfo - } - - // If not found, decode the bytes to strings and try again - ns, decErr := r.namespaceDecoder.Decode(nsBytes.Data, nsBytes.Encoding) - if decErr != nil { - err.SetError(fmt.Errorf("namespace decode failed: %w", decErr)) - return nil - } - - typeName, decErr := r.typeNameDecoder.Decode(typeBytes.Data, typeBytes.Encoding) - if decErr != nil { - err.SetError(fmt.Errorf("typename decode failed: %w", decErr)) - return nil - } - - nameKey := [2]string{ns, typeName} - if typeInfo, exists := r.namedTypeToTypeInfo[nameKey]; exists { - r.nsTypeToTypeInfo[compositeKey] = typeInfo - return typeInfo - } - // Type not found - fullName := typeName - if ns != "" { - fullName = ns + "." + typeName - } - err.SetError(fmt.Errorf("unregistered type: %s (typeID: %d)", fullName, typeID)) - return nil - - case NAMED_COMPATIBLE_STRUCT, COMPATIBLE_STRUCT: - // Meta share must be enabled for compatible mode - if r.metaShareEnabled() { - return r.readSharedTypeMeta(buffer, err) - } + return r.resolveTypeInfoByMetaBytes(nsBytes, typeBytes, compositeKey, typeID, err) + default: + break } // Handle simple type IDs (non-namespaced types) @@ -2024,119 +2087,11 @@ func (r *TypeResolver) ReadTypeInfo(buffer *ByteBuffer, err *Error) *TypeInfo { return typeInfo } - // Handle collection types (LIST, SET, MAP) that don't have specific registration - // Use generic types that can hold any element type - switch TypeId(typeID) { - case LIST, -LIST: - return &TypeInfo{ - Type: interfaceSliceType, - TypeID: typeID, - Serializer: r.typeToSerializers[interfaceSliceType], - DispatchId: UnknownDispatchId, - } - case SET, -SET: - return &TypeInfo{ - Type: genericSetType, - TypeID: typeID, - Serializer: r.typeToSerializers[genericSetType], - DispatchId: UnknownDispatchId, - } - case MAP, -MAP: - return &TypeInfo{ - Type: interfaceMapType, - TypeID: typeID, - Serializer: r.typeToSerializers[interfaceMapType], - DispatchId: UnknownDispatchId, - } - case BOOL: - return &TypeInfo{ - Type: reflect.TypeOf(false), - TypeID: typeID, - Serializer: r.typeToSerializers[reflect.TypeOf(false)], - DispatchId: PrimitiveBoolDispatchId, - } - case INT8: - return &TypeInfo{ - Type: reflect.TypeOf(int8(0)), - TypeID: typeID, - Serializer: r.typeToSerializers[reflect.TypeOf(int8(0))], - DispatchId: PrimitiveInt8DispatchId, - } - case UINT8: - return &TypeInfo{ - Type: reflect.TypeOf(uint8(0)), - TypeID: typeID, - Serializer: r.typeToSerializers[reflect.TypeOf(uint8(0))], - DispatchId: PrimitiveInt8DispatchId, // Use Int8 static ID for uint8 - } - case INT16: - return &TypeInfo{ - Type: reflect.TypeOf(int16(0)), - TypeID: typeID, - Serializer: r.typeToSerializers[reflect.TypeOf(int16(0))], - DispatchId: PrimitiveInt16DispatchId, - } - case UINT16: - return &TypeInfo{ - Type: reflect.TypeOf(uint16(0)), - TypeID: typeID, - Serializer: r.typeToSerializers[reflect.TypeOf(uint16(0))], - DispatchId: PrimitiveInt16DispatchId, // Use Int16 static ID for uint16 - } - case INT32, VARINT32: - return &TypeInfo{ - Type: reflect.TypeOf(int32(0)), - TypeID: typeID, - Serializer: r.typeToSerializers[reflect.TypeOf(int32(0))], - DispatchId: PrimitiveInt32DispatchId, - } - case UINT32: - return &TypeInfo{ - Type: reflect.TypeOf(uint32(0)), - TypeID: typeID, - Serializer: r.typeToSerializers[reflect.TypeOf(uint32(0))], - DispatchId: PrimitiveInt32DispatchId, // Use Int32 static ID for uint32 - } - case INT64, VARINT64, TAGGED_INT64: - return &TypeInfo{ - Type: reflect.TypeOf(int64(0)), - TypeID: typeID, - Serializer: r.typeToSerializers[reflect.TypeOf(int64(0))], - DispatchId: PrimitiveInt64DispatchId, - } - case UINT64: - return &TypeInfo{ - Type: reflect.TypeOf(uint64(0)), - TypeID: typeID, - Serializer: r.typeToSerializers[reflect.TypeOf(uint64(0))], - DispatchId: PrimitiveInt64DispatchId, // Use Int64 static ID for uint64 - } - case FLOAT32: - return &TypeInfo{ - Type: reflect.TypeOf(float32(0)), - TypeID: typeID, - Serializer: r.typeToSerializers[reflect.TypeOf(float32(0))], - DispatchId: PrimitiveFloat32DispatchId, - } - case FLOAT64: - return &TypeInfo{ - Type: reflect.TypeOf(float64(0)), - TypeID: typeID, - Serializer: r.typeToSerializers[reflect.TypeOf(float64(0))], - DispatchId: PrimitiveFloat64DispatchId, - } - case STRING: - return &TypeInfo{ - Type: reflect.TypeOf(""), - TypeID: typeID, - Serializer: r.typeToSerializers[reflect.TypeOf("")], - DispatchId: StringDispatchId, - } - case BINARY: + // Handle UNKNOWN type (0) - used for polymorphic types + if typeID == 0 { return &TypeInfo{ - Type: reflect.TypeOf([]byte(nil)), + Type: interfaceType, TypeID: typeID, - Serializer: r.typeToSerializers[reflect.TypeOf([]byte(nil))], DispatchId: UnknownDispatchId, } } @@ -2148,46 +2103,34 @@ func (r *TypeResolver) ReadTypeInfo(buffer *ByteBuffer, err *Error) *TypeInfo { // readTypeInfoWithTypeID reads type info when the typeID has already been read from buffer. // This is used by collection serializers that read typeID separately before deciding how to proceed. func (r *TypeResolver) readTypeInfoWithTypeID(buffer *ByteBuffer, typeID uint32, err *Error) *TypeInfo { - internalTypeID := TypeId(typeID & 0xFF) + internalTypeID := TypeId(typeID) - if IsNamespacedType(TypeId(typeID)) { + switch internalTypeID { + case ENUM, STRUCT, EXT, TYPED_UNION: + userTypeID := buffer.ReadVarUint32(err) + if err.HasError() { + return nil + } + if typeInfo, exists := r.userTypeIdToTypeInfo[userTypeID]; exists { + return typeInfo + } + case COMPATIBLE_STRUCT, NAMED_COMPATIBLE_STRUCT: + return r.readSharedTypeMeta(buffer, err) + case NAMED_ENUM, NAMED_STRUCT, NAMED_EXT, NAMED_UNION: if r.metaShareEnabled() { return r.readSharedTypeMeta(buffer, err) } // ReadData namespace and type name metadata bytes nsBytes, _ := r.metaStringResolver.ReadMetaStringBytes(buffer, err) typeBytes, _ := r.metaStringResolver.ReadMetaStringBytes(buffer, err) - - compositeKey := nsTypeKey{nsBytes.Hashcode, typeBytes.Hashcode} - if typeInfo, exists := r.nsTypeToTypeInfo[compositeKey]; exists { - return typeInfo - } - - // If not found, decode the bytes to strings and try again - ns, nsErr := r.namespaceDecoder.Decode(nsBytes.Data, nsBytes.Encoding) - if nsErr != nil { - err.SetError(fmt.Errorf("namespace decode failed: %w", nsErr)) - return nil - } - - typeName, tnErr := r.typeNameDecoder.Decode(typeBytes.Data, typeBytes.Encoding) - if tnErr != nil { - err.SetError(fmt.Errorf("typename decode failed: %w", tnErr)) + if err.HasError() { return nil } - nameKey := [2]string{ns, typeName} - if typeInfo, exists := r.namedTypeToTypeInfo[nameKey]; exists { - r.nsTypeToTypeInfo[compositeKey] = typeInfo - return typeInfo - } - err.SetError(fmt.Errorf("namespaced type not found: %s.%s", ns, typeName)) - return nil - } - - // Handle COMPATIBLE_STRUCT and STRUCT types - they also need to read shared type meta - if (internalTypeID == COMPATIBLE_STRUCT || internalTypeID == STRUCT) && r.metaShareEnabled() { - return r.readSharedTypeMeta(buffer, err) + compositeKey := nsTypeKey{nsBytes.Hashcode, typeBytes.Hashcode} + return r.resolveTypeInfoByMetaBytes(nsBytes, typeBytes, compositeKey, typeID, err) + default: + break } // Handle simple type IDs (non-namespaced types) @@ -2195,59 +2138,6 @@ func (r *TypeResolver) readTypeInfoWithTypeID(buffer *ByteBuffer, typeID uint32, return typeInfo } - // Handle collection types (LIST, SET, MAP) that don't have specific registration - // Use generic types that can hold any element type - switch TypeId(typeID) { - case LIST: - return &TypeInfo{ - Type: interfaceSliceType, - TypeID: typeID, - Serializer: r.typeToSerializers[interfaceSliceType], - DispatchId: UnknownDispatchId, - } - case SET: - return &TypeInfo{ - Type: genericSetType, - TypeID: typeID, - Serializer: r.typeToSerializers[genericSetType], - DispatchId: UnknownDispatchId, - } - case MAP: - return &TypeInfo{ - Type: interfaceMapType, - TypeID: typeID, - Serializer: r.typeToSerializers[interfaceMapType], - DispatchId: UnknownDispatchId, - } - // Handle primitive types that may not be explicitly registered - case BOOL: - return &TypeInfo{Type: boolType, TypeID: typeID, Serializer: r.typeToSerializers[boolType], DispatchId: PrimitiveBoolDispatchId} - case INT8: - return &TypeInfo{Type: int8Type, TypeID: typeID, Serializer: r.typeToSerializers[int8Type], DispatchId: PrimitiveInt8DispatchId} - case INT16: - return &TypeInfo{Type: int16Type, TypeID: typeID, Serializer: r.typeToSerializers[int16Type], DispatchId: PrimitiveInt16DispatchId} - case INT32, VARINT32: - return &TypeInfo{Type: int32Type, TypeID: typeID, Serializer: r.typeToSerializers[int32Type], DispatchId: PrimitiveInt32DispatchId} - case INT64, VARINT64, TAGGED_INT64: - return &TypeInfo{Type: int64Type, TypeID: typeID, Serializer: r.typeToSerializers[int64Type], DispatchId: PrimitiveInt64DispatchId} - case UINT8: - return &TypeInfo{Type: uint8Type, TypeID: typeID, Serializer: r.typeToSerializers[uint8Type], DispatchId: PrimitiveInt8DispatchId} - case UINT16: - return &TypeInfo{Type: uint16Type, TypeID: typeID, Serializer: r.typeToSerializers[uint16Type], DispatchId: PrimitiveInt16DispatchId} - case UINT32, VAR_UINT32: - return &TypeInfo{Type: uint32Type, TypeID: typeID, Serializer: r.typeToSerializers[uint32Type], DispatchId: PrimitiveInt32DispatchId} - case UINT64, VAR_UINT64, TAGGED_UINT64: - return &TypeInfo{Type: uint64Type, TypeID: typeID, Serializer: r.typeToSerializers[uint64Type], DispatchId: PrimitiveInt64DispatchId} - case FLOAT32: - return &TypeInfo{Type: float32Type, TypeID: typeID, Serializer: r.typeToSerializers[float32Type], DispatchId: PrimitiveFloat32DispatchId} - case FLOAT64: - return &TypeInfo{Type: float64Type, TypeID: typeID, Serializer: r.typeToSerializers[float64Type], DispatchId: PrimitiveFloat64DispatchId} - case STRING: - return &TypeInfo{Type: stringType, TypeID: typeID, Serializer: r.typeToSerializers[stringType], DispatchId: StringDispatchId} - case BINARY: - return &TypeInfo{Type: byteSliceType, TypeID: typeID, Serializer: r.typeToSerializers[byteSliceType], DispatchId: ByteSliceDispatchId} - } - // Handle UNKNOWN type (0) - used for polymorphic types if typeID == 0 { return &TypeInfo{ @@ -2268,43 +2158,46 @@ func (r *TypeResolver) readTypeInfoWithTypeID(buffer *ByteBuffer, typeID uint32, // For STRUCT/NAMED_STRUCT: Gets serializer directly by the passed type (skips type resolution) // For COMPATIBLE_STRUCT/NAMED_COMPATIBLE_STRUCT: Reads type def and creates serializer with passed type func (r *TypeResolver) ReadTypeInfoForType(buffer *ByteBuffer, expectedType reflect.Type, err *Error) Serializer { - typeID := buffer.ReadVarUint32Small7(err) - internalTypeID := TypeId(typeID & 0xFF) - + typeID := uint32(buffer.ReadUint8(err)) + internalTypeID := TypeId(typeID) switch internalTypeID { - case STRUCT, NAMED_STRUCT: - // Non-compatible mode: skip namespace/typename meta strings if present - if IsNamespacedType(TypeId(typeID)) { - // Skip namespace meta string - r.metaStringResolver.ReadMetaStringBytes(buffer, err) - // Skip typename meta string - r.metaStringResolver.ReadMetaStringBytes(buffer, err) + case ENUM, STRUCT, EXT, TYPED_UNION: + buffer.ReadVarUint32(err) + if internalTypeID == STRUCT { + return r.typeToSerializers[expectedType] } - // Get serializer directly by the expected type - no map lookup needed - return r.typeToSerializers[expectedType] - - case COMPATIBLE_STRUCT, NAMED_COMPATIBLE_STRUCT: - // Compatible mode: read type def from shared meta + return nil + case NAMED_ENUM, NAMED_STRUCT, NAMED_EXT, NAMED_UNION: if r.metaShareEnabled() { typeInfo := r.readSharedTypeMeta(buffer, err) if err.HasError() { return nil } - return typeInfo.Serializer + if internalTypeID == NAMED_STRUCT { + return typeInfo.Serializer + } + return nil } - // Fallback: skip namespace/typename and use expected type's serializer - if IsNamespacedType(TypeId(typeID)) { - r.metaStringResolver.ReadMetaStringBytes(buffer, err) - r.metaStringResolver.ReadMetaStringBytes(buffer, err) + r.metaStringResolver.ReadMetaStringBytes(buffer, err) + r.metaStringResolver.ReadMetaStringBytes(buffer, err) + if internalTypeID == NAMED_STRUCT { + return r.typeToSerializers[expectedType] } - return r.typeToSerializers[expectedType] + return nil + case COMPATIBLE_STRUCT, NAMED_COMPATIBLE_STRUCT: + // Compatible mode: read type def from shared meta + typeInfo := r.readSharedTypeMeta(buffer, err) + if err.HasError() { + return nil + } + return typeInfo.Serializer default: // For other types, return nil - caller should handle return nil } } -func (r *TypeResolver) getTypeById(id int16) (reflect.Type, error) { +func (r *TypeResolver) getTypeById(id TypeId) (reflect.Type, error) { type_, ok := r.typeIdToType[id] if !ok { return nil, fmt.Errorf("type of id %d not supported, supported types: %v", id, r.typeIdToType) @@ -2320,19 +2213,37 @@ func (r *TypeResolver) getTypeInfoById(id uint32) (*TypeInfo, error) { } } +func (r *TypeResolver) getUserTypeInfoById(userTypeID uint32) *TypeInfo { + if userTypeID == invalidUserTypeID { + return nil + } + if typeInfo, exists := r.userTypeIdToTypeInfo[userTypeID]; exists { + return typeInfo + } + return nil +} + func (r *TypeResolver) writeMetaString(buffer *ByteBuffer, str string, err *Error) { if id, ok := r.dynamicStringToId[str]; !ok { dynamicStringId := r.dynamicStringId r.dynamicStringId += 1 r.dynamicStringToId[str] = dynamicStringId - length := len(str) + encodedMeta, encErr := r.namespaceEncoder.EncodeWithEncoding(str, meta.UTF_8) + if encErr != nil { + err.SetError(encErr) + return + } + encoded := encodedMeta.GetEncodedBytes() + length := len(encoded) buffer.WriteVarUint32(uint32(length << 1)) if length <= SMALL_STRING_THRESHOLD { - buffer.WriteByte_(uint8(meta.UTF_8)) + if length != 0 { + buffer.WriteByte_(uint8(meta.UTF_8)) + } } else { // TODO this hash should be unique, since we don't compare data equality for performance h := fnv.New64a() - if _, hashErr := h.Write([]byte(str)); hashErr != nil { + if _, hashErr := h.Write(encoded); hashErr != nil { err.SetError(hashErr) return } @@ -2343,7 +2254,7 @@ func (r *TypeResolver) writeMetaString(buffer *ByteBuffer, str string, err *Erro err.SetError(fmt.Errorf("too long string: %s", str)) return } - buffer.WriteBinary(unsafeGetBytes(str)) + buffer.WriteBinary(encoded) } else { buffer.WriteVarUint32(uint32(((id + 1) << 1) | 1)) } @@ -2353,13 +2264,26 @@ func (r *TypeResolver) readMetaString(buffer *ByteBuffer, err *Error) string { header := buffer.ReadVarUint32(err) var length = int(header >> 1) if header&0b1 == 0 { + encoding := meta.UTF_8 if length <= SMALL_STRING_THRESHOLD { - buffer.ReadByte(err) + if length != 0 { + encoding = meta.Encoding(buffer.ReadByte(err)) + } } else { // TODO support use computed hash - buffer.ReadInt64(err) + hash := buffer.ReadInt64(err) + encoding = meta.Encoding(hash & 0xFF) + } + raw := buffer.ReadBinary(length, err) + if length == 0 { + raw = nil + } + decoder := meta.NewDecoder('.', '_') + str, decErr := decoder.Decode(raw, encoding) + if decErr != nil { + err.SetError(decErr) + return "" } - str := string(buffer.ReadBinary(length, err)) dynamicStringId := r.dynamicStringId r.dynamicStringId += 1 r.dynamicIdToString[dynamicStringId] = str diff --git a/go/fory/types.go b/go/fory/types.go index b72d27db57..c561903659 100644 --- a/go/fory/types.go +++ b/go/fory/types.go @@ -22,7 +22,7 @@ import ( "strings" ) -type TypeId = int16 +type TypeId = uint8 const ( // UNKNOWN Unknown/polymorphic type marker @@ -135,7 +135,7 @@ const ( // IsNamespacedType checks whether the given type ID is a namespace type func IsNamespacedType(typeID TypeId) bool { - switch typeID & 0xFF { + switch typeID { case NAMED_EXT, NAMED_ENUM, NAMED_STRUCT, NAMED_COMPATIBLE_STRUCT, NAMED_UNION: return true default: @@ -146,8 +146,7 @@ func IsNamespacedType(typeID TypeId) bool { // NeedsTypeMetaWrite checks whether a type needs additional type meta written after type ID // This includes namespaced types and struct types that need meta share in compatible mode func NeedsTypeMetaWrite(typeID TypeId) bool { - internalID := typeID & 0xFF - switch TypeId(internalID) { + switch typeID { case NAMED_EXT, NAMED_ENUM, NAMED_STRUCT, NAMED_COMPATIBLE_STRUCT, NAMED_UNION, COMPATIBLE_STRUCT, STRUCT: return true default: @@ -155,7 +154,16 @@ func NeedsTypeMetaWrite(typeID TypeId) bool { } } -func isPrimitiveType(typeID int16) bool { +func isUserTypeRegisteredById(typeID TypeId) bool { + switch typeID { + case ENUM, STRUCT, COMPATIBLE_STRUCT, EXT, TYPED_UNION: + return true + default: + return false + } +} + +func isPrimitiveType(typeID TypeId) bool { switch typeID { case BOOL, INT8, @@ -195,23 +203,23 @@ func NeedWriteRef(typeID TypeId) bool { } } -func isListType(typeID int16) bool { +func isListType(typeID TypeId) bool { return typeID == LIST } -func isSetType(typeID int16) bool { +func isSetType(typeID TypeId) bool { return typeID == SET } -func isMapType(typeID int16) bool { +func isMapType(typeID TypeId) bool { return typeID == MAP } -func isCollectionType(typeID int16) bool { +func isCollectionType(typeID TypeId) bool { return typeID == LIST || typeID == SET || typeID == MAP } -func isPrimitiveArrayType(typeID int16) bool { +func isPrimitiveArrayType(typeID TypeId) bool { switch typeID { case BOOL_ARRAY, INT8_ARRAY, @@ -231,7 +239,7 @@ func isPrimitiveArrayType(typeID int16) bool { } } -var primitiveTypeSizes = map[int16]int{ +var primitiveTypeSizes = map[TypeId]int{ BOOL: 1, INT8: 1, UINT8: 1, @@ -261,26 +269,25 @@ const MinInt31 int64 = -0x40000000 // -2^30 // MaxInt31Signed is MaxInt31 as a signed int64 for TAGGED_INT64 encoding const MaxInt31Signed int64 = 0x3FFFFFFF // 2^30 - 1 -func getPrimitiveTypeSize(typeID int16) int { +func getPrimitiveTypeSize(typeID TypeId) int { if sz, ok := primitiveTypeSizes[typeID]; ok { return sz } return -1 } -func isUserDefinedType(typeID int16) bool { - id := int(typeID & 0xff) - return id == STRUCT || - id == COMPATIBLE_STRUCT || - id == NAMED_STRUCT || - id == NAMED_COMPATIBLE_STRUCT || - id == EXT || - id == NAMED_EXT || - id == ENUM || - id == NAMED_ENUM || - id == UNION || - id == TYPED_UNION || - id == NAMED_UNION +func isUserDefinedType(typeID TypeId) bool { + return typeID == STRUCT || + typeID == COMPATIBLE_STRUCT || + typeID == NAMED_STRUCT || + typeID == NAMED_COMPATIBLE_STRUCT || + typeID == EXT || + typeID == NAMED_EXT || + typeID == ENUM || + typeID == NAMED_ENUM || + typeID == UNION || + typeID == TYPED_UNION || + typeID == NAMED_UNION } // ============================================================================ @@ -728,8 +735,7 @@ func getVarintMaxSizeByDispatchId(dispatchId DispatchId) int { // getEncodingFromTypeId returns the encoding string ("fixed", "varint", "tagged") from a TypeId. func getEncodingFromTypeId(typeId TypeId) string { - internalId := typeId & 0xFF - switch TypeId(internalId) { + switch typeId { case INT32, INT64, UINT32, UINT64: return "fixed" case VARINT32, VARINT64, VAR_UINT32, VAR_UINT64: diff --git a/go/fory/union.go b/go/fory/union.go index 8b2dc02167..5bcafa2b66 100644 --- a/go/fory/union.go +++ b/go/fory/union.go @@ -382,7 +382,7 @@ func writeUnionOverrideValue(ctx *WriteContext, info *unionCaseInfo, value refle if refWritten { return } - ctx.Buffer().WriteVarUint32Small7(uint32(info.typeID)) + ctx.Buffer().WriteUint8(uint8(info.typeID)) if value.Kind() == reflect.Ptr { value = value.Elem() } @@ -404,7 +404,7 @@ func readUnionOverrideValue(ctx *ReadContext, info *unionCaseInfo) (any, bool) { return nil, true } - typeID := TypeId(buf.ReadVarUint32Small7(ctx.Err())) + typeID := TypeId(buf.ReadUint8(ctx.Err())) if ctx.HasError() { return nil, false } diff --git a/go/fory/writer.go b/go/fory/writer.go index 47849e93a8..4fccf37112 100644 --- a/go/fory/writer.go +++ b/go/fory/writer.go @@ -176,8 +176,7 @@ func (c *WriteContext) WriteBinary(v []byte) { } func (c *WriteContext) WriteTypeId(id TypeId) { - // Use VarUint32Small7 encoding to match Java's xlang serialization - c.buffer.WriteVarUint32Small7(uint32(id)) + c.buffer.WriteUint8(uint8(id)) } // writeFast writes a value using fast path based on DispatchId diff --git a/integration_tests/idl_tests/cpp/CMakeLists.txt b/integration_tests/idl_tests/cpp/CMakeLists.txt index 29841ddd42..cc62c14fc7 100644 --- a/integration_tests/idl_tests/cpp/CMakeLists.txt +++ b/integration_tests/idl_tests/cpp/CMakeLists.txt @@ -45,7 +45,7 @@ if(DEFINED ENV{ENABLE_FORY_DEBUG_OUTPUT} AND "$ENV{ENABLE_FORY_DEBUG_OUTPUT}" ST endif() target_include_directories(idl_roundtrip PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/generated + ${CMAKE_CURRENT_SOURCE_DIR} ) target_link_libraries(idl_roundtrip PRIVATE fory::serialization) diff --git a/integration_tests/idl_tests/cpp/main.cc b/integration_tests/idl_tests/cpp/main.cc index a429bb3322..f1ed316da6 100644 --- a/integration_tests/idl_tests/cpp/main.cc +++ b/integration_tests/idl_tests/cpp/main.cc @@ -28,18 +28,19 @@ #include #include -#include "addressbook.h" -#include "any_example.h" -#include "collection.h" -#include "complex_fbs.h" -#include "complex_pb.h" #include "fory/serialization/any_serializer.h" #include "fory/serialization/fory.h" -#include "graph.h" -#include "monster.h" -#include "optional_types.h" -#include "root.h" -#include "tree.h" +#include "generated/addressbook.h" +#include "generated/any_example.h" +#include "generated/auto_id.h" +#include "generated/collection.h" +#include "generated/complex_fbs.h" +#include "generated/complex_pb.h" +#include "generated/graph.h" +#include "generated/monster.h" +#include "generated/optional_types.h" +#include "generated/root.h" +#include "generated/tree.h" namespace { @@ -170,6 +171,7 @@ fory::Result RunRoundTrip(bool compatible) { complex_pb::register_types(fory); addressbook::register_types(fory); + auto_id::register_types(fory); monster::register_types(fory); complex_fbs::register_types(fory); collection::register_types(fory); @@ -243,6 +245,43 @@ fory::Result RunRoundTrip(bool compatible) { fory::Error::invalid("animal to_bytes roundtrip mismatch")); } + auto_id::Envelope::Payload auto_payload; + auto_payload.set_value(42); + auto_id::Envelope::Detail auto_detail = + auto_id::Envelope::Detail::payload(auto_payload); + auto_id::Envelope auto_envelope; + auto_envelope.set_id("env-1"); + *auto_envelope.mutable_payload() = auto_payload; + *auto_envelope.mutable_detail() = auto_detail; + auto_envelope.set_status(auto_id::Status::OK); + + FORY_TRY(auto_bytes, fory.serialize(auto_envelope)); + FORY_TRY(auto_roundtrip, fory.deserialize( + auto_bytes.data(), auto_bytes.size())); + if (!(auto_roundtrip == auto_envelope)) { + return fory::Unexpected( + fory::Error::invalid("auto_id envelope roundtrip mismatch")); + } + + auto_id::Envelope auto_envelope_for_wrapper; + auto_envelope_for_wrapper.set_id(auto_envelope.id()); + if (auto_envelope.has_payload()) { + *auto_envelope_for_wrapper.mutable_payload() = auto_envelope.payload(); + } + *auto_envelope_for_wrapper.mutable_detail() = auto_envelope.detail(); + auto_envelope_for_wrapper.set_status(auto_envelope.status()); + + auto_id::Wrapper auto_wrapper = + auto_id::Wrapper::envelope(std::move(auto_envelope_for_wrapper)); + FORY_TRY(auto_wrapper_bytes, fory.serialize(auto_wrapper)); + FORY_TRY(auto_wrapper_roundtrip, + fory.deserialize(auto_wrapper_bytes.data(), + auto_wrapper_bytes.size())); + if (!(auto_wrapper_roundtrip == auto_wrapper)) { + return fory::Unexpected( + fory::Error::invalid("auto_id wrapper roundtrip mismatch")); + } + addressbook::Person multi_owner; multi_owner.set_name("Alice"); multi_owner.set_id(123); @@ -537,6 +576,19 @@ fory::Result RunRoundTrip(bool compatible) { FORY_RETURN_IF_ERROR(WriteFile(data_file, peer_bytes)); } + const char *auto_id_file = std::getenv("DATA_FILE_AUTO_ID"); + if (auto_id_file != nullptr && auto_id_file[0] != '\0') { + FORY_TRY(payload, ReadFile(auto_id_file)); + FORY_TRY(peer_envelope, fory.deserialize( + payload.data(), payload.size())); + if (!(peer_envelope == auto_envelope)) { + return fory::Unexpected( + fory::Error::invalid("peer auto_id payload mismatch")); + } + FORY_TRY(peer_bytes, fory.serialize(peer_envelope)); + FORY_RETURN_IF_ERROR(WriteFile(auto_id_file, peer_bytes)); + } + const char *primitive_file = std::getenv("DATA_FILE_PRIMITIVES"); if (primitive_file != nullptr && primitive_file[0] != '\0') { FORY_TRY(payload, ReadFile(primitive_file)); diff --git a/integration_tests/idl_tests/generate_idl.py b/integration_tests/idl_tests/generate_idl.py index d6d0da11da..e65628149f 100755 --- a/integration_tests/idl_tests/generate_idl.py +++ b/integration_tests/idl_tests/generate_idl.py @@ -35,28 +35,30 @@ IDL_DIR / "idl" / "any_example.proto", IDL_DIR / "idl" / "monster.fbs", IDL_DIR / "idl" / "complex_fbs.fbs", + IDL_DIR / "idl" / "auto_id.fdl", ] LANG_OUTPUTS = { - "java": REPO_ROOT / "integration_tests/idl_tests/java/src/main/java", - "python": REPO_ROOT / "integration_tests/idl_tests/python/src", + "java": REPO_ROOT / "integration_tests/idl_tests/java/src/main/java/generated", + "python": REPO_ROOT / "integration_tests/idl_tests/python/idl_tests/generated", "cpp": REPO_ROOT / "integration_tests/idl_tests/cpp/generated", - "go": REPO_ROOT / "integration_tests/idl_tests/go", - "rust": REPO_ROOT / "integration_tests/idl_tests/rust/src", + "go": REPO_ROOT / "integration_tests/idl_tests/go/generated", + "rust": REPO_ROOT / "integration_tests/idl_tests/rust/src/generated", } GO_OUTPUT_OVERRIDES = { - "addressbook.fdl": IDL_DIR / "go" / "addressbook", - "collection.fdl": IDL_DIR / "go" / "collection", - "monster.fbs": IDL_DIR / "go" / "monster", - "complex_fbs.fbs": IDL_DIR / "go" / "complex_fbs", - "optional_types.fdl": IDL_DIR / "go" / "optional_types", - "tree.fdl": IDL_DIR / "go" / "tree", - "graph.fdl": IDL_DIR / "go" / "graph", - "root.idl": IDL_DIR / "go" / "root", - "any_example.fdl": IDL_DIR / "go" / "any_example", - "any_example.proto": IDL_DIR / "go" / "any_example_pb", - "complex_pb.proto": IDL_DIR / "go" / "complex_pb", + "addressbook.fdl": IDL_DIR / "go" / "addressbook" / "generated", + "collection.fdl": IDL_DIR / "go" / "collection" / "generated", + "monster.fbs": IDL_DIR / "go" / "monster" / "generated", + "complex_fbs.fbs": IDL_DIR / "go" / "complex_fbs" / "generated", + "optional_types.fdl": IDL_DIR / "go" / "optional_types" / "generated", + "tree.fdl": IDL_DIR / "go" / "tree" / "generated", + "graph.fdl": IDL_DIR / "go" / "graph" / "generated", + "root.idl": IDL_DIR / "go" / "root" / "generated", + "any_example.fdl": IDL_DIR / "go" / "any_example" / "generated", + "any_example.proto": IDL_DIR / "go" / "any_example_pb" / "generated", + "complex_pb.proto": IDL_DIR / "go" / "complex_pb" / "generated", + "auto_id.fdl": IDL_DIR / "go" / "auto_id" / "generated", } @@ -87,6 +89,30 @@ def main() -> int: compiler_path = str(REPO_ROOT / "compiler") env["PYTHONPATH"] = compiler_path + os.pathsep + env.get("PYTHONPATH", "") + generated_roots = set() + for lang in langs: + out_dir = LANG_OUTPUTS[lang] + if lang == "go": + for schema in SCHEMAS: + generated_roots.add(GO_OUTPUT_OVERRIDES.get(schema.name, out_dir)) + else: + generated_roots.add(out_dir) + + for root in sorted(generated_roots): + Path(root).mkdir(parents=True, exist_ok=True) + subprocess.check_call( + [ + sys.executable, + "-m", + "fory_compiler", + "--scan-generated", + "--delete", + "--root", + str(root), + ], + env=env, + ) + for schema in SCHEMAS: cmd = [ sys.executable, diff --git a/integration_tests/idl_tests/go/idl_roundtrip_test.go b/integration_tests/idl_tests/go/idl_roundtrip_test.go index f1853766a7..acd5d39739 100644 --- a/integration_tests/idl_tests/go/idl_roundtrip_test.go +++ b/integration_tests/idl_tests/go/idl_roundtrip_test.go @@ -25,15 +25,16 @@ import ( fory "github.com/apache/fory/go/fory" "github.com/apache/fory/go/fory/optional" - addressbook "github.com/apache/fory/integration_tests/idl_tests/go/addressbook" - anyexample "github.com/apache/fory/integration_tests/idl_tests/go/any_example" - collection "github.com/apache/fory/integration_tests/idl_tests/go/collection" - complexfbs "github.com/apache/fory/integration_tests/idl_tests/go/complex_fbs" - complexpb "github.com/apache/fory/integration_tests/idl_tests/go/complex_pb" - graphpkg "github.com/apache/fory/integration_tests/idl_tests/go/graph" - monster "github.com/apache/fory/integration_tests/idl_tests/go/monster" - optionaltypes "github.com/apache/fory/integration_tests/idl_tests/go/optional_types" - treepkg "github.com/apache/fory/integration_tests/idl_tests/go/tree" + addressbook "github.com/apache/fory/integration_tests/idl_tests/go/addressbook/generated" + anyexample "github.com/apache/fory/integration_tests/idl_tests/go/any_example/generated" + autoid "github.com/apache/fory/integration_tests/idl_tests/go/auto_id/generated" + collection "github.com/apache/fory/integration_tests/idl_tests/go/collection/generated" + complexfbs "github.com/apache/fory/integration_tests/idl_tests/go/complex_fbs/generated" + complexpb "github.com/apache/fory/integration_tests/idl_tests/go/complex_pb/generated" + graphpkg "github.com/apache/fory/integration_tests/idl_tests/go/graph/generated" + monster "github.com/apache/fory/integration_tests/idl_tests/go/monster/generated" + optionaltypes "github.com/apache/fory/integration_tests/idl_tests/go/optional_types/generated" + treepkg "github.com/apache/fory/integration_tests/idl_tests/go/tree/generated" ) func buildAddressBook() addressbook.AddressBook { @@ -72,6 +73,21 @@ func buildAddressBook() addressbook.AddressBook { } } +func buildAutoIdEnvelope() autoid.Envelope { + payload := autoid.Envelope_Payload{Value: 42} + detail := autoid.PayloadEnvelope_Detail(&payload) + return autoid.Envelope{ + Id: "env-1", + Payload: &payload, + Detail: detail, + Status: autoid.StatusOk, + } +} + +func buildAutoIdWrapper(envelope autoid.Envelope) autoid.Wrapper { + return autoid.EnvelopeWrapper(&envelope) +} + func TestAddressBookRoundTripCompatible(t *testing.T) { runAddressBookRoundTrip(t, true) } @@ -80,6 +96,14 @@ func TestAddressBookRoundTripSchemaConsistent(t *testing.T) { runAddressBookRoundTrip(t, false) } +func TestAutoIdRoundTripCompatible(t *testing.T) { + runAutoIdRoundTrip(t, true) +} + +func TestAutoIdRoundTripSchemaConsistent(t *testing.T) { + runAutoIdRoundTrip(t, false) +} + func runAddressBookRoundTrip(t *testing.T, compatible bool) { f := fory.NewFory( fory.WithXlang(true), @@ -157,6 +181,23 @@ func runAddressBookRoundTrip(t *testing.T, compatible bool) { runFileGraphRoundTrip(t, refFory, graphValue) } +func runAutoIdRoundTrip(t *testing.T, compatible bool) { + f := fory.NewFory( + fory.WithXlang(true), + fory.WithRefTracking(false), + fory.WithCompatible(compatible), + ) + if err := autoid.RegisterTypes(f); err != nil { + t.Fatalf("register auto_id types: %v", err) + } + + envelope := buildAutoIdEnvelope() + wrapper := buildAutoIdWrapper(envelope) + runLocalAutoIdEnvelopeRoundTrip(t, f, envelope) + runLocalAutoIdWrapperRoundTrip(t, f, wrapper) + runFileAutoIdRoundTrip(t, f, envelope) +} + func TestToBytesFromBytes(t *testing.T) { book := buildAddressBook() data, err := book.ToBytes() @@ -427,6 +468,65 @@ func runFileRoundTrip(t *testing.T, f *fory.Fory, book addressbook.AddressBook) } } +func runLocalAutoIdEnvelopeRoundTrip(t *testing.T, f *fory.Fory, env autoid.Envelope) { + data, err := f.Serialize(&env) + if err != nil { + t.Fatalf("serialize auto_id envelope: %v", err) + } + + var out autoid.Envelope + if err := f.Deserialize(data, &out); err != nil { + t.Fatalf("deserialize auto_id envelope: %v", err) + } + + if !reflect.DeepEqual(env, out) { + t.Fatalf("auto_id envelope mismatch: %#v != %#v", env, out) + } +} + +func runLocalAutoIdWrapperRoundTrip(t *testing.T, f *fory.Fory, wrapper autoid.Wrapper) { + data, err := f.Serialize(&wrapper) + if err != nil { + t.Fatalf("serialize auto_id wrapper: %v", err) + } + + var out autoid.Wrapper + if err := f.Deserialize(data, &out); err != nil { + t.Fatalf("deserialize auto_id wrapper: %v", err) + } + + if !reflect.DeepEqual(wrapper, out) { + t.Fatalf("auto_id wrapper mismatch: %#v != %#v", wrapper, out) + } +} + +func runFileAutoIdRoundTrip(t *testing.T, f *fory.Fory, env autoid.Envelope) { + dataFile := os.Getenv("DATA_FILE_AUTO_ID") + if dataFile == "" { + return + } + payload, err := os.ReadFile(dataFile) + if err != nil { + t.Fatalf("read auto_id data file: %v", err) + } + + var decoded autoid.Envelope + if err := f.Deserialize(payload, &decoded); err != nil { + t.Fatalf("deserialize auto_id peer payload: %v", err) + } + if !reflect.DeepEqual(env, decoded) { + t.Fatalf("auto_id peer payload mismatch: %#v != %#v", env, decoded) + } + + out, err := f.Serialize(&decoded) + if err != nil { + t.Fatalf("serialize auto_id peer payload: %v", err) + } + if err := os.WriteFile(dataFile, out, 0o644); err != nil { + t.Fatalf("write auto_id data file: %v", err) + } +} + func buildPrimitiveTypes() complexpb.PrimitiveTypes { contact := complexpb.EmailPrimitiveTypes_Contact("alice@example.com") contact = complexpb.PhonePrimitiveTypes_Contact(12345) diff --git a/integration_tests/idl_tests/go/root_roundtrip_test.go b/integration_tests/idl_tests/go/root_roundtrip_test.go index ea8cd7fb58..ef5ec794da 100644 --- a/integration_tests/idl_tests/go/root_roundtrip_test.go +++ b/integration_tests/idl_tests/go/root_roundtrip_test.go @@ -21,9 +21,9 @@ import ( "reflect" "testing" - addressbook "github.com/apache/fory/integration_tests/idl_tests/go/addressbook" - rootpkg "github.com/apache/fory/integration_tests/idl_tests/go/root" - treepkg "github.com/apache/fory/integration_tests/idl_tests/go/tree" + addressbook "github.com/apache/fory/integration_tests/idl_tests/go/addressbook/generated" + rootpkg "github.com/apache/fory/integration_tests/idl_tests/go/root/generated" + treepkg "github.com/apache/fory/integration_tests/idl_tests/go/tree/generated" ) func buildRootHolder() rootpkg.MultiHolder { diff --git a/integration_tests/idl_tests/idl/addressbook.fdl b/integration_tests/idl_tests/idl/addressbook.fdl index bb6f8226dd..756a2b2d6b 100644 --- a/integration_tests/idl_tests/idl/addressbook.fdl +++ b/integration_tests/idl_tests/idl/addressbook.fdl @@ -17,7 +17,7 @@ package addressbook; -option go_package = "github.com/apache/fory/integration_tests/idl_tests/go/addressbook;addressbook"; +option go_package = "github.com/apache/fory/integration_tests/idl_tests/go/addressbook/generated;addressbook"; import "complex_pb.proto"; diff --git a/integration_tests/idl_tests/idl/auto_id.fdl b/integration_tests/idl_tests/idl/auto_id.fdl new file mode 100644 index 0000000000..f5a8a83ac6 --- /dev/null +++ b/integration_tests/idl_tests/idl/auto_id.fdl @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package auto_id; + +enum Status { + UNKNOWN = 0; + OK = 1; +} + +message Envelope { + string id = 1; + + message Payload { + int32 value = 1; + } + + union Detail { + Payload payload = 1; + string note = 2; + } + + Payload payload = 2; + Detail detail = 3; + Status status = 4; +} + +union Wrapper { + Envelope envelope = 1; + string raw = 2; +} diff --git a/integration_tests/idl_tests/idl/complex_pb.proto b/integration_tests/idl_tests/idl/complex_pb.proto index 79b1efa561..214243cc84 100644 --- a/integration_tests/idl_tests/idl/complex_pb.proto +++ b/integration_tests/idl_tests/idl/complex_pb.proto @@ -21,7 +21,7 @@ syntax = "proto3"; package complex_pb; -option go_package = "github.com/apache/fory/integration_tests/idl_tests/go/complex_pb;complex_pb"; +option go_package = "github.com/apache/fory/integration_tests/idl_tests/go/complex_pb/generated;complex_pb"; message PrimitiveTypes { option (fory).id = 200; diff --git a/integration_tests/idl_tests/idl/tree.fdl b/integration_tests/idl_tests/idl/tree.fdl index 205b9804f3..2ab8491ae3 100644 --- a/integration_tests/idl_tests/idl/tree.fdl +++ b/integration_tests/idl_tests/idl/tree.fdl @@ -17,7 +17,7 @@ package tree; -option go_package = "github.com/apache/fory/integration_tests/idl_tests/go/tree;tree"; +option go_package = "github.com/apache/fory/integration_tests/idl_tests/go/tree/generated;tree"; message TreeNode { string id = 1; diff --git a/integration_tests/idl_tests/java/src/test/java/org/apache/fory/idl_tests/IdlRoundTripTest.java b/integration_tests/idl_tests/java/src/test/java/org/apache/fory/idl_tests/IdlRoundTripTest.java index dfec1d79a6..87bc6b7200 100644 --- a/integration_tests/idl_tests/java/src/test/java/org/apache/fory/idl_tests/IdlRoundTripTest.java +++ b/integration_tests/idl_tests/java/src/test/java/org/apache/fory/idl_tests/IdlRoundTripTest.java @@ -27,6 +27,9 @@ import addressbook.Person; import addressbook.Person.PhoneNumber; import addressbook.Person.PhoneType; +import auto_id.AutoIdForyRegistration; +import auto_id.Envelope; +import auto_id.Wrapper; import collection.CollectionForyRegistration; import collection.NumericCollectionArrayUnion; import collection.NumericCollectionUnion; @@ -101,6 +104,16 @@ public void testAddressBookRoundTripSchemaConsistent() throws Exception { runAddressBookRoundTrip(false); } + @Test + public void testAutoIdRoundTripCompatible() throws Exception { + runAutoIdRoundTrip(true); + } + + @Test + public void testAutoIdRoundTripSchemaConsistent() throws Exception { + runAutoIdRoundTrip(false); + } + private void runAddressBookRoundTrip(boolean compatible) throws Exception { Fory fory = buildFory(compatible); AddressbookForyRegistration.register(fory); @@ -129,6 +142,40 @@ private void runAddressBookRoundTrip(boolean compatible) throws Exception { } } + private void runAutoIdRoundTrip(boolean compatible) throws Exception { + Fory fory = buildFory(compatible); + AutoIdForyRegistration.register(fory); + + Envelope envelope = buildAutoIdEnvelope(); + byte[] bytes = fory.serialize(envelope); + Object decoded = fory.deserialize(bytes); + + Assert.assertTrue(decoded instanceof Envelope); + Assert.assertEquals(decoded, envelope); + + Wrapper wrapper = buildAutoIdWrapper(envelope); + byte[] wrapperBytes = fory.serialize(wrapper); + Object decodedWrapper = fory.deserialize(wrapperBytes); + Assert.assertTrue(decodedWrapper instanceof Wrapper); + Assert.assertEquals(decodedWrapper, wrapper); + + for (String peer : resolvePeers()) { + Path dataFile = Files.createTempFile("idl-auto-id-" + peer + "-", ".bin"); + dataFile.toFile().deleteOnExit(); + Files.write(dataFile, bytes); + + Map env = new HashMap<>(); + env.put("DATA_FILE_AUTO_ID", dataFile.toAbsolutePath().toString()); + PeerCommand command = buildPeerCommand(peer, env, compatible); + runPeer(command, peer); + + byte[] peerBytes = Files.readAllBytes(dataFile); + Object roundTrip = fory.deserialize(peerBytes); + Assert.assertTrue(roundTrip instanceof Envelope); + Assert.assertEquals(roundTrip, envelope); + } + } + @Test public void testToBytesFromBytes() { AddressBook book = buildAddressBook(); @@ -544,8 +591,11 @@ private PeerCommand buildPeerCommand( switch (peer) { case "python": command = Arrays.asList("python", "-m", "idl_tests.roundtrip"); + Path pythonRoot = idlRoot.resolve("python"); String pythonPath = - idlRoot.resolve("python").resolve("src") + pythonRoot.resolve("idl_tests").resolve("generated") + + File.pathSeparator + + pythonRoot + File.pathSeparator + repoRoot.resolve("python"); String existingPythonPath = System.getenv("PYTHONPATH"); @@ -666,6 +716,24 @@ private AddressBook buildAddressBook() { return book; } + private Envelope buildAutoIdEnvelope() { + Envelope.Payload payload = new Envelope.Payload(); + payload.setValue(42); + + Envelope.Detail detail = Envelope.Detail.ofPayload(payload); + + Envelope envelope = new Envelope(); + envelope.setId("env-1"); + envelope.setPayload(payload); + envelope.setDetail(detail); + envelope.setStatus(auto_id.Status.OK); + return envelope; + } + + private Wrapper buildAutoIdWrapper(Envelope envelope) { + return Wrapper.ofEnvelope(envelope); + } + private PrimitiveTypes buildPrimitiveTypes() { PrimitiveTypes types = new PrimitiveTypes(); types.setBoolValue(true); diff --git a/integration_tests/idl_tests/python/src/idl_tests/__init__.py b/integration_tests/idl_tests/python/idl_tests/__init__.py similarity index 100% rename from integration_tests/idl_tests/python/src/idl_tests/__init__.py rename to integration_tests/idl_tests/python/idl_tests/__init__.py diff --git a/integration_tests/idl_tests/python/src/idl_tests/roundtrip.py b/integration_tests/idl_tests/python/idl_tests/roundtrip.py similarity index 94% rename from integration_tests/idl_tests/python/src/idl_tests/roundtrip.py rename to integration_tests/idl_tests/python/idl_tests/roundtrip.py index 869d079e0e..52d5039859 100644 --- a/integration_tests/idl_tests/python/src/idl_tests/roundtrip.py +++ b/integration_tests/idl_tests/python/idl_tests/roundtrip.py @@ -22,6 +22,7 @@ from pathlib import Path import addressbook +import auto_id import any_example import complex_fbs import complex_pb @@ -93,6 +94,21 @@ def build_root_holder() -> "root.MultiHolder": ) +def build_auto_id_envelope() -> "auto_id.Envelope": + payload = auto_id.Envelope.Payload(value=42) + detail = auto_id.Envelope.Detail.payload(payload) + return auto_id.Envelope( + id="env-1", + payload=payload, + detail=detail, + status=auto_id.Status.OK, + ) + + +def build_auto_id_wrapper(envelope: "auto_id.Envelope") -> "auto_id.Wrapper": + return auto_id.Wrapper.envelope(envelope) + + def local_roundtrip(fory: pyfory.Fory, book: "addressbook.AddressBook") -> None: data = fory.serialize(book) decoded = fory.deserialize(data) @@ -129,6 +145,31 @@ def file_roundtrip(fory: pyfory.Fory, book: "addressbook.AddressBook") -> None: Path(data_file).write_bytes(fory.serialize(decoded)) +def local_roundtrip_auto_id(fory: pyfory.Fory, envelope: "auto_id.Envelope") -> None: + data = fory.serialize(envelope) + decoded = fory.deserialize(data) + assert isinstance(decoded, auto_id.Envelope) + assert decoded == envelope + + +def local_roundtrip_auto_wrapper(fory: pyfory.Fory, wrapper: "auto_id.Wrapper") -> None: + data = fory.serialize(wrapper) + decoded = fory.deserialize(data) + assert isinstance(decoded, auto_id.Wrapper) + assert decoded == wrapper + + +def file_roundtrip_auto_id(fory: pyfory.Fory, envelope: "auto_id.Envelope") -> None: + data_file = os.environ.get("DATA_FILE_AUTO_ID") + if not data_file: + return + payload = Path(data_file).read_bytes() + decoded = fory.deserialize(payload) + assert isinstance(decoded, auto_id.Envelope) + assert decoded == envelope + Path(data_file).write_bytes(fory.serialize(decoded)) + + def build_primitive_types() -> "complex_pb.PrimitiveTypes": contact = complex_pb.PrimitiveTypes.Contact.email("alice@example.com") contact = complex_pb.PrimitiveTypes.Contact.phone(12345) @@ -669,6 +710,7 @@ def run_roundtrip(compatible: bool) -> None: fory = pyfory.Fory(xlang=True, compatible=compatible) complex_pb.register_complex_pb_types(fory) addressbook.register_addressbook_types(fory) + auto_id.register_auto_id_types(fory) monster.register_monster_types(fory) complex_fbs.register_complex_fbs_types(fory) collection.register_collection_types(fory) @@ -681,6 +723,12 @@ def run_roundtrip(compatible: bool) -> None: local_roundtrip(fory, book) file_roundtrip(fory, book) + auto_envelope = build_auto_id_envelope() + auto_wrapper = build_auto_id_wrapper(auto_envelope) + local_roundtrip_auto_id(fory, auto_envelope) + local_roundtrip_auto_wrapper(fory, auto_wrapper) + file_roundtrip_auto_id(fory, auto_envelope) + primitives = build_primitive_types() local_roundtrip_primitives(fory, primitives) file_roundtrip_primitives(fory, primitives) diff --git a/integration_tests/idl_tests/python/pyproject.toml b/integration_tests/idl_tests/python/pyproject.toml index 6521976270..e7ea05ac1a 100644 --- a/integration_tests/idl_tests/python/pyproject.toml +++ b/integration_tests/idl_tests/python/pyproject.toml @@ -29,5 +29,5 @@ license = {text = "Apache-2.0"} dependencies = ["pyfory"] [tool.setuptools.packages.find] -where = ["src"] +where = ["."] include = ["idl_tests"] diff --git a/integration_tests/idl_tests/run_python_tests.sh b/integration_tests/idl_tests/run_python_tests.sh index dc30750676..2237613cd5 100755 --- a/integration_tests/idl_tests/run_python_tests.sh +++ b/integration_tests/idl_tests/run_python_tests.sh @@ -24,7 +24,7 @@ ROOT_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)" python "${SCRIPT_DIR}/generate_idl.py" --lang python -ENABLE_FORY_DEBUG_OUTPUT=1 PYTHONPATH="${SCRIPT_DIR}/python/src:${ROOT_DIR}/python:${PYTHONPATH:-}" \ +ENABLE_FORY_DEBUG_OUTPUT=1 PYTHONPATH="${SCRIPT_DIR}/python/idl_tests/generated:${SCRIPT_DIR}/python:${ROOT_DIR}/python:${PYTHONPATH:-}" \ ENABLE_FORY_CYTHON_SERIALIZATION=0 \ python -m idl_tests.roundtrip diff --git a/integration_tests/idl_tests/rust/src/lib.rs b/integration_tests/idl_tests/rust/src/lib.rs index 47e0e4b701..2a78cf5e2e 100644 --- a/integration_tests/idl_tests/rust/src/lib.rs +++ b/integration_tests/idl_tests/rust/src/lib.rs @@ -15,14 +15,30 @@ // specific language governing permissions and limitations // under the License. -pub mod addressbook; -pub mod any_example; -pub mod any_example_pb; -pub mod complex_fbs; -pub mod complex_pb; -pub mod collection; -pub mod monster; -pub mod root; -pub mod optional_types; -pub mod graph; -pub mod tree; +pub mod generated { + #[path = "../generated/addressbook.rs"] + pub mod addressbook; + #[path = "../generated/any_example.rs"] + pub mod any_example; + #[path = "../generated/any_example_pb.rs"] + pub mod any_example_pb; + #[path = "../generated/auto_id.rs"] + pub mod auto_id; + #[path = "../generated/collection.rs"] + pub mod collection; + #[path = "../generated/complex_fbs.rs"] + pub mod complex_fbs; + #[path = "../generated/complex_pb.rs"] + pub mod complex_pb; + #[path = "../generated/graph.rs"] + pub mod graph; + #[path = "../generated/monster.rs"] + pub mod monster; + #[path = "../generated/optional_types.rs"] + pub mod optional_types; + #[path = "../generated/root.rs"] + pub mod root; + #[path = "../generated/tree.rs"] + pub mod tree; +} +pub use generated::*; diff --git a/integration_tests/idl_tests/rust/tests/idl_roundtrip.rs b/integration_tests/idl_tests/rust/tests/idl_roundtrip.rs index 82c549938e..523e451273 100644 --- a/integration_tests/idl_tests/rust/tests/idl_roundtrip.rs +++ b/integration_tests/idl_tests/rust/tests/idl_roundtrip.rs @@ -21,19 +21,23 @@ use std::{env, fs}; use chrono::NaiveDate; use fory::{ArcWeak, Fory}; -use idl_tests::addressbook::{ +use idl_tests::generated::addressbook::{ self, person::{PhoneNumber, PhoneType}, AddressBook, Animal, Cat, Dog, Person, }; -use idl_tests::complex_pb::{self, PrimitiveTypes}; -use idl_tests::complex_fbs::{self, Container, Note, Payload, ScalarPack, Status}; -use idl_tests::collection::{self, NumericCollectionArrayUnion, NumericCollectionUnion, NumericCollections, NumericCollectionsArray}; -use idl_tests::monster::{self, Color, Monster, Vec3}; -use idl_tests::optional_types::{self, AllOptionalTypes, OptionalHolder, OptionalUnion}; -use idl_tests::any_example::{self, AnyHolder, AnyInner, AnyUnion}; -use idl_tests::root; -use idl_tests::{graph, tree}; +use idl_tests::generated::any_example::{self, AnyHolder, AnyInner, AnyUnion}; +use idl_tests::generated::auto_id; +use idl_tests::generated::collection::{ + self, NumericCollectionArrayUnion, NumericCollectionUnion, NumericCollections, + NumericCollectionsArray, +}; +use idl_tests::generated::complex_fbs::{self, Container, Note, Payload, ScalarPack, Status}; +use idl_tests::generated::complex_pb::{self, PrimitiveTypes}; +use idl_tests::generated::monster::{self, Color, Monster, Vec3}; +use idl_tests::generated::optional_types::{self, AllOptionalTypes, OptionalHolder, OptionalUnion}; +use idl_tests::generated::root; +use idl_tests::generated::{graph, tree}; fn build_address_book() -> AddressBook { let mobile = PhoneNumber { @@ -105,6 +109,21 @@ fn build_root_holder() -> root::MultiHolder { } } +fn build_auto_id_envelope() -> auto_id::Envelope { + let payload = auto_id::envelope::Payload { value: 42 }; + let detail = auto_id::envelope::Detail::Payload(payload.clone()); + auto_id::Envelope { + id: "env-1".to_string(), + payload: Some(payload), + detail, + status: auto_id::Status::Ok, + } +} + +fn build_auto_id_wrapper(envelope: auto_id::Envelope) -> auto_id::Wrapper { + auto_id::Wrapper::Envelope(envelope) +} + #[test] fn test_to_bytes_from_bytes() { let book = build_address_book(); @@ -488,6 +507,7 @@ fn run_address_book_roundtrip(compatible: bool) { let mut fory = Fory::default().xlang(true).compatible(compatible); complex_pb::register_types(&mut fory).expect("register complex pb types"); addressbook::register_types(&mut fory).expect("register types"); + auto_id::register_types(&mut fory).expect("register auto_id types"); monster::register_types(&mut fory).expect("register monster types"); complex_fbs::register_types(&mut fory).expect("register flatbuffers types"); collection::register_types(&mut fory).expect("register collection types"); @@ -500,6 +520,20 @@ fn run_address_book_roundtrip(compatible: bool) { assert_eq!(book, roundtrip); + let auto_env = build_auto_id_envelope(); + let auto_bytes = fory.serialize(&auto_env).expect("serialize auto_id"); + let auto_roundtrip: auto_id::Envelope = + fory.deserialize(&auto_bytes).expect("deserialize auto_id"); + assert_eq!(auto_env, auto_roundtrip); + + let auto_wrapper = build_auto_id_wrapper(auto_env.clone()); + let wrapper_bytes = fory + .serialize(&auto_wrapper) + .expect("serialize auto_id wrapper"); + let wrapper_roundtrip: auto_id::Wrapper = + fory.deserialize(&wrapper_bytes).expect("deserialize auto_id wrapper"); + assert_eq!(auto_wrapper, wrapper_roundtrip); + let data_file = match env::var("DATA_FILE") { Ok(path) => path, Err(_) => return, @@ -512,6 +546,16 @@ fn run_address_book_roundtrip(compatible: bool) { let encoded = fory.serialize(&peer_book).expect("serialize peer payload"); fs::write(data_file, encoded).expect("write data file"); + if let Ok(data_file) = env::var("DATA_FILE_AUTO_ID") { + let payload = fs::read(&data_file).expect("read auto_id data file"); + let peer_env: auto_id::Envelope = fory + .deserialize(&payload) + .expect("deserialize auto_id peer payload"); + assert_eq!(auto_env, peer_env); + let encoded = fory.serialize(&peer_env).expect("serialize auto_id payload"); + fs::write(data_file, encoded).expect("write auto_id data file"); + } + let types = build_primitive_types(); let bytes = fory.serialize(&types).expect("serialize"); let roundtrip: PrimitiveTypes = fory.deserialize(&bytes).expect("deserialize"); diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java index 4b7d727cb4..a744815f36 100644 --- a/java/fory-core/src/main/java/org/apache/fory/Fory.java +++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java @@ -49,19 +49,20 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.MemoryUtils; import org.apache.fory.meta.MetaCompressor; -import org.apache.fory.resolver.ClassInfo; -import org.apache.fory.resolver.ClassInfoHolder; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.MapRefResolver; import org.apache.fory.resolver.MetaStringResolver; import org.apache.fory.resolver.NoRefResolver; import org.apache.fory.resolver.RefResolver; import org.apache.fory.resolver.SerializationContext; +import org.apache.fory.resolver.TypeInfo; +import org.apache.fory.resolver.TypeInfoHolder; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.resolver.XtypeResolver; import org.apache.fory.serializer.ArraySerializers; import org.apache.fory.serializer.BufferCallback; import org.apache.fory.serializer.BufferObject; +import org.apache.fory.serializer.NonexistentClass.NonexistentMetaShared; import org.apache.fory.serializer.PrimitiveSerializers.LongSerializer; import org.apache.fory.serializer.Serializer; import org.apache.fory.serializer.SerializerFactory; @@ -174,7 +175,7 @@ public void register(Class cls) { @Override public void register(Class cls, int id) { - getTypeResolver().register(cls, id); + getTypeResolver().register(cls, Integer.toUnsignedLong(id)); } @Deprecated @@ -186,7 +187,7 @@ public void register(Class cls, boolean createSerializer) { @Deprecated @Override public void register(Class cls, int id, boolean createSerializer) { - getTypeResolver().register(cls, id); + getTypeResolver().register(cls, Integer.toUnsignedLong(id)); } /** @@ -215,7 +216,7 @@ public void register(String className) { @Override public void register(String className, int classId) { - getTypeResolver().register(className, classId); + getTypeResolver().register(className, Integer.toUnsignedLong(classId)); } @Override @@ -225,7 +226,7 @@ public void register(String className, String namespace, String typeName) { @Override public void registerUnion(Class cls, int id, Serializer serializer) { - getTypeResolver().registerUnion(cls, id, serializer); + getTypeResolver().registerUnion(cls, Integer.toUnsignedLong(id), serializer); } @Override @@ -280,7 +281,7 @@ public Serializer getSerializer(Class cls) { if (!crossLanguage) { return classResolver.getSerializer(cls); } else { - return xtypeResolver.getClassInfo(cls).getSerializer(); + return xtypeResolver.getTypeInfo(cls).getSerializer(); } } @@ -412,9 +413,9 @@ public void resetBuffer() { private void write(MemoryBuffer buffer, Object obj) { // reduce caller stack if (!refResolver.writeRefOrNull(buffer, obj)) { - ClassInfo classInfo = classResolver.getOrUpdateClassInfo(obj.getClass()); - classResolver.writeClassInfo(buffer, classInfo); - writeData(buffer, classInfo, obj); + TypeInfo typeInfo = classResolver.getOrUpdateTypeInfo(obj.getClass()); + classResolver.writeTypeInfo(buffer, typeInfo); + writeData(buffer, typeInfo, obj); } } @@ -426,25 +427,25 @@ private void xwrite(MemoryBuffer buffer, Object obj) { /** Serialize a nullable referencable object to buffer. */ public void writeRef(MemoryBuffer buffer, Object obj) { if (!refResolver.writeRefOrNull(buffer, obj)) { - ClassInfo classInfo = classResolver.getOrUpdateClassInfo(obj.getClass()); - classResolver.writeClassInfo(buffer, classInfo); - writeData(buffer, classInfo, obj); + TypeInfo typeInfo = classResolver.getOrUpdateTypeInfo(obj.getClass()); + classResolver.writeTypeInfo(buffer, typeInfo); + writeData(buffer, typeInfo, obj); } } - public void writeRef(MemoryBuffer buffer, Object obj, ClassInfoHolder classInfoHolder) { + public void writeRef(MemoryBuffer buffer, Object obj, TypeInfoHolder classInfoHolder) { if (!refResolver.writeRefOrNull(buffer, obj)) { - ClassInfo classInfo = classResolver.getClassInfo(obj.getClass(), classInfoHolder); - classResolver.writeClassInfo(buffer, classInfo); - writeData(buffer, classInfo, obj); + TypeInfo typeInfo = classResolver.getTypeInfo(obj.getClass(), classInfoHolder); + classResolver.writeTypeInfo(buffer, typeInfo); + writeData(buffer, typeInfo, obj); } } - public void writeRef(MemoryBuffer buffer, Object obj, ClassInfo classInfo) { - Serializer serializer = classInfo.getSerializer(); + public void writeRef(MemoryBuffer buffer, Object obj, TypeInfo typeInfo) { + Serializer serializer = typeInfo.getSerializer(); if (serializer.needToWriteRef()) { if (!refResolver.writeRefOrNull(buffer, obj)) { - classResolver.writeClassInfo(buffer, classInfo); + classResolver.writeTypeInfo(buffer, typeInfo); depth++; serializer.write(buffer, obj); depth--; @@ -454,7 +455,7 @@ public void writeRef(MemoryBuffer buffer, Object obj, ClassInfo classInfo) { buffer.writeByte(Fory.NULL_FLAG); } else { buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); - classResolver.writeClassInfo(buffer, classInfo); + classResolver.writeTypeInfo(buffer, typeInfo); depth++; serializer.write(buffer, obj); depth--; @@ -488,9 +489,9 @@ public void writeRef(MemoryBuffer buffer, T obj, Serializer serializer) { * object graph. */ public void writeNonRef(MemoryBuffer buffer, Object obj) { - ClassInfo classInfo = classResolver.getOrUpdateClassInfo(obj.getClass()); - classResolver.writeClassInfo(buffer, classInfo); - writeData(buffer, classInfo, obj); + TypeInfo typeInfo = classResolver.getOrUpdateTypeInfo(obj.getClass()); + classResolver.writeTypeInfo(buffer, typeInfo); + writeData(buffer, typeInfo, obj); } public void writeNonRef(MemoryBuffer buffer, Object obj, Serializer serializer) { @@ -499,15 +500,15 @@ public void writeNonRef(MemoryBuffer buffer, Object obj, Serializer serializer) depth--; } - public void writeNonRef(MemoryBuffer buffer, Object obj, ClassInfoHolder holder) { - ClassInfo classInfo = classResolver.getClassInfo(obj.getClass(), holder); - classResolver.writeClassInfo(buffer, classInfo); - writeData(buffer, classInfo, obj); + public void writeNonRef(MemoryBuffer buffer, Object obj, TypeInfoHolder holder) { + TypeInfo typeInfo = classResolver.getTypeInfo(obj.getClass(), holder); + classResolver.writeTypeInfo(buffer, typeInfo); + writeData(buffer, typeInfo, obj); } - public void writeNonRef(MemoryBuffer buffer, Object obj, ClassInfo classInfo) { - classResolver.writeClassInfo(buffer, classInfo); - Serializer serializer = classInfo.getSerializer(); + public void writeNonRef(MemoryBuffer buffer, Object obj, TypeInfo typeInfo) { + classResolver.writeTypeInfo(buffer, typeInfo); + Serializer serializer = typeInfo.getSerializer(); depth++; serializer.write(buffer, obj); depth--; @@ -515,23 +516,42 @@ public void writeNonRef(MemoryBuffer buffer, Object obj, ClassInfo classInfo) { public void xwriteRef(MemoryBuffer buffer, Object obj) { if (!refResolver.writeRefOrNull(buffer, obj)) { - ClassInfo classInfo = xtypeResolver.writeClassInfo(buffer, obj); - xwriteData(buffer, classInfo, obj); + TypeInfo typeInfo = xtypeResolver.getTypeInfo(obj.getClass()); + if (typeInfo.getCls() == NonexistentMetaShared.class) { + depth++; + typeInfo.getSerializer().xwrite(buffer, obj); + depth--; + return; + } + xtypeResolver.writeTypeInfo(buffer, typeInfo); + xwriteData(buffer, typeInfo, obj); } } - public void xwriteRef(MemoryBuffer buffer, Object obj, ClassInfoHolder classInfoHolder) { + public void xwriteRef(MemoryBuffer buffer, Object obj, TypeInfoHolder classInfoHolder) { if (!refResolver.writeRefOrNull(buffer, obj)) { - ClassInfo classInfo = xtypeResolver.getClassInfo(obj.getClass(), classInfoHolder); - xtypeResolver.writeClassInfo(buffer, obj); - xwriteData(buffer, classInfo, obj); + TypeInfo typeInfo = xtypeResolver.getTypeInfo(obj.getClass(), classInfoHolder); + if (typeInfo.getCls() == NonexistentMetaShared.class) { + depth++; + typeInfo.getSerializer().xwrite(buffer, obj); + depth--; + return; + } + xtypeResolver.writeTypeInfo(buffer, typeInfo); + xwriteData(buffer, typeInfo, obj); } } - public void xwriteRef(MemoryBuffer buffer, Object obj, ClassInfo classInfo) { + public void xwriteRef(MemoryBuffer buffer, Object obj, TypeInfo typeInfo) { if (!refResolver.writeRefOrNull(buffer, obj)) { - xtypeResolver.writeClassInfo(buffer, obj); - xwriteData(buffer, classInfo, obj); + if (typeInfo.getCls() == NonexistentMetaShared.class) { + depth++; + typeInfo.getSerializer().xwrite(buffer, obj); + depth--; + return; + } + xtypeResolver.writeTypeInfo(buffer, typeInfo); + xwriteData(buffer, typeInfo, obj); } } @@ -555,19 +575,38 @@ public void xwriteRef(MemoryBuffer buffer, T obj, Serializer serializer) } public void xwriteNonRef(MemoryBuffer buffer, Object obj) { - ClassInfo classInfo = xtypeResolver.writeClassInfo(buffer, obj); - xwriteData(buffer, classInfo, obj); + TypeInfo typeInfo = xtypeResolver.getTypeInfo(obj.getClass()); + if (typeInfo.getCls() == NonexistentMetaShared.class) { + depth++; + typeInfo.getSerializer().xwrite(buffer, obj); + depth--; + return; + } + xtypeResolver.writeTypeInfo(buffer, typeInfo); + xwriteData(buffer, typeInfo, obj); } - public void xwriteNonRef(MemoryBuffer buffer, Object obj, ClassInfoHolder holder) { - ClassInfo classInfo = xtypeResolver.getClassInfo(obj.getClass(), holder); - xtypeResolver.writeClassInfo(buffer, obj); - xwriteData(buffer, classInfo, obj); + public void xwriteNonRef(MemoryBuffer buffer, Object obj, TypeInfoHolder holder) { + TypeInfo typeInfo = xtypeResolver.getTypeInfo(obj.getClass(), holder); + if (typeInfo.getCls() == NonexistentMetaShared.class) { + depth++; + typeInfo.getSerializer().xwrite(buffer, obj); + depth--; + return; + } + xtypeResolver.writeTypeInfo(buffer, typeInfo); + xwriteData(buffer, typeInfo, obj); } - public void xwriteNonRef(MemoryBuffer buffer, Object obj, ClassInfo classInfo) { - xtypeResolver.writeClassInfo(buffer, classInfo); - xwriteData(buffer, classInfo, obj); + public void xwriteNonRef(MemoryBuffer buffer, Object obj, TypeInfo typeInfo) { + if (typeInfo.getCls() == NonexistentMetaShared.class) { + depth++; + typeInfo.getSerializer().xwrite(buffer, obj); + depth--; + return; + } + xtypeResolver.writeTypeInfo(buffer, typeInfo); + xwriteData(buffer, typeInfo, obj); } public void xwriteNonRef(MemoryBuffer buffer, Object obj, Serializer serializer) { @@ -577,9 +616,9 @@ public void xwriteNonRef(MemoryBuffer buffer, Object obj, Serializer serializer) ; } - public void xwriteData(MemoryBuffer buffer, ClassInfo classInfo, Object obj) { - int internalTypeId = classInfo.getTypeId() & 0xff; - switch (internalTypeId) { + public void xwriteData(MemoryBuffer buffer, TypeInfo typeInfo, Object obj) { + int typeId = typeInfo.getTypeId(); + switch (typeId) { case Types.BOOL: buffer.writeBoolean((Boolean) obj); break; @@ -607,15 +646,15 @@ public void xwriteData(MemoryBuffer buffer, ClassInfo classInfo, Object obj) { // TODO(add fastpath for other types) default: depth++; - classInfo.getSerializer().xwrite(buffer, obj); + typeInfo.getSerializer().xwrite(buffer, obj); depth--; } } /** Write not null data to buffer. */ - private void writeData(MemoryBuffer buffer, ClassInfo classInfo, Object obj) { - int internalTypeId = classInfo.getTypeId() & 0xff; - switch (internalTypeId) { + private void writeData(MemoryBuffer buffer, TypeInfo typeInfo, Object obj) { + int typeId = typeInfo.getTypeId(); + switch (typeId) { case Types.BOOL: buffer.writeBoolean((Boolean) obj); break; @@ -649,7 +688,7 @@ private void writeData(MemoryBuffer buffer, ClassInfo classInfo, Object obj) { break; default: depth++; - classInfo.getSerializer().write(buffer, obj); + typeInfo.getSerializer().write(buffer, obj); depth--; } } @@ -912,7 +951,7 @@ public Object readRef(MemoryBuffer buffer) { int nextReadRefId = refResolver.tryPreserveRefId(buffer); if (nextReadRefId >= NOT_NULL_VALUE_FLAG) { // ref value or not-null value - Object o = readDataInternal(buffer, classResolver.readClassInfo(buffer)); + Object o = readDataInternal(buffer, classResolver.readTypeInfo(buffer)); refResolver.setReadObject(nextReadRefId, o); return o; } else { @@ -920,12 +959,12 @@ public Object readRef(MemoryBuffer buffer) { } } - public Object readRef(MemoryBuffer buffer, ClassInfo classInfo) { + public Object readRef(MemoryBuffer buffer, TypeInfo typeInfo) { RefResolver refResolver = this.refResolver; int nextReadRefId = refResolver.tryPreserveRefId(buffer); if (nextReadRefId >= NOT_NULL_VALUE_FLAG) { // ref value or not-null value - Object o = readDataInternal(buffer, classInfo); + Object o = readDataInternal(buffer, typeInfo); refResolver.setReadObject(nextReadRefId, o); return o; } else { @@ -933,12 +972,12 @@ public Object readRef(MemoryBuffer buffer, ClassInfo classInfo) { } } - public Object readRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { + public Object readRef(MemoryBuffer buffer, TypeInfoHolder classInfoHolder) { RefResolver refResolver = this.refResolver; int nextReadRefId = refResolver.tryPreserveRefId(buffer); if (nextReadRefId >= NOT_NULL_VALUE_FLAG) { // ref value or not-null value - Object o = readDataInternal(buffer, classResolver.readClassInfo(buffer, classInfoHolder)); + Object o = readDataInternal(buffer, classResolver.readTypeInfo(buffer, classInfoHolder)); refResolver.setReadObject(nextReadRefId, o); return o; } else { @@ -970,15 +1009,15 @@ public T readRef(MemoryBuffer buffer, Serializer serializer) { /** Deserialize not-null and non-reference object from buffer. */ public Object readNonRef(MemoryBuffer buffer) { - return readDataInternal(buffer, classResolver.readClassInfo(buffer)); + return readDataInternal(buffer, classResolver.readTypeInfo(buffer)); } - public Object readNonRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { - return readDataInternal(buffer, classResolver.readClassInfo(buffer, classInfoHolder)); + public Object readNonRef(MemoryBuffer buffer, TypeInfoHolder classInfoHolder) { + return readDataInternal(buffer, classResolver.readTypeInfo(buffer, classInfoHolder)); } - public Object readNonRef(MemoryBuffer buffer, ClassInfo classInfo) { - return readDataInternal(buffer, classInfo); + public Object readNonRef(MemoryBuffer buffer, TypeInfo typeInfo) { + return readDataInternal(buffer, typeInfo); } /** Read object class and data without tracking ref. */ @@ -1000,7 +1039,7 @@ public Object readNullable(MemoryBuffer buffer, Serializer serializer) { } } - public Object readNullable(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { + public Object readNullable(MemoryBuffer buffer, TypeInfoHolder classInfoHolder) { byte headFlag = buffer.readByte(); if (headFlag == Fory.NULL_FLAG) { return null; @@ -1010,17 +1049,17 @@ public Object readNullable(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) } /** Class should be read already. */ - public Object readData(MemoryBuffer buffer, ClassInfo classInfo) { + public Object readData(MemoryBuffer buffer, TypeInfo typeInfo) { incReadDepth(); - Serializer serializer = classInfo.getSerializer(); + Serializer serializer = typeInfo.getSerializer(); Object read = serializer.read(buffer); depth--; return read; } - private Object readDataInternal(MemoryBuffer buffer, ClassInfo classInfo) { - int internalTypeId = classInfo.getTypeId() & 0xff; - switch (internalTypeId) { + private Object readDataInternal(MemoryBuffer buffer, TypeInfo typeInfo) { + int typeId = typeInfo.getTypeId(); + switch (typeId) { case Types.BOOL: return buffer.readBoolean(); case Types.INT8: @@ -1045,15 +1084,15 @@ private Object readDataInternal(MemoryBuffer buffer, ClassInfo classInfo) { return stringSerializer.readJavaString(buffer); default: incReadDepth(); - Object read = classInfo.getSerializer().read(buffer); + Object read = typeInfo.getSerializer().read(buffer); depth--; return read; } } - private Object xreadDataInternal(MemoryBuffer buffer, ClassInfo classInfo) { - int internalTypeId = classInfo.getTypeId() & 0xff; - switch (internalTypeId) { + private Object xreadDataInternal(MemoryBuffer buffer, TypeInfo typeInfo) { + int typeId = typeInfo.getTypeId(); + switch (typeId) { case Types.BOOL: return buffer.readBoolean(); case Types.INT8: @@ -1078,7 +1117,7 @@ private Object xreadDataInternal(MemoryBuffer buffer, ClassInfo classInfo) { return stringSerializer.readJavaString(buffer); default: incReadDepth(); - Object read = classInfo.getSerializer().xread(buffer); + Object read = typeInfo.getSerializer().xread(buffer); depth--; return read; } @@ -1088,7 +1127,7 @@ public Object xreadRef(MemoryBuffer buffer) { RefResolver refResolver = this.refResolver; int nextReadRefId = refResolver.tryPreserveRefId(buffer); if (nextReadRefId >= NOT_NULL_VALUE_FLAG) { - Object o = xreadNonRef(buffer, xtypeResolver.readClassInfo(buffer)); + Object o = xreadNonRef(buffer, xtypeResolver.readTypeInfo(buffer)); refResolver.setReadObject(nextReadRefId, o); return o; } else { @@ -1096,12 +1135,12 @@ public Object xreadRef(MemoryBuffer buffer) { } } - public Object xreadRef(MemoryBuffer buffer, ClassInfo classInfo) { + public Object xreadRef(MemoryBuffer buffer, TypeInfo typeInfo) { RefResolver refResolver = this.refResolver; int nextReadRefId = refResolver.tryPreserveRefId(buffer); if (nextReadRefId >= NOT_NULL_VALUE_FLAG) { // ref value or not-null value - Object o = xreadDataInternal(buffer, classInfo); + Object o = xreadDataInternal(buffer, typeInfo); refResolver.setReadObject(nextReadRefId, o); return o; } else { @@ -1109,12 +1148,12 @@ public Object xreadRef(MemoryBuffer buffer, ClassInfo classInfo) { } } - public Object xreadRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { + public Object xreadRef(MemoryBuffer buffer, TypeInfoHolder classInfoHolder) { RefResolver refResolver = this.refResolver; int nextReadRefId = refResolver.tryPreserveRefId(buffer); if (nextReadRefId >= NOT_NULL_VALUE_FLAG) { // ref value or not-null value - Object o = xreadDataInternal(buffer, xtypeResolver.readClassInfo(buffer, classInfoHolder)); + Object o = xreadDataInternal(buffer, xtypeResolver.readTypeInfo(buffer, classInfoHolder)); refResolver.setReadObject(nextReadRefId, o); return o; } else { @@ -1144,7 +1183,7 @@ public Object xreadRef(MemoryBuffer buffer, Serializer serializer) { } public Object xreadNonRef(MemoryBuffer buffer) { - return xreadNonRef(buffer, xtypeResolver.readClassInfo(buffer)); + return xreadNonRef(buffer, xtypeResolver.readTypeInfo(buffer)); } public Object xreadNonRef(MemoryBuffer buffer, Serializer serializer) { @@ -1154,10 +1193,10 @@ public Object xreadNonRef(MemoryBuffer buffer, Serializer serializer) { return o; } - public Object xreadNonRef(MemoryBuffer buffer, ClassInfo classInfo) { - assert classInfo != null; - int internalTypeId = classInfo.getTypeId() & 0xff; - switch (internalTypeId) { + public Object xreadNonRef(MemoryBuffer buffer, TypeInfo typeInfo) { + assert typeInfo != null; + int typeId = typeInfo.getTypeId(); + switch (typeId) { case Types.BOOL: return buffer.readBoolean(); case Types.INT8: @@ -1180,20 +1219,20 @@ public Object xreadNonRef(MemoryBuffer buffer, ClassInfo classInfo) { // TODO(add fastpath for other types) default: incReadDepth(); - Object o = classInfo.getSerializer().xread(buffer); + Object o = typeInfo.getSerializer().xread(buffer); depth--; return o; } } - public Object xreadNonRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { - ClassInfo classInfo = xtypeResolver.readClassInfo(buffer, classInfoHolder); - return xreadNonRef(buffer, classInfo); + public Object xreadNonRef(MemoryBuffer buffer, TypeInfoHolder classInfoHolder) { + TypeInfo typeInfo = xtypeResolver.readTypeInfo(buffer, classInfoHolder); + return xreadNonRef(buffer, typeInfo); } - public Object xreadNullable(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { - ClassInfo classInfo = xtypeResolver.readClassInfo(buffer, classInfoHolder); - return xreadNullable(buffer, classInfo.getSerializer()); + public Object xreadNullable(MemoryBuffer buffer, TypeInfoHolder classInfoHolder) { + TypeInfo typeInfo = xtypeResolver.readTypeInfo(buffer, classInfoHolder); + return xreadNullable(buffer, typeInfo.getSerializer()); } public Object xreadNullable(MemoryBuffer buffer, Serializer serializer) { @@ -1224,14 +1263,14 @@ public void serializeJavaObject(MemoryBuffer buffer, Object obj) { } if (config.isMetaShareEnabled()) { if (!refResolver.writeRefOrNull(buffer, obj)) { - ClassInfo classInfo = classResolver.getOrUpdateClassInfo(obj.getClass()); - classResolver.writeClassInfo(buffer, classInfo); - writeData(buffer, classInfo, obj); + TypeInfo typeInfo = classResolver.getOrUpdateTypeInfo(obj.getClass()); + classResolver.writeTypeInfo(buffer, typeInfo); + writeData(buffer, typeInfo, obj); } } else { if (!refResolver.writeRefOrNull(buffer, obj)) { - ClassInfo classInfo = classResolver.getOrUpdateClassInfo(obj.getClass()); - writeData(buffer, classInfo, obj); + TypeInfo typeInfo = classResolver.getOrUpdateTypeInfo(obj.getClass()); + writeData(buffer, typeInfo, obj); } } } catch (Throwable t) { @@ -1267,13 +1306,13 @@ public T deserializeJavaObject(MemoryBuffer buffer, Class cls) { T obj; int nextReadRefId = refResolver.tryPreserveRefId(buffer); if (nextReadRefId >= NOT_NULL_VALUE_FLAG) { - ClassInfo classInfo; + TypeInfo typeInfo; if (shareMeta) { - classInfo = classResolver.readClassInfo(buffer, cls); + typeInfo = classResolver.readTypeInfo(buffer, cls); } else { - classInfo = classResolver.getClassInfo(cls); + typeInfo = classResolver.getTypeInfo(cls); } - obj = (T) readDataInternal(buffer, classInfo); + obj = (T) readDataInternal(buffer, typeInfo); return obj; } else { return null; @@ -1438,9 +1477,9 @@ public T copyObject(T obj) { return null; } Object copy; - ClassInfo classInfo = classResolver.getOrUpdateClassInfo(obj.getClass()); - int internalTypeId = classInfo.getTypeId() & 0xff; - switch (internalTypeId) { + TypeInfo typeInfo = classResolver.getOrUpdateTypeInfo(obj.getClass()); + int typeId = typeInfo.getTypeId(); + switch (typeId) { case Types.BOOL: case Types.INT8: case ClassResolver.CHAR_ID: @@ -1486,7 +1525,7 @@ public T copyObject(T obj) { break; // todo: add fastpath for other types. default: - copy = copyObject(obj, classInfo.getSerializer()); + copy = copyObject(obj, typeInfo.getSerializer()); } return (T) copy; } @@ -1516,7 +1555,7 @@ public T copyObject(T obj, int classId) { case Types.STRING: return obj; default: - return copyObject(obj, classResolver.getOrUpdateClassInfo(obj.getClass()).getSerializer()); + return copyObject(obj, classResolver.getOrUpdateTypeInfo(obj.getClass()).getSerializer()); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java index 408272dfbb..259dd88201 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java @@ -117,11 +117,11 @@ import org.apache.fory.memory.Platform; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.reflect.TypeRef; -import org.apache.fory.resolver.ClassInfo; -import org.apache.fory.resolver.ClassInfoHolder; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.RefMode; import org.apache.fory.resolver.RefResolver; +import org.apache.fory.resolver.TypeInfo; +import org.apache.fory.resolver.TypeInfoHolder; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.DeferedLazySerializer.DeferredLazyObjectSerializer; import org.apache.fory.serializer.EnumSerializer; @@ -139,6 +139,7 @@ import org.apache.fory.type.DispatchId; import org.apache.fory.type.GenericType; import org.apache.fory.type.TypeUtils; +import org.apache.fory.type.Types; import org.apache.fory.util.GraalvmSupport; import org.apache.fory.util.Preconditions; import org.apache.fory.util.StringUtils; @@ -376,7 +377,7 @@ protected void registerJITNotifyCallback() { protected void addCommonImports() { ctx.addImports( Fory.class, MemoryBuffer.class, fory.getRefResolver().getClass(), Platform.class); - ctx.addImports(ClassInfo.class, ClassInfoHolder.class, ClassResolver.class); + ctx.addImports(TypeInfo.class, TypeInfoHolder.class, ClassResolver.class); ctx.addImport(Generated.class); ctx.addImports(LazyInitBeanSerializer.class, EnumSerializer.class); ctx.addImports(Serializer.class, StringSerializer.class); @@ -708,7 +709,7 @@ protected Expression writeForNotNullNonFinalObject( Class clz = getRawType(typeRef); Expression clsExpr = new Invoke(inputObject, "getClass", "cls", CLASS_TYPE); ListExpression writeClassAndObject = new ListExpression(); - Tuple2 classInfoRef = addClassInfoField(clz); + Tuple2 classInfoRef = addTypeInfoField(clz); Expression classInfo = classInfoRef.f0; if (classInfoRef.f1) { writeClassAndObject.add( @@ -716,7 +717,7 @@ protected Expression writeForNotNullNonFinalObject( neq(new Invoke(classInfo, "getCls", CLASS_TYPE), clsExpr), new Assign( classInfo, - inlineInvoke(typeResolverRef, "getClassInfo", classInfoTypeRef, clsExpr)))); + inlineInvoke(typeResolverRef, "getTypeInfo", classInfoTypeRef, clsExpr)))); } writeClassAndObject.add( typeResolver(r -> r.writeClassExpr(typeResolverRef, buffer, classInfo))); @@ -731,17 +732,17 @@ protected Expression writeForNotNullNonFinalObject( ctx, ofHashSet(buffer, inputObject), writeClassAndObject, "writeClassAndObject", false); } - protected Expression writeClassInfo( + protected Expression writeTypeInfo( Expression buffer, Expression clsExpr, Class declaredClass, boolean returnSerializer) { ListExpression writeClassAction = new ListExpression(); - Tuple2 classInfoRef = addClassInfoField(declaredClass); + Tuple2 classInfoRef = addTypeInfoField(declaredClass); Expression classInfo = classInfoRef.f0; writeClassAction.add( new If( neq(inlineInvoke(classInfo, "getCls", CLASS_TYPE), clsExpr), new Assign( classInfo, - inlineInvoke(typeResolverRef, "getClassInfo", classInfoTypeRef, clsExpr)))); + inlineInvoke(typeResolverRef, "getTypeInfo", classInfoTypeRef, clsExpr)))); writeClassAction.add(typeResolver(r -> r.writeClassExpr(typeResolverRef, buffer, classInfo))); if (returnSerializer) { writeClassAction.add( @@ -895,7 +896,7 @@ protected Expression getGenericTypeField(TypeRef typeRef) { * * @return false for tuple field1 if the classinfo doesn't need update. */ - protected Tuple2 addClassInfoField(Class cls) { + protected Tuple2 addTypeInfoField(Class cls) { Expression classInfoExpr; boolean needUpdate = !ReflectionUtils.isMonomorphic(cls); String key; @@ -910,15 +911,15 @@ protected Tuple2 addClassInfoField(Class cls) { } if (!needUpdate) { Expression clsExpr = getClassExpr(cls); - classInfoExpr = inlineInvoke(typeResolverRef, "getClassInfo", classInfoTypeRef, clsExpr); + classInfoExpr = inlineInvoke(typeResolverRef, "getTypeInfo", classInfoTypeRef, clsExpr); // Use `ctx.freshName(cls)` to avoid wrong name for arr type. - String name = ctx.newName(ctx.newName(cls) + "ClassInfo"); - ctx.addField(true, ctx.type(ClassInfo.class), name, classInfoExpr); + String name = ctx.newName(ctx.newName(cls) + "TypeInfo"); + ctx.addField(true, ctx.type(TypeInfo.class), name, classInfoExpr); classInfoRef = Tuple2.of(fieldRef(name, classInfoTypeRef), false); } else { - classInfoExpr = inlineInvoke(typeResolverRef, "nilClassInfo", classInfoTypeRef); - String name = ctx.newName(cls, "ClassInfo"); - ctx.addField(false, ctx.type(ClassInfo.class), name, classInfoExpr); + classInfoExpr = inlineInvoke(typeResolverRef, "nilTypeInfo", classInfoTypeRef); + String name = ctx.newName(cls, "TypeInfo"); + ctx.addField(false, ctx.type(TypeInfo.class), name, classInfoExpr); // Can't use fieldRef, since the field is not final. classInfoRef = Tuple2.of(new Reference(name, classInfoTypeRef), true); } @@ -926,7 +927,7 @@ protected Tuple2 addClassInfoField(Class cls) { return classInfoRef; } - protected Reference addClassInfoHolderField(Class cls) { + protected Reference addTypeInfoHolderField(Class cls) { // Final type need to write classinfo when meta share enabled. String key; if (ReflectionUtils.isMonomorphic(cls)) { @@ -939,36 +940,36 @@ protected Reference addClassInfoHolderField(Class cls) { return reference; } Expression classInfoHolderExpr = - inlineInvoke(typeResolverRef, "nilClassInfoHolder", classInfoHolderTypeRef); - String name = ctx.newName(cls, "ClassInfoHolder"); - ctx.addField(true, ctx.type(ClassInfoHolder.class), name, classInfoHolderExpr); + inlineInvoke(typeResolverRef, "nilTypeInfoHolder", classInfoHolderTypeRef); + String name = ctx.newName(cls, "TypeInfoHolder"); + ctx.addField(true, ctx.type(TypeInfoHolder.class), name, classInfoHolderExpr); // The class info field read only once, no need to shallow. reference = new Reference(name, classInfoHolderTypeRef); sharedFieldMap.put(key, reference); return reference; } - protected Expression readClassInfo(Class cls, Expression buffer) { - return readClassInfo(cls, buffer, true); + protected Expression readTypeInfo(Class cls, Expression buffer) { + return readTypeInfo(cls, buffer, true); } - protected Expression readClassInfo(Class cls, Expression buffer, boolean inlineReadClassInfo) { + protected Expression readTypeInfo(Class cls, Expression buffer, boolean inlineReadTypeInfo) { if (ReflectionUtils.isMonomorphic(cls)) { - Reference classInfoRef = addClassInfoField(cls).f0; - if (inlineReadClassInfo) { + Reference classInfoRef = addTypeInfoField(cls).f0; + if (inlineReadTypeInfo) { return inlineInvoke( - typeResolverRef, "readClassInfo", classInfoTypeRef, buffer, classInfoRef); + typeResolverRef, "readTypeInfo", classInfoTypeRef, buffer, classInfoRef); } else { - return new Invoke(typeResolverRef, "readClassInfo", classInfoTypeRef, buffer, classInfoRef); + return new Invoke(typeResolverRef, "readTypeInfo", classInfoTypeRef, buffer, classInfoRef); } } - Reference classInfoHolderRef = addClassInfoHolderField(cls); - if (inlineReadClassInfo) { + Reference classInfoHolderRef = addTypeInfoHolderField(cls); + if (inlineReadTypeInfo) { return inlineInvoke( - typeResolverRef, "readClassInfo", classInfoTypeRef, buffer, classInfoHolderRef); + typeResolverRef, "readTypeInfo", classInfoTypeRef, buffer, classInfoHolderRef); } else { return new Invoke( - typeResolverRef, "readClassInfo", classInfoTypeRef, buffer, classInfoHolderRef); + typeResolverRef, "readTypeInfo", classInfoTypeRef, buffer, classInfoHolderRef); } } @@ -1006,7 +1007,7 @@ protected Expression serializeForCollection( serializer = getOrCreateSerializer(clz); } else { ListExpression writeClassAction = new ListExpression(); - Tuple2 classInfoRef = addClassInfoField(clz); + Tuple2 classInfoRef = addTypeInfoField(clz); Expression classInfo = classInfoRef.f0; Expression clsExpr = new Invoke(collection, "getClass", "cls", CLASS_TYPE); writeClassAction.add( @@ -1014,7 +1015,7 @@ protected Expression serializeForCollection( neq(new Invoke(classInfo, "getCls", CLASS_TYPE), clsExpr), new Assign( classInfo, - inlineInvoke(typeResolverRef, "getClassInfo", classInfoTypeRef, clsExpr)))); + inlineInvoke(typeResolverRef, "getTypeInfo", classInfoTypeRef, clsExpr)))); writeClassAction.add( typeResolver(r -> r.writeClassExpr(typeResolverRef, buffer, classInfo))); writeClassAction.add( @@ -1025,7 +1026,7 @@ protected Expression serializeForCollection( ctx, ofHashSet(buffer, collection), writeClassAction, - "writeCollectionClassInfo", + "writeCollectionTypeInfo", false); } } else if (!TypeRef.of(CollectionLikeSerializer.class).isSupertypeOf(serializer.type())) { @@ -1204,7 +1205,7 @@ private Tuple2 writeElementsHeader( return Tuple2.of(bitmap, null); } else { Expression elementTypeExpr = getClassExpr(elementType); - Expression classInfoHolder = addClassInfoHolderField(elementType); + Expression classInfoHolder = addTypeInfoHolderField(elementType); Expression bitmap; if (trackingRef) { if (elementType == Object.class) { @@ -1359,7 +1360,7 @@ protected Expression serializeForMap( serializer = getOrCreateSerializer(clz); } else { ListExpression writeClassAction = new ListExpression(); - Tuple2 classInfoRef = addClassInfoField(clz); + Tuple2 classInfoRef = addTypeInfoField(clz); Expression classInfo = classInfoRef.f0; Expression clsExpr = new Invoke(map, "getClass", "cls", CLASS_TYPE); writeClassAction.add( @@ -1367,7 +1368,7 @@ protected Expression serializeForMap( neq(new Invoke(classInfo, "getCls", CLASS_TYPE), clsExpr), new Assign( classInfo, - inlineInvoke(typeResolverRef, "getClassInfo", classInfoTypeRef, clsExpr)))); + inlineInvoke(typeResolverRef, "getTypeInfo", classInfoTypeRef, clsExpr)))); // Note: writeClassExpr is thread safe. writeClassAction.add( typeResolver(r -> r.writeClassExpr(typeResolverRef, buffer, classInfo))); @@ -1376,7 +1377,7 @@ protected Expression serializeForMap( // Spit this into a separate method to avoid method too big to inline. serializer = invokeGenerated( - ctx, ofHashSet(buffer, map), writeClassAction, "writeMapClassInfo", false); + ctx, ofHashSet(buffer, map), writeClassAction, "writeMapTypeInfo", false); } } else if (!MapLikeSerializer.class.isAssignableFrom(serializer.type().getRawType())) { serializer = cast(serializer, TypeRef.of(MapLikeSerializer.class), "mapSerializer"); @@ -1569,7 +1570,7 @@ protected Expression writeChunk( } keySerializer = getOrCreateSerializer(keyTypeRawType); walkPath.add("value:" + valueType); - valueSerializer = writeClassInfo(buffer, valueTypeExpr, valueTypeRawType, true); + valueSerializer = writeTypeInfo(buffer, valueTypeExpr, valueTypeRawType, true); walkPath.removeLast(); chunkHeader = ExpressionUtils.ofInt("chunkHeader", header); expressions.add(chunkHeader); @@ -1584,7 +1585,7 @@ protected Expression writeChunk( } } else if (valueMonomorphic) { walkPath.add("key:" + keyType); - keySerializer = writeClassInfo(buffer, keyTypeExpr, keyTypeRawType, true); + keySerializer = writeTypeInfo(buffer, keyTypeExpr, keyTypeRawType, true); walkPath.removeLast(); valueSerializer = getOrCreateSerializer(valueTypeRawType); int header = VALUE_DECL_TYPE; @@ -1603,10 +1604,10 @@ protected Expression writeChunk( } } else { walkPath.add("key:" + keyType); - keySerializer = writeClassInfo(buffer, keyTypeExpr, keyTypeRawType, true); + keySerializer = writeTypeInfo(buffer, keyTypeExpr, keyTypeRawType, true); walkPath.removeLast(); walkPath.add("value:" + valueType); - valueSerializer = writeClassInfo(buffer, valueTypeExpr, valueTypeRawType, true); + valueSerializer = writeTypeInfo(buffer, valueTypeExpr, valueTypeRawType, true); walkPath.removeLast(); chunkHeader = ExpressionUtils.ofInt("chunkHeader", 0); expressions.add(chunkHeader); @@ -2120,7 +2121,23 @@ protected Expression read( protected Expression readForNotNullNonFinal( Expression buffer, TypeRef typeRef, Expression serializer) { if (serializer == null) { - Expression classInfo = readClassInfo(getRawType(typeRef), buffer); + Expression classInfo; + Class rawType = typeRef.getRawType(); + if (fory.isCompatible() && rawType != Object.class) { + TypeInfo typeInfo = typeResolver(r -> r.getTypeInfo(rawType, false)); + if (typeInfo == null + || (typeInfo.getTypeId() == Types.COMPATIBLE_STRUCT + || typeInfo.getTypeId() == Types.NAMED_COMPATIBLE_STRUCT)) { + String name = ctx.newName(StringUtils.uncapitalize(rawType.getSimpleName()) + "Class"); + Expression clsExpr = staticClassFieldExpr(rawType, name); + classInfo = + inlineInvoke(typeResolverRef, "readTypeInfo", classInfoTypeRef, buffer, clsExpr); + } else { + classInfo = readTypeInfo(getRawType(typeRef), buffer); + } + } else { + classInfo = readTypeInfo(getRawType(typeRef), buffer); + } serializer = inlineInvoke(classInfo, "getSerializer", SERIALIZER_TYPE); } return read(serializer, buffer, OBJECT_TYPE); @@ -2138,7 +2155,7 @@ protected Expression deserializeForCollection( if (isMonomorphic(cls)) { serializer = getOrCreateSerializer(cls); } else { - Expression classInfo = readClassInfo(cls, buffer); + Expression classInfo = readTypeInfo(cls, buffer); serializer = invoke(classInfo, "getSerializer", "collectionSerializer", COLLECTION_SERIALIZER_TYPE); } @@ -2208,7 +2225,7 @@ protected Expression readCollectionCodegen( Literal isDeclTypeFlag = ofInt(CollectionFlags.IS_DECL_ELEMENT_TYPE); Expression isDeclType = eq(new BitAnd(flags, isDeclTypeFlag), isDeclTypeFlag); Invoke serializer = - inlineInvoke(readClassInfo(elemClass, buffer), "getSerializer", SERIALIZER_TYPE); + inlineInvoke(readTypeInfo(elemClass, buffer), "getSerializer", SERIALIZER_TYPE); TypeRef serializerType = getSerializerType(elementType); Expression elemSerializer; // make it in scope of `if(sameElementClass)` boolean maybeDecl = typeResolver(r -> r.isSerializable(elemClass)); @@ -2384,7 +2401,7 @@ protected Expression deserializeForMap( if (isMonomorphic(cls)) { serializer = getOrCreateSerializer(cls); } else { - Expression classInfo = readClassInfo(cls, buffer); + Expression classInfo = readTypeInfo(cls, buffer); serializer = invoke(classInfo, "getSerializer", "mapSerializer", MAP_SERIALIZER_TYPE); } } else { @@ -2634,12 +2651,12 @@ private Expression readOrGetSerializerForDeclaredType( } TypeRef serializerType = getSerializerType(type); if (ReflectionUtils.isAbstract(type) || type.isInterface()) { - return invoke(readClassInfo(type, buffer), "getSerializer", "serializer", serializerType); + return invoke(readTypeInfo(type, buffer), "getSerializer", "serializer", serializerType); } else { return new If( isDeclaredType, getOrCreateSerializer(type), - invokeInline(readClassInfo(type, buffer), "getSerializer", serializerType), + invokeInline(readTypeInfo(type, buffer), "getSerializer", serializerType), false); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java index cce3953c5e..ff81a2511f 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java @@ -60,8 +60,8 @@ import org.apache.fory.reflect.ObjectCreators; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.reflect.TypeRef; -import org.apache.fory.resolver.ClassInfo; -import org.apache.fory.resolver.ClassInfoHolder; +import org.apache.fory.resolver.TypeInfo; +import org.apache.fory.resolver.TypeInfoHolder; import org.apache.fory.type.Descriptor; import org.apache.fory.util.GraalvmSupport; import org.apache.fory.util.Preconditions; @@ -87,8 +87,8 @@ public abstract class CodecBuilder { protected static final String FORY_NAME = "_f_fory"; static TypeRef objectArrayTypeRef = TypeRef.of(Object[].class); static TypeRef bufferTypeRef = TypeRef.of(MemoryBuffer.class); - static TypeRef classInfoTypeRef = TypeRef.of(ClassInfo.class); - static TypeRef classInfoHolderTypeRef = TypeRef.of(ClassInfoHolder.class); + static TypeRef classInfoTypeRef = TypeRef.of(TypeInfo.class); + static TypeRef classInfoHolderTypeRef = TypeRef.of(TypeInfoHolder.class); protected final CodegenContext ctx; protected final TypeRef beanType; @@ -575,8 +575,9 @@ protected Expression staticBeanClassExpr() { } protected Expression staticClassFieldExpr(Class cls, String fieldName) { - Preconditions.checkArgument( - !sourcePublicAccessible(cls), "Public class %s should use class literal instead", cls); + if (sourcePublicAccessible(cls)) { + return Literal.ofClass(cls); + } return getOrCreateField( true, Class.class, diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java b/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java index 3f940e5291..5bce2db3be 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java @@ -26,7 +26,7 @@ import org.apache.fory.codegen.CodeGenerator; import org.apache.fory.codegen.CompileUnit; import org.apache.fory.collection.Tuple3; -import org.apache.fory.meta.ClassDef; +import org.apache.fory.meta.TypeDef; import org.apache.fory.reflect.TypeRef; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.serializer.Serializer; @@ -55,7 +55,7 @@ public static Class> loadOrGenObjectCodecClass( } public static Class> loadOrGenMetaSharedCodecClass( - Fory fory, Class cls, ClassDef classDef) { + Fory fory, Class cls, TypeDef typeDef) { Preconditions.checkNotNull(fory); return loadSerializer( "loadOrGenMetaSharedCodecClass", @@ -63,7 +63,7 @@ public static Class> loadOrGenMetaSharedCodecClass( fory, () -> loadOrGenCodecClass( - cls, fory, new MetaSharedCodecBuilder(TypeRef.of(cls), fory, classDef))); + cls, fory, new MetaSharedCodecBuilder(TypeRef.of(cls), fory, typeDef))); } /** @@ -71,12 +71,12 @@ public static Class> loadOrGenMetaSharedCodecClass( * * @param cls the target class * @param fory the Fory instance - * @param layerClassDef the ClassDef for this layer only + * @param layerTypeDef the TypeDef for this layer only * @param layerMarkerClass the marker class for this layer * @return the generated serializer class */ public static Class> loadOrGenMetaSharedLayerCodecClass( - Class cls, Fory fory, ClassDef layerClassDef, Class layerMarkerClass) { + Class cls, Fory fory, TypeDef layerTypeDef, Class layerMarkerClass) { Preconditions.checkNotNull(fory); return loadSerializer( "loadOrGenMetaSharedLayerCodecClass", @@ -87,7 +87,7 @@ public static Class> loadOrGenMetaSharedLayerCodecCl cls, fory, new MetaSharedLayerCodecBuilder( - TypeRef.of(cls), fory, layerClassDef, layerMarkerClass))); + TypeRef.of(cls), fory, layerTypeDef, layerMarkerClass))); } @SuppressWarnings("unchecked") diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/Generated.java b/java/fory-core/src/main/java/org/apache/fory/builder/Generated.java index 4dadbff23e..740eaed846 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/Generated.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/Generated.java @@ -25,7 +25,7 @@ import java.util.Objects; import org.apache.fory.Fory; import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.meta.ClassDef; +import org.apache.fory.meta.TypeDef; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.serializer.AbstractObjectSerializer; import org.apache.fory.serializer.Serializer; @@ -100,7 +100,7 @@ public GeneratedObjectSerializer(Fory fory, Class cls) { } } - /** Base class for all serializers with meta shared by {@link ClassDef}. */ + /** Base class for all serializers with meta shared by {@link TypeDef}. */ abstract class GeneratedMetaSharedSerializer extends GeneratedSerializer implements Generated { public static final String SERIALIZER_FIELD_NAME = "serializer"; @@ -118,7 +118,7 @@ public void write(MemoryBuffer buffer, Object value) { } /** - * Base class for layer serializers with meta shared by {@link ClassDef}. Unlike {@link + * Base class for layer serializers with meta shared by {@link TypeDef}. Unlike {@link * GeneratedMetaSharedSerializer}, this serializer only handles fields from a single class layer * and does not include parent class fields. */ diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java index f52359eacd..3703503d00 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java @@ -41,7 +41,7 @@ import org.apache.fory.logging.Logger; import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.meta.ClassDef; +import org.apache.fory.meta.TypeDef; import org.apache.fory.reflect.TypeRef; import org.apache.fory.serializer.CodegenSerializer; import org.apache.fory.serializer.MetaSharedSerializer; @@ -61,11 +61,11 @@ import org.apache.fory.util.record.RecordUtils; /** - * A meta-shared compatible deserializer builder based on {@link ClassDef}. This builder will - * compare fields between {@link ClassDef} and class fields, then create serializer to read and - * set/skip corresponding fields to support type forward/backward compatibility. Serializer are - * forward to {@link ObjectCodecBuilder} for now. We can consolidate fields between peers to create - * better serializers to serialize common fields between peers for efficiency. + * A meta-shared compatible deserializer builder based on {@link TypeDef}. This builder will compare + * fields between {@link TypeDef} and class fields, then create serializer to read and set/skip + * corresponding fields to support type forward/backward compatibility. Serializer are forward to + * {@link ObjectCodecBuilder} for now. We can consolidate fields between peers to create better + * serializers to serialize common fields between peers for efficiency. * *

With meta context share enabled and compatible mode, the {@link ObjectCodecBuilder} will take * all non-inner final types as non-final, so that fory can write class definition when write class @@ -79,25 +79,25 @@ public class MetaSharedCodecBuilder extends ObjectCodecBuilder { private static final Logger LOG = LoggerFactory.getLogger(MetaSharedCodecBuilder.class); - private final ClassDef classDef; + private final TypeDef typeDef; private final String defaultValueLanguage; private final DefaultValueUtils.DefaultValueField[] defaultValueFields; - public MetaSharedCodecBuilder(TypeRef beanType, Fory fory, ClassDef classDef) { + public MetaSharedCodecBuilder(TypeRef beanType, Fory fory, TypeDef typeDef) { super(beanType, fory, GeneratedMetaSharedSerializer.class); Preconditions.checkArgument( !fory.getConfig().checkClassVersion(), "Class version check should be disabled when compatible mode is enabled."); - this.classDef = classDef; + this.typeDef = typeDef; Collection descriptors = - fory(f -> MetaSharedSerializer.consolidateFields(f.getTypeResolver(), beanClass, classDef)); + fory(f -> MetaSharedSerializer.consolidateFields(f.getTypeResolver(), beanClass, typeDef)); DescriptorGrouper grouper = typeResolver(r -> r.createDescriptorGrouper(descriptors, false)); List sortedDescriptors = grouper.getSortedDescriptors(); if (org.apache.fory.util.Utils.DEBUG_OUTPUT_ENABLED) { LOG.info( "========== {} sorted descriptors for {} ==========", - classDef.getFieldCount(), - classDef.getClassName()); + typeDef.getFieldCount(), + typeDef.getClassName()); for (Descriptor d : sortedDescriptors) { LOG.info( " {} -> {}, ref {}, nullable {}", @@ -143,10 +143,10 @@ public MetaSharedCodecBuilder(TypeRef beanType, Fory fory, ClassDef classDef) protected String codecSuffix() { // For every class def sent from different peer, if the class def are different, then // a new serializer needs being generated. - Integer id = idGenerator.get(classDef.getId()); + Integer id = idGenerator.get(typeDef.getId()); if (id == null) { synchronized (idGenerator) { - id = idGenerator.computeIfAbsent(classDef.getId(), k -> idGenerator.size()); + id = idGenerator.computeIfAbsent(typeDef.getId(), k -> idGenerator.size()); } } return "MetaShared" + id; diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedLayerCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedLayerCodecBuilder.java index d7f6f15c22..1c0d6faf84 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedLayerCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedLayerCodecBuilder.java @@ -30,7 +30,7 @@ import org.apache.fory.codegen.Expression; import org.apache.fory.codegen.Expression.StaticInvoke; import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.meta.ClassDef; +import org.apache.fory.meta.TypeDef; import org.apache.fory.reflect.TypeRef; import org.apache.fory.serializer.CodegenSerializer; import org.apache.fory.serializer.MetaSharedLayerSerializer; @@ -56,18 +56,18 @@ * @see GeneratedMetaSharedLayerSerializer */ public class MetaSharedLayerCodecBuilder extends ObjectCodecBuilder { - private final ClassDef layerClassDef; + private final TypeDef layerTypeDef; private final Class layerMarkerClass; public MetaSharedLayerCodecBuilder( - TypeRef beanType, Fory fory, ClassDef layerClassDef, Class layerMarkerClass) { + TypeRef beanType, Fory fory, TypeDef layerTypeDef, Class layerMarkerClass) { super(beanType, fory, GeneratedMetaSharedLayerSerializer.class); Preconditions.checkArgument( !fory.getConfig().checkClassVersion(), "Class version check should be disabled when compatible mode is enabled."); - this.layerClassDef = layerClassDef; + this.layerTypeDef = layerTypeDef; this.layerMarkerClass = layerMarkerClass; - Collection descriptors = layerClassDef.getDescriptors(typeResolver, beanClass); + Collection descriptors = layerTypeDef.getDescriptors(typeResolver, beanClass); DescriptorGrouper grouper = typeResolver(r -> r.createDescriptorGrouper(descriptors, false)); objectCodecOptimizer = new ObjectCodecOptimizer(beanClass, grouper, false, ctx); } @@ -79,10 +79,10 @@ public MetaSharedLayerCodecBuilder( protected String codecSuffix() { // For every class def sent from different peer, if the class def are different, then // a new serializer needs being generated. - Integer id = idGenerator.get(layerClassDef.getId()); + Integer id = idGenerator.get(layerTypeDef.getId()); if (id == null) { synchronized (idGenerator) { - id = idGenerator.computeIfAbsent(layerClassDef.getId(), k -> idGenerator.size()); + id = idGenerator.computeIfAbsent(layerTypeDef.getId(), k -> idGenerator.size()); } } return "MetaSharedLayer" + id; diff --git a/java/fory-core/src/main/java/org/apache/fory/collection/LongMap.java b/java/fory-core/src/main/java/org/apache/fory/collection/LongMap.java index fc4f59a8d3..ff8820074d 100644 --- a/java/fory-core/src/main/java/org/apache/fory/collection/LongMap.java +++ b/java/fory-core/src/main/java/org/apache/fory/collection/LongMap.java @@ -307,7 +307,7 @@ public void clear() { hasZeroValue = false; } - public boolean containsKey(int key) { + public boolean containsKey(long key) { if (key == 0) { return hasZeroValue; } diff --git a/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java b/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java index fded0c5ab9..aed359c6aa 100644 --- a/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java @@ -500,7 +500,7 @@ private void finish() { /** * Create Fory and print exception when failed. Many application will create fory as a static - * variable, Fory creation exception will be swallowed by {@link NoClassDefFoundError}. We print + * variable, Fory creation exception will be swallowed by {@link NoTypeDefFoundError}. We print * exception explicitly for better debugging. */ private static Fory newFory(ForyBuilder builder, ClassLoader classLoader) { diff --git a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java index 41fbc2b48b..3f03b08956 100644 --- a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java +++ b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java @@ -673,6 +673,10 @@ public void _unsafeWriteByte(byte value) { writerIndex = newIdx; } + public void writeUint8(int value) { + writeByte((byte) value); + } + public void writeByte(byte value) { final int writerIdx = writerIndex; final int newIdx = writerIdx + 1; @@ -1423,7 +1427,7 @@ public boolean readBoolean() { return UNSAFE.getByte(heapMemory, address + readerIdx) != 0; } - public int readUnsignedByte() { + public int readUint8() { int readerIdx = readerIndex; if (readerIdx > size - 1) { streamReader.fillBuffer(1); @@ -1434,6 +1438,10 @@ public int readUnsignedByte() { return v; } + public int readUnsignedByte() { + return readUint8(); + } + public byte readByte() { int readerIdx = readerIndex; if (readerIdx > size - 1) { diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/ClassSpec.java b/java/fory-core/src/main/java/org/apache/fory/meta/ClassSpec.java index 62f82e4cc3..3d7ab77db5 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/ClassSpec.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/ClassSpec.java @@ -30,6 +30,8 @@ public class ClassSpec { public final boolean isArray; public final int dimension; public final int typeId; + // Stored as unsigned int. -1 (0xffffffff) means "unset". + public final int userTypeId; public Class type; public ClassSpec(Class cls) { @@ -38,6 +40,7 @@ public ClassSpec(Class cls) { cls.isEnum(), cls.isArray(), cls.isArray() ? TypeUtils.getArrayDimensions(cls) : 0, + -1, -1); type = cls; } @@ -48,20 +51,37 @@ public ClassSpec(Class cls, int typeId) { cls.isEnum(), cls.isArray(), cls.isArray() ? TypeUtils.getArrayDimensions(cls) : 0, - typeId); + typeId, + -1); + } + + public ClassSpec(Class cls, int typeId, int userTypeId) { + this( + cls.getName(), + cls.isEnum(), + cls.isArray(), + cls.isArray() ? TypeUtils.getArrayDimensions(cls) : 0, + typeId, + userTypeId); } public ClassSpec(String entireClassName, boolean isEnum, boolean isArray, int dimension) { - this(entireClassName, isEnum, isArray, dimension, -1); + this(entireClassName, isEnum, isArray, dimension, -1, -1); } public ClassSpec( - String entireClassName, boolean isEnum, boolean isArray, int dimension, int typeId) { + String entireClassName, + boolean isEnum, + boolean isArray, + int dimension, + int typeId, + int userTypeId) { this.entireClassName = entireClassName; this.isEnum = isEnum; this.isArray = isArray; this.dimension = dimension; this.typeId = typeId; + this.userTypeId = userTypeId; } @Override diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java index 07753aee22..d42b5460fa 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java @@ -48,8 +48,8 @@ import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.reflect.TypeRef; -import org.apache.fory.resolver.ClassInfo; import org.apache.fory.resolver.ClassResolver; +import org.apache.fory.resolver.TypeInfo; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.resolver.XtypeResolver; import org.apache.fory.serializer.NonexistentClass; @@ -106,10 +106,13 @@ private static FieldType buildFieldType( // This allows @Uint8ArrayType etc. to override the default INT8_ARRAY type typeId = Types.getDescriptorTypeId(resolver.getFory(), field); } else { - ClassInfo info = - isXlang && rawType == Object.class ? null : resolver.getClassInfo(rawType, false); + TypeInfo info = + isXlang && rawType == Object.class ? null : resolver.getTypeInfo(rawType, false); if (info != null) { typeId = info.getTypeId(); + if (Types.isEnumType(typeId)) { + typeId = Types.ENUM; + } } else if (isXlang) { if (rawType.isArray()) { Class componentType = rawType.getComponentType(); @@ -131,7 +134,7 @@ private static FieldType buildFieldType( typeId = Types.UNKNOWN; } } else if (resolver instanceof ClassResolver) { - typeId = ((ClassResolver) resolver).getTypeIdForClassDef(rawType); + typeId = ((ClassResolver) resolver).getTypeIdForTypeDef(rawType); } else { typeId = Types.UNKNOWN; } @@ -163,13 +166,13 @@ private static FieldType buildFieldType( } } - boolean isUnionType = Types.isUnionType(typeId & 0xff); + boolean isUnionType = Types.isUnionType(typeId); if (isUnionType) { typeId = Types.UNION; } - if (Types.isPrimitiveArray(typeId & 0xff)) { - return new RegisteredFieldType(nullable, trackingRef, typeId); + if (Types.isPrimitiveArray(typeId)) { + return new RegisteredFieldType(nullable, trackingRef, typeId, -1); } if (COLLECTION_TYPE.isSupertypeOf(genericType.getTypeRef())) { @@ -204,10 +207,10 @@ private static FieldType buildFieldType( return new UnionFieldType(nullable, trackingRef); } else if (TypeUtils.unwrap(rawType).isPrimitive()) { // unified basic types for xlang and native mode - return new RegisteredFieldType(nullable, trackingRef, typeId); + return new RegisteredFieldType(nullable, trackingRef, typeId, -1); } else { if (rawType.isEnum()) { - return new EnumFieldType(nullable, typeId); + return new EnumFieldType(nullable, Types.ENUM, -1); } if (rawType.isArray()) { Class elemType = rawType.getComponentType(); @@ -215,7 +218,7 @@ private static FieldType buildFieldType( if (elemType.isPrimitive()) { // For xlang mode, use the typeId we already computed above // which respects @Uint8ArrayType etc. annotations - return new RegisteredFieldType(nullable, trackingRef, typeId); + return new RegisteredFieldType(nullable, trackingRef, typeId, -1); } return new CollectionFieldType( typeId, @@ -225,7 +228,7 @@ private static FieldType buildFieldType( } else { // For native mode, use Java class IDs for arrays if (resolver.isRegisteredById(rawType)) { - return new RegisteredFieldType(nullable, trackingRef, typeId); + return new RegisteredFieldType(nullable, trackingRef, typeId, -1); } Tuple2, Integer> arrayComponentInfo = getArrayComponentInfo(rawType); return new ArrayFieldType( @@ -236,12 +239,8 @@ private static FieldType buildFieldType( arrayComponentInfo.f1); } } - if (isXlang - && !Types.isUserDefinedType((byte) typeId) - && resolver.isRegisteredById(rawType)) { - return new RegisteredFieldType(nullable, trackingRef, typeId); - } else if (!isXlang && resolver.isRegisteredById(rawType)) { - return new RegisteredFieldType(nullable, trackingRef, typeId); + if (resolver.isRegisteredById(rawType)) { + return new RegisteredFieldType(nullable, trackingRef, typeId, -1); } else { return new ObjectFieldType(typeId, nullable, trackingRef); } @@ -249,14 +248,23 @@ private static FieldType buildFieldType( } public abstract static class FieldType implements Serializable { + private static final int KIND_OBJECT = 0; + private static final int KIND_MAP = 1; + private static final int KIND_COLLECTION = 2; + private static final int KIND_ARRAY = 3; + private static final int KIND_ENUM = 4; + private static final int KIND_REGISTERED = 5; + protected final int typeId; + protected final int userTypeId; protected final boolean nullable; protected final boolean trackingRef; - public FieldType(int typeId, boolean nullable, boolean trackingRef) { + public FieldType(int typeId, int userTypeId, boolean nullable, boolean trackingRef) { this.trackingRef = trackingRef; this.nullable = nullable; this.typeId = typeId; + this.userTypeId = userTypeId; } public boolean trackingRef() { @@ -267,6 +275,25 @@ public boolean nullable() { return nullable; } + private int typeKind() { + if (this instanceof RegisteredFieldType) { + return KIND_REGISTERED; + } + if (this instanceof EnumFieldType) { + return KIND_ENUM; + } + if (this instanceof ArrayFieldType) { + return KIND_ARRAY; + } + if (this instanceof CollectionFieldType) { + return KIND_COLLECTION; + } + if (this instanceof MapFieldType) { + return KIND_MAP; + } + return KIND_OBJECT; + } + /** * Convert a serializable field type to type token. If field type is a generic type with * generics, the generics will be built up recursively. The final leaf object type will be built @@ -297,34 +324,36 @@ public int hashCode() { /** Write field type info. */ public void write(MemoryBuffer buffer, boolean writeHeader) { - // Header format for nested types (writeHeader=true): + // Header format: // - bit 0: trackingRef // - bit 1: nullable - // - bits 2+: typeId + // - bits 2+: type kind byte header = (byte) ((nullable ? 0b10 : 0) | (trackingRef ? 0b1 : 0)); + byte kindHeader = (byte) ((typeKind() << 2) | (writeHeader ? header : 0)); if (this instanceof RegisteredFieldType) { int typeId = ((RegisteredFieldType) this).getTypeId(); - buffer.writeVarUint32Small7(writeHeader ? ((5 + typeId) << 2) | header : 5 + typeId); + buffer.writeUint8(kindHeader); + buffer.writeUint8(typeId); } else if (this instanceof EnumFieldType) { - buffer.writeVarUint32Small7(writeHeader ? ((4) << 2) | header : 4); + buffer.writeUint8(kindHeader); } else if (this instanceof ArrayFieldType) { ArrayFieldType arrayFieldType = (ArrayFieldType) this; - buffer.writeVarUint32Small7(writeHeader ? ((3) << 2) | header : 3); + buffer.writeUint8(kindHeader); buffer.writeVarUint32Small7(arrayFieldType.getDimensions()); (arrayFieldType).getComponentType().write(buffer); } else if (this instanceof CollectionFieldType) { - buffer.writeVarUint32Small7(writeHeader ? ((2) << 2) | header : 2); + buffer.writeUint8(kindHeader); // TODO remove it when new collection deserialization jit finished. ((CollectionFieldType) this).getElementType().write(buffer); } else if (this instanceof MapFieldType) { - buffer.writeVarUint32Small7(writeHeader ? ((1) << 2) | header : 1); + buffer.writeUint8(kindHeader); // TODO remove it when new map deserialization jit finished. MapFieldType mapFieldType = (MapFieldType) this; mapFieldType.getKeyType().write(buffer); mapFieldType.getValueType().write(buffer); } else { Preconditions.checkArgument(this instanceof ObjectFieldType); - buffer.writeVarUint32Small7(writeHeader ? header : 0); + buffer.writeUint8(kindHeader); } } @@ -333,15 +362,15 @@ public void write(MemoryBuffer buffer) { } public static FieldType read(MemoryBuffer buffer, TypeResolver resolver) { - // Header format for nested types: + // Header format: // - bit 0: trackingRef // - bit 1: nullable - // - bits 2+: typeId - int header = buffer.readVarUint32Small7(); + // - bits 2+: type kind + int header = buffer.readUint8(); boolean trackingRef = (header & 0b1) != 0; boolean nullable = (header & 0b10) != 0; - int typeId = header >>> 2; - return read(buffer, resolver, nullable, trackingRef, typeId); + int kind = header >>> 2; + return read(buffer, resolver, nullable, trackingRef, kind); } /** Read field type info. */ @@ -350,38 +379,42 @@ public static FieldType read( TypeResolver resolver, boolean nullable, boolean trackingRef, - int typeId) { - if (typeId == 0) { - return new ObjectFieldType(-1, nullable, trackingRef); - } else if (typeId == 1) { + int kind) { + if (kind == 0) { + return new ObjectFieldType(Types.UNKNOWN, nullable, trackingRef); + } else if (kind == 1) { return new MapFieldType( -1, nullable, trackingRef, read(buffer, resolver), read(buffer, resolver)); - } else if (typeId == 2) { + } else if (kind == 2) { return new CollectionFieldType(-1, nullable, trackingRef, read(buffer, resolver)); - } else if (typeId == 3) { + } else if (kind == 3) { int dims = buffer.readVarUint32Small7(); return new ArrayFieldType(-1, nullable, trackingRef, read(buffer, resolver), dims); - } else if (typeId == 4) { - return new EnumFieldType(nullable, -1); + } else if (kind == 4) { + return new EnumFieldType(nullable, -1, -1); + } else if (kind == 5) { + int actualTypeId = buffer.readUint8(); + return new RegisteredFieldType(nullable, trackingRef, actualTypeId, -1); } else { - return new RegisteredFieldType(nullable, trackingRef, (typeId - 5)); + throw new IllegalStateException("Unexpected field type kind: " + kind); } } public final void xwrite(MemoryBuffer buffer, boolean writeFlags) { - int typeId = this.typeId; if (writeFlags) { - typeId = (typeId << 2); + int typeId = (this.typeId << 2); if (nullable) { typeId |= 0b10; } if (trackingRef) { typeId |= 0b1; } + buffer.writeVarUint32Small7(typeId); + } else { + buffer.writeUint8(this.typeId); } - buffer.writeVarUint32Small7(typeId); // Use the original typeId for the switch (not the one with flags) - switch (this.typeId & 0xff) { + switch (this.typeId) { case Types.LIST: case Types.SET: ((CollectionFieldType) this).getElementType().xwrite(buffer, true); @@ -411,7 +444,7 @@ public static FieldType xread( int typeId, boolean nullable, boolean trackingRef) { - switch (typeId & 0xff) { + switch (typeId) { case Types.LIST: case Types.SET: return new CollectionFieldType(typeId, nullable, trackingRef, xread(buffer, resolver)); @@ -419,8 +452,7 @@ public static FieldType xread( return new MapFieldType( typeId, nullable, trackingRef, xread(buffer, resolver), xread(buffer, resolver)); case Types.ENUM: - case Types.NAMED_ENUM: - return new EnumFieldType(nullable, typeId); + return new EnumFieldType(nullable, typeId, -1); case Types.UNION: return new UnionFieldType(nullable, trackingRef); case Types.UNKNOWN: @@ -430,18 +462,18 @@ public static FieldType xread( if (Types.isPrimitiveType(typeId)) { // unsigned types share same class with signed numeric types, so unsigned types are // not registered. - return new RegisteredFieldType(nullable, trackingRef, typeId); + return new RegisteredFieldType(nullable, trackingRef, typeId, -1); } if (!Types.isUserDefinedType((byte) typeId)) { - ClassInfo classInfo = resolver.getXtypeInfo(typeId); - if (classInfo == null) { + TypeInfo typeInfo = resolver.getXtypeInfo(typeId); + if (typeInfo == null) { // Type not registered locally - this can happen in compatible mode // when remote sends a type ID that's not registered here. // Fall back to ObjectFieldType to handle gracefully. LOG.warn("Type {} not registered locally, treating as ObjectFieldType", typeId); return new ObjectFieldType(typeId, nullable, trackingRef); } - return new RegisteredFieldType(nullable, trackingRef, typeId); + return new RegisteredFieldType(nullable, trackingRef, typeId, -1); } else { return new ObjectFieldType(typeId, nullable, trackingRef); } @@ -452,8 +484,8 @@ public static FieldType xread( /** Class for field type which is registered. */ public static class RegisteredFieldType extends FieldType { - public RegisteredFieldType(boolean nullable, boolean trackingRef, int typeId) { - super(typeId, nullable, trackingRef); + public RegisteredFieldType(boolean nullable, boolean trackingRef, int typeId, int userTypeId) { + super(typeId, userTypeId, nullable, trackingRef); Preconditions.checkArgument(typeId > 0); } @@ -464,10 +496,13 @@ public int getTypeId() { @Override public TypeRef toTypeToken(TypeResolver resolver, TypeRef declared) { Class cls; - int internalTypeId = typeId & 0xff; + int internalTypeId = typeId; + if (declared != null && internalTypeId == Types.ENUM && declared.getRawType().isEnum()) { + return TypeRef.of(declared.getRawType(), new TypeExtMeta(typeId, nullable, trackingRef)); + } if (Types.isPrimitiveType(internalTypeId)) { if (declared != null) { - ClassInfo declaredInfo = resolver.getClassInfo(declared.getRawType(), false); + TypeInfo declaredInfo = resolver.getTypeInfo(declared.getRawType(), false); if (declaredInfo != null && declaredInfo.getTypeId() == typeId) { return TypeRef.of( declared.getRawType(), new TypeExtMeta(typeId, nullable, trackingRef)); @@ -509,8 +544,17 @@ public TypeRef toTypeToken(TypeResolver resolver, TypeRef declared) { return TypeRef.of(cls, new TypeExtMeta(typeId, nullable, trackingRef)); } } + if (Types.isUserDefinedType((byte) internalTypeId)) { + if (declared != null) { + return TypeRef.of(declared.getRawType(), new TypeExtMeta(typeId, nullable, trackingRef)); + } + LOG.warn("Class {} not registered, take it as Struct type for deserialization.", typeId); + boolean isEnum = internalTypeId == Types.ENUM; + cls = NonexistentClass.getNonexistentClass(isEnum, 0, resolver.getFory().isShareMeta()); + return TypeRef.of(cls, new TypeExtMeta(typeId, nullable, trackingRef)); + } if (resolver instanceof XtypeResolver) { - ClassInfo xtypeInfo = ((XtypeResolver) resolver).getXtypeInfo(typeId); + TypeInfo xtypeInfo = ((XtypeResolver) resolver).getXtypeInfo(typeId); Preconditions.checkNotNull(xtypeInfo); cls = xtypeInfo.getCls(); } else { @@ -518,7 +562,8 @@ public TypeRef toTypeToken(TypeResolver resolver, TypeRef declared) { } if (cls == null) { LOG.warn("Class {} not registered, take it as Struct type for deserialization.", typeId); - cls = NonexistentClass.NonexistentMetaShared.class; + boolean isEnum = internalTypeId == Types.ENUM; + cls = NonexistentClass.getNonexistentClass(isEnum, 0, resolver.getFory().isShareMeta()); } return TypeRef.of(cls, new TypeExtMeta(typeId, nullable, trackingRef)); } @@ -553,12 +598,12 @@ public boolean equals(Object o) { return false; } RegisteredFieldType that = (RegisteredFieldType) o; - return typeId == that.typeId; + return typeId == that.typeId && userTypeId == that.userTypeId; } @Override public int hashCode() { - return Objects.hash(super.hashCode(), typeId); + return Objects.hash(super.hashCode(), typeId, userTypeId); } @Override @@ -570,6 +615,8 @@ public String toString() { + trackingRef() + ", typeId=" + typeId + + ", userTypeId=" + + userTypeId + '}'; } } @@ -641,7 +688,7 @@ public static class CollectionFieldType extends FieldType { public CollectionFieldType( int typeId, boolean nullable, boolean trackingRef, FieldType elementType) { - super(typeId, nullable, trackingRef); + super(typeId, -1, nullable, trackingRef); this.elementType = elementType; } @@ -747,7 +794,7 @@ public static class MapFieldType extends FieldType { public MapFieldType( int typeId, boolean nullable, boolean trackingRef, FieldType keyType, FieldType valueType) { - super(typeId, nullable, trackingRef); + super(typeId, -1, nullable, trackingRef); this.keyType = keyType; this.valueType = valueType; } @@ -825,8 +872,8 @@ public String toString() { } public static class EnumFieldType extends FieldType { - public EnumFieldType(boolean nullable, int typeId) { - super(typeId, nullable, false); + public EnumFieldType(boolean nullable, int typeId, int userTypeId) { + super(typeId, userTypeId, nullable, false); } @Override @@ -844,7 +891,14 @@ public String getTypeName(TypeResolver resolver, TypeRef typeRef) { @Override public String toString() { - return "EnumFieldType{" + "typeId=" + typeId + ", nullable=" + nullable + '}'; + return "EnumFieldType{" + + "typeId=" + + typeId + + ", userTypeId=" + + userTypeId + + ", nullable=" + + nullable + + '}'; } } @@ -858,7 +912,7 @@ public ArrayFieldType( boolean trackingRef, FieldType componentType, int dimensions) { - super(typeId, nullable, trackingRef); + super(typeId, -1, nullable, trackingRef); this.componentType = componentType; this.dimensions = dimensions; } @@ -938,7 +992,7 @@ public String toString() { public static class ObjectFieldType extends FieldType { public ObjectFieldType(int typeId, boolean nullable, boolean trackingRef) { - super(typeId, nullable, trackingRef); + super(typeId, -1, nullable, trackingRef); } @Override @@ -981,7 +1035,7 @@ public String toString() { public static class UnionFieldType extends FieldType { public UnionFieldType(boolean nullable, boolean trackingRef) { - super(Types.UNION, nullable, trackingRef); + super(Types.UNION, -1, nullable, trackingRef); } @Override diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDefDecoder.java b/java/fory-core/src/main/java/org/apache/fory/meta/NativeTypeDefDecoder.java similarity index 74% rename from java/fory-core/src/main/java/org/apache/fory/meta/ClassDefDecoder.java rename to java/fory-core/src/main/java/org/apache/fory/meta/NativeTypeDefDecoder.java index a756fb246e..49c7c5dd18 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDefDecoder.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/NativeTypeDefDecoder.java @@ -19,14 +19,14 @@ package org.apache.fory.meta; -import static org.apache.fory.meta.ClassDef.COMPRESS_META_FLAG; -import static org.apache.fory.meta.ClassDef.HAS_FIELDS_META_FLAG; -import static org.apache.fory.meta.ClassDef.META_SIZE_MASKS; -import static org.apache.fory.meta.ClassDefEncoder.BIG_NAME_THRESHOLD; -import static org.apache.fory.meta.ClassDefEncoder.NUM_CLASS_THRESHOLD; import static org.apache.fory.meta.Encoders.fieldNameEncodings; import static org.apache.fory.meta.Encoders.pkgEncodings; import static org.apache.fory.meta.Encoders.typeNameEncodings; +import static org.apache.fory.meta.NativeTypeDefEncoder.BIG_NAME_THRESHOLD; +import static org.apache.fory.meta.NativeTypeDefEncoder.NUM_CLASS_THRESHOLD; +import static org.apache.fory.meta.TypeDef.COMPRESS_META_FLAG; +import static org.apache.fory.meta.TypeDef.HAS_FIELDS_META_FLAG; +import static org.apache.fory.meta.TypeDef.META_SIZE_MASKS; import java.util.ArrayList; import java.util.List; @@ -41,12 +41,12 @@ import org.apache.fory.util.Preconditions; /** - * An decoder which decode binary into {@link ClassDef}. See spec documentation: + * An decoder which decode binary into {@link TypeDef}. See spec documentation: * docs/specification/java_serialization_spec.md ... */ -class ClassDefDecoder { - static Tuple2 decodeClassDefBuf( +class NativeTypeDefDecoder { + static Tuple2 decodeTypeDefBuf( MemoryBuffer inputBuffer, TypeResolver resolver, long id) { MemoryBuffer encoded = MemoryBuffer.newHeapBuffer(64); encoded.writeInt64(id); @@ -56,21 +56,21 @@ static Tuple2 decodeClassDefBuf( encoded.writeVarUint32(moreSize); size += moreSize; } - byte[] encodedClassDef = inputBuffer.readBytes(size); - encoded.writeBytes(encodedClassDef); + byte[] encodedTypeDef = inputBuffer.readBytes(size); + encoded.writeBytes(encodedTypeDef); if ((id & COMPRESS_META_FLAG) != 0) { - encodedClassDef = - resolver.getFory().getConfig().getMetaCompressor().decompress(encodedClassDef, 0, size); + encodedTypeDef = + resolver.getFory().getConfig().getMetaCompressor().decompress(encodedTypeDef, 0, size); } - return Tuple2.of(encodedClassDef, encoded.getBytes(0, encoded.writerIndex())); + return Tuple2.of(encodedTypeDef, encoded.getBytes(0, encoded.writerIndex())); } - public static ClassDef decodeClassDef(ClassResolver resolver, MemoryBuffer buffer, long id) { - Tuple2 decoded = decodeClassDefBuf(buffer, resolver, id); - MemoryBuffer classDefBuf = MemoryBuffer.fromByteArray(decoded.f0); - int numClasses = classDefBuf.readByte(); + public static TypeDef decodeTypeDef(ClassResolver resolver, MemoryBuffer buffer, long id) { + Tuple2 decoded = decodeTypeDefBuf(buffer, resolver, id); + MemoryBuffer typeDefBuf = MemoryBuffer.fromByteArray(decoded.f0); + int numClasses = typeDefBuf.readByte(); if (numClasses == NUM_CLASS_THRESHOLD) { - numClasses += classDefBuf.readVarUint32Small7(); + numClasses += typeDefBuf.readVarUint32Small7(); } numClasses += 1; String className; @@ -79,28 +79,35 @@ public static ClassDef decodeClassDef(ClassResolver resolver, MemoryBuffer buffe for (int i = 0; i < numClasses; i++) { // | num fields + register flag | header + package name | header + class name // | header + type id + field name | next field info | ... | - int currentClassHeader = classDefBuf.readVarUint32Small7(); + int currentClassHeader = typeDefBuf.readVarUint32Small7(); boolean isRegistered = (currentClassHeader & 0b1) != 0; int numFields = currentClassHeader >>> 1; if (isRegistered) { - int typeId = classDefBuf.readVarUint32Small7(); - Class cls = resolver.getRegisteredClassByTypeId(typeId); + int typeId = typeDefBuf.readUint8(); + int userTypeId = -1; + if (Types.isUserTypeRegisteredById(typeId)) { + userTypeId = typeDefBuf.readVarUint32(); + } + Class cls = resolver.getRegisteredClassByTypeId(typeId, userTypeId); if (cls == null) { - classSpec = new ClassSpec(NonexistentClass.NonexistentMetaShared.class, typeId); + classSpec = + new ClassSpec(NonexistentClass.NonexistentMetaShared.class, typeId, userTypeId); className = classSpec.entireClassName; } else { className = cls.getName(); - classSpec = new ClassSpec(cls, typeId); + classSpec = new ClassSpec(cls, typeId, userTypeId); } } else { - String pkg = readPkgName(classDefBuf); - String typeName = readTypeName(classDefBuf); + String pkg = readPkgName(typeDefBuf); + String typeName = readTypeName(typeDefBuf); ClassSpec decodedSpec = Encoders.decodePkgAndClass(pkg, typeName); className = decodedSpec.entireClassName; if (resolver.isRegisteredByName(className)) { Class cls = resolver.getRegisteredClass(className); className = cls.getName(); - classSpec = new ClassSpec(cls, resolver.getTypeIdForClassDef(cls)); + classSpec = + new ClassSpec( + cls, resolver.getTypeIdForTypeDef(cls), resolver.getUserTypeIdForTypeDef(cls)); } else { Class cls = resolver.loadClassForMeta( @@ -121,22 +128,23 @@ public static ClassDef decodeClassDef(ClassResolver resolver, MemoryBuffer buffe decodedSpec.isEnum, decodedSpec.isArray, decodedSpec.dimension, - typeId); + typeId, + -1); classSpec.type = cls; className = classSpec.entireClassName; } else { - int typeId = resolver.getTypeIdForClassDef(cls); - classSpec = new ClassSpec(cls, typeId); + int typeId = resolver.getTypeIdForTypeDef(cls); + classSpec = new ClassSpec(cls, typeId, resolver.getUserTypeIdForTypeDef(cls)); className = classSpec.entireClassName; } } } - List fieldInfos = readFieldsInfo(classDefBuf, resolver, className, numFields); + List fieldInfos = readFieldsInfo(typeDefBuf, resolver, className, numFields); classFields.addAll(fieldInfos); } Preconditions.checkNotNull(classSpec); boolean hasFieldsMeta = (id & HAS_FIELDS_META_FLAG) != 0; - return new ClassDef(classSpec, classFields, hasFieldsMeta, id, decoded.f1); + return new TypeDef(classSpec, classFields, hasFieldsMeta, id, decoded.f1); } private static List readFieldsInfo( @@ -168,9 +176,10 @@ private static List readFieldsInfo( boolean nullable = (header & 0b010) != 0; boolean trackingRef = (header & 0b001) != 0; - int typeId = buffer.readVarUint32Small14(); + int kindHeader = buffer.readUint8(); + int kind = kindHeader >>> 2; FieldType fieldType = - FieldTypes.FieldType.read(buffer, resolver, nullable, trackingRef, typeId); + FieldTypes.FieldType.read(buffer, resolver, nullable, trackingRef, kind); if (useTagID) { fieldInfos.add(new FieldInfo(className, fieldName, fieldType, tagId)); @@ -183,7 +192,7 @@ private static List readFieldsInfo( static String readPkgName(MemoryBuffer buffer) { // - Package name encoding(omitted when class is registered): - // - encoding algorithm: `UTF8/ALL_TO_LOWER_SPECIAL/LOWER_UPPER_DIGIT_SPECIAL` + // - encoding algorithm: `UTF_8/ALL_TO_LOWER_SPECIAL/LOWER_UPPER_DIGIT_SPECIAL` // - Header: `6 bits size | 2 bits encoding flags`. // The `6 bits size: 0~63` will be used to indicate size `0~63`, // the value `63` the size need more byte to read, the encoding will encode `size - 63` as @@ -194,7 +203,7 @@ static String readPkgName(MemoryBuffer buffer) { static String readTypeName(MemoryBuffer buffer) { // - Class name encoding(omitted when class is registered): // - encoding algorithm: - // `UTF8/LOWER_UPPER_DIGIT_SPECIAL/FIRST_TO_LOWER_SPECIAL/ALL_TO_LOWER_SPECIAL` + // `UTF_8/LOWER_UPPER_DIGIT_SPECIAL/FIRST_TO_LOWER_SPECIAL/ALL_TO_LOWER_SPECIAL` // - header: `6 bits size | 2 bits encoding flags`. // The `6 bits size: 0~63` will be used to indicate size `0~63`, // the value `63` the size need more byte to read, the encoding will encode `size - 63` as diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDefEncoder.java b/java/fory-core/src/main/java/org/apache/fory/meta/NativeTypeDefEncoder.java similarity index 82% rename from java/fory-core/src/main/java/org/apache/fory/meta/ClassDefEncoder.java rename to java/fory-core/src/main/java/org/apache/fory/meta/NativeTypeDefEncoder.java index 49cab8c4f0..2933488b69 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDefEncoder.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/NativeTypeDefEncoder.java @@ -19,13 +19,13 @@ package org.apache.fory.meta; -import static org.apache.fory.meta.ClassDef.COMPRESS_META_FLAG; -import static org.apache.fory.meta.ClassDef.HAS_FIELDS_META_FLAG; -import static org.apache.fory.meta.ClassDef.META_SIZE_MASKS; -import static org.apache.fory.meta.ClassDef.NUM_HASH_BITS; import static org.apache.fory.meta.Encoders.fieldNameEncodingsList; import static org.apache.fory.meta.Encoders.pkgEncodingsList; import static org.apache.fory.meta.Encoders.typeNameEncodingsList; +import static org.apache.fory.meta.TypeDef.COMPRESS_META_FLAG; +import static org.apache.fory.meta.TypeDef.HAS_FIELDS_META_FLAG; +import static org.apache.fory.meta.TypeDef.META_SIZE_MASKS; +import static org.apache.fory.meta.TypeDef.NUM_HASH_BITS; import java.lang.reflect.Field; import java.util.ArrayList; @@ -47,18 +47,32 @@ import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.type.DescriptorGrouper; +import org.apache.fory.type.Types; import org.apache.fory.util.MurmurHash3; /** - * An encoder which encode {@link ClassDef} into binary. See spec documentation: + * An encoder which encode {@link TypeDef} into binary. See spec documentation: * docs/specification/java_serialization_spec.md ... */ @Internal -public class ClassDefEncoder { +public class NativeTypeDefEncoder { // a flag to mark a type is not struct. static final int NUM_CLASS_THRESHOLD = 0b1111; + private static boolean needsUserTypeId(int typeId) { + switch (typeId) { + case Types.ENUM: + case Types.STRUCT: + case Types.COMPATIBLE_STRUCT: + case Types.EXT: + case Types.TYPED_UNION: + return true; + default: + return false; + } + } + static List buildFields(Fory fory, Class cls, boolean resolveParent) { DescriptorGrouper descriptorGrouper = fory.getClassResolver() @@ -131,13 +145,13 @@ public static List buildFieldsInfo(TypeResolver resolver, List } /** Build class definition from fields of class. */ - static ClassDef buildClassDef( + static TypeDef buildTypeDef( ClassResolver classResolver, Class type, List fields, boolean hasFieldsMeta) { - return buildClassDefWithFieldInfos( + return buildTypeDefWithFieldInfos( classResolver, type, buildFieldsInfo(classResolver, fields), hasFieldsMeta); } - public static ClassDef buildClassDefWithFieldInfos( + public static TypeDef buildTypeDefWithFieldInfos( ClassResolver classResolver, Class type, List fieldInfos, @@ -145,28 +159,29 @@ public static ClassDef buildClassDefWithFieldInfos( Map> classLayers = getClassFields(type, fieldInfos); fieldInfos = new ArrayList<>(fieldInfos.size()); classLayers.values().forEach(fieldInfos::addAll); - MemoryBuffer encodeClassDef = encodeClassDef(classResolver, type, classLayers, hasFieldsMeta); - byte[] classDefBytes = encodeClassDef.getBytes(0, encodeClassDef.writerIndex()); - int typeId = classResolver.getTypeIdForClassDef(type); - ClassSpec classSpec = new ClassSpec(type, typeId); - return new ClassDef( - classSpec, fieldInfos, hasFieldsMeta, encodeClassDef.getInt64(0), classDefBytes); + MemoryBuffer encodeTypeDef = encodeTypeDef(classResolver, type, classLayers, hasFieldsMeta); + byte[] typeDefBytes = encodeTypeDef.getBytes(0, encodeTypeDef.writerIndex()); + int typeId = classResolver.getTypeIdForTypeDef(type); + int userTypeId = classResolver.getUserTypeIdForTypeDef(type); + ClassSpec classSpec = new ClassSpec(type, typeId, userTypeId); + return new TypeDef( + classSpec, fieldInfos, hasFieldsMeta, encodeTypeDef.getInt64(0), typeDefBytes); } // see spec documentation: docs/specification/java_serialization_spec.md // https://fory.apache.org/docs/specification/fory_java_serialization_spec - public static MemoryBuffer encodeClassDef( + public static MemoryBuffer encodeTypeDef( ClassResolver classResolver, Class type, Map> classLayers, boolean hasFieldsMeta) { - MemoryBuffer classDefBuf = MemoryBuffer.newHeapBuffer(128); + MemoryBuffer typeDefBuf = MemoryBuffer.newHeapBuffer(128); int numClasses = classLayers.size() - 1; // num class must be greater than 0 if (numClasses >= NUM_CLASS_THRESHOLD) { - classDefBuf.writeByte(NUM_CLASS_THRESHOLD); - classDefBuf.writeVarUint32Small7(numClasses - NUM_CLASS_THRESHOLD); + typeDefBuf.writeByte(NUM_CLASS_THRESHOLD); + typeDefBuf.writeVarUint32Small7(numClasses - NUM_CLASS_THRESHOLD); } else { - classDefBuf.writeByte(numClasses); + typeDefBuf.writeByte(numClasses); } for (Map.Entry> entry : classLayers.entrySet()) { String className = entry.getKey(); @@ -177,10 +192,15 @@ public static MemoryBuffer encodeClassDef( int currentClassHeader = (fields.size() << 1); if (classResolver.isRegisteredById(currentType)) { currentClassHeader |= 1; - classDefBuf.writeVarUint32Small7(currentClassHeader); - classDefBuf.writeVarUint32Small7(classResolver.getTypeIdForClassDef(currentType)); + typeDefBuf.writeVarUint32Small7(currentClassHeader); + int typeId = classResolver.getTypeIdForTypeDef(currentType); + typeDefBuf.writeUint8(typeId); + if (needsUserTypeId(typeId)) { + int userTypeId = classResolver.getUserTypeIdForTypeDef(currentType); + typeDefBuf.writeVarUint32(userTypeId); + } } else { - classDefBuf.writeVarUint32Small7(currentClassHeader); + typeDefBuf.writeVarUint32Small7(currentClassHeader); String ns, typename; if (classResolver.isRegisteredByName(currentType)) { Tuple2 nameTuple = classResolver.getRegisteredNameTuple(currentType); @@ -191,24 +211,24 @@ public static MemoryBuffer encodeClassDef( ns = encoded.f0; typename = encoded.f1; } - writePkgName(classDefBuf, ns); - writeTypeName(classDefBuf, typename); + writePkgName(typeDefBuf, ns); + writeTypeName(typeDefBuf, typename); } - writeFieldsInfo(classDefBuf, fields); + writeFieldsInfo(typeDefBuf, fields); } byte[] compressed = classResolver .getFory() .getConfig() .getMetaCompressor() - .compress(classDefBuf.getHeapMemory(), 0, classDefBuf.writerIndex()); + .compress(typeDefBuf.getHeapMemory(), 0, typeDefBuf.writerIndex()); boolean isCompressed = false; - if (compressed.length < classDefBuf.writerIndex()) { + if (compressed.length < typeDefBuf.writerIndex()) { isCompressed = true; - classDefBuf = MemoryBuffer.fromByteArray(compressed); - classDefBuf.writerIndex(compressed.length); + typeDefBuf = MemoryBuffer.fromByteArray(compressed); + typeDefBuf.writerIndex(compressed.length); } - return prependHeader(classDefBuf, isCompressed, hasFieldsMeta); + return prependHeader(typeDefBuf, isCompressed, hasFieldsMeta); } static MemoryBuffer prependHeader( @@ -282,7 +302,7 @@ static void writeFieldsInfo(MemoryBuffer buffer, List fields) { // `3 bits size + 2 bits field name encoding + nullability flag + ref tracking flag` int header = ((fieldType.nullable() ? 1 : 0) << 1); header |= ((fieldType.trackingRef() ? 1 : 0)); - // Encoding `UTF8/ALL_TO_LOWER_SPECIAL/LOWER_UPPER_DIGIT_SPECIAL/TAG_ID` + // Encoding `UTF_8/ALL_TO_LOWER_SPECIAL/LOWER_UPPER_DIGIT_SPECIAL/TAG_ID` MetaString metaString = Encoders.encodeFieldName(fieldInfo.getFieldName()); int encodingFlags = fieldNameEncodingsList.indexOf(metaString.getEncoding()); byte[] encoded = metaString.getBytes(); @@ -310,7 +330,7 @@ static void writeFieldsInfo(MemoryBuffer buffer, List fields) { static void writePkgName(MemoryBuffer buffer, String pkg) { // - Package name encoding(omitted when class is registered): - // - encoding algorithm: `UTF8/ALL_TO_LOWER_SPECIAL/LOWER_UPPER_DIGIT_SPECIAL` + // - encoding algorithm: `UTF_8/ALL_TO_LOWER_SPECIAL/LOWER_UPPER_DIGIT_SPECIAL` // - Header: `6 bits size | 2 bits encoding flags`. // The `6 bits size: 0~63` will be used to indicate size `0~62`, // the value `63` the size need more byte to read, the encoding will encode `size - 62` as @@ -323,7 +343,7 @@ static void writePkgName(MemoryBuffer buffer, String pkg) { static void writeTypeName(MemoryBuffer buffer, String typeName) { // - Class name encoding(omitted when class is registered): // - encoding algorithm: - // `UTF8/LOWER_UPPER_DIGIT_SPECIAL/FIRST_TO_LOWER_SPECIAL/ALL_TO_LOWER_SPECIAL` + // `UTF_8/LOWER_UPPER_DIGIT_SPECIAL/FIRST_TO_LOWER_SPECIAL/ALL_TO_LOWER_SPECIAL` // - header: `6 bits size | 2 bits encoding flags`. // The `6 bits size: 0~63` will be used to indicate size `1~64`, // the value `63` the size need more byte to read, the encoding will encode `size - 63` as diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java similarity index 84% rename from java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java rename to java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java index 821d19c13b..fdf2b9a809 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java @@ -19,7 +19,7 @@ package org.apache.fory.meta; -import static org.apache.fory.meta.ClassDefEncoder.buildFields; +import static org.apache.fory.meta.NativeTypeDefEncoder.buildFields; import java.io.ObjectStreamClass; import java.io.Serializable; @@ -66,8 +66,8 @@ * @see ForyBuilder#withMetaShare * @see ReflectionUtils#getFieldOffset */ -public class ClassDef implements Serializable { - private static final Logger LOG = LoggerFactory.getLogger(ClassDef.class); +public class TypeDef implements Serializable { + private static final Logger LOG = LoggerFactory.getLogger(TypeDef.class); static final int COMPRESS_META_FLAG = 0b1 << 9; static final int HAS_FIELDS_META_FLAG = 0b1 << 8; @@ -108,7 +108,7 @@ public class ClassDef implements Serializable { private final byte[] encoded; private transient List descriptors; - ClassDef( + TypeDef( ClassSpec classSpec, List fieldsInfo, boolean hasFieldsMeta, @@ -121,7 +121,7 @@ public class ClassDef implements Serializable { this.encoded = encoded; } - public static void skipClassDef(MemoryBuffer buffer, long id) { + public static void skipTypeDef(MemoryBuffer buffer, long id) { int size = (int) (id & META_SIZE_MASKS); if (size == META_SIZE_MASKS) { size += buffer.readVarUint32Small14(); @@ -169,21 +169,20 @@ public int getFieldCount() { } public boolean isNamed() { - return classSpec.typeId < 0 || Types.isNamedType(classSpec.typeId & 0xff); + return classSpec.typeId < 0 || Types.isNamedType(classSpec.typeId); } public boolean isCompatible() { if (classSpec.typeId < 0) { return false; } - int internalTypeId = classSpec.typeId & 0xff; - return internalTypeId == Types.COMPATIBLE_STRUCT - || internalTypeId == Types.NAMED_COMPATIBLE_STRUCT; + return classSpec.typeId == Types.COMPATIBLE_STRUCT + || classSpec.typeId == Types.NAMED_COMPATIBLE_STRUCT; } public int getUserTypeId() { Preconditions.checkArgument(!isNamed(), "Named types don't have user type id"); - return classSpec.typeId >>> 8; + return classSpec.userTypeId; } @Override @@ -191,11 +190,11 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) { return false; } - ClassDef classDef = (ClassDef) o; - return hasFieldsMeta == classDef.hasFieldsMeta - && id == classDef.id - && Objects.equals(classSpec, classDef.classSpec) - && Objects.equals(fieldsInfo, classDef.fieldsInfo); + TypeDef typeDef = (TypeDef) o; + return hasFieldsMeta == typeDef.hasFieldsMeta + && id == typeDef.id + && Objects.equals(classSpec, typeDef.classSpec) + && Objects.equals(fieldsInfo, typeDef.fieldsInfo); } @Override @@ -205,7 +204,7 @@ public int hashCode() { @Override public String toString() { - return "ClassDef{" + return "TypeDef{" + "className='" + classSpec.entireClassName + '\'' @@ -219,10 +218,10 @@ public String toString() { } /** - * Compute diff between this (decoded/remote) ClassDef and a local ClassDef. Returns a string + * Compute diff between this (decoded/remote) TypeDef and a local TypeDef. Returns a string * describing the differences, or null if they are identical. */ - public String computeDiff(ClassDef localDef) { + public String computeDiff(TypeDef localDef) { if (localDef == null) { return "Local TypeDef is null (type not registered locally)"; } @@ -341,30 +340,30 @@ private static String fieldLabel(FieldInfo fieldInfo) { } /** Write class definition to buffer. */ - public void writeClassDef(MemoryBuffer buffer) { + public void writeTypeDef(MemoryBuffer buffer) { buffer.writeBytes(encoded, 0, encoded.length); } /** Read class definition from buffer. */ - public static ClassDef readClassDef(Fory fory, MemoryBuffer buffer) { + public static TypeDef readTypeDef(Fory fory, MemoryBuffer buffer) { if (fory.isCrossLanguage()) { - return TypeDefDecoder.decodeClassDef(fory.getXtypeResolver(), buffer, buffer.readInt64()); + return TypeDefDecoder.decodeTypeDef(fory.getXtypeResolver(), buffer, buffer.readInt64()); } - return ClassDefDecoder.decodeClassDef(fory.getClassResolver(), buffer, buffer.readInt64()); + return NativeTypeDefDecoder.decodeTypeDef(fory.getClassResolver(), buffer, buffer.readInt64()); } /** Read class definition from buffer. */ - public static ClassDef readClassDef(Fory fory, MemoryBuffer buffer, long header) { + public static TypeDef readTypeDef(Fory fory, MemoryBuffer buffer, long header) { if (fory.isCrossLanguage()) { - return TypeDefDecoder.decodeClassDef(fory.getXtypeResolver(), buffer, header); + return TypeDefDecoder.decodeTypeDef(fory.getXtypeResolver(), buffer, header); } - return ClassDefDecoder.decodeClassDef(fory.getClassResolver(), buffer, header); + return NativeTypeDefDecoder.decodeTypeDef(fory.getClassResolver(), buffer, header); } /** - * Consolidate fields of classDef with cls. If some field exists in - * cls but not in classDef, it won't be returned in final collection. If - * some field exists in classDef but not in cls, it will be added to + * Consolidate fields of typeDef with cls. If some field exists in + * cls but not in typeDef, it won't be returned in final collection. If + * some field exists in typeDef but not in cls, it will be added to * final collection. * * @param cls class load in current process. @@ -424,29 +423,29 @@ public List getDescriptors(TypeResolver resolver, Class cls) { return descriptors; } - public static ClassDef buildClassDef(Fory fory, Class cls) { - return buildClassDef(fory, cls, true); + public static TypeDef buildTypeDef(Fory fory, Class cls) { + return buildTypeDef(fory, cls, true); } - public static ClassDef buildClassDef(Fory fory, Class cls, boolean resolveParent) { + public static TypeDef buildTypeDef(Fory fory, Class cls, boolean resolveParent) { if (fory.isCrossLanguage()) { return TypeDefEncoder.buildTypeDef(fory, cls); } - return ClassDefEncoder.buildClassDef( + return NativeTypeDefEncoder.buildTypeDef( fory.getClassResolver(), cls, buildFields(fory, cls, resolveParent), true); } /** Build class definition from fields of class. */ - static ClassDef buildClassDef(ClassResolver classResolver, Class type, List fields) { - return buildClassDef(classResolver, type, fields, true); + static TypeDef buildTypeDef(ClassResolver classResolver, Class type, List fields) { + return buildTypeDef(classResolver, type, fields, true); } - public static ClassDef buildClassDef( + public static TypeDef buildTypeDef( ClassResolver classResolver, Class type, List fields, boolean hasFieldsMeta) { - return ClassDefEncoder.buildClassDef(classResolver, type, fields, hasFieldsMeta); + return NativeTypeDefEncoder.buildTypeDef(classResolver, type, fields, hasFieldsMeta); } - public ClassDef replaceRootClassTo(ClassResolver classResolver, Class targetCls) { + public TypeDef replaceRootClassTo(TypeResolver resolver, Class targetCls) { String name = targetCls.getName(); List fieldInfos = fieldsInfo.stream() @@ -459,7 +458,11 @@ public ClassDef replaceRootClassTo(ClassResolver classResolver, Class targetC } }) .collect(Collectors.toList()); - return ClassDefEncoder.buildClassDefWithFieldInfos( - classResolver, targetCls, fieldInfos, hasFieldsMeta); + if (resolver.getFory().isCrossLanguage()) { + return TypeDefEncoder.buildTypeDefWithFieldInfos( + (org.apache.fory.resolver.XtypeResolver) resolver, targetCls, fieldInfos); + } + return NativeTypeDefEncoder.buildTypeDefWithFieldInfos( + (ClassResolver) resolver, targetCls, fieldInfos, hasFieldsMeta); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefDecoder.java b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefDecoder.java index 12a5a40b80..cb4b38066a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefDecoder.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefDecoder.java @@ -19,11 +19,11 @@ package org.apache.fory.meta; -import static org.apache.fory.meta.ClassDef.HAS_FIELDS_META_FLAG; -import static org.apache.fory.meta.ClassDefDecoder.decodeClassDefBuf; -import static org.apache.fory.meta.ClassDefDecoder.readPkgName; -import static org.apache.fory.meta.ClassDefDecoder.readTypeName; import static org.apache.fory.meta.Encoders.fieldNameEncodings; +import static org.apache.fory.meta.NativeTypeDefDecoder.decodeTypeDefBuf; +import static org.apache.fory.meta.NativeTypeDefDecoder.readPkgName; +import static org.apache.fory.meta.NativeTypeDefDecoder.readTypeName; +import static org.apache.fory.meta.TypeDef.HAS_FIELDS_META_FLAG; import static org.apache.fory.meta.TypeDefEncoder.FIELD_NAME_SIZE_THRESHOLD; import static org.apache.fory.meta.TypeDefEncoder.REGISTER_BY_NAME_FLAG; import static org.apache.fory.meta.TypeDefEncoder.SMALL_NUM_FIELDS_THRESHOLD; @@ -36,14 +36,14 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.meta.FieldTypes.FieldType; import org.apache.fory.meta.MetaString.Encoding; -import org.apache.fory.resolver.ClassInfo; +import org.apache.fory.resolver.TypeInfo; import org.apache.fory.resolver.XtypeResolver; import org.apache.fory.serializer.NonexistentClass; import org.apache.fory.util.StringUtils; import org.apache.fory.util.Utils; /** - * A decoder which decode binary into {@link ClassDef}. Global header layout follows the xlang spec + * A decoder which decode binary into {@link TypeDef}. Global header layout follows the xlang spec * with an 8-bit meta size and flags at bits 8/9. See spec documentation: * docs/specification/fory_xlang_serialization_spec.md ... @@ -51,8 +51,8 @@ class TypeDefDecoder { private static final Logger LOG = LoggerFactory.getLogger(TypeDefDecoder.class); - public static ClassDef decodeClassDef(XtypeResolver resolver, MemoryBuffer inputBuffer, long id) { - Tuple2 decoded = decodeClassDefBuf(inputBuffer, resolver, id); + public static TypeDef decodeTypeDef(XtypeResolver resolver, MemoryBuffer inputBuffer, long id) { + Tuple2 decoded = decodeTypeDefBuf(inputBuffer, resolver, id); MemoryBuffer buffer = MemoryBuffer.fromByteArray(decoded.f0); byte header = buffer.readByte(); int numFields = header & SMALL_NUM_FIELDS_THRESHOLD; @@ -66,32 +66,33 @@ public static ClassDef decodeClassDef(XtypeResolver resolver, MemoryBuffer input if (Utils.DEBUG_OUTPUT_ENABLED) { LOG.info("Decode class {} using namespace {}", typeName, namespace); } - ClassInfo userTypeInfo = resolver.getUserTypeInfo(namespace, typeName); + TypeInfo userTypeInfo = resolver.getUserTypeInfo(namespace, typeName); if (userTypeInfo == null) { classSpec = new ClassSpec(NonexistentClass.NonexistentMetaShared.class); } else { classSpec = new ClassSpec(userTypeInfo.getCls()); } } else { - int typeId = buffer.readVarUint32Small7(); - ClassInfo userTypeInfo = resolver.getUserTypeInfo(typeId); + int typeId = buffer.readUint8(); + int userTypeId = buffer.readVarUint32(); + TypeInfo userTypeInfo = resolver.getUserTypeInfo(userTypeId); if (userTypeInfo == null) { - classSpec = new ClassSpec(NonexistentClass.NonexistentMetaShared.class, typeId); + classSpec = new ClassSpec(NonexistentClass.NonexistentMetaShared.class, typeId, userTypeId); } else { - classSpec = new ClassSpec(userTypeInfo.getCls(), typeId); + classSpec = new ClassSpec(userTypeInfo.getCls(), typeId, userTypeId); } } List classFields = readFieldsInfo(buffer, resolver, classSpec.entireClassName, numFields); boolean hasFieldsMeta = (id & HAS_FIELDS_META_FLAG) != 0; - ClassDef classDef = new ClassDef(classSpec, classFields, hasFieldsMeta, id, decoded.f1); + TypeDef typeDef = new TypeDef(classSpec, classFields, hasFieldsMeta, id, decoded.f1); if (Utils.DEBUG_OUTPUT_ENABLED) { - LOG.info("[Java TypeDef DECODED] " + classDef); + LOG.info("[Java TypeDef DECODED] " + typeDef); // Compute and print diff with local TypeDef Class cls = classSpec.type; if (cls != null && cls != NonexistentClass.NonexistentMetaShared.class) { - ClassDef localDef = ClassDef.buildClassDef(resolver.getFory(), cls); - String diff = classDef.computeDiff(localDef); + TypeDef localDef = TypeDef.buildTypeDef(resolver.getFory(), cls); + String diff = typeDef.computeDiff(localDef); if (diff != null) { LOG.info("[Java TypeDef DIFF] " + classSpec.entireClassName + ":\n" + diff); } else { @@ -99,7 +100,7 @@ public static ClassDef decodeClassDef(XtypeResolver resolver, MemoryBuffer input } } } - return classDef; + return typeDef; } // | header + type info + field name | ... | header + type info + field name | @@ -118,7 +119,7 @@ private static List readFieldsInfo( fieldNameSize += 1; boolean nullable = (header & 0b10) != 0; boolean trackingRef = (header & 0b1) != 0; - int typeId = buffer.readVarUint32Small14(); + int typeId = buffer.readUint8(); FieldType fieldType = FieldTypes.FieldType.xread(buffer, resolver, typeId, nullable, trackingRef); diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefEncoder.java b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefEncoder.java index bbdee20215..34d302a25d 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefEncoder.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefEncoder.java @@ -19,10 +19,10 @@ package org.apache.fory.meta; -import static org.apache.fory.meta.ClassDefEncoder.prependHeader; -import static org.apache.fory.meta.ClassDefEncoder.writePkgName; -import static org.apache.fory.meta.ClassDefEncoder.writeTypeName; import static org.apache.fory.meta.Encoders.fieldNameEncodingsList; +import static org.apache.fory.meta.NativeTypeDefEncoder.prependHeader; +import static org.apache.fory.meta.NativeTypeDefEncoder.writePkgName; +import static org.apache.fory.meta.NativeTypeDefEncoder.writeTypeName; import java.lang.reflect.Field; import java.util.ArrayList; @@ -40,7 +40,7 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.meta.FieldTypes.FieldType; import org.apache.fory.reflect.ReflectionUtils; -import org.apache.fory.resolver.ClassInfo; +import org.apache.fory.resolver.TypeInfo; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.resolver.XtypeResolver; import org.apache.fory.type.Descriptor; @@ -51,7 +51,7 @@ import org.apache.fory.util.Utils; /** - * An encoder which encode {@link ClassDef} into binary. Global header layout follows the xlang spec + * An encoder which encode {@link TypeDef} into binary. Global header layout follows the xlang spec * with an 8-bit meta size and flags at bits 8/9. See spec documentation: * docs/specification/fory_xlang_serialization_spec.md ... @@ -60,17 +60,17 @@ class TypeDefEncoder { private static final Logger LOG = LoggerFactory.getLogger(TypeDefEncoder.class); /** Build class definition from fields of class. */ - static ClassDef buildTypeDef(Fory fory, Class type) { + static TypeDef buildTypeDef(Fory fory, Class type) { DescriptorGrouper descriptorGrouper = fory.getXtypeResolver() .createDescriptorGrouper( fory.getXtypeResolver().getFieldDescriptors(type, true), false, Function.identity()); - ClassInfo classInfo = fory.getTypeResolver().getClassInfo(type); + TypeInfo typeInfo = fory.getTypeResolver().getTypeInfo(type); List fields; - int typeId = classInfo.getTypeId(); - if (Types.isStructType(typeId & 0xff)) { + int typeId = typeInfo.getTypeId(); + if (Types.isStructType(typeId)) { fields = descriptorGrouper.getSortedDescriptors().stream() .map(Descriptor::getField) @@ -78,7 +78,7 @@ static ClassDef buildTypeDef(Fory fory, Class type) { } else { fields = new ArrayList<>(); } - return buildClassDefWithFieldInfos( + return buildTypeDefWithFieldInfos( fory.getXtypeResolver(), type, buildFieldsInfo(fory.getXtypeResolver(), type, fields)); } @@ -110,22 +110,23 @@ static List buildFieldsInfo(TypeResolver resolver, Class type, Lis .collect(Collectors.toList()); } - static ClassDef buildClassDefWithFieldInfos( + static TypeDef buildTypeDefWithFieldInfos( XtypeResolver resolver, Class type, List fieldInfos) { fieldInfos = new ArrayList<>(getClassFields(type, fieldInfos).values()); - MemoryBuffer encodeClassDef = encodeClassDef(resolver, type, fieldInfos); - byte[] classDefBytes = encodeClassDef.getBytes(0, encodeClassDef.writerIndex()); - ClassDef classDef = - new ClassDef( - Encoders.buildClassSpec(type), + TypeInfo typeInfo = resolver.getTypeInfo(type); + MemoryBuffer encodeTypeDef = encodeTypeDef(resolver, type, fieldInfos); + byte[] typeDefBytes = encodeTypeDef.getBytes(0, encodeTypeDef.writerIndex()); + TypeDef typeDef = + new TypeDef( + new ClassSpec(type, typeInfo.getTypeId(), typeInfo.getUserTypeId()), fieldInfos, true, - encodeClassDef.getInt64(0), - classDefBytes); + encodeTypeDef.getInt64(0), + typeDefBytes); if (Utils.DEBUG_OUTPUT_ENABLED) { - LOG.info("[Java TypeDef BUILT] " + classDef); + LOG.info("[Java TypeDef BUILT] " + typeDef); } - return classDef; + return typeDef; } static final int SMALL_NUM_FIELDS_THRESHOLD = 0b11111; @@ -134,9 +135,8 @@ static ClassDef buildClassDefWithFieldInfos( // see spec documentation: docs/specification/xlang_serialization_spec.md // https://fory.apache.org/docs/specification/fory_xlang_serialization_spec - static MemoryBuffer encodeClassDef( - XtypeResolver resolver, Class type, List fields) { - ClassInfo classInfo = resolver.getClassInfo(type); + static MemoryBuffer encodeTypeDef(XtypeResolver resolver, Class type, List fields) { + TypeInfo typeInfo = resolver.getTypeInfo(type); MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(128); buffer.writeByte(-1); // placeholder for header, update later int currentClassHeader = fields.size(); @@ -145,12 +145,17 @@ static MemoryBuffer encodeClassDef( buffer.writeVarUint32(fields.size() - SMALL_NUM_FIELDS_THRESHOLD); } if (resolver.isRegisteredById(type)) { - buffer.writeVarUint32(classInfo.getTypeId()); + buffer.writeUint8(typeInfo.getTypeId()); + Preconditions.checkArgument( + typeInfo.getUserTypeId() != -1, + "User type id is required for typeId %s", + typeInfo.getTypeId()); + buffer.writeVarUint32(typeInfo.getUserTypeId()); } else { Preconditions.checkArgument(resolver.isRegisteredByName(type)); currentClassHeader |= REGISTER_BY_NAME_FLAG; - String ns = classInfo.decodeNamespace(); - String typename = classInfo.decodeTypeName(); + String ns = typeInfo.decodeNamespace(); + String typename = typeInfo.decodeTypeName(); writePkgName(buffer, ns); writeTypeName(buffer, typename); } @@ -173,7 +178,7 @@ static MemoryBuffer encodeClassDef( static Map getClassFields(Class type, List fieldsInfo) { Map sortedClassFields = new LinkedHashMap<>(); - Map> classFields = ClassDefEncoder.groupClassFields(fieldsInfo); + Map> classFields = NativeTypeDefEncoder.groupClassFields(fieldsInfo); for (Class clz : ReflectionUtils.getAllClasses(type, true)) { List fieldInfos = classFields.get(clz.getName()); if (fieldInfos != null) { @@ -201,7 +206,7 @@ static void writeFieldsInfo(XtypeResolver resolver, MemoryBuffer buffer, ListClass ID Space * - *

Fory separates internal IDs (built-in types) from user IDs by encoding user-registered types - * with their internal type tag (ENUM/STRUCT/EXT). User IDs start from 0 and are encoded into the - * unified type ID as {@code (userId << 8) | internalTypeId}. + *

Fory separates internal IDs (built-in types) from user IDs by storing the internal type tag + * (ENUM/STRUCT/EXT) in {@code typeId} and keeping the user-registered ID in {@code userTypeId}. + * User IDs start from 0. * *

Registration Methods

* *
    *
  • {@link #register(Class)} - Auto-assigns the next available user ID - *
  • {@link #register(Class, int)} - Registers with a user-specified ID (0-based) + *
  • {@link #register(Class, long)} - Registers with a user-specified ID (0-based) *
  • {@link #register(Class, String, String)} - Registers with namespace and type name *
* * @see #register(Class) - * @see #register(Class, int) + * @see #register(Class, long) */ @SuppressWarnings({"rawtypes", "unchecked"}) public class ClassResolver extends TypeResolver { @@ -225,18 +227,18 @@ public class ClassResolver extends TypeResolver { public static final int NONEXISTENT_META_SHARED_ID = REPLACE_STUB_ID + 1; private final Fory fory; - private ClassInfo classInfoCache; + private TypeInfo typeInfoCache; // Every deserialization for unregistered class will query it, performance is important. - private final ObjectMap compositeNameBytes2ClassInfo = + private final ObjectMap compositeNameBytes2TypeInfo = new ObjectMap<>(16, foryMapLoadFactor); - // classDefMap is inherited from TypeResolver + // typeDefMap is inherited from TypeResolver private Class currentReadClass; private final ShimDispatcher shimDispatcher; public ClassResolver(Fory fory) { super(fory); this.fory = fory; - classInfoCache = NIL_CLASS_INFO; + typeInfoCache = NIL_TYPE_INFO; extRegistry.classIdGenerator = NONEXISTENT_META_SHARED_ID + 1; shimDispatcher = new ShimDispatcher(fory); _addGraalvmClassRegistry(fory.getConfig().getConfigHash(), this); @@ -305,7 +307,7 @@ public void initialize() { classInfoMap.forEach( (cls, classInfo) -> { if (classInfo.serializer != null) { - extRegistry.registeredClassInfos.add(classInfo); + extRegistry.registeredTypeInfos.add(classInfo); } }); } @@ -441,33 +443,26 @@ public void register(String className) { * * @param className the fully qualified class name * @param classId the user ID to assign (0-based, in user ID space) - * @see #register(Class, int) + * @see #register(Class, long) */ @Override - public void register(String className, int classId) { + public void register(String className, long classId) { register(loadClass(className, false, 0, false), classId); } /** * Registers a class with a user-specified ID. * - *

The ID is in the user ID space, starting from 0. The unified type ID is encoded as {@code - * (userId << 8) | internalTypeId}. - * - *

Example: - * - * 

{@code
-   * fory.register(MyClass.class, 0);      // User ID 0 -> typeId 0x000019 (STRUCT)
-   * fory.register(AnotherClass.class, 1); // User ID 1 -> typeId 0x000119 (STRUCT)
-   * }
+ *

The ID is in the user ID space, starting from 0. The class will store the internal type tag + * (STRUCT/ENUM/EXT/UNION) in {@code typeId} and the provided value in {@code userTypeId}. * * @param cls the class to register - * @param id the user ID to assign (0-based) + * @param id the user ID to assign (0-based, range [0, 0xfffffffe], 0xffffffff reserved) * @throws IllegalArgumentException if the ID is out of valid range or already in use */ @Override - public void register(Class cls, int id) { - registerUserImpl(cls, id); + public void register(Class cls, long id) { + registerUserImpl(cls, toUserTypeId(id)); } /** @@ -488,43 +483,41 @@ public void register(Class cls, String namespace, String name) { if (!StringUtils.isBlank(namespace)) { fullname = namespace + "." + name; } - checkRegistration(cls, (short) -1, fullname, false); + checkRegistration(cls, -1, fullname, false); MetaStringBytes fullNameBytes = metaStringResolver.getOrCreateMetaStringBytes( GENERIC_ENCODER.encode(fullname, MetaString.Encoding.UTF_8)); MetaStringBytes nsBytes = metaStringResolver.getOrCreateMetaStringBytes(encodePackage(namespace)); MetaStringBytes nameBytes = metaStringResolver.getOrCreateMetaStringBytes(encodeTypeName(name)); - ClassInfo existingInfo = classInfoMap.get(cls); + TypeInfo existingInfo = classInfoMap.get(cls); int typeId = buildUnregisteredTypeId(cls, existingInfo == null ? null : existingInfo.serializer); - ClassInfo classInfo = - new ClassInfo(cls, fullNameBytes, nsBytes, nameBytes, false, null, typeId); - classInfoMap.put(cls, classInfo); - compositeNameBytes2ClassInfo.put( - new TypeNameBytes(nsBytes.hashCode, nameBytes.hashCode), classInfo); + TypeInfo typeInfo = + new TypeInfo(cls, fullNameBytes, nsBytes, nameBytes, false, null, typeId, -1); + classInfoMap.put(cls, typeInfo); + compositeNameBytes2TypeInfo.put( + new TypeNameBytes(nsBytes.hashCode, nameBytes.hashCode), typeInfo); extRegistry.registeredClasses.put(fullname, cls); GraalvmSupport.registerClass(cls, fory.getConfig().getConfigHash()); } @Override - public void registerUnion(Class cls, int userId, Serializer serializer) { + public void registerUnion(Class cls, long userId, Serializer serializer) { checkRegisterAllowed(); - Preconditions.checkArgument(userId >= 0 && userId < Short.MAX_VALUE); + int checkedUserId = toUserTypeId(userId); Preconditions.checkNotNull(serializer); - short id = (short) userId; - checkRegistration(cls, id, cls.getName(), false); - extRegistry.registeredClassIdMap.put(cls, id); - int typeId = (userId << 8) | Types.TYPED_UNION; - ClassInfo classInfo = classInfoMap.get(cls); - if (classInfo == null) { - classInfo = new ClassInfo(this, cls, serializer, typeId); - classInfoMap.put(cls, classInfo); + checkRegistration(cls, checkedUserId, cls.getName(), false); + extRegistry.registeredClassIdMap.put(cls, checkedUserId); + int typeId = Types.TYPED_UNION; + TypeInfo typeInfo = classInfoMap.get(cls); + if (typeInfo == null) { + typeInfo = new TypeInfo(this, cls, serializer, typeId, checkedUserId); } else { - classInfo.typeId = typeId; - classInfo.setSerializer(this, serializer); + typeInfo = typeInfo.copy(typeId, checkedUserId); + typeInfo.setSerializer(this, serializer); } - putUserTypeInfo(id, classInfo); + updateTypeInfo(cls, typeInfo); extRegistry.registeredClasses.put(cls.getName(), cls); GraalvmSupport.registerClass(cls, fory.getConfig().getConfigHash()); } @@ -543,7 +536,7 @@ public void registerUnion(Class cls, String namespace, String name, Serialize if (!StringUtils.isBlank(namespace)) { fullname = namespace + "." + name; } - checkRegistration(cls, (short) -1, fullname, false); + checkRegistration(cls, -1, fullname, false); MetaStringBytes fullNameBytes = metaStringResolver.getOrCreateMetaStringBytes( GENERIC_ENCODER.encode(fullname, MetaString.Encoding.UTF_8)); @@ -551,12 +544,12 @@ public void registerUnion(Class cls, String namespace, String name, Serialize metaStringResolver.getOrCreateMetaStringBytes(encodePackage(namespace)); MetaStringBytes nameBytes = metaStringResolver.getOrCreateMetaStringBytes(encodeTypeName(name)); int typeId = Types.NAMED_UNION; - ClassInfo classInfo = - new ClassInfo(cls, fullNameBytes, nsBytes, nameBytes, false, serializer, typeId); - classInfo.setSerializer(this, serializer); - classInfoMap.put(cls, classInfo); - compositeNameBytes2ClassInfo.put( - new TypeNameBytes(nsBytes.hashCode, nameBytes.hashCode), classInfo); + TypeInfo typeInfo = + new TypeInfo(cls, fullNameBytes, nsBytes, nameBytes, false, serializer, typeId, -1); + typeInfo.setSerializer(this, serializer); + classInfoMap.put(cls, typeInfo); + compositeNameBytes2TypeInfo.put( + new TypeNameBytes(nsBytes.hashCode, nameBytes.hashCode), typeInfo); extRegistry.registeredClasses.put(fullname, cls); GraalvmSupport.registerClass(cls, fory.getConfig().getConfigHash()); } @@ -589,8 +582,8 @@ public void registerInternal(Class cls) { extRegistry.classIdGenerator < INTERNAL_ID_LIMIT, "Internal type id overflow: %s", extRegistry.classIdGenerator); - while (extRegistry.classIdGenerator < typeIdToClassInfo.length - && typeIdToClassInfo[extRegistry.classIdGenerator] != null) { + while (extRegistry.classIdGenerator < typeIdToTypeInfo.length + && typeIdToTypeInfo[extRegistry.classIdGenerator] != null) { extRegistry.classIdGenerator++; } Preconditions.checkArgument( @@ -605,7 +598,7 @@ public void registerInternal(Class cls) { * Registers a class for internal use with a specified internal ID. * *

Internal API: This method is for Fory's internal use only. Users should use {@link - * #register(Class, int)} instead. + * #register(Class, long)} instead. * *

Internal IDs are reserved for Fory's built-in types and must be in the range [0, 255]. * @@ -618,57 +611,47 @@ public void registerInternal(Class cls, int classId) { registerInternalImpl(cls, classId); } - private void registerInternalImpl(Class cls, int classId) { + private void registerInternalImpl(Class cls, int typeId) { checkRegisterAllowed(); - Preconditions.checkArgument(classId >= 0 && classId < INTERNAL_ID_LIMIT); - short id = (short) classId; - checkRegistration(cls, id, cls.getName(), true); - extRegistry.registeredClassIdMap.put(cls, id); - int typeId = classId; - ClassInfo classInfo = classInfoMap.get(cls); - if (classInfo != null) { - classInfo.typeId = typeId; + Preconditions.checkArgument(typeId >= 0 && typeId < INTERNAL_ID_LIMIT); + checkRegistration(cls, typeId, cls.getName(), true); + extRegistry.registeredClassIdMap.put(cls, typeId); + TypeInfo typeInfo = classInfoMap.get(cls); + if (typeInfo != null) { + typeInfo = typeInfo.copy(typeId, INVALID_USER_TYPE_ID); } else { - classInfo = new ClassInfo(this, cls, null, typeId); - // make `extRegistry.registeredClassIdMap` and `classInfoMap` share same classInfo - // instances. - classInfoMap.put(cls, classInfo); + typeInfo = new TypeInfo(this, cls, null, typeId, INVALID_USER_TYPE_ID); } - // serializer will be set lazily in `addSerializer` method if it's null. - putInternalTypeInfo(id, classInfo); + updateTypeInfo(cls, typeInfo); extRegistry.registeredClasses.put(cls.getName(), cls); GraalvmSupport.registerClass(cls, fory.getConfig().getConfigHash()); } private void registerUserImpl(Class cls, int userId) { checkRegisterAllowed(); - Preconditions.checkArgument(userId >= 0 && userId < Short.MAX_VALUE); - short id = (short) userId; - checkRegistration(cls, id, cls.getName(), false); - extRegistry.registeredClassIdMap.put(cls, id); - int typeId = buildUserTypeId(cls, userId, null); - ClassInfo classInfo = classInfoMap.get(cls); - if (classInfo != null) { - classInfo.typeId = typeId; + Preconditions.checkArgument(userId != -1, "User type id 0xffffffff is reserved"); + checkRegistration(cls, userId, cls.getName(), false); + extRegistry.registeredClassIdMap.put(cls, userId); + int typeId = buildUserTypeId(cls, null); + TypeInfo typeInfo = classInfoMap.get(cls); + if (typeInfo != null) { + typeInfo = typeInfo.copy(typeId, userId); } else { - classInfo = new ClassInfo(this, cls, null, typeId); - classInfoMap.put(cls, classInfo); + typeInfo = new TypeInfo(this, cls, null, typeId, userId); } - putUserTypeInfo(id, classInfo); + updateTypeInfo(cls, typeInfo); extRegistry.registeredClasses.put(cls.getName(), cls); GraalvmSupport.registerClass(cls, fory.getConfig().getConfigHash()); } - private int buildUserTypeId(Class cls, int userId, Serializer serializer) { - int internalTypeId; + private int buildUserTypeId(Class cls, Serializer serializer) { if (cls.isEnum()) { - internalTypeId = Types.ENUM; + return Types.ENUM; } else if (serializer != null && !isStructSerializer(serializer)) { - internalTypeId = Types.EXT; + return Types.EXT; } else { - internalTypeId = metaContextShareEnabled ? Types.COMPATIBLE_STRUCT : Types.STRUCT; + return metaContextShareEnabled ? Types.COMPATIBLE_STRUCT : Types.STRUCT; } - return (userId << 8) | internalTypeId; } @Override @@ -679,23 +662,23 @@ protected int buildUnregisteredTypeId(Class cls, Serializer serializer) { return super.buildUnregisteredTypeId(cls, serializer); } - private void checkRegistration(Class cls, short classId, String name, boolean internal) { + private void checkRegistration(Class cls, int classId, String name, boolean internal) { if (extRegistry.registeredClassIdMap.containsKey(cls)) { throw new IllegalArgumentException( String.format( "Class %s already registered with id %s.", cls, extRegistry.registeredClassIdMap.get(cls))); } - if (classId >= 0) { + if (classId != -1) { if (internal) { - if (classId < typeIdToClassInfo.length && typeIdToClassInfo[classId] != null) { + if (classId < typeIdToTypeInfo.length && typeIdToTypeInfo[classId] != null) { throw new IllegalArgumentException( String.format( "Class %s with id %s has been registered, registering class %s with same id are not allowed.", - typeIdToClassInfo[classId].getCls(), classId, cls.getName())); + typeIdToTypeInfo[classId].getCls(), classId, cls.getName())); } } else { - ClassInfo existingInfo = userTypeIdToClassInfo.get(classId); + TypeInfo existingInfo = userTypeIdToTypeInfo.get(classId); if (existingInfo != null) { throw new IllegalArgumentException( String.format( @@ -713,12 +696,12 @@ private void checkRegistration(Class cls, short classId, String name, boolean } } - private boolean isInternalRegisteredClassId(Class cls, short classId) { - if (classId < 0 || classId >= typeIdToClassInfo.length) { + private boolean isInternalRegisteredClassId(Class cls, int classId) { + if (classId < 0 || classId >= typeIdToTypeInfo.length) { return false; } - ClassInfo classInfo = typeIdToClassInfo[classId]; - return classInfo != null && classInfo.cls == cls; + TypeInfo typeInfo = typeIdToTypeInfo[classId]; + return typeInfo != null && typeInfo.cls == cls; } @Override @@ -756,15 +739,15 @@ public boolean isRegisteredById(Class cls) { return extRegistry.registeredClassIdMap.get(cls) != null; } - public Short getRegisteredClassId(Class cls) { + public Integer getRegisteredClassId(Class cls) { return extRegistry.registeredClassIdMap.get(cls); } public Class getRegisteredClass(short id) { - if (id < typeIdToClassInfo.length) { - ClassInfo classInfo = typeIdToClassInfo[id]; - if (classInfo != null) { - return classInfo.cls; + if (id < typeIdToTypeInfo.length) { + TypeInfo typeInfo = typeIdToTypeInfo[id]; + if (typeInfo != null) { + return typeInfo.cls; } } return null; @@ -775,31 +758,36 @@ public Class getRegisteredClass(String className) { } public Class getRegisteredClassByTypeId(int typeId) { - ClassInfo classInfo = getRegisteredClassInfoByTypeId(typeId); - return classInfo == null ? null : classInfo.cls; + TypeInfo typeInfo = getRegisteredTypeInfoByTypeId(typeId, -1); + return typeInfo == null ? null : typeInfo.cls; + } + + public Class getRegisteredClassByTypeId(int typeId, int userTypeId) { + TypeInfo typeInfo = getRegisteredTypeInfoByTypeId(typeId, userTypeId); + return typeInfo == null ? null : typeInfo.cls; } - public ClassInfo getRegisteredClassInfoByTypeId(int typeId) { - int internalTypeId = typeId & 0xff; - if (Types.isNamedType(internalTypeId)) { + public TypeInfo getRegisteredTypeInfoByTypeId(int typeId, int userTypeId) { + if (Types.isNamedType(typeId)) { return null; } - if (Types.isUserDefinedType((byte) internalTypeId)) { - int userId = typeId >>> 8; - ClassInfo classInfo = userTypeIdToClassInfo.get(userId); - if (classInfo == null) { + if (isUserTypeRegisteredById(typeId)) { + if (userTypeId == -1) { return null; } - int existingInternalTypeId = classInfo.typeId & 0xff; - if (existingInternalTypeId != internalTypeId) { + TypeInfo typeInfo = userTypeIdToTypeInfo.get(userTypeId); + if (typeInfo == null) { return null; } - return classInfo; + if (typeInfo.typeId != typeId) { + return null; + } + return typeInfo; } - if (typeId < 0 || typeId >= typeIdToClassInfo.length) { + if (typeId < 0 || typeId >= typeIdToTypeInfo.length) { return null; } - return typeIdToClassInfo[typeId]; + return typeIdToTypeInfo[typeId]; } public List> getRegisteredClasses() { @@ -809,9 +797,9 @@ public List> getRegisteredClasses() { } public String getTypeAlias(Class cls) { - Short id = extRegistry.registeredClassIdMap.get(cls); + Integer id = extRegistry.registeredClassIdMap.get(cls); if (id != null) { - return String.valueOf(id); + return Integer.toUnsignedString(id); } String name = extRegistry.registeredClasses.inverse().get(cls); if (name != null) { @@ -821,33 +809,49 @@ public String getTypeAlias(Class cls) { } /** - * Compute the typeId used in ClassDef without forcing serializer creation. This avoids recursive + * Compute the typeId used in TypeDef without forcing serializer creation. This avoids recursive * serializer construction while building class metadata. */ - public int getTypeIdForClassDef(Class cls) { - ClassInfo classInfo = classInfoMap.get(cls); - if (classInfo != null) { - return classInfo.typeId; + public int getTypeIdForTypeDef(Class cls) { + TypeInfo typeInfo = classInfoMap.get(cls); + if (typeInfo != null) { + return typeInfo.typeId; } - Short classId = extRegistry.registeredClassIdMap.get(cls); + Integer classId = extRegistry.registeredClassIdMap.get(cls); if (classId != null) { - classInfo = classInfoMap.get(cls); - if (classInfo == null) { - classInfo = getClassInfo(cls); + typeInfo = classInfoMap.get(cls); + if (typeInfo == null) { + typeInfo = getTypeInfo(cls); } - return classInfo.typeId; + return typeInfo.typeId; } int typeId = buildUnregisteredTypeId(cls, null); - classInfo = new ClassInfo(this, cls, null, typeId); - classInfoMap.put(cls, classInfo); - if (classInfo.namespaceBytes != null && classInfo.typeNameBytes != null) { + typeInfo = new TypeInfo(this, cls, null, typeId, INVALID_USER_TYPE_ID); + classInfoMap.put(cls, typeInfo); + if (typeInfo.namespaceBytes != null && typeInfo.typeNameBytes != null) { TypeNameBytes typeNameBytes = - new TypeNameBytes(classInfo.namespaceBytes.hashCode, classInfo.typeNameBytes.hashCode); - compositeNameBytes2ClassInfo.put(typeNameBytes, classInfo); + new TypeNameBytes(typeInfo.namespaceBytes.hashCode, typeInfo.typeNameBytes.hashCode); + compositeNameBytes2TypeInfo.put(typeNameBytes, typeInfo); } return typeId; } + /** + * Compute the user type id used in TypeDef without forcing serializer creation. Returns -1 when + * the class isn't registered by numeric id. + */ + public int getUserTypeIdForTypeDef(Class cls) { + TypeInfo typeInfo = classInfoMap.get(cls); + if (typeInfo != null) { + return typeInfo.userTypeId; + } + Integer classId = extRegistry.registeredClassIdMap.get(cls); + if (classId != null && !isInternalRegisteredClassId(cls, classId)) { + return classId; + } + return -1; + } + @Override public boolean isMonomorphic(Descriptor descriptor) { ForyField foryField = descriptor.getForyField(); @@ -903,17 +907,16 @@ public boolean isBuildIn(Descriptor descriptor) { } public boolean isInternalRegistered(int classId) { - int internalTypeId = classId & 0xff; - if (Types.isUserDefinedType((byte) internalTypeId)) { + if (Types.isUserDefinedType((byte) classId)) { return false; } - return classId > 0 && classId < typeIdToClassInfo.length && typeIdToClassInfo[classId] != null; + return classId > 0 && classId < typeIdToTypeInfo.length && typeIdToTypeInfo[classId] != null; } /** Returns true if cls is fory inner registered class. */ public boolean isInternalRegistered(Class cls) { - ClassInfo classInfo = classInfoMap.get(cls); - return classInfo != null && isInternalRegistered(classInfo.typeId); + TypeInfo typeInfo = classInfoMap.get(cls); + return typeInfo != null && isInternalRegistered(typeInfo.typeId); } /** @@ -994,7 +997,7 @@ public void registerSerializer(Class type, Serializer serializer) { */ @Override public void registerInternalSerializer(Class type, Serializer serializer) { - Short classId = extRegistry.registeredClassIdMap.get(type); + Integer classId = extRegistry.registeredClassIdMap.get(type); if (classId != null && !isInternalRegisteredClassId(type, classId)) { throw new IllegalArgumentException( String.format( @@ -1016,13 +1019,13 @@ private void registerSerializerImpl(Class type, Serializer serializer) { SerializationUtils.validate(type, serializer.getClass()); } addSerializer(type, serializer); - ClassInfo classInfo = classInfoMap.get(type); - classInfoMap.put(type, classInfo); - extRegistry.registeredClassInfos.add(classInfo); + TypeInfo typeInfo = classInfoMap.get(type); + classInfoMap.put(type, typeInfo); + extRegistry.registeredTypeInfos.add(typeInfo); // in order to support customized serializer for abstract or interface. if (!type.isPrimitive() && (ReflectionUtils.isAbstract(type) || type.isInterface())) { - extRegistry.absClassInfo.put(type, classInfo); - extRegistry.registeredClassInfos.add(classInfo); + extRegistry.absTypeInfo.put(type, typeInfo); + extRegistry.registeredTypeInfos.add(typeInfo); } } @@ -1047,7 +1050,7 @@ public void setSerializer(Class cls, Serializer serializer) { /** Set serializer for class whose name is {@code className}. */ public void setSerializer(String className, Class serializer) { - for (Map.Entry, ClassInfo> entry : classInfoMap.iterable()) { + for (Map.Entry, TypeInfo> entry : classInfoMap.iterable()) { if (extRegistry.registeredClasses.containsKey(className)) { LOG.warn("Skip clear serializer for registered class {}", className); return; @@ -1056,7 +1059,7 @@ public void setSerializer(String className, Class serializ if (cls.getName().equals(className)) { LOG.info("Clear serializer for class {}.", className); entry.getValue().setSerializer(this, Serializers.newSerializer(fory, cls, serializer)); - classInfoCache = NIL_CLASS_INFO; + typeInfoCache = NIL_TYPE_INFO; return; } } @@ -1064,7 +1067,7 @@ public void setSerializer(String className, Class serializ /** Set serializer for classes starts with {@code classNamePrefix}. */ public void setSerializers(String classNamePrefix, Class serializer) { - for (Map.Entry, ClassInfo> entry : classInfoMap.iterable()) { + for (Map.Entry, TypeInfo> entry : classInfoMap.iterable()) { Class cls = entry.getKey(); String className = cls.getName(); if (extRegistry.registeredClasses.containsKey(className)) { @@ -1073,7 +1076,7 @@ public void setSerializers(String classNamePrefix, Class s if (className.startsWith(classNamePrefix)) { LOG.info("Clear serializer for class {}.", className); entry.getValue().setSerializer(this, Serializers.newSerializer(fory, cls, serializer)); - classInfoCache = NIL_CLASS_INFO; + typeInfoCache = NIL_TYPE_INFO; } } } @@ -1110,55 +1113,50 @@ public void setSerializerIfAbsent(Class cls, Serializer serializer) { /** Clear serializer associated with cls if not null. */ public void clearSerializer(Class cls) { - ClassInfo classInfo = classInfoMap.get(cls); - if (classInfo != null) { - classInfo.setSerializer(this, null); + TypeInfo typeInfo = classInfoMap.get(cls); + if (typeInfo != null) { + typeInfo.setSerializer(this, null); } } /** Add serializer for specified class. */ public void addSerializer(Class type, Serializer serializer) { Preconditions.checkNotNull(serializer); - ClassInfo classInfo; - Short classId = extRegistry.registeredClassIdMap.get(type); + TypeInfo typeInfo; + Integer classId = extRegistry.registeredClassIdMap.get(type); boolean registered = classId != null; if (registered) { int id = classId; - boolean internal = isInternalRegisteredClassId(type, (short) id); - int typeId = internal ? id : buildUserTypeId(type, id, serializer); - classInfo = classInfoMap.get(type); - if (classInfo == null) { - classInfo = new ClassInfo(this, type, null, typeId); - classInfoMap.put(type, classInfo); + boolean internal = isInternalRegisteredClassId(type, id); + int typeId = internal ? id : buildUserTypeId(type, serializer); + typeInfo = classInfoMap.get(type); + if (typeInfo == null) { + typeInfo = new TypeInfo(this, type, null, typeId, internal ? INVALID_USER_TYPE_ID : id); } else { - classInfo.typeId = typeId; - } - if (internal) { - putInternalTypeInfo(id, classInfo); - } else { - putUserTypeInfo(id, classInfo); + typeInfo = typeInfo.copy(typeId); } + updateTypeInfo(type, typeInfo); } else { int typeId = buildUnregisteredTypeId(type, serializer); - classInfo = classInfoMap.get(type); - if (classInfo == null) { - classInfo = new ClassInfo(this, type, null, typeId); - classInfoMap.put(type, classInfo); + typeInfo = classInfoMap.get(type); + if (typeInfo == null) { + typeInfo = new TypeInfo(this, type, null, typeId, INVALID_USER_TYPE_ID); } else { - classInfo.typeId = typeId; + typeInfo = typeInfo.copy(typeId); } - // Add to compositeNameBytes2ClassInfo for unregistered classes so that - // readClassInfo can find the ClassInfo by name bytes during deserialization. + updateTypeInfo(type, typeInfo); + // Add to compositeNameBytes2TypeInfo for unregistered classes so that + // readTypeInfo can find the TypeInfo by name bytes during deserialization. // This is important for dynamically created classes that can't be loaded by name. - if (classInfo.namespaceBytes != null && classInfo.typeNameBytes != null) { + if (typeInfo.namespaceBytes != null && typeInfo.typeNameBytes != null) { TypeNameBytes typeNameBytes = - new TypeNameBytes(classInfo.namespaceBytes.hashCode, classInfo.typeNameBytes.hashCode); - compositeNameBytes2ClassInfo.put(typeNameBytes, classInfo); + new TypeNameBytes(typeInfo.namespaceBytes.hashCode, typeInfo.typeNameBytes.hashCode); + compositeNameBytes2TypeInfo.put(typeNameBytes, typeInfo); } } - // 2. Set `Serializer` for `ClassInfo`. - classInfo.setSerializer(this, serializer); + // 2. Set `Serializer` for `TypeInfo`. + typeInfo.setSerializer(this, serializer); } @SuppressWarnings("unchecked") @@ -1167,8 +1165,8 @@ public Serializer getSerializer(Class cls, boolean createIfNotExist) { if (createIfNotExist) { return getSerializer(cls); } - ClassInfo classInfo = classInfoMap.get(cls); - return classInfo == null ? null : (Serializer) classInfo.serializer; + TypeInfo typeInfo = classInfoMap.get(cls); + return typeInfo == null ? null : (Serializer) typeInfo.serializer; } /** Get or create serializer for cls. */ @@ -1176,7 +1174,7 @@ public Serializer getSerializer(Class cls, boolean createIfNotExist) { @SuppressWarnings("unchecked") public Serializer getSerializer(Class cls) { Preconditions.checkNotNull(cls); - return (Serializer) getOrUpdateClassInfo(cls).serializer; + return (Serializer) getOrUpdateTypeInfo(cls).serializer; } /** @@ -1188,7 +1186,7 @@ public Serializer getSerializer(Class cls) { @Override public Serializer getRawSerializer(Class cls) { Preconditions.checkNotNull(cls); - return getOrUpdateClassInfo(cls).serializer; + return getOrUpdateTypeInfo(cls).serializer; } @Override @@ -1208,11 +1206,11 @@ public Class getSerializerClass(Class cls, boolean code return serializerClass; } cls = TypeUtils.boxedType(cls); - ClassInfo classInfo = classInfoMap.get(cls); - if (classInfo != null && classInfo.serializer != null) { + TypeInfo typeInfo = classInfoMap.get(cls); + if (typeInfo != null && typeInfo.serializer != null) { // Note: need to check `classInfo.serializer != null`, because sometimes `cls` is already // serialized, which will create a class info with serializer null, see `#writeClassInternal` - return classInfo.serializer.getClass(); + return typeInfo.serializer.getClass(); } else { if (getSerializerFactory() != null) { Serializer serializer = getSerializerFactory().createSerializer(fory, cls); @@ -1325,8 +1323,8 @@ public Class getSerializerClass(Class cls, boolean code @Override public void onSuccess(Class result) { setSerializer(clz, Serializers.newSerializer(fory, clz, result)); - if (classInfoCache.cls == clz) { - classInfoCache = NIL_CLASS_INFO; // clear class info cache + if (typeInfoCache.cls == clz) { + typeInfoCache = NIL_TYPE_INFO; // clear class info cache } Preconditions.checkState(getSerializer(clz).getClass() == result); } @@ -1424,52 +1422,52 @@ public void setClassChecker(ClassChecker classChecker) { // Invoked by fory JIT. @Override - public ClassInfo getClassInfo(Class cls) { - ClassInfo classInfo = classInfoMap.get(cls); - if (classInfo == null || classInfo.serializer == null) { + public TypeInfo getTypeInfo(Class cls) { + TypeInfo typeInfo = classInfoMap.get(cls); + if (typeInfo == null || typeInfo.serializer == null) { addSerializer(cls, createSerializer(cls)); - classInfo = classInfoMap.get(cls); + typeInfo = classInfoMap.get(cls); } - return classInfo; + return typeInfo; } - public ClassInfo getClassInfo(short classId) { - ClassInfo classInfo = typeIdToClassInfo[classId]; - assert classInfo != null : classId; - if (classInfo.serializer == null) { - addSerializer(classInfo.cls, createSerializer(classInfo.cls)); - classInfo = classInfoMap.get(classInfo.cls); + public TypeInfo getTypeInfo(short classId) { + TypeInfo typeInfo = typeIdToTypeInfo[classId]; + assert typeInfo != null : classId; + if (typeInfo.serializer == null) { + addSerializer(typeInfo.cls, createSerializer(typeInfo.cls)); + typeInfo = classInfoMap.get(typeInfo.cls); } - return classInfo; + return typeInfo; } /** Get classinfo by cache, update cache if miss. */ @Override - public ClassInfo getClassInfo(Class cls, ClassInfoHolder classInfoHolder) { - ClassInfo classInfo = classInfoHolder.classInfo; - if (classInfo.getCls() != cls) { - classInfo = classInfoMap.get(cls); - if (classInfo == null || classInfo.serializer == null) { + public TypeInfo getTypeInfo(Class cls, TypeInfoHolder classInfoHolder) { + TypeInfo typeInfo = classInfoHolder.typeInfo; + if (typeInfo.getCls() != cls) { + typeInfo = classInfoMap.get(cls); + if (typeInfo == null || typeInfo.serializer == null) { addSerializer(cls, createSerializer(cls)); - classInfo = Objects.requireNonNull(classInfoMap.get(cls)); + typeInfo = Objects.requireNonNull(classInfoMap.get(cls)); } - classInfoHolder.classInfo = classInfo; + classInfoHolder.typeInfo = typeInfo; } - assert classInfo.serializer != null; - return classInfo; + assert typeInfo.serializer != null; + return typeInfo; } /** - * Get class information, create class info if not found and `createClassInfoIfNotFound` is true. + * Get class information, create class info if not found and `createTypeInfoIfNotFound` is true. * * @param cls which class to get class info. - * @param createClassInfoIfNotFound whether create class info if not found. + * @param createTypeInfoIfNotFound whether create class info if not found. * @return Class info. */ @Override - public ClassInfo getClassInfo(Class cls, boolean createClassInfoIfNotFound) { - if (createClassInfoIfNotFound) { - return getOrUpdateClassInfo(cls); + public TypeInfo getTypeInfo(Class cls, boolean createTypeInfoIfNotFound) { + if (createTypeInfoIfNotFound) { + return getOrUpdateTypeInfo(cls); } if (extRegistry.getClassCtx.contains(cls)) { return null; @@ -1479,33 +1477,33 @@ public ClassInfo getClassInfo(Class cls, boolean createClassInfoIfNotFound) { } @Internal - public ClassInfo getOrUpdateClassInfo(Class cls) { - ClassInfo classInfo = classInfoCache; - if (classInfo.cls != cls) { - classInfo = classInfoMap.get(cls); - if (classInfo == null || classInfo.serializer == null) { + public TypeInfo getOrUpdateTypeInfo(Class cls) { + TypeInfo typeInfo = typeInfoCache; + if (typeInfo.cls != cls) { + typeInfo = classInfoMap.get(cls); + if (typeInfo == null || typeInfo.serializer == null) { addSerializer(cls, createSerializer(cls)); - classInfo = classInfoMap.get(cls); + typeInfo = classInfoMap.get(cls); } - classInfoCache = classInfo; + typeInfoCache = typeInfo; } - return classInfo; + return typeInfo; } - private ClassInfo getOrUpdateClassInfo(short classId) { - ClassInfo classInfo = classInfoCache; - ClassInfo internalInfo = classId < typeIdToClassInfo.length ? typeIdToClassInfo[classId] : null; + private TypeInfo getOrUpdateTypeInfo(short classId) { + TypeInfo typeInfo = typeInfoCache; + TypeInfo internalInfo = classId < typeIdToTypeInfo.length ? typeIdToTypeInfo[classId] : null; Preconditions.checkArgument( internalInfo != null, "Internal class id %s is not registered", classId); - if (classInfo != internalInfo) { - classInfo = internalInfo; - if (classInfo.serializer == null) { - addSerializer(classInfo.cls, createSerializer(classInfo.cls)); - classInfo = classInfoMap.get(classInfo.cls); + if (typeInfo != internalInfo) { + typeInfo = internalInfo; + if (typeInfo.serializer == null) { + addSerializer(typeInfo.cls, createSerializer(typeInfo.cls)); + typeInfo = classInfoMap.get(typeInfo.cls); } - classInfoCache = classInfo; + typeInfoCache = typeInfo; } - return classInfo; + return typeInfo; } public Serializer createSerializerSafe(Class cls, Supplier> func) { @@ -1559,15 +1557,15 @@ private Serializer createSerializer(Class cls) { } // support customized serializer for abstract or interface. - if (!extRegistry.absClassInfo.isEmpty()) { + if (!extRegistry.absTypeInfo.isEmpty()) { Class tmpCls = cls; while (tmpCls != null && tmpCls != Object.class) { - ClassInfo absClass; - if ((absClass = extRegistry.absClassInfo.get(tmpCls.getSuperclass())) != null) { + TypeInfo absClass; + if ((absClass = extRegistry.absTypeInfo.get(tmpCls.getSuperclass())) != null) { return absClass.serializer; } for (Class tmpI : tmpCls.getInterfaces()) { - if ((absClass = extRegistry.absClassInfo.get(tmpI)) != null) { + if ((absClass = extRegistry.absTypeInfo.get(tmpI)) != null) { return absClass.serializer; } } @@ -1584,29 +1582,29 @@ private Serializer createSerializer(Class cls) { } private void createSerializer0(Class cls) { - ClassInfo classInfo = getClassInfo(cls); - ClassInfo deserializationClassInfo; - if (metaContextShareEnabled && needToWriteClassDef(classInfo.serializer)) { - ClassDef classDef = classInfo.classDef; - if (classDef == null) { - classDef = buildClassDef(classInfo); + TypeInfo typeInfo = getTypeInfo(cls); + TypeInfo deserializationTypeInfo; + if (metaContextShareEnabled && needToWriteTypeDef(typeInfo.serializer)) { + TypeDef typeDef = typeInfo.typeDef; + if (typeDef == null) { + typeDef = buildTypeDef(typeInfo); } - deserializationClassInfo = buildMetaSharedClassInfo(Tuple2.of(classDef, null), classDef); - if (deserializationClassInfo != null && GraalvmSupport.isGraalBuildtime()) { + deserializationTypeInfo = buildMetaSharedTypeInfo(Tuple2.of(typeDef, null), typeDef); + if (deserializationTypeInfo != null && GraalvmSupport.isGraalBuildtime()) { getGraalvmClassRegistry() .deserializerClassMap - .put(classDef.getId(), getGraalvmSerializerClass(deserializationClassInfo.serializer)); - Tuple2 classDefTuple = extRegistry.classIdToDef.get(classDef.getId()); - classInfoCache = NIL_CLASS_INFO; - extRegistry.classIdToDef.put(classDef.getId(), Tuple2.of(classDefTuple.f0, null)); + .put(typeDef.getId(), getGraalvmSerializerClass(deserializationTypeInfo.serializer)); + Tuple2 typeDefTuple = extRegistry.classIdToDef.get(typeDef.getId()); + typeInfoCache = NIL_TYPE_INFO; + extRegistry.classIdToDef.put(typeDef.getId(), Tuple2.of(typeDefTuple.f0, null)); } } if (GraalvmSupport.isGraalBuildtime()) { // Instance for generated class should be hold at graalvm runtime only. getGraalvmClassRegistry() .serializerClassMap - .put(cls, getGraalvmSerializerClass(classInfo.serializer)); - classInfoCache = NIL_CLASS_INFO; + .put(cls, getGraalvmSerializerClass(typeInfo.serializer)); + typeInfoCache = NIL_TYPE_INFO; if (RecordUtils.isRecord(cls)) { RecordUtils.getRecordConstructor(cls); RecordUtils.getRecordComponents(cls); @@ -1658,7 +1656,7 @@ private boolean isSecure(Class cls) { /** * Write class info to buffer. TODO(chaokunyang): The method should try to write - * aligned data to reduce cpu instruction overhead. `writeClassInfo` is the last step before + * aligned data to reduce cpu instruction overhead. `writeTypeInfo` is the last step before * serializing object, if this writes are aligned, then later serialization will be more * efficient. */ @@ -1669,147 +1667,168 @@ public void writeClassAndUpdateCache(MemoryBuffer buffer, Class cls) { } else if (cls == Long.class) { buffer.writeVarUint32Small7(Types.INT64); } else { - writeClassInfo(buffer, getOrUpdateClassInfo(cls)); + writeTypeInfo(buffer, getOrUpdateTypeInfo(cls)); } } // The jit-compiled native code for this method will be too big for inline, so we generated - // `getClassInfo` - // in fory-jit, see `BaseSeqCodecBuilder#writeAndGetClassInfo` - // public ClassInfo writeClassInfo(MemoryBuffer buffer, Class cls, ClassInfoHolder + // `getTypeInfo` + // in fory-jit, see `BaseSeqCodecBuilder#writeAndGetTypeInfo` + // public TypeInfo writeTypeInfo(MemoryBuffer buffer, Class cls, TypeInfoHolder // classInfoHolder) // { - // ClassInfo classInfo = getClassInfo(cls, classInfoHolder); - // writeClassInfo(buffer, classInfo); + // TypeInfo classInfo = getTypeInfo(cls, classInfoHolder); + // writeTypeInfo(buffer, classInfo); // return classInfo; // } @Override - protected ClassDef buildClassDef(ClassInfo classInfo) { - ClassDef classDef; - Serializer serializer = classInfo.serializer; + protected TypeDef buildTypeDef(TypeInfo typeInfo) { + TypeDef typeDef; + Serializer serializer = typeInfo.serializer; Preconditions.checkArgument(serializer.getClass() != NonexistentClassSerializer.class); - if (needToWriteClassDef(serializer)) { - classDef = - classDefMap.computeIfAbsent(classInfo.cls, cls -> ClassDef.buildClassDef(fory, cls)); + if (needToWriteTypeDef(serializer)) { + typeDef = typeDefMap.computeIfAbsent(typeInfo.cls, cls -> TypeDef.buildTypeDef(fory, cls)); } else { // Some type will use other serializers such MapSerializer and so on. - classDef = - classDefMap.computeIfAbsent( - classInfo.cls, cls -> ClassDef.buildClassDef(this, cls, new ArrayList<>(), false)); + typeDef = + typeDefMap.computeIfAbsent( + typeInfo.cls, cls -> TypeDef.buildTypeDef(this, cls, new ArrayList<>(), false)); } - classInfo.classDef = classDef; - return classDef; + typeInfo.typeDef = typeDef; + return typeDef; } /** * Write classname for java serialization. Note that the object of provided class can be * non-serializable, and class with writeReplace/readResolve defined won't be skipped. For - * serializable object, {@link #writeClassInfo(MemoryBuffer, ClassInfo)} should be invoked. + * serializable object, {@link #writeTypeInfo(MemoryBuffer, TypeInfo)} should be invoked. */ public void writeClassInternal(MemoryBuffer buffer, Class cls) { - ClassInfo classInfo = classInfoMap.get(cls); - if (classInfo == null) { - Short classId = extRegistry.registeredClassIdMap.get(cls); - // Don't create serializer in case the object for class is non-serializable, - // Or class is abstract or interface. - int typeId; - if (classId == null) { - typeId = buildUnregisteredTypeId(cls, null); - } else { - boolean internal = isInternalRegisteredClassId(cls, classId); - typeId = internal ? classId : buildUserTypeId(cls, classId, null); - } - classInfo = new ClassInfo(this, cls, null, typeId); - classInfoMap.put(cls, classInfo); + TypeInfo typeInfo = classInfoMap.get(cls); + if (typeInfo == null) { + typeInfo = buildClassInfo(cls); } - writeClassInternal(buffer, classInfo); + writeClassInternal(buffer, typeInfo); } - public void writeClassInternal(MemoryBuffer buffer, ClassInfo classInfo) { - int typeId = classInfo.typeId; - int internalTypeId = typeId & 0xff; - boolean writeById = typeId != REPLACE_STUB_ID && !Types.isNamedType(internalTypeId); + public void writeClassInternal(MemoryBuffer buffer, TypeInfo typeInfo) { + int typeId = typeInfo.typeId; + boolean writeById = typeId != REPLACE_STUB_ID && !Types.isNamedType(typeId); if (writeById) { - buffer.writeVarUint32(typeId << 1); + buffer.writeVarUint32Small7(typeId << 1); + switch (typeId) { + case Types.ENUM: + case Types.STRUCT: + case Types.COMPATIBLE_STRUCT: + case Types.EXT: + case Types.TYPED_UNION: + buffer.writeVarUint32(typeInfo.userTypeId); + break; + default: + break; + } } else { // let the lowermost bit of next byte be set, so the deserialization can know // whether need to read class by name in advance - metaStringResolver.writeMetaStringBytesWithFlag(buffer, classInfo.namespaceBytes); - metaStringResolver.writeMetaStringBytes(buffer, classInfo.typeNameBytes); + metaStringResolver.writeMetaStringBytesWithFlag(buffer, typeInfo.namespaceBytes); + metaStringResolver.writeMetaStringBytes(buffer, typeInfo.typeNameBytes); + } + } + + private TypeInfo buildClassInfo(Class cls) { + TypeInfo typeInfo; + Integer classId = extRegistry.registeredClassIdMap.get(cls); + // Don't create serializer in case the object for class is non-serializable, + // Or class is abstract or interface. + int typeId; + if (classId == null) { + typeId = buildUnregisteredTypeId(cls, null); + } else { + boolean internal = isInternalRegisteredClassId(cls, classId); + typeId = internal ? classId : buildUserTypeId(cls, null); } + typeInfo = new TypeInfo(this, cls, null, typeId, INVALID_USER_TYPE_ID); + classInfoMap.put(cls, typeInfo); + return typeInfo; } /** * Read serialized java classname. Note that the object of the class can be non-serializable. For - * serializable object, {@link #readClassInfo(MemoryBuffer)} or {@link - * #readClassInfo(MemoryBuffer, ClassInfoHolder)} should be invoked. + * serializable object, {@link #readTypeInfo(MemoryBuffer)} or {@link #readTypeInfo(MemoryBuffer, + * TypeInfoHolder)} should be invoked. */ public Class readClassInternal(MemoryBuffer buffer) { int header = buffer.readVarUint32Small14(); - final ClassInfo classInfo; if ((header & 0b1) != 0) { // let the lowermost bit of next byte be set, so the deserialization can know // whether need to read class by name in advance MetaStringBytes packageBytes = metaStringResolver.readMetaStringBytesWithFlag(buffer, header); MetaStringBytes simpleClassNameBytes = metaStringResolver.readMetaStringBytes(buffer); - classInfo = loadBytesToClassInfo(packageBytes, simpleClassNameBytes); - } else { - classInfo = getClassInfoByTypeIdForReadClassInternal(header >> 1); - } - final Class cls = classInfo.cls; - currentReadClass = cls; - return cls; - } - - private ClassInfo getClassInfoByTypeIdForReadClassInternal(int typeId) { - int internalTypeId = typeId & 0xff; - ClassInfo classInfo; - if (Types.isUserDefinedType((byte) internalTypeId) && !Types.isNamedType(internalTypeId)) { - classInfo = getUserTypeInfoByTypeId(typeId); + return loadBytesToTypeInfo(packageBytes, simpleClassNameBytes).cls; + } + int typeId = header >>> 1; + switch (typeId) { + case Types.ENUM: + case Types.STRUCT: + case Types.COMPATIBLE_STRUCT: + case Types.EXT: + case Types.TYPED_UNION: + return getTypeInfoByTypeIdForReadClassInternal(typeId, buffer.readVarUint32()).cls; + default: + TypeInfo internalTypeInfoByTypeId = getInternalTypeInfoByTypeId(typeId); + Preconditions.checkNotNull(internalTypeInfoByTypeId); + return internalTypeInfoByTypeId.cls; + } + } + + private TypeInfo getTypeInfoByTypeIdForReadClassInternal(int typeId, int userTypeId) { + TypeInfo typeInfo; + if (userTypeId != INVALID_USER_TYPE_ID) { + typeInfo = userTypeIdToTypeInfo.get(userTypeId); } else { - classInfo = getInternalTypeInfoByTypeId(typeId); + typeInfo = getInternalTypeInfoByTypeId(typeId); } - Preconditions.checkArgument(classInfo != null, "Type id %s not registered", typeId); - return classInfo; + Preconditions.checkArgument(typeInfo != null, "Type id %s not registered", typeId); + return typeInfo; } @Override - protected ClassInfo loadBytesToClassInfo( + protected TypeInfo loadBytesToTypeInfo( MetaStringBytes packageBytes, MetaStringBytes simpleClassNameBytes) { TypeNameBytes typeNameBytes = new TypeNameBytes(packageBytes.hashCode, simpleClassNameBytes.hashCode); - ClassInfo classInfo = compositeNameBytes2ClassInfo.get(typeNameBytes); - if (classInfo == null) { - classInfo = populateBytesToClassInfo(typeNameBytes, packageBytes, simpleClassNameBytes); + TypeInfo typeInfo = compositeNameBytes2TypeInfo.get(typeNameBytes); + if (typeInfo == null) { + typeInfo = populateBytesToTypeInfo(typeNameBytes, packageBytes, simpleClassNameBytes); } - // Note: Don't create serializer here - this method is used by both readClassInfo + // Note: Don't create serializer here - this method is used by both readTypeInfo // (which needs serializer) and readClassInternal (which doesn't need serializer). - // Serializer creation is handled by ensureSerializerForClassInfo in TypeResolver. - return classInfo; + // Serializer creation is handled by ensureSerializerForTypeInfo in TypeResolver. + return typeInfo; } @Override - protected ClassInfo ensureSerializerForClassInfo(ClassInfo classInfo) { - if (classInfo.serializer == null) { - Class cls = classInfo.cls; + protected TypeInfo ensureSerializerForTypeInfo(TypeInfo typeInfo) { + if (typeInfo.serializer == null) { + Class cls = typeInfo.cls; if (cls != null && (ReflectionUtils.isAbstract(cls) || cls.isInterface())) { - return classInfo; + return typeInfo; } - // Get or create ClassInfo with serializer - ClassInfo newClassInfo = getClassInfo(classInfo.cls); - // Update the cache with the correct ClassInfo that has a serializer - if (classInfo.typeNameBytes != null) { + // Get or create TypeInfo with serializer + TypeInfo newTypeInfo = getTypeInfo(typeInfo.cls); + // Update the cache with the correct TypeInfo that has a serializer + if (typeInfo.typeNameBytes != null) { TypeNameBytes typeNameBytes = - new TypeNameBytes(classInfo.namespaceBytes.hashCode, classInfo.typeNameBytes.hashCode); - compositeNameBytes2ClassInfo.put(typeNameBytes, newClassInfo); + new TypeNameBytes(typeInfo.namespaceBytes.hashCode, typeInfo.typeNameBytes.hashCode); + compositeNameBytes2TypeInfo.put(typeNameBytes, newTypeInfo); } - return newClassInfo; + return newTypeInfo; } - return classInfo; + return typeInfo; } - private ClassInfo populateBytesToClassInfo( + private TypeInfo populateBytesToTypeInfo( TypeNameBytes typeNameBytes, MetaStringBytes packageBytes, MetaStringBytes simpleClassNameBytes) { @@ -1821,21 +1840,28 @@ private ClassInfo populateBytesToClassInfo( PACKAGE_ENCODER.encode(classSpec.entireClassName, MetaString.Encoding.UTF_8)); Class cls = loadClass(classSpec.entireClassName, classSpec.isEnum, classSpec.dimension); int typeId = buildUnregisteredTypeId(cls, null); - ClassInfo classInfo = - new ClassInfo( - cls, fullClassNameBytes, packageBytes, simpleClassNameBytes, false, null, typeId); + TypeInfo typeInfo = + new TypeInfo( + cls, + fullClassNameBytes, + packageBytes, + simpleClassNameBytes, + false, + null, + typeId, + INVALID_USER_TYPE_ID); if (NonexistentClass.class.isAssignableFrom(TypeUtils.getComponentIfArray(cls))) { - classInfo.serializer = + typeInfo.serializer = NonexistentClassSerializers.getSerializer(fory, classSpec.entireClassName, cls); } else { // don't create serializer here, if the class is an interface, // there won't be serializer since interface has no instance. if (!classInfoMap.containsKey(cls)) { - classInfoMap.put(cls, classInfo); + classInfoMap.put(cls, typeInfo); } } - compositeNameBytes2ClassInfo.put(typeNameBytes, classInfo); - return classInfo; + compositeNameBytes2TypeInfo.put(typeNameBytes, typeInfo); + return typeInfo; } public Class loadClassForMeta(String className, boolean isEnum, int arrayDims) { @@ -1844,8 +1870,8 @@ public Class loadClassForMeta(String className, boolean isEnum, int arrayDims MetaStringBytes pkgBytes = metaStringResolver.getOrCreateMetaStringBytes(encodePackage(pkg)); MetaStringBytes typeBytes = metaStringResolver.getOrCreateMetaStringBytes(encodeTypeName(typeName)); - ClassInfo cachedInfo = - compositeNameBytes2ClassInfo.get(new TypeNameBytes(pkgBytes.hashCode, typeBytes.hashCode)); + TypeInfo cachedInfo = + compositeNameBytes2TypeInfo.get(new TypeNameBytes(pkgBytes.hashCode, typeBytes.hashCode)); if (cachedInfo != null) { return cachedInfo.cls; } @@ -1865,17 +1891,17 @@ public void resetRead() {} public void resetWrite() {} - // buildGenericType, nilClassInfo, nilClassInfoHolder are inherited from TypeResolver + // buildGenericType, nilTypeInfo, nilTypeInfoHolder are inherited from TypeResolver public GenericType getObjectGenericType() { return extRegistry.objectGenericType; } - public ClassInfo newClassInfo(Class cls, Serializer serializer, int typeId) { - return new ClassInfo(this, cls, serializer, typeId); + public TypeInfo newTypeInfo(Class cls, Serializer serializer, int typeId) { + return new TypeInfo(this, cls, serializer, typeId, getUserTypeIdForTypeDef(cls)); } - public boolean isPrimitive(short classId) { + public boolean isPrimitive(int classId) { return classId >= PRIMITIVE_VOID_ID && classId <= PRIMITIVE_FLOAT64_ID; } @@ -2017,11 +2043,11 @@ public void ensureSerializersCompiled() { } }); if (GraalvmSupport.isGraalBuildtime()) { - classInfoCache = NIL_CLASS_INFO; + typeInfoCache = NIL_TYPE_INFO; classInfoMap.forEach( (cls, classInfo) -> { if (classInfo.serializer != null - && !extRegistry.registeredClassInfos.contains(classInfo)) { + && !extRegistry.registeredTypeInfos.contains(classInfo)) { classInfo.serializer = null; } }); diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/MetaContext.java b/java/fory-core/src/main/java/org/apache/fory/resolver/MetaContext.java index 83f95f49c4..ececa3c691 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/MetaContext.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/MetaContext.java @@ -30,6 +30,6 @@ public class MetaContext { /** Classes which has sent definitions to peer. */ public final IdentityObjectIntMap> classMap = new IdentityObjectIntMap<>(1, 0.5f); - /** ClassInfos read from peer for reference lookup during deserialization. */ - public final ObjectArray readClassInfos = new ObjectArray<>(); + /** TypeInfos read from peer for reference lookup during deserialization. */ + public final ObjectArray readTypeInfos = new ObjectArray<>(); } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/MetaStringResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/MetaStringResolver.java index 3f070e9736..9ced4b1e82 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/MetaStringResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/MetaStringResolver.java @@ -20,6 +20,7 @@ package org.apache.fory.resolver; import java.util.Arrays; +import java.util.Objects; import org.apache.fory.collection.LongLongByteMap; import org.apache.fory.collection.LongMap; import org.apache.fory.collection.ObjectMap; @@ -71,6 +72,7 @@ public MetaStringBytes getOrCreateMetaStringBytes(MetaString str) { } public void writeMetaStringBytesWithFlag(MemoryBuffer buffer, MetaStringBytes byteString) { + Objects.requireNonNull(byteString); short id = byteString.dynamicWriteStringId; if (id == MetaStringBytes.DEFAULT_DYNAMIC_WRITE_STRING_ID) { // noinspection Duplicates @@ -86,7 +88,7 @@ public void writeMetaStringBytesWithFlag(MemoryBuffer buffer, MetaStringBytes by buffer.writeVarUint32Small7(length << 2 | 0b1); if (length > SMALL_STRING_THRESHOLD) { buffer.writeInt64(byteString.hashCode); - } else { + } else if (length != 0) { buffer.writeByte(byteString.encoding.getValue()); } buffer.writeBytes(byteString.bytes); @@ -111,7 +113,7 @@ public void writeMetaStringBytes(MemoryBuffer buffer, MetaStringBytes byteString buffer.writeVarUint32Small7(length << 1); if (length > SMALL_STRING_THRESHOLD) { buffer.writeInt64(byteString.hashCode); - } else { + } else if (length != 0) { buffer.writeByte(byteString.encoding.getValue()); } buffer.writeBytes(byteString.bytes); @@ -222,11 +224,10 @@ private MetaStringBytes readBigMetaStringBytes(MemoryBuffer buffer, int len, lon } private MetaStringBytes readSmallMetaStringBytes(MemoryBuffer buffer, int len) { - byte encoding = buffer.readByte(); if (len == 0) { - assert encoding == MetaString.Encoding.UTF_8.getValue(); return MetaStringBytes.EMPTY; } + byte encoding = buffer.readByte(); long v1, v2 = 0; if (len <= 8) { v1 = buffer.readBytesAsInt64(len); @@ -243,11 +244,10 @@ private MetaStringBytes readSmallMetaStringBytes(MemoryBuffer buffer, int len) { private MetaStringBytes readSmallMetaStringBytes( MemoryBuffer buffer, MetaStringBytes cache, int len) { - byte encoding = buffer.readByte(); if (len == 0) { - assert encoding == MetaString.Encoding.UTF_8.getValue(); return MetaStringBytes.EMPTY; } + byte encoding = buffer.readByte(); long v1, v2 = 0; if (len <= 8) { v1 = buffer.readBytesAsInt64(len); diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/SerializationContext.java b/java/fory-core/src/main/java/org/apache/fory/resolver/SerializationContext.java index 852bb686da..86b69b12b6 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/SerializationContext.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/SerializationContext.java @@ -84,7 +84,7 @@ public void resetRead() { objects.clear(); } if (scopedMetaShareEnabled) { - metaContext.readClassInfos.size = 0; + metaContext.readTypeInfos.size = 0; } else { metaContext = null; } @@ -96,7 +96,7 @@ public void reset() { } if (scopedMetaShareEnabled) { metaContext.classMap.clear(); - metaContext.readClassInfos.size = 0; + metaContext.readTypeInfos.size = 0; } else { metaContext = null; } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassInfo.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeInfo.java similarity index 70% rename from java/fory-core/src/main/java/org/apache/fory/resolver/ClassInfo.java rename to java/fory-core/src/main/java/org/apache/fory/resolver/TypeInfo.java index f72086fbdf..de5ce49a43 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassInfo.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeInfo.java @@ -24,9 +24,9 @@ import static org.apache.fory.meta.Encoders.TYPE_NAME_DECODER; import org.apache.fory.collection.Tuple2; -import org.apache.fory.meta.ClassDef; import org.apache.fory.meta.Encoders; import org.apache.fory.meta.MetaString.Encoding; +import org.apache.fory.meta.TypeDef; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.serializer.Serializer; import org.apache.fory.type.Types; @@ -36,60 +36,69 @@ * This class put together object type related information to reduce array/map loop up when * serialization. */ -public class ClassInfo { +public class TypeInfo { final Class cls; final MetaStringBytes fullNameBytes; final MetaStringBytes namespaceBytes; final MetaStringBytes typeNameBytes; final boolean isDynamicGeneratedClass; - // Unified type ID for both native and xlang modes. + // Fory type ID for both native and xlang modes. // - Types 0-30: Shared internal types (Types.BOOL, Types.STRING, etc.) // - Types 31-255: Native-only internal types (VOID_ID, CHAR_ID, etc.) - // - Types 256+: User-registered types encoded as (userTypeId << 8) | internalTypeId - int typeId; + final int typeId; + // User-registered type ID stored as unsigned int. -1 (0xffffffff) means "unset". + final int userTypeId; Serializer serializer; - ClassDef classDef; - boolean needToWriteClassDef; + TypeDef typeDef; + boolean needToWriteTypeDef; - ClassInfo( + TypeInfo( Class cls, MetaStringBytes fullNameBytes, MetaStringBytes namespaceBytes, MetaStringBytes typeNameBytes, boolean isDynamicGeneratedClass, Serializer serializer, - int typeId) { + int typeId, + int userTypeId) { this.cls = cls; this.fullNameBytes = fullNameBytes; this.namespaceBytes = namespaceBytes; this.typeNameBytes = typeNameBytes; this.isDynamicGeneratedClass = isDynamicGeneratedClass; this.typeId = typeId; + this.userTypeId = userTypeId; this.serializer = serializer; } /** - * Creates a ClassInfo for deserialization with a ClassDef. Used when reading class meta from - * stream where the ClassDef specifies the field layout. + * Creates a TypeInfo for deserialization with a TypeDef. Used when reading class meta from stream + * where the TypeDef specifies the field layout. * * @param cls the class - * @param classDef the class definition from stream + * @param typeDef the class definition from stream */ - public ClassInfo(Class cls, ClassDef classDef) { + public TypeInfo(Class cls, TypeDef typeDef) { this.cls = cls; - this.classDef = classDef; + this.typeDef = typeDef; this.fullNameBytes = null; this.namespaceBytes = null; this.typeNameBytes = null; this.isDynamicGeneratedClass = false; this.serializer = null; - this.typeId = classDef == null ? Types.UNKNOWN : classDef.getClassSpec().typeId; + this.typeId = typeDef == null ? Types.UNKNOWN : typeDef.getClassSpec().typeId; + this.userTypeId = typeDef == null ? -1 : typeDef.getClassSpec().userTypeId; } - ClassInfo(TypeResolver classResolver, Class cls, Serializer serializer, int typeId) { + TypeInfo( + TypeResolver classResolver, + Class cls, + Serializer serializer, + int typeId, + int userTypeId) { this.cls = cls; this.serializer = serializer; - needToWriteClassDef = serializer != null && classResolver.needToWriteClassDef(serializer); + needToWriteTypeDef = serializer != null && classResolver.needToWriteTypeDef(serializer); MetaStringResolver metaStringResolver = classResolver.getMetaStringResolver(); if (cls != null && classResolver.getFory().isCrossLanguage()) { this.fullNameBytes = @@ -119,51 +128,74 @@ public ClassInfo(Class cls, ClassDef classDef) { this.namespaceBytes = null; this.typeNameBytes = null; } - this.typeId = typeId; if (cls != null) { boolean isLambda = Functions.isLambda(cls); boolean isProxy = typeId != ClassResolver.REPLACE_STUB_ID && ReflectionUtils.isJdkProxy(cls); this.isDynamicGeneratedClass = isLambda || isProxy; if (isLambda) { - this.typeId = ClassResolver.LAMBDA_STUB_ID; + typeId = ClassResolver.LAMBDA_STUB_ID; } if (isProxy) { - this.typeId = ClassResolver.JDK_PROXY_STUB_ID; + typeId = ClassResolver.JDK_PROXY_STUB_ID; } } else { this.isDynamicGeneratedClass = false; } + this.typeId = typeId; + this.userTypeId = userTypeId; + } + + public TypeInfo copy(int typeId) { + if (typeId == this.typeId) { + return this; + } + return new TypeInfo( + cls, + fullNameBytes, + namespaceBytes, + typeNameBytes, + isDynamicGeneratedClass, + serializer, + typeId, + userTypeId); + } + + public TypeInfo copy(int typeId, int userTypeId) { + if (typeId == this.typeId && userTypeId == this.userTypeId) { + return this; + } + return new TypeInfo( + cls, + fullNameBytes, + namespaceBytes, + typeNameBytes, + isDynamicGeneratedClass, + serializer, + typeId, + userTypeId); } public Class getCls() { return cls; } - public ClassDef getClassDef() { - return classDef; + public TypeDef getTypeDef() { + return typeDef; } - void setClassDef(ClassDef classDef) { - this.classDef = classDef; + void setTypeDef(TypeDef typeDef) { + this.typeDef = typeDef; } - /** - * Returns the unified type ID for this class. - * - *

Type ID encoding: - * - *

    - *
  • 0-30: Shared internal types (Types.BOOL, Types.STRING, etc.) - *
  • 31-255: Native-only internal types (VOID_ID, CHAR_ID, etc.) - *
  • 256+: User-registered types encoded as (userTypeId << 8) | internalTypeId - *
- * - * @return the unified type ID - */ + /** Returns the fory type ID for this class. */ public int getTypeId() { return typeId; } + public int getUserTypeId() { + return userTypeId; + } + @SuppressWarnings("unchecked") public Serializer getSerializer() { return (Serializer) serializer; @@ -175,7 +207,7 @@ public void setSerializer(Serializer serializer) { void setSerializer(TypeResolver resolver, Serializer serializer) { this.serializer = serializer; - needToWriteClassDef = serializer != null && resolver.needToWriteClassDef(serializer); + needToWriteTypeDef = serializer != null && resolver.needToWriteTypeDef(serializer); } public String decodeNamespace() { @@ -188,7 +220,7 @@ public String decodeTypeName() { @Override public String toString() { - return "ClassInfo{" + return "TypeInfo{" + "cls=" + cls + ", fullClassNameBytes=" @@ -199,6 +231,8 @@ public String toString() { + serializer + ", typeId=" + typeId + + ", userTypeId=" + + userTypeId + '}'; } } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassInfoHolder.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeInfoHolder.java similarity index 77% rename from java/fory-core/src/main/java/org/apache/fory/resolver/ClassInfoHolder.java rename to java/fory-core/src/main/java/org/apache/fory/resolver/TypeInfoHolder.java index de4fe92f58..36d2149be5 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassInfoHolder.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeInfoHolder.java @@ -21,21 +21,21 @@ import org.apache.fory.serializer.Serializer; -/** A helper class for holding and update {@link ClassInfo} to reduce map look-up. */ +/** A helper class for holding and update {@link TypeInfo} to reduce map look-up. */ @SuppressWarnings("rawtypes") -public class ClassInfoHolder { - public ClassInfo classInfo; +public class TypeInfoHolder { + public TypeInfo typeInfo; - public ClassInfoHolder(ClassInfo classInfo) { - this.classInfo = classInfo; + public TypeInfoHolder(TypeInfo typeInfo) { + this.typeInfo = typeInfo; } public Serializer getSerializer() { - return classInfo.serializer; + return typeInfo.serializer; } @Override public String toString() { - return "Holder{" + classInfo + '}'; + return "Holder{" + typeInfo + '}'; } } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index c973b50b78..d55d6167bf 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -20,6 +20,7 @@ package org.apache.fory.resolver; import static org.apache.fory.type.TypeUtils.getSizeOfPrimitiveType; +import static org.apache.fory.type.Types.INVALID_USER_TYPE_ID; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; @@ -34,6 +35,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.SortedMap; import java.util.concurrent.ConcurrentHashMap; @@ -58,8 +60,8 @@ import org.apache.fory.logging.Logger; import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.meta.ClassDef; import org.apache.fory.meta.ClassSpec; +import org.apache.fory.meta.TypeDef; import org.apache.fory.meta.TypeExtMeta; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.reflect.TypeRef; @@ -94,8 +96,8 @@ public abstract class TypeResolver { private static final Logger LOG = LoggerFactory.getLogger(ClassResolver.class); - static final ClassInfo NIL_CLASS_INFO = - new ClassInfo(null, null, null, null, false, null, Types.UNKNOWN); + static final TypeInfo NIL_TYPE_INFO = + new TypeInfo(null, null, null, null, false, null, Types.UNKNOWN, INVALID_USER_TYPE_ID); // use a lower load factor to minimize hash collision static final float foryMapLoadFactor = 0.25f; static final int estimatedNumRegistered = 150; @@ -104,21 +106,22 @@ public abstract class TypeResolver { + "please set meta context by SerializationContext.setMetaContext"; private static final GenericType OBJECT_GENERIC_TYPE = GenericType.build(Object.class); private static final float TYPE_ID_MAP_LOAD_FACTOR = 0.5f; + static final long MAX_USER_TYPE_ID = 0xffff_fffEL; final Fory fory; final boolean metaContextShareEnabled; final MetaStringResolver metaStringResolver; // IdentityMap has better lookup performance, when loadFactor is 0.05f, performance is better - final IdentityMap, ClassInfo> classInfoMap = new IdentityMap<>(64, foryMapLoadFactor); + final IdentityMap, TypeInfo> classInfoMap = new IdentityMap<>(64, foryMapLoadFactor); final ExtRegistry extRegistry; - final Map, ClassDef> classDefMap = new HashMap<>(); + final Map, TypeDef> typeDefMap = new HashMap<>(); // Map for internal type ids (non-user-defined). - ClassInfo[] typeIdToClassInfo = new ClassInfo[] {}; + TypeInfo[] typeIdToTypeInfo = new TypeInfo[] {}; // Map for user-registered type ids, keyed by user id. - final LongMap userTypeIdToClassInfo = new LongMap<>(4, TYPE_ID_MAP_LOAD_FACTOR); - // Cache for readClassInfo(MemoryBuffer) - persists between calls to avoid reloading + final LongMap userTypeIdToTypeInfo = new LongMap<>(4, TYPE_ID_MAP_LOAD_FACTOR); + // Cache for readTypeInfo(MemoryBuffer) - persists between calls to avoid reloading // dynamically created classes that can't be found by Class.forName - private ClassInfo classInfoCache; + private TypeInfo typeInfoCache; protected TypeResolver(Fory fory) { this.fory = fory; @@ -143,12 +146,13 @@ protected final void checkRegisterAllowed() { public abstract void register(Class type); /** - * Registers a class with a user-specified ID. Valid ID range is [0, 32510]. + * Registers a class with a user-specified ID. Valid ID range is [0, 0xfffffffe] (0xffffffff is + * reserved for "unset"). * * @param type the class to register * @param id the user ID to assign (0-based) */ - public abstract void register(Class type, int id); + public abstract void register(Class type, long id); /** * Registers a class with a namespace and type name for cross-language serialization. @@ -165,7 +169,7 @@ public void register(String className) { } /** Registers a class by name with a user-specified ID. */ - public void register(String className, int classId) { + public void register(String className, long classId) { register(loadClass(className), classId); } @@ -181,7 +185,7 @@ public void register(String className, String namespace, String typeName) { * @param id the user ID to assign (0-based) * @param serializer serializer for the union */ - public abstract void registerUnion(Class type, int id, Serializer serializer); + public abstract void registerUnion(Class type, long id, Serializer serializer); /** * Registers a union type with a namespace and type name and serializer. @@ -266,16 +270,16 @@ public final boolean needToWriteRef(TypeRef typeRef) { return meta.trackingRef(); } - ClassInfo classInfo = classInfoMap.get(cls); - if (classInfo == null || classInfo.serializer == null) { + TypeInfo typeInfo = classInfoMap.get(cls); + if (typeInfo == null || typeInfo.serializer == null) { // TODO group related logic together for extendability and consistency. return !cls.isEnum(); } else { - return classInfo.serializer.needToWriteRef(); + return typeInfo.serializer.needToWriteRef(); } } - public final boolean needToWriteClassDef(Serializer serializer) { + public final boolean needToWriteTypeDef(Serializer serializer) { if (fory.getConfig().getCompatibleMode() != CompatibleMode.COMPATIBLE) { return false; } @@ -305,11 +309,11 @@ public final boolean needToWriteClassDef(Serializer serializer) { public abstract boolean isMonomorphic(Class clz); - public abstract ClassInfo getClassInfo(Class cls); + public abstract TypeInfo getTypeInfo(Class cls); - public abstract ClassInfo getClassInfo(Class cls, boolean createIfAbsent); + public abstract TypeInfo getTypeInfo(Class cls, boolean createIfAbsent); - public abstract ClassInfo getClassInfo(Class cls, ClassInfoHolder classInfoHolder); + public abstract TypeInfo getTypeInfo(Class cls, TypeInfoHolder classInfoHolder); /** * Writes class info to buffer using the unified type system. This is the single implementation @@ -325,29 +329,31 @@ public final boolean needToWriteClassDef(Serializer serializer) { *
  • Other types: just the type ID * */ - public final void writeClassInfo(MemoryBuffer buffer, ClassInfo classInfo) { - int typeId = classInfo.getTypeId(); - int internalTypeId = typeId & 0xff; - buffer.writeVarUint32Small7(typeId); - - switch (internalTypeId) { + public final void writeTypeInfo(MemoryBuffer buffer, TypeInfo typeInfo) { + int typeId = typeInfo.getTypeId(); + buffer.writeUint8(typeId); + switch (typeId) { + case Types.ENUM: + case Types.STRUCT: + case Types.EXT: + case Types.TYPED_UNION: + buffer.writeVarUint32(typeInfo.userTypeId); + break; + case Types.COMPATIBLE_STRUCT: + case Types.NAMED_COMPATIBLE_STRUCT: + writeSharedClassMeta(buffer, typeInfo); + break; case Types.NAMED_ENUM: case Types.NAMED_STRUCT: case Types.NAMED_EXT: case Types.NAMED_UNION: - case Types.NAMED_COMPATIBLE_STRUCT: if (!metaContextShareEnabled) { - Preconditions.checkNotNull(classInfo.namespaceBytes); - metaStringResolver.writeMetaStringBytes(buffer, classInfo.namespaceBytes); - Preconditions.checkNotNull(classInfo.typeNameBytes); - metaStringResolver.writeMetaStringBytes(buffer, classInfo.typeNameBytes); + Preconditions.checkNotNull(typeInfo.namespaceBytes); + metaStringResolver.writeMetaStringBytes(buffer, typeInfo.namespaceBytes); + Preconditions.checkNotNull(typeInfo.typeNameBytes); + metaStringResolver.writeMetaStringBytes(buffer, typeInfo.typeNameBytes); } else { - writeSharedClassMeta(buffer, classInfo); - } - break; - case Types.COMPATIBLE_STRUCT: - if (metaContextShareEnabled && classInfo.cls != NonexistentMetaShared.class) { - writeSharedClassMeta(buffer, classInfo); + writeSharedClassMeta(buffer, typeInfo); } break; default: @@ -355,8 +361,15 @@ public final void writeClassInfo(MemoryBuffer buffer, ClassInfo classInfo) { } } + static int toUserTypeId(long userTypeId) { + Preconditions.checkArgument( + userTypeId >= 0 && userTypeId <= MAX_USER_TYPE_ID, + "User type id must be in range [0, 0xfffffffe]"); + return (int) userTypeId; + } + /** - * Native code for ClassResolver.writeClassInfo is too big to inline, so inline it manually. + * Native code for ClassResolver.writeTypeInfo is too big to inline, so inline it manually. * *

    See `already compiled into a big method` in Server+Compiler+Inlining+Messages @@ -364,290 +377,265 @@ public final void writeClassInfo(MemoryBuffer buffer, ClassInfo classInfo) { // Note: Thread safe for jit thread to call. public Expression writeClassExpr( Expression classResolverRef, Expression buffer, Expression classInfo) { - return new Invoke(classResolverRef, "writeClassInfo", buffer, classInfo); + return new Invoke(classResolverRef, "writeTypeInfo", buffer, classInfo); } /** * Writes shared class metadata using the meta-share protocol. Protocol: If class already written, - * writes (index << 1) | 1 (reference). If new class, writes (index << 1) followed by ClassDef + * writes (index << 1) | 1 (reference). If new class, writes (index << 1) followed by TypeDef * bytes. * *

    This method is shared between XtypeResolver and ClassResolver. */ - protected final void writeSharedClassMeta(MemoryBuffer buffer, ClassInfo classInfo) { + protected final void writeSharedClassMeta(MemoryBuffer buffer, TypeInfo typeInfo) { MetaContext metaContext = fory.getSerializationContext().getMetaContext(); assert metaContext != null : SET_META__CONTEXT_MSG; IdentityObjectIntMap> classMap = metaContext.classMap; int newId = classMap.size; - int id = classMap.putOrGet(classInfo.cls, newId); + int id = classMap.putOrGet(typeInfo.cls, newId); if (id >= 0) { // Reference to previously written type: (index << 1) | 1, LSB=1 buffer.writeVarUint32((id << 1) | 1); } else { - // New type: index << 1, LSB=0, followed by ClassDef bytes inline + // New type: index << 1, LSB=0, followed by TypeDef bytes inline buffer.writeVarUint32(newId << 1); - ClassDef classDef = classInfo.classDef; - if (classDef == null) { - classDef = buildClassDef(classInfo); + TypeDef typeDef = typeInfo.typeDef; + if (typeDef == null) { + typeDef = buildTypeDef(typeInfo); } - buffer.writeBytes(classDef.getEncoded()); + buffer.writeBytes(typeDef.getEncoded()); } } /** - * Build ClassDef for the given ClassInfo. Used by writeSharedClassMeta when the classDef is not - * yet created. + * Build TypeDef for the given TypeInfo. Used by writeSharedClassMeta when the typeDef is not yet + * created. */ - protected abstract ClassDef buildClassDef(ClassInfo classInfo); + protected abstract TypeDef buildTypeDef(TypeInfo typeInfo); /** * Reads class info from buffer using the unified type system. This is the single implementation * shared by both ClassResolver and XtypeResolver. * - *

    Note: {@link #readClassInfo(MemoryBuffer, ClassInfo)} is faster since it uses a non-global + *

    Note: {@link #readTypeInfo(MemoryBuffer, TypeInfo)} is faster since it uses a non-global * class info cache. */ - public final ClassInfo readClassInfo(MemoryBuffer buffer) { - int header = buffer.readVarUint32Small14(); - int internalTypeId = header & 0xff; - ClassInfo classInfo; - switch (internalTypeId) { + public final TypeInfo readTypeInfo(MemoryBuffer buffer) { + int typeId = buffer.readUint8(); + TypeInfo typeInfo; + switch (typeId) { + case Types.ENUM: + case Types.STRUCT: + case Types.EXT: + case Types.TYPED_UNION: + typeInfo = Objects.requireNonNull(userTypeIdToTypeInfo.get(buffer.readVarUint32())); + break; + case Types.COMPATIBLE_STRUCT: + case Types.NAMED_COMPATIBLE_STRUCT: + typeInfo = readSharedClassMeta(buffer); + break; case Types.NAMED_ENUM: case Types.NAMED_STRUCT: case Types.NAMED_EXT: case Types.NAMED_UNION: - case Types.NAMED_COMPATIBLE_STRUCT: if (!metaContextShareEnabled) { - classInfo = readClassInfoFromBytes(buffer, classInfoCache, header); - } else { - classInfo = readSharedClassMeta(buffer); - } - break; - case Types.COMPATIBLE_STRUCT: - if (metaContextShareEnabled) { - classInfo = readSharedClassMeta(buffer); + typeInfo = readTypeInfoFromBytes(buffer, typeInfoCache, typeId); } else { - classInfo = requireUserTypeInfoByTypeId(header); + typeInfo = readSharedClassMeta(buffer); } break; - case Types.ENUM: - case Types.STRUCT: - case Types.EXT: - case Types.UNION: - case Types.TYPED_UNION: - classInfo = requireUserTypeInfoByTypeId(header); - break; case Types.LIST: - classInfo = getListClassInfo(); + typeInfo = getListTypeInfo(); break; case Types.TIMESTAMP: - classInfo = getTimestampClassInfo(); + typeInfo = getTimestampTypeInfo(); break; default: - classInfo = requireInternalTypeInfoByTypeId(header); + typeInfo = Objects.requireNonNull(getInternalTypeInfoByTypeId(typeId)); } - if (classInfo.serializer == null) { - classInfo = ensureSerializerForClassInfo(classInfo); + if (typeInfo.serializer == null) { + typeInfo = ensureSerializerForTypeInfo(typeInfo); } - classInfoCache = classInfo; - return classInfo; + typeInfoCache = typeInfo; + return typeInfo; } /** * Read class info from buffer using a target class. This is used by java serialization APIs that * pass an expected class for meta share resolution. */ - public final ClassInfo readClassInfo(MemoryBuffer buffer, Class targetClass) { - int header = buffer.readVarUint32Small14(); - int internalTypeId = header & 0xff; - ClassInfo classInfo; - switch (internalTypeId) { + public final TypeInfo readTypeInfo(MemoryBuffer buffer, Class targetClass) { + int typeId = buffer.readUint8(); + TypeInfo typeInfo; + switch (typeId) { + case Types.ENUM: + case Types.STRUCT: + case Types.EXT: + case Types.TYPED_UNION: + typeInfo = Objects.requireNonNull(userTypeIdToTypeInfo.get(buffer.readVarUint32())); + break; + case Types.COMPATIBLE_STRUCT: + case Types.NAMED_COMPATIBLE_STRUCT: + typeInfo = readSharedClassMeta(buffer, targetClass); + break; case Types.NAMED_ENUM: case Types.NAMED_STRUCT: case Types.NAMED_EXT: case Types.NAMED_UNION: - case Types.NAMED_COMPATIBLE_STRUCT: if (!metaContextShareEnabled) { - classInfo = readClassInfoFromBytes(buffer, classInfoCache, header); + typeInfo = readTypeInfoFromBytes(buffer, typeInfoCache, typeId); } else { - classInfo = readSharedClassMeta(buffer, targetClass); + typeInfo = readSharedClassMeta(buffer, targetClass); } break; - case Types.COMPATIBLE_STRUCT: - if (metaContextShareEnabled) { - classInfo = readSharedClassMeta(buffer, targetClass); - } else { - classInfo = requireUserTypeInfoByTypeId(header); - } - break; - case Types.ENUM: - case Types.STRUCT: - case Types.EXT: - case Types.UNION: - case Types.TYPED_UNION: - classInfo = requireUserTypeInfoByTypeId(header); - break; case Types.LIST: - classInfo = getListClassInfo(); + typeInfo = getListTypeInfo(); break; case Types.TIMESTAMP: - classInfo = getTimestampClassInfo(); + typeInfo = getTimestampTypeInfo(); break; default: - classInfo = requireInternalTypeInfoByTypeId(header); + typeInfo = Objects.requireNonNull(getInternalTypeInfoByTypeId(typeId)); } - if (classInfo.serializer == null) { - classInfo = ensureSerializerForClassInfo(classInfo); + if (typeInfo.serializer == null) { + typeInfo = ensureSerializerForTypeInfo(typeInfo); } - classInfoCache = classInfo; - return classInfo; + typeInfoCache = typeInfo; + return typeInfo; } /** - * Read class info from buffer with ClassInfo cache. This version is faster than {@link - * #readClassInfo(MemoryBuffer)} because it uses the provided classInfoCache to reduce map lookups + * Read class info from buffer with TypeInfo cache. This version is faster than {@link + * #readTypeInfo(MemoryBuffer)} because it uses the provided classInfoCache to reduce map lookups * when reading class from binary. * * @param buffer the buffer to read from - * @param classInfoCache cache for class info to speed up repeated reads - * @return the ClassInfo read from buffer + * @param typeInfoCache cache for class info to speed up repeated reads + * @return the TypeInfo read from buffer */ @CodegenInvoke - public final ClassInfo readClassInfo(MemoryBuffer buffer, ClassInfo classInfoCache) { - int header = buffer.readVarUint32Small14(); - int internalTypeId = header & 0xff; - ClassInfo classInfo; - switch (internalTypeId) { + public final TypeInfo readTypeInfo(MemoryBuffer buffer, TypeInfo typeInfoCache) { + int typeId = buffer.readUint8(); + TypeInfo typeInfo; + switch (typeId) { + case Types.ENUM: + case Types.STRUCT: + case Types.EXT: + case Types.TYPED_UNION: + typeInfo = Objects.requireNonNull(userTypeIdToTypeInfo.get(buffer.readVarUint32())); + break; + case Types.COMPATIBLE_STRUCT: + case Types.NAMED_COMPATIBLE_STRUCT: + typeInfo = readSharedClassMeta(buffer); + break; case Types.NAMED_ENUM: case Types.NAMED_STRUCT: case Types.NAMED_EXT: case Types.NAMED_UNION: - case Types.NAMED_COMPATIBLE_STRUCT: if (!metaContextShareEnabled) { - classInfo = readClassInfoFromBytes(buffer, classInfoCache, header); - } else { - classInfo = readSharedClassMeta(buffer); - } - break; - case Types.COMPATIBLE_STRUCT: - if (metaContextShareEnabled) { - classInfo = readSharedClassMeta(buffer); + typeInfo = readTypeInfoFromBytes(buffer, typeInfoCache, typeId); } else { - classInfo = requireUserTypeInfoByTypeId(header); + typeInfo = readSharedClassMeta(buffer); } break; - case Types.ENUM: - case Types.STRUCT: - case Types.EXT: - case Types.UNION: - case Types.TYPED_UNION: - classInfo = requireUserTypeInfoByTypeId(header); - break; case Types.LIST: - classInfo = getListClassInfo(); + typeInfo = getListTypeInfo(); break; case Types.TIMESTAMP: - classInfo = getTimestampClassInfo(); + typeInfo = getTimestampTypeInfo(); break; default: - classInfo = requireInternalTypeInfoByTypeId(header); + typeInfo = Objects.requireNonNull(getInternalTypeInfoByTypeId(typeId)); } - if (classInfo.serializer == null) { - classInfo = ensureSerializerForClassInfo(classInfo); + if (typeInfo.serializer == null) { + typeInfo = ensureSerializerForTypeInfo(typeInfo); } - return classInfo; + return typeInfo; } /** - * Read class info from buffer with ClassInfoHolder cache. This version updates the - * classInfoHolder if the cache doesn't hit, allowing callers to maintain the cache across calls. + * Read class info from buffer with TypeInfoHolder cache. This version updates the classInfoHolder + * if the cache doesn't hit, allowing callers to maintain the cache across calls. * * @param buffer the buffer to read from * @param classInfoHolder holder containing cache, will be updated on cache miss - * @return the ClassInfo read from buffer + * @return the TypeInfo read from buffer */ @CodegenInvoke - public final ClassInfo readClassInfo(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { - int header = buffer.readVarUint32Small14(); - int internalTypeId = header & 0xff; - ClassInfo classInfo; + public final TypeInfo readTypeInfo(MemoryBuffer buffer, TypeInfoHolder classInfoHolder) { + int typeId = buffer.readUint8(); + TypeInfo typeInfo; boolean updateCache = false; - switch (internalTypeId) { + switch (typeId) { + case Types.ENUM: + case Types.STRUCT: + case Types.EXT: + case Types.TYPED_UNION: + typeInfo = Objects.requireNonNull(userTypeIdToTypeInfo.get(buffer.readVarUint32())); + break; + case Types.COMPATIBLE_STRUCT: + case Types.NAMED_COMPATIBLE_STRUCT: + typeInfo = readSharedClassMeta(buffer); + break; case Types.NAMED_ENUM: case Types.NAMED_STRUCT: case Types.NAMED_EXT: case Types.NAMED_UNION: - case Types.NAMED_COMPATIBLE_STRUCT: if (!metaContextShareEnabled) { - classInfo = readClassInfoFromBytes(buffer, classInfoHolder.classInfo, header); + typeInfo = readTypeInfoFromBytes(buffer, classInfoHolder.typeInfo, typeId); updateCache = true; } else { - classInfo = readSharedClassMeta(buffer); + typeInfo = readSharedClassMeta(buffer); } break; - case Types.COMPATIBLE_STRUCT: - if (metaContextShareEnabled) { - classInfo = readSharedClassMeta(buffer); - } else { - classInfo = requireUserTypeInfoByTypeId(header); - } - break; - case Types.ENUM: - case Types.STRUCT: - case Types.EXT: - case Types.UNION: - case Types.TYPED_UNION: - classInfo = requireUserTypeInfoByTypeId(header); - break; case Types.LIST: - classInfo = getListClassInfo(); + typeInfo = getListTypeInfo(); break; case Types.TIMESTAMP: - classInfo = getTimestampClassInfo(); + typeInfo = getTimestampTypeInfo(); break; default: - classInfo = requireInternalTypeInfoByTypeId(header); + typeInfo = Objects.requireNonNull(getInternalTypeInfoByTypeId(typeId)); } - if (classInfo.serializer == null) { - classInfo = ensureSerializerForClassInfo(classInfo); + if (typeInfo.serializer == null) { + typeInfo = ensureSerializerForTypeInfo(typeInfo); } if (updateCache) { - classInfoHolder.classInfo = classInfo; + classInfoHolder.typeInfo = typeInfo; } - return classInfo; + return typeInfo; } /** - * Read class info using the provided cache. Returns cached ClassInfo if the namespace and type + * Read class info using the provided cache. Returns cached TypeInfo if the namespace and type * name bytes match. */ - protected final ClassInfo readClassInfoByCache( - MemoryBuffer buffer, ClassInfo classInfoCache, int header) { - return readClassInfoFromBytes(buffer, classInfoCache, header); + protected final TypeInfo readTypeInfoByCache( + MemoryBuffer buffer, TypeInfo typeInfoCache, int header) { + return readTypeInfoFromBytes(buffer, typeInfoCache, header); } /** * Read class info from bytes with cache optimization. Uses the cached namespace and type name * bytes to avoid map lookups when the class is the same as the cached one (hash comparison). */ - protected final ClassInfo readClassInfoFromBytes( - MemoryBuffer buffer, ClassInfo classInfoCache, int header) { - MetaStringBytes typeNameBytesCache = - classInfoCache != null ? classInfoCache.typeNameBytes : null; + protected final TypeInfo readTypeInfoFromBytes( + MemoryBuffer buffer, TypeInfo typeInfoCache, int header) { + MetaStringBytes typeNameBytesCache = typeInfoCache != null ? typeInfoCache.typeNameBytes : null; MetaStringBytes namespaceBytes; MetaStringBytes simpleClassNameBytes; if (typeNameBytesCache != null) { // Use cache for faster comparison - MetaStringBytes packageNameBytesCache = classInfoCache.namespaceBytes; + MetaStringBytes packageNameBytesCache = typeInfoCache.namespaceBytes; namespaceBytes = metaStringResolver.readMetaStringBytes(buffer, packageNameBytesCache); assert packageNameBytesCache != null; simpleClassNameBytes = metaStringResolver.readMetaStringBytes(buffer, typeNameBytesCache); - // Fast path: if hashes match, return cached ClassInfo (already has serializer) + // Fast path: if hashes match, return cached TypeInfo (already has serializer) if (typeNameBytesCache.hashCode == simpleClassNameBytes.hashCode && packageNameBytesCache.hashCode == namespaceBytes.hashCode) { - return classInfoCache; + return typeInfoCache; } } else { // No cache available, read fresh @@ -656,246 +644,262 @@ protected final ClassInfo readClassInfoFromBytes( } // Load class info from bytes (subclass-specific). - return loadBytesToClassInfo(namespaceBytes, simpleClassNameBytes); + return loadBytesToTypeInfo(header, namespaceBytes, simpleClassNameBytes); } /** * Reads shared class metadata from buffer. This is the shared implementation used by both * ClassResolver and XtypeResolver. */ - protected final ClassInfo readSharedClassMeta(MemoryBuffer buffer) { + protected final TypeInfo readSharedClassMeta(MemoryBuffer buffer) { MetaContext metaContext = fory.getSerializationContext().getMetaContext(); assert metaContext != null : SET_META__CONTEXT_MSG; int indexMarker = buffer.readVarUint32Small14(); boolean isRef = (indexMarker & 1) == 1; int index = indexMarker >>> 1; - ClassInfo classInfo; + TypeInfo typeInfo; if (isRef) { // Reference to previously read type in this stream - classInfo = metaContext.readClassInfos.get(index); + typeInfo = metaContext.readTypeInfos.get(index); } else { // New type in stream - but may already be known from registry long id = buffer.readInt64(); - Tuple2 tuple2 = extRegistry.classIdToDef.get(id); + Tuple2 tuple2 = extRegistry.classIdToDef.get(id); if (tuple2 != null) { - // Already known - skip the ClassDef bytes, reuse existing ClassInfo - ClassDef.skipClassDef(buffer, id); - classInfo = tuple2.f1; - if (classInfo == null) { - classInfo = buildMetaSharedClassInfo(tuple2, tuple2.f0); + // Already known - skip the TypeDef bytes, reuse existing TypeInfo + TypeDef.skipTypeDef(buffer, id); + typeInfo = tuple2.f1; + if (typeInfo == null) { + typeInfo = buildMetaSharedTypeInfo(tuple2, tuple2.f0); } } else { - // Unknown - read ClassDef and create ClassInfo - tuple2 = readClassDef(buffer, id); - classInfo = tuple2.f1; - if (classInfo == null) { - classInfo = buildMetaSharedClassInfo(tuple2, tuple2.f0); + // Unknown - read TypeDef and create TypeInfo + tuple2 = readTypeDef(buffer, id); + typeInfo = tuple2.f1; + if (typeInfo == null) { + typeInfo = buildMetaSharedTypeInfo(tuple2, tuple2.f0); } } - // index == readClassInfos.size() since types are written sequentially - metaContext.readClassInfos.add(classInfo); + // index == readTypeInfos.size() since types are written sequentially + metaContext.readTypeInfos.add(typeInfo); } - return classInfo; + return typeInfo; } - public final ClassInfo readSharedClassMeta(MemoryBuffer buffer, Class targetClass) { - ClassInfo classInfo = readSharedClassMeta(buffer); - Class readClass = classInfo.getCls(); + public final TypeInfo readSharedClassMeta(MemoryBuffer buffer, Class targetClass) { + TypeInfo typeInfo = readSharedClassMeta(buffer); + Class readClass = typeInfo.getCls(); // replace target class if needed if (targetClass != readClass) { - Tuple2, Class> key = Tuple2.of(readClass, targetClass); - ClassInfo newClassInfo = extRegistry.transformedClassInfo.get(key); - if (newClassInfo == null) { - // similar to create serializer for `NonexistentMetaShared` - newClassInfo = - getMetaSharedClassInfo( - classInfo.classDef.replaceRootClassTo((ClassResolver) this, targetClass), - targetClass); - extRegistry.transformedClassInfo.put(key, newClassInfo); + return getTargetTypeInfo(typeInfo, targetClass); + } + return typeInfo; + } + + private TypeInfo getTargetTypeInfo(TypeInfo typeInfo, Class targetClass) { + Tuple2, TypeInfo>[] infos = extRegistry.transformedTypeInfo.get(targetClass); + Class readClass = typeInfo.getCls(); + if (infos != null) { + // It's ok to use loop here since most of case the array size will be 1. + for (Tuple2, TypeInfo> info : infos) { + if (info.f0 == readClass) { + return info.f1; + } } - return newClassInfo; } - return classInfo; + return transformTypeInfo(typeInfo, targetClass); + } + + private TypeInfo transformTypeInfo(TypeInfo typeInfo, Class targetClass) { + Class readClass = typeInfo.getCls(); + TypeInfo newTypeInfo; + if (targetClass.isAssignableFrom(readClass)) { + newTypeInfo = typeInfo; + } else { + // similar to create serializer for `NonexistentMetaShared` + newTypeInfo = + getMetaSharedTypeInfo( + typeInfo.typeDef.replaceRootClassTo(this, targetClass), targetClass); + } + Tuple2, TypeInfo>[] infos = extRegistry.transformedTypeInfo.get(targetClass); + int size = infos == null ? 0 : infos.length; + @SuppressWarnings("unchecked") + Tuple2, TypeInfo>[] newInfos = (Tuple2, TypeInfo>[]) new Tuple2[size + 1]; + if (size > 0) { + System.arraycopy(infos, 0, newInfos, 0, size); + } + newInfos[size] = Tuple2.of(readClass, newTypeInfo); + extRegistry.transformedTypeInfo.put(targetClass, newInfos); + return newTypeInfo; } /** * Load class info from namespace and type name bytes. Subclasses implement this to resolve the - * class and create/lookup ClassInfo. + * class and create/lookup TypeInfo. * - *

    Note: This method should NOT create serializers. It's used by both readClassInfo (which - * needs serializers) and readClassInternal (which doesn't need serializers). Use {@link - * #ensureSerializerForClassInfo} after calling this if a serializer is needed. + *

    Note: This method should NOT create serializers. It's used by both readTypeInfo (which needs + * serializers) and readClassInternal (which doesn't need serializers). Use {@link + * #ensureSerializerForTypeInfo} after calling this if a serializer is needed. */ - protected abstract ClassInfo loadBytesToClassInfo( + protected abstract TypeInfo loadBytesToTypeInfo( MetaStringBytes namespaceBytes, MetaStringBytes simpleClassNameBytes); /** - * Ensure the ClassInfo has a serializer set. Called after loading class info for deserialization. + * Load class info from namespace and type name bytes, with the type id from the stream. + * Subclasses can override this to use the incoming type id (e.g. NAMED_EXT/NAMED_ENUM). + */ + protected TypeInfo loadBytesToTypeInfo( + int typeId, MetaStringBytes namespaceBytes, MetaStringBytes simpleClassNameBytes) { + return loadBytesToTypeInfo(namespaceBytes, simpleClassNameBytes); + } + + /** + * Ensure the TypeInfo has a serializer set. Called after loading class info for deserialization. * If the class is abstract/interface or can't be serialized, this may throw an exception. * - * @param classInfo the class info to ensure has a serializer - * @return the ClassInfo with serializer set (may be the same instance or a different one) + * @param typeInfo the class info to ensure has a serializer + * @return the TypeInfo with serializer set (may be the same instance or a different one) */ - protected abstract ClassInfo ensureSerializerForClassInfo(ClassInfo classInfo); + protected abstract TypeInfo ensureSerializerForTypeInfo(TypeInfo typeInfo); - protected ClassInfo getListClassInfo() { - return requireInternalTypeInfoByTypeId(Types.LIST); + protected TypeInfo getListTypeInfo() { + return getInternalTypeInfoByTypeId(Types.LIST); } - protected ClassInfo getTimestampClassInfo() { - return requireInternalTypeInfoByTypeId(Types.TIMESTAMP); + protected TypeInfo getTimestampTypeInfo() { + return getInternalTypeInfoByTypeId(Types.TIMESTAMP); } - protected final ClassInfo getInternalTypeInfoByTypeId(int typeId) { - if (typeId < 0 || typeId >= typeIdToClassInfo.length) { + protected final TypeInfo getInternalTypeInfoByTypeId(int typeId) { + if (typeId < 0 || typeId >= typeIdToTypeInfo.length) { return null; } - return typeIdToClassInfo[typeId]; + return typeIdToTypeInfo[typeId]; } - protected final ClassInfo getUserTypeInfoByTypeId(int typeId) { - int userId = typeId >>> 8; - ClassInfo classInfo = userTypeIdToClassInfo.get(userId); - if (classInfo == null) { - return null; - } - int internalTypeId = typeId & 0xff; - int registeredInternalTypeId = classInfo.typeId & 0xff; - if (registeredInternalTypeId != internalTypeId) { - if (classInfo.serializer == null) { - classInfo.typeId = typeId; - } else { - return null; + protected void updateTypeInfo(Class cls, TypeInfo typeInfo) { + // make `extRegistry.registeredClassIdMap` and `classInfoMap` share same classInfo + // instances. + classInfoMap.put(cls, typeInfo); + if (typeInfo.userTypeId != INVALID_USER_TYPE_ID) { + // serializer will be set lazily in `addSerializer` method if it's null. + putUserTypeInfo(typeInfo.userTypeId, typeInfo); + } else { + if (Types.isUserDefinedType((byte) typeInfo.typeId)) { + return; } + putInternalTypeInfo(typeInfo.typeId, typeInfo); } - return classInfo; } - protected final ClassInfo requireUserTypeInfoByTypeId(int typeId) { - ClassInfo classInfo = getUserTypeInfoByTypeId(typeId); - if (classInfo == null) { - throw new IllegalStateException(String.format("Type id %s not registered", typeId)); + protected final void putInternalTypeInfo(int typeId, TypeInfo typeInfo) { + if (typeIdToTypeInfo.length <= typeId) { + TypeInfo[] tmp = new TypeInfo[(typeId + 1) * 2]; + System.arraycopy(typeIdToTypeInfo, 0, tmp, 0, typeIdToTypeInfo.length); + typeIdToTypeInfo = tmp; } - return classInfo; + typeIdToTypeInfo[typeId] = typeInfo; } - protected final ClassInfo requireInternalTypeInfoByTypeId(int typeId) { - ClassInfo classInfo = getInternalTypeInfoByTypeId(typeId); - if (classInfo == null) { - throw new IllegalStateException(String.format("Type id %s not registered", typeId)); - } - return classInfo; - } - - protected final void putInternalTypeInfo(int typeId, ClassInfo classInfo) { - if (typeIdToClassInfo.length <= typeId) { - ClassInfo[] tmp = new ClassInfo[(typeId + 1) * 2]; - System.arraycopy(typeIdToClassInfo, 0, tmp, 0, typeIdToClassInfo.length); - typeIdToClassInfo = tmp; - } - typeIdToClassInfo[typeId] = classInfo; - } - - protected final void putUserTypeInfo(int userId, ClassInfo classInfo) { - userTypeIdToClassInfo.put(userId, classInfo); + protected final void putUserTypeInfo(int userId, TypeInfo typeInfo) { + userTypeIdToTypeInfo.put(userId, typeInfo); } protected final boolean containsUserTypeId(int userId) { - return userTypeIdToClassInfo.containsKey(userId); + return userTypeIdToTypeInfo.containsKey(userId); } - final ClassInfo buildMetaSharedClassInfo( - Tuple2 classDefTuple, ClassDef classDef) { - ClassInfo classInfo; - if (classDefTuple != null) { - classDef = classDefTuple.f0; + final TypeInfo buildMetaSharedTypeInfo(Tuple2 typeDefTuple, TypeDef typeDef) { + TypeInfo typeInfo; + if (typeDefTuple != null) { + typeDef = typeDefTuple.f0; } - Class cls = loadClass(classDef.getClassSpec()); - // For nonexistent classes, always create a new ClassInfo with the correct classDef, - // even if the classDef has no fields meta. This ensures the NonexistentClassSerializer - // has access to the classDef for proper deserialization. - if (!classDef.hasFieldsMeta() + Class cls = loadClass(typeDef.getClassSpec()); + // For nonexistent classes, always create a new TypeInfo with the correct typeDef, + // even if the typeDef has no fields meta. This ensures the NonexistentClassSerializer + // has access to the typeDef for proper deserialization. + if (!typeDef.hasFieldsMeta() && !NonexistentClass.class.isAssignableFrom(TypeUtils.getComponentIfArray(cls))) { - classInfo = getClassInfo(cls); + typeInfo = getTypeInfo(cls); } else if (ClassResolver.useReplaceResolveSerializer(cls)) { // For classes with writeReplace/readResolve, use their natural serializer // (ReplaceResolveSerializer) instead of MetaSharedSerializer - classInfo = getClassInfo(cls); + typeInfo = getTypeInfo(cls); } else { - classInfo = getMetaSharedClassInfo(classDef, cls); + typeInfo = getMetaSharedTypeInfo(typeDef, cls); } // Share serializer for same version class def to avoid too much different meta // context take up too much memory. - putClassDef(classDef, classInfo); - return classInfo; + putTypeDef(typeDef, typeInfo); + return typeInfo; } - // TODO(chaokunyang) if ClassDef is consistent with class in this process, + // TODO(chaokunyang) if TypeDef is consistent with class in this process, // use existing serializer instead. - private ClassInfo getMetaSharedClassInfo(ClassDef classDef, Class clz) { + private TypeInfo getMetaSharedTypeInfo(TypeDef typeDef, Class clz) { if (clz == NonexistentSkip.class) { clz = NonexistentMetaShared.class; } Class cls = clz; - Short classId = extRegistry.registeredClassIdMap.get(cls); + Integer classId = extRegistry.registeredClassIdMap.get(cls); int typeId; if (classId != null) { - ClassInfo registeredInfo = classInfoMap.get(cls); + TypeInfo registeredInfo = classInfoMap.get(cls); if (registeredInfo == null) { - registeredInfo = getClassInfo(cls); + registeredInfo = getTypeInfo(cls); } typeId = registeredInfo.typeId; } else { - ClassInfo cachedInfo = classInfoMap.get(cls); + TypeInfo cachedInfo = classInfoMap.get(cls); if (cachedInfo != null) { typeId = cachedInfo.typeId; } else { typeId = buildUnregisteredTypeId(cls, null); } } - ClassInfo classInfo = new ClassInfo(this, cls, null, typeId); - classInfo.classDef = classDef; + TypeInfo typeInfo = new TypeInfo(this, cls, null, typeId, typeDef.getClassSpec().userTypeId); + typeInfo.typeDef = typeDef; if (NonexistentClass.class.isAssignableFrom(TypeUtils.getComponentIfArray(cls))) { if (cls == NonexistentMetaShared.class) { - classInfo.setSerializer(this, new NonexistentClassSerializer(fory, classDef)); - // Ensure NonexistentMetaShared is registered so writeClassInfo emits a placeholder typeId + typeInfo.setSerializer(this, new NonexistentClassSerializer(fory, typeDef)); + // Ensure NonexistentMetaShared is registered so writeTypeInfo emits a placeholder typeId // that NonexistentClassSerializer can rewrite to the original typeId. if (!fory.isCrossLanguage()) { Preconditions.checkNotNull(classId); } } else { - classInfo.serializer = - NonexistentClassSerializers.getSerializer(fory, classDef.getClassName(), cls); + typeInfo.serializer = + NonexistentClassSerializers.getSerializer(fory, typeDef.getClassName(), cls); } - return classInfo; + return typeInfo; } if (clz.isArray() || cls.isEnum()) { - return getClassInfo(cls); + return getTypeInfo(cls); } Class sc = - getMetaSharedDeserializerClassFromGraalvmRegistry(cls, classDef); + getMetaSharedDeserializerClassFromGraalvmRegistry(cls, typeDef); if (sc == null) { if (GraalvmSupport.isGraalRuntime()) { sc = MetaSharedSerializer.class; LOG.warn( "Can't generate class at runtime in graalvm for class def {}, use {} instead", - classDef, + typeDef, sc); } else { sc = fory.getJITContext() .registerSerializerJITCallback( () -> MetaSharedSerializer.class, - () -> CodecUtils.loadOrGenMetaSharedCodecClass(fory, cls, classDef), - c -> classInfo.setSerializer(this, Serializers.newSerializer(fory, cls, c))); + () -> CodecUtils.loadOrGenMetaSharedCodecClass(fory, cls, typeDef), + c -> typeInfo.setSerializer(this, Serializers.newSerializer(fory, cls, c))); } } if (sc == MetaSharedSerializer.class) { - classInfo.setSerializer(this, new MetaSharedSerializer(fory, cls, classDef)); + typeInfo.setSerializer(this, new MetaSharedSerializer(fory, cls, typeDef)); } else { - classInfo.setSerializer(this, Serializers.newSerializer(fory, cls, sc)); + typeInfo.setSerializer(this, Serializers.newSerializer(fory, cls, sc)); } - return classInfo; + return typeInfo; } protected int buildUnregisteredTypeId(Class cls, Serializer serializer) { @@ -906,7 +910,7 @@ protected int buildUnregisteredTypeId(Class cls, Serializer serializer) { return Types.NAMED_EXT; } if (fory.isCompatible()) { - return Types.NAMED_COMPATIBLE_STRUCT; + return metaContextShareEnabled ? Types.NAMED_COMPATIBLE_STRUCT : Types.NAMED_STRUCT; } return Types.NAMED_STRUCT; } @@ -919,18 +923,18 @@ protected static boolean isStructSerializer(Serializer serializer) { || serializer instanceof MetaSharedSerializer; } - protected Tuple2 readClassDef(MemoryBuffer buffer, long header) { - ClassDef readClassDef = ClassDef.readClassDef(fory, buffer, header); - Tuple2 tuple2 = extRegistry.classIdToDef.get(readClassDef.getId()); + protected Tuple2 readTypeDef(MemoryBuffer buffer, long header) { + TypeDef readTypeDef = TypeDef.readTypeDef(fory, buffer, header); + Tuple2 tuple2 = extRegistry.classIdToDef.get(readTypeDef.getId()); if (tuple2 == null) { - tuple2 = putClassDef(readClassDef, null); + tuple2 = putTypeDef(readTypeDef, null); } return tuple2; } - private Tuple2 putClassDef(ClassDef classDef, ClassInfo classInfo) { - Tuple2 tuple2 = Tuple2.of(classDef, classInfo); - extRegistry.classIdToDef.put(classDef.getId(), tuple2); + private Tuple2 putTypeDef(TypeDef typeDef, TypeInfo typeInfo) { + Tuple2 tuple2 = Tuple2.of(typeDef, typeInfo); + extRegistry.classIdToDef.put(typeDef.getId(), tuple2); return tuple2; } @@ -992,27 +996,24 @@ public final Serializer getSerializer(TypeRef typeRef) { public abstract void setSerializerIfAbsent(Class cls, Serializer serializer); - public final ClassInfo getClassInfoByTypeId(int typeId) { - int internalTypeId = typeId & 0xFF; - if (Types.isUserDefinedType((byte) internalTypeId)) { - int userId = typeId >>> 8; - if (userId != 0) { - return requireUserTypeInfoByTypeId(userId); - } + public final TypeInfo getTypeInfoByTypeId(int typeId) { + if (Types.isUserDefinedType((byte) typeId)) { + throw new IllegalArgumentException( + "User type id must be provided to resolve user-defined type " + typeId); } - return requireInternalTypeInfoByTypeId(internalTypeId); + return getInternalTypeInfoByTypeId(typeId); } public final Serializer getSerializerByTypeId(int typeId) { - return getClassInfoByTypeId(typeId).getSerializer(); + return getTypeInfoByTypeId(typeId).getSerializer(); } - public final ClassInfo nilClassInfo() { - return NIL_CLASS_INFO; + public final TypeInfo nilTypeInfo() { + return NIL_TYPE_INFO; } - public final ClassInfoHolder nilClassInfoHolder() { - return new ClassInfoHolder(NIL_CLASS_INFO); + public final TypeInfoHolder nilTypeInfoHolder() { + return new TypeInfoHolder(NIL_TYPE_INFO); } public final GenericType buildGenericType(TypeRef typeRef) { @@ -1057,16 +1058,16 @@ public GenericType getGenericTypeInStruct(Class cls, String genericTypeStr) { public abstract void ensureSerializersCompiled(); - public final ClassDef getTypeDef(Class cls, boolean resolveParent) { + public final TypeDef getTypeDef(Class cls, boolean resolveParent) { if (resolveParent) { - return classDefMap.computeIfAbsent(cls, k -> ClassDef.buildClassDef(fory, cls)); + return typeDefMap.computeIfAbsent(cls, k -> TypeDef.buildTypeDef(fory, cls)); } - ClassDef classDef = extRegistry.currentLayerClassDef.get(cls); - if (classDef == null) { - classDef = ClassDef.buildClassDef(fory, cls, false); - extRegistry.currentLayerClassDef.put(cls, classDef); + TypeDef typeDef = extRegistry.currentLayerTypeDef.get(cls); + if (typeDef == null) { + typeDef = TypeDef.buildTypeDef(fory, cls, false); + extRegistry.currentLayerTypeDef.put(cls, typeDef); } - return classDef; + return typeDef; } public final boolean isSerializable(Class cls) { @@ -1078,14 +1079,14 @@ public final boolean isSerializable(Class cls) { return false; } try { - ClassInfo classInfo = classInfoMap.get(cls); + TypeInfo typeInfo = classInfoMap.get(cls); Serializer serializer = null; - if (classInfo != null) { - serializer = classInfo.serializer; + if (typeInfo != null) { + serializer = typeInfo.serializer; } getSerializerClass(cls, false); - if (classInfo != null && serializer == null) { - classInfo.serializer = null; + if (typeInfo != null && serializer == null) { + typeInfo.serializer = null; } return true; } catch (Throwable t) { @@ -1441,9 +1442,9 @@ final Class getSerializerClassFromGraalvmRegistry(Class } for (TypeResolver resolver : resolvers) { if (resolver != this) { - ClassInfo classInfo = getClassInfo(cls, false); - if (classInfo != null && classInfo.serializer != null) { - return classInfo.serializer.getClass(); + TypeInfo typeInfo = getTypeInfo(cls, false); + if (typeInfo != null && typeInfo.serializer != null) { + return typeInfo.serializer.getClass(); } } } @@ -1462,14 +1463,14 @@ final Class getSerializerClassFromGraalvmRegistry(Class } private Class getMetaSharedDeserializerClassFromGraalvmRegistry( - Class cls, ClassDef classDef) { + Class cls, TypeDef typeDef) { GraalvmSupport.GraalvmClassRegistry registry = getGraalvmClassRegistry(); List resolvers = registry.resolvers; if (resolvers.isEmpty()) { return null; } Class deserializerClass = - registry.deserializerClassMap.get(classDef.getId()); + registry.deserializerClassMap.get(typeDef.getId()); // noinspection Duplicates if (deserializerClass != null) { return deserializerClass; @@ -1496,22 +1497,23 @@ public final MetaStringResolver getMetaStringResolver() { static class ExtRegistry { // Here we set it to 1 to avoid calculating it again in `register(Class cls)`. - short classIdGenerator = 1; - short userIdGenerator = 0; + int classIdGenerator = 1; + int userIdGenerator = 0; SerializerFactory serializerFactory; - final IdentityMap, Short> registeredClassIdMap = + final IdentityMap, Integer> registeredClassIdMap = new IdentityMap<>(estimatedNumRegistered); final BiMap> registeredClasses = HashBiMap.create(estimatedNumRegistered); - // cache absClassInfo, support customized serializer for abstract or interface. - final IdentityMap, ClassInfo> absClassInfo = + // cache absTypeInfo, support customized serializer for abstract or interface. + final IdentityMap, TypeInfo> absTypeInfo = new IdentityMap<>(estimatedNumRegistered, foryMapLoadFactor); // avoid potential recursive call for seq codec generation. // ex. A->field1: B, B.field1: A final Set> getClassCtx = new HashSet<>(); - final LongMap> classIdToDef = new LongMap<>(); - final Map, ClassDef> currentLayerClassDef = new HashMap<>(); + final LongMap> classIdToDef = new LongMap<>(); + final Map, TypeDef> currentLayerTypeDef = new HashMap<>(); // Tuple2: Tuple2 - final Map, Class>, ClassInfo> transformedClassInfo = new HashMap<>(); + final IdentityMap, Tuple2, TypeInfo>[]> transformedTypeInfo = + new IdentityMap<>(); // TODO(chaokunyang) Better to use soft reference, see ObjectStreamClass. final ConcurrentHashMap, Boolean>, SortedMap> descriptorsCache = new ConcurrentHashMap<>(); @@ -1521,7 +1523,7 @@ static class ExtRegistry { final IdentityMap genericTypes = new IdentityMap<>(); final Map> classGenericTypes = new HashMap<>(); final Map, CodeGenerator> codeGeneratorMap = new HashMap<>(); - final Set registeredClassInfos = new HashSet<>(); + final Set registeredTypeInfos = new HashSet<>(); boolean ensureSerializersCompiled; public boolean isTypeCheckerSet() { diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java index 785dd2266c..c6103866ca 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java @@ -27,6 +27,7 @@ import static org.apache.fory.meta.Encoders.TYPE_NAME_DECODER; import static org.apache.fory.serializer.collection.MapSerializers.HashMapSerializer; import static org.apache.fory.type.TypeUtils.qualifiedName; +import static org.apache.fory.type.Types.INVALID_USER_TYPE_ID; import java.math.BigDecimal; import java.math.BigInteger; @@ -74,9 +75,9 @@ import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.Platform; -import org.apache.fory.meta.ClassDef; import org.apache.fory.meta.Encoders; import org.apache.fory.meta.MetaString; +import org.apache.fory.meta.TypeDef; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.serializer.ArraySerializers; import org.apache.fory.serializer.DeferedLazySerializer; @@ -130,19 +131,18 @@ public class XtypeResolver extends TypeResolver { private final Config config; private final Fory fory; - private final ClassInfoHolder classInfoCache = new ClassInfoHolder(NIL_CLASS_INFO); + private final TypeInfoHolder classInfoCache = new TypeInfoHolder(NIL_TYPE_INFO); private final MetaStringResolver metaStringResolver; // Every deserialization for unregistered class will query it, performance is important. - private final ObjectMap compositeClassNameBytes2ClassInfo = + private final ObjectMap compositeClassNameBytes2TypeInfo = new ObjectMap<>(16, loadFactor); - private final ObjectMap qualifiedType2ClassInfo = + private final ObjectMap qualifiedType2TypeInfo = new ObjectMap<>(16, loadFactor); - // classDefMap is inherited from TypeResolver + // typeDefMap is inherited from TypeResolver private final boolean shareMeta; private int xtypeIdGenerator = 64; - private final Set registeredTypeIds = new HashSet<>(); private final Generics generics; public XtypeResolver(Fory fory) { @@ -160,7 +160,12 @@ public void initialize() { if (shareMeta) { Serializer serializer = new NonexistentClassSerializer(fory, null); register( - NonexistentMetaShared.class, serializer, "", "unknown_struct", Types.COMPATIBLE_STRUCT); + NonexistentMetaShared.class, + serializer, + "", + "unknown_struct", + Types.COMPATIBLE_STRUCT, + -1); } } @@ -173,28 +178,26 @@ public void register(Class type) { } @Override - public void register(Class type, int userTypeId) { + public void register(Class type, long userTypeId) { checkRegisterAllowed(); - // ClassInfo[] has length of max type id. If the type id is too big, Fory will waste many - // memory. We can relax this limit in the future. - Preconditions.checkArgument(userTypeId < MAX_TYPE_ID, "Too big type id %s", userTypeId); + int checkedUserTypeId = toUserTypeId(userTypeId); Preconditions.checkArgument( - !containsUserTypeId(userTypeId), "Type id %s has been registered", userTypeId); - ClassInfo classInfo = classInfoMap.get(type); + !containsUserTypeId(checkedUserTypeId), "Type id %s has been registered", userTypeId); + TypeInfo typeInfo = classInfoMap.get(type); if (type.isArray()) { - buildClassInfo(type); + buildTypeInfo(type); GraalvmSupport.registerClass(type, fory.getConfig().getConfigHash()); return; } Serializer serializer = null; - if (classInfo != null) { - serializer = classInfo.serializer; - if (classInfo.typeId != 0) { + if (typeInfo != null) { + serializer = typeInfo.serializer; + if (typeInfo.typeId != 0) { throw new IllegalArgumentException( - String.format("Type %s has been registered with id %s", type, classInfo.typeId)); + String.format("Type %s has been registered with id %s", type, typeInfo.typeId)); } - String prevNamespace = classInfo.decodeNamespace(); - String prevTypeName = classInfo.decodeTypeName(); + String prevNamespace = typeInfo.decodeNamespace(); + String prevTypeName = typeInfo.decodeTypeName(); if (!type.getSimpleName().equals(prevTypeName)) { throw new IllegalArgumentException( String.format( @@ -202,19 +205,19 @@ public void register(Class type, int userTypeId) { type, prevNamespace, prevTypeName)); } } - int xtypeId = userTypeId; + int typeId; if (type.isEnum()) { - xtypeId = (xtypeId << 8) + Types.ENUM; + typeId = Types.ENUM; } else { - int id = (xtypeId << 8) + (shareMeta ? Types.COMPATIBLE_STRUCT : Types.STRUCT); + int structTypeId = shareMeta ? Types.COMPATIBLE_STRUCT : Types.STRUCT; if (serializer != null) { if (isStructType(serializer)) { - xtypeId = id; + typeId = structTypeId; } else { - xtypeId = (xtypeId << 8) + Types.EXT; + typeId = Types.EXT; } } else { - xtypeId = id; + typeId = structTypeId; } } register( @@ -222,7 +225,8 @@ public void register(Class type, int userTypeId) { serializer, ReflectionUtils.getPackage(type), ReflectionUtils.getClassNameWithoutPackage(type), - xtypeId); + typeId, + checkedUserTypeId); } @Override @@ -232,13 +236,13 @@ public void register(Class type, String namespace, String typeName) { !typeName.contains("."), "Typename %s should not contains `.`, please put it into namespace", typeName); - ClassInfo classInfo = classInfoMap.get(type); + TypeInfo typeInfo = classInfoMap.get(type); Serializer serializer = null; - if (classInfo != null) { - serializer = classInfo.serializer; - if (classInfo.typeNameBytes != null) { - String prevNamespace = classInfo.decodeNamespace(); - String prevTypeName = classInfo.decodeTypeName(); + if (typeInfo != null) { + serializer = typeInfo.serializer; + if (typeInfo.typeNameBytes != null) { + String prevNamespace = typeInfo.decodeNamespace(); + String prevTypeName = typeInfo.decodeTypeName(); if (!namespace.equals(prevNamespace) || typeName.equals(prevTypeName)) { throw new IllegalArgumentException( String.format( @@ -250,8 +254,7 @@ public void register(Class type, String namespace, String typeName) { short xtypeId; if (serializer != null) { if (isStructType(serializer)) { - xtypeId = - (short) (fory.isCompatible() ? Types.NAMED_COMPATIBLE_STRUCT : Types.NAMED_STRUCT); + xtypeId = (short) (shareMeta ? Types.NAMED_COMPATIBLE_STRUCT : Types.NAMED_STRUCT); } else if (serializer instanceof EnumSerializer) { xtypeId = Types.NAMED_ENUM; } else { @@ -261,27 +264,31 @@ public void register(Class type, String namespace, String typeName) { if (type.isEnum()) { xtypeId = Types.NAMED_ENUM; } else { - xtypeId = - (short) (fory.isCompatible() ? Types.NAMED_COMPATIBLE_STRUCT : Types.NAMED_STRUCT); + xtypeId = (short) (shareMeta ? Types.NAMED_COMPATIBLE_STRUCT : Types.NAMED_STRUCT); } } - register(type, serializer, namespace, typeName, xtypeId); + register(type, serializer, namespace, typeName, xtypeId, -1); } private void register( - Class type, Serializer serializer, String namespace, String typeName, int xtypeId) { - ClassInfo classInfo = newClassInfo(type, serializer, namespace, typeName, xtypeId); + Class type, + Serializer serializer, + String namespace, + String typeName, + int typeId, + int userTypeId) { + TypeInfo typeInfo = newTypeInfo(type, serializer, namespace, typeName, typeId, userTypeId); String qualifiedName = qualifiedName(namespace, typeName); - qualifiedType2ClassInfo.put(qualifiedName, classInfo); + qualifiedType2TypeInfo.put(qualifiedName, typeInfo); extRegistry.registeredClasses.put(qualifiedName, type); GraalvmSupport.registerClass(type, fory.getConfig().getConfigHash()); if (serializer == null) { if (type.isEnum()) { - classInfo.serializer = new EnumSerializer(fory, (Class) type); + typeInfo.serializer = new EnumSerializer(fory, (Class) type); } else { AtomicBoolean updated = new AtomicBoolean(false); AtomicReference ref = new AtomicReference(null); - classInfo.serializer = + typeInfo.serializer = new DeferedLazySerializer.DeferredLazyObjectSerializer( fory, type, @@ -302,37 +309,29 @@ private void register( }); } } - classInfoMap.put(type, classInfo); - registeredTypeIds.add(xtypeId); - int internalTypeId = xtypeId & 0xff; - if (Types.isUserDefinedType((byte) internalTypeId) && !Types.isNamedType(internalTypeId)) { - putUserTypeInfo(xtypeId >>> 8, classInfo); - } else if (!Types.isNamedType(internalTypeId)) { - if (getInternalTypeInfoByTypeId(xtypeId) == null) { - putInternalTypeInfo(xtypeId, classInfo); - } - } + updateTypeInfo(type, typeInfo); } @Override - public void registerUnion(Class type, int userTypeId, Serializer serializer) { + public void registerUnion(Class type, long userTypeId, Serializer serializer) { checkRegisterAllowed(); Preconditions.checkNotNull(serializer); - Preconditions.checkArgument(userTypeId < MAX_TYPE_ID, "Too big type id %s", userTypeId); + int checkedUserTypeId = toUserTypeId(userTypeId); Preconditions.checkArgument( - !containsUserTypeId(userTypeId), "Type id %s has been registered", userTypeId); - ClassInfo classInfo = classInfoMap.get(type); - if (classInfo != null && classInfo.typeId != 0) { + !containsUserTypeId(checkedUserTypeId), "Type id %s has been registered", userTypeId); + TypeInfo typeInfo = classInfoMap.get(type); + if (typeInfo != null && typeInfo.typeId != 0) { throw new IllegalArgumentException( - String.format("Type %s has been registered with id %s", type, classInfo.typeId)); + String.format("Type %s has been registered with id %s", type, typeInfo.typeId)); } - int xtypeId = (userTypeId << 8) + Types.TYPED_UNION; + int xtypeId = Types.TYPED_UNION; register( type, serializer, ReflectionUtils.getPackage(type), ReflectionUtils.getClassNameWithoutPackage(type), - xtypeId); + xtypeId, + checkedUserTypeId); } @Override @@ -344,10 +343,10 @@ public void registerUnion( !typeName.contains("."), "Typename %s should not contains `.`, please put it into namespace", typeName); - ClassInfo classInfo = classInfoMap.get(type); - if (classInfo != null && classInfo.typeNameBytes != null) { - String prevNamespace = classInfo.decodeNamespace(); - String prevTypeName = classInfo.decodeTypeName(); + TypeInfo typeInfo = classInfoMap.get(type); + if (typeInfo != null && typeInfo.typeNameBytes != null) { + String prevNamespace = typeInfo.decodeNamespace(); + String prevTypeName = typeInfo.decodeTypeName(); if (!namespace.equals(prevNamespace) || typeName.equals(prevTypeName)) { throw new IllegalArgumentException( String.format( @@ -356,14 +355,14 @@ public void registerUnion( } } int xtypeId = Types.NAMED_UNION; - register(type, serializer, namespace, typeName, xtypeId); + register(type, serializer, namespace, typeName, xtypeId, -1); } /** * Register type with given type id and serializer for type in fory type system. * *

    Do not use this method to register custom type in java type system. Use {@link - * #register(Class, String, String)} or {@link #register(Class, int)} instead. + * #register(Class, String, String)} or {@link #register(Class, long)} instead. * * @param type type to register. * @param serializer serializer to register. @@ -378,7 +377,8 @@ public void registerForyType(Class type, Serializer serializer, int typeId) { serializer, ReflectionUtils.getPackage(type), ReflectionUtils.getClassNameWithoutPackage(type), - typeId); + typeId, + -1); } private boolean isStructType(Serializer serializer) { @@ -388,17 +388,33 @@ private boolean isStructType(Serializer serializer) { return serializer instanceof DeferredLazyObjectSerializer; } - private ClassInfo newClassInfo(Class type, Serializer serializer, int typeId) { - return newClassInfo( + private TypeInfo newTypeInfo(Class type, Serializer serializer, int typeId) { + return newTypeInfo(type, serializer, typeId, INVALID_USER_TYPE_ID); + } + + private TypeInfo newTypeInfo( + Class type, Serializer serializer, int typeId, int userTypeId) { + return newTypeInfo( type, serializer, ReflectionUtils.getPackage(type), ReflectionUtils.getClassNameWithoutPackage(type), - typeId); + typeId, + userTypeId); } - private ClassInfo newClassInfo( + private TypeInfo newTypeInfo( Class type, Serializer serializer, String namespace, String typeName, int typeId) { + return newTypeInfo(type, serializer, namespace, typeName, typeId, INVALID_USER_TYPE_ID); + } + + private TypeInfo newTypeInfo( + Class type, + Serializer serializer, + String namespace, + String typeName, + int typeId, + int userTypeId) { MetaStringBytes fullClassNameBytes = metaStringResolver.getOrCreateMetaStringBytes( GENERIC_ENCODER.encode(type.getName(), MetaString.Encoding.UTF_8)); @@ -406,8 +422,8 @@ private ClassInfo newClassInfo( metaStringResolver.getOrCreateMetaStringBytes(Encoders.encodePackage(namespace)); MetaStringBytes classNameBytes = metaStringResolver.getOrCreateMetaStringBytes(Encoders.encodeTypeName(typeName)); - return new ClassInfo( - type, fullClassNameBytes, nsBytes, classNameBytes, false, serializer, typeId); + return new TypeInfo( + type, fullClassNameBytes, nsBytes, classNameBytes, false, serializer, typeId, userTypeId); } public void registerSerializer(Class type, Class serializerClass) { @@ -417,32 +433,32 @@ public void registerSerializer(Class type, Class se public void registerSerializer(Class type, Serializer serializer) { checkRegisterAllowed(); - ClassInfo classInfo = checkClassRegistration(type); + TypeInfo typeInfo = checkClassRegistration(type); if (!serializer.getClass().getPackage().getName().startsWith("org.apache.fory")) { SerializationUtils.validate(type, serializer.getClass()); } - int oldTypeId = classInfo.typeId; - int foryId = oldTypeId & 0xff; - - if (oldTypeId != 0) { - registeredTypeIds.remove(oldTypeId); - } + int oldTypeId = typeInfo.typeId; + int foryId = oldTypeId; if (foryId != Types.EXT && foryId != Types.NAMED_EXT) { if (foryId == Types.STRUCT || foryId == Types.COMPATIBLE_STRUCT) { - classInfo.typeId = (oldTypeId & 0xffffff00) | Types.EXT; + foryId = Types.EXT; } else if (foryId == Types.NAMED_STRUCT || foryId == Types.NAMED_COMPATIBLE_STRUCT) { - classInfo.typeId = (oldTypeId & 0xffffff00) | Types.NAMED_EXT; + foryId = Types.NAMED_EXT; } else { throw new IllegalArgumentException( String.format("Can't register serializer for type %s with id %s", type, oldTypeId)); } } - classInfo.serializer = serializer; - - int newTypeId = classInfo.typeId; - if (newTypeId != 0) { - registeredTypeIds.add(newTypeId); + typeInfo = typeInfo.copy(foryId); + typeInfo.serializer = serializer; + updateTypeInfo(type, typeInfo); + if (typeInfo.typeNameBytes != null) { + String qualifiedName = qualifiedName(typeInfo.decodeNamespace(), typeInfo.decodeTypeName()); + qualifiedType2TypeInfo.put(qualifiedName, typeInfo); + TypeNameBytes typeNameBytes = + new TypeNameBytes(typeInfo.namespaceBytes.hashCode, typeInfo.typeNameBytes.hashCode); + compositeClassNameBytes2TypeInfo.put(typeNameBytes, typeInfo); } } @@ -456,17 +472,17 @@ public void registerInternalSerializer(Class type, Serializer serializer) || type == Character[].class) { return; } - ClassInfo classInfo = classInfoMap.get(type); - if (classInfo != null) { - if (classInfo.serializer == null) { - classInfo.serializer = serializer; + TypeInfo typeInfo = classInfoMap.get(type); + if (typeInfo != null) { + if (typeInfo.serializer == null) { + typeInfo.serializer = serializer; } return; } // Determine appropriate type ID based on the type int typeId = determineTypeIdForClass(type); - classInfo = newClassInfo(type, serializer, typeId); - classInfoMap.put(type, classInfo); + typeInfo = newTypeInfo(type, serializer, typeId); + classInfoMap.put(type, typeInfo); } /** @@ -497,14 +513,14 @@ private int determineTypeIdForClass(Class type) { } } - private ClassInfo checkClassRegistration(Class type) { - ClassInfo classInfo = classInfoMap.get(type); + private TypeInfo checkClassRegistration(Class type) { + TypeInfo typeInfo = classInfoMap.get(type); Preconditions.checkArgument( - classInfo != null - && (classInfo.typeId != 0 || !type.getSimpleName().equals(classInfo.decodeTypeName())), + typeInfo != null + && (typeInfo.typeId != 0 || !type.getSimpleName().equals(typeInfo.decodeTypeName())), "Type %s should be registered with id or namespace+typename before register serializer", type); - return classInfo; + return typeInfo; } @Override @@ -514,11 +530,11 @@ public boolean isRegistered(Class cls) { @Override public boolean isRegisteredById(Class cls) { - ClassInfo classInfo = classInfoMap.get(cls); - if (classInfo == null) { + TypeInfo typeInfo = classInfoMap.get(cls); + if (typeInfo == null) { return false; } - int typeId = classInfo.typeId & 0xff; + int typeId = typeInfo.typeId; switch (typeId) { case Types.NAMED_COMPATIBLE_STRUCT: case Types.NAMED_ENUM: @@ -533,11 +549,11 @@ public boolean isRegisteredById(Class cls) { @Override public boolean isRegisteredByName(Class cls) { - ClassInfo classInfo = classInfoMap.get(cls); - if (classInfo == null) { + TypeInfo typeInfo = classInfoMap.get(cls); + if (typeInfo == null) { return false; } - int typeId = classInfo.typeId & 0xff; + int typeId = typeInfo.typeId; switch (typeId) { case Types.NAMED_COMPATIBLE_STRUCT: case Types.NAMED_ENUM: @@ -574,7 +590,7 @@ public boolean isMonomorphic(Descriptor descriptor) { return true; } if (rawType == NonexistentMetaShared.class) { - return true; + return false; } byte typeIdByte = getInternalTypeId(rawType); if (fory.isCompatible()) { @@ -604,9 +620,9 @@ public boolean isMonomorphic(Class clz) { if (clz == NonexistentMetaShared.class) { return false; } - ClassInfo classInfo = getClassInfo(clz, false); - if (classInfo != null) { - Serializer s = classInfo.serializer; + TypeInfo typeInfo = getTypeInfo(clz, false); + if (typeInfo != null) { + Serializer s = typeInfo.serializer; if (s instanceof TimeSerializers.TimeSerializer || s instanceof MapLikeSerializer || s instanceof CollectionLikeSerializer @@ -635,51 +651,51 @@ public boolean isBuildIn(Descriptor descriptor) { } @Override - public ClassInfo getClassInfo(Class cls) { - ClassInfo classInfo = classInfoMap.get(cls); - if (classInfo == null) { - classInfo = buildClassInfo(cls); + public TypeInfo getTypeInfo(Class cls) { + TypeInfo typeInfo = classInfoMap.get(cls); + if (typeInfo == null) { + typeInfo = buildTypeInfo(cls); } - return classInfo; + return typeInfo; } @Override - public ClassInfo getClassInfo(Class cls, boolean createIfAbsent) { + public TypeInfo getTypeInfo(Class cls, boolean createIfAbsent) { if (createIfAbsent) { - return getClassInfo(cls); + return getTypeInfo(cls); } return classInfoMap.get(cls); } - public ClassInfo getClassInfo(Class cls, ClassInfoHolder classInfoHolder) { - ClassInfo classInfo = classInfoHolder.classInfo; - if (classInfo.getCls() != cls) { - classInfo = classInfoMap.get(cls); - if (classInfo == null) { - classInfo = buildClassInfo(cls); + public TypeInfo getTypeInfo(Class cls, TypeInfoHolder classInfoHolder) { + TypeInfo typeInfo = classInfoHolder.typeInfo; + if (typeInfo.getCls() != cls) { + typeInfo = classInfoMap.get(cls); + if (typeInfo == null) { + typeInfo = buildTypeInfo(cls); } - classInfoHolder.classInfo = classInfo; + classInfoHolder.typeInfo = typeInfo; } - assert classInfo.serializer != null; - return classInfo; + assert typeInfo.serializer != null; + return typeInfo; } - public ClassInfo getXtypeInfo(int typeId) { + public TypeInfo getXtypeInfo(int typeId) { return getInternalTypeInfoByTypeId(typeId); } - public ClassInfo getUserTypeInfo(String namespace, String typeName) { + public TypeInfo getUserTypeInfo(String namespace, String typeName) { String name = qualifiedName(namespace, typeName); - return qualifiedType2ClassInfo.get(name); + return qualifiedType2TypeInfo.get(name); } - public ClassInfo getUserTypeInfo(int userTypeId) { - return getUserTypeInfoByTypeId(userTypeId); + public TypeInfo getUserTypeInfo(int userTypeId) { + return userTypeIdToTypeInfo.get(userTypeId); } // buildGenericType methods are inherited from TypeResolver - private ClassInfo buildClassInfo(Class cls) { + private TypeInfo buildTypeInfo(Class cls) { Serializer serializer; int typeId; if (isSet(cls)) { @@ -706,12 +722,12 @@ private ClassInfo buildClassInfo(Class cls) { cls = HashMap.class; serializer = new HashMapSerializer(fory); } else { - ClassInfo classInfo = classInfoMap.get(cls); - if (classInfo != null - && classInfo.serializer != null - && classInfo.serializer instanceof MapLikeSerializer - && ((MapLikeSerializer) classInfo.serializer).supportCodegenHook()) { - serializer = classInfo.serializer; + TypeInfo typeInfo = classInfoMap.get(cls); + if (typeInfo != null + && typeInfo.serializer != null + && typeInfo.serializer instanceof MapLikeSerializer + && ((MapLikeSerializer) typeInfo.serializer).supportCodegenHook()) { + serializer = typeInfo.serializer; } else { serializer = new MapSerializer(fory, cls); } @@ -726,28 +742,29 @@ private ClassInfo buildClassInfo(Class cls) { } } else if (cls == Object.class) { // Object.class is handled as unknown type in xlang - return getClassInfo(cls); + return getTypeInfo(cls); } else { Class enclosingClass = (Class) cls.getEnclosingClass(); if (enclosingClass != null && enclosingClass.isEnum()) { - serializer = new EnumSerializer(fory, (Class) cls); - typeId = getClassInfo(enclosingClass).typeId; + TypeInfo enumInfo = getTypeInfo(enclosingClass); + classInfoMap.put(cls, enumInfo); + return enumInfo; } else { throw new ClassUnregisteredException(cls); } } - ClassInfo info = newClassInfo(cls, serializer, typeId); + TypeInfo info = newTypeInfo(cls, serializer, typeId); classInfoMap.put(cls, info); return info; } private Serializer getCollectionSerializer(Class cls) { - ClassInfo classInfo = classInfoMap.get(cls); - if (classInfo != null - && classInfo.serializer != null - && classInfo.serializer instanceof CollectionLikeSerializer - && ((CollectionLikeSerializer) (classInfo.serializer)).supportCodegenHook()) { - return classInfo.serializer; + TypeInfo typeInfo = classInfoMap.get(cls); + if (typeInfo != null + && typeInfo.serializer != null + && typeInfo.serializer instanceof CollectionLikeSerializer + && ((CollectionLikeSerializer) (typeInfo.serializer)).supportCodegenHook()) { + return typeInfo.serializer; } return new CollectionSerializer(fory, cls); } @@ -956,10 +973,10 @@ private void registerDefaultTypes() { } private void registerType(int xtypeId, Class type, Serializer serializer) { - ClassInfo classInfo = newClassInfo(type, serializer, (short) xtypeId); - classInfoMap.put(type, classInfo); + TypeInfo typeInfo = newTypeInfo(type, serializer, xtypeId, INVALID_USER_TYPE_ID); + classInfoMap.put(type, typeInfo); if (getInternalTypeInfoByTypeId(xtypeId) == null) { - putInternalTypeInfo(xtypeId, classInfo); + putInternalTypeInfo(xtypeId, typeInfo); } } @@ -978,122 +995,128 @@ private void registerUnionTypes() { Class unionCls = (Class) cls; UnionSerializer serializer = new UnionSerializer(fory, unionCls); - ClassInfo classInfo = newClassInfo(cls, serializer, (short) Types.UNION); - classInfoMap.put(cls, classInfo); + TypeInfo typeInfo = newTypeInfo(cls, serializer, Types.UNION, INVALID_USER_TYPE_ID); + classInfoMap.put(cls, typeInfo); } putInternalTypeInfo(Types.UNION, classInfoMap.get(org.apache.fory.type.union.Union.class)); } - public ClassInfo writeClassInfo(MemoryBuffer buffer, Object obj) { - ClassInfo classInfo = getClassInfo(obj.getClass(), classInfoCache); - writeClassInfo(buffer, classInfo); - return classInfo; + public TypeInfo writeTypeInfo(MemoryBuffer buffer, Object obj) { + TypeInfo typeInfo = getTypeInfo(obj.getClass(), classInfoCache); + writeTypeInfo(buffer, typeInfo); + return typeInfo; } @Override - protected ClassDef buildClassDef(ClassInfo classInfo) { - ClassDef classDef = - classDefMap.computeIfAbsent(classInfo.cls, cls -> ClassDef.buildClassDef(fory, cls)); - classInfo.classDef = classDef; - return classDef; + protected TypeDef buildTypeDef(TypeInfo typeInfo) { + TypeDef typeDef = + typeDefMap.computeIfAbsent(typeInfo.cls, cls -> TypeDef.buildTypeDef(fory, cls)); + typeInfo.typeDef = typeDef; + return typeDef; } @Override public Serializer getSerializer(Class cls) { - return (Serializer) getClassInfo(cls).serializer; + return (Serializer) getTypeInfo(cls).serializer; } @Override public Serializer getRawSerializer(Class cls) { - return getClassInfo(cls).serializer; + return getTypeInfo(cls).serializer; } @Override public void setSerializer(Class cls, Serializer serializer) { - getClassInfo(cls).serializer = serializer; + getTypeInfo(cls).serializer = serializer; } @Override public void setSerializerIfAbsent(Class cls, Serializer serializer) { - ClassInfo classInfo = classInfoMap.get(cls); - Preconditions.checkNotNull(classInfo); - Preconditions.checkNotNull(classInfo.serializer); + TypeInfo typeInfo = classInfoMap.get(cls); + Preconditions.checkNotNull(typeInfo); + Preconditions.checkNotNull(typeInfo.serializer); } - // nilClassInfo and nilClassInfoHolder are inherited from TypeResolver + // nilTypeInfo and nilTypeInfoHolder are inherited from TypeResolver @Override - protected ClassInfo getListClassInfo() { + protected TypeInfo getListTypeInfo() { fory.incReadDepth(); GenericType genericType = generics.nextGenericType(); fory.decDepth(); if (genericType != null) { - return getOrBuildClassInfo(genericType.getCls()); + return getOrBuildTypeInfo(genericType.getCls()); } - return requireInternalTypeInfoByTypeId(Types.LIST); + return getInternalTypeInfoByTypeId(Types.LIST); } @Override - protected ClassInfo getTimestampClassInfo() { + protected TypeInfo getTimestampTypeInfo() { fory.incReadDepth(); GenericType genericType = generics.nextGenericType(); fory.decDepth(); if (genericType != null) { - return getOrBuildClassInfo(genericType.getCls()); + return getOrBuildTypeInfo(genericType.getCls()); } - return requireInternalTypeInfoByTypeId(Types.TIMESTAMP); + return getInternalTypeInfoByTypeId(Types.TIMESTAMP); } - private ClassInfo getOrBuildClassInfo(Class cls) { - ClassInfo classInfo = classInfoMap.get(cls); - if (classInfo == null) { - classInfo = buildClassInfo(cls); - classInfoMap.put(cls, classInfo); + private TypeInfo getOrBuildTypeInfo(Class cls) { + TypeInfo typeInfo = classInfoMap.get(cls); + if (typeInfo == null) { + typeInfo = buildTypeInfo(cls); + classInfoMap.put(cls, typeInfo); } - return classInfo; + return typeInfo; } @Override - protected ClassInfo loadBytesToClassInfo( + protected TypeInfo loadBytesToTypeInfo( MetaStringBytes packageBytes, MetaStringBytes simpleClassNameBytes) { // Default to NAMED_STRUCT when called without internalTypeId - return loadBytesToClassInfoWithTypeId(Types.NAMED_STRUCT, packageBytes, simpleClassNameBytes); + return loadBytesToTypeInfoWithTypeId(Types.NAMED_STRUCT, packageBytes, simpleClassNameBytes); + } + + @Override + protected TypeInfo loadBytesToTypeInfo( + int typeId, MetaStringBytes packageBytes, MetaStringBytes simpleClassNameBytes) { + return loadBytesToTypeInfoWithTypeId(typeId, packageBytes, simpleClassNameBytes); } @Override - protected ClassInfo ensureSerializerForClassInfo(ClassInfo classInfo) { - if (classInfo.serializer == null) { - Class cls = classInfo.cls; + protected TypeInfo ensureSerializerForTypeInfo(TypeInfo typeInfo) { + if (typeInfo.serializer == null) { + Class cls = typeInfo.cls; if (cls != null && (ReflectionUtils.isAbstract(cls) || cls.isInterface())) { - return classInfo; + return typeInfo; } - // Get or create ClassInfo with serializer - ClassInfo newClassInfo = getClassInfo(classInfo.cls); - // Update the cache with the correct ClassInfo that has a serializer - if (classInfo.typeNameBytes != null) { + // Get or create TypeInfo with serializer + TypeInfo newTypeInfo = getTypeInfo(typeInfo.cls); + // Update the cache with the correct TypeInfo that has a serializer + if (typeInfo.typeNameBytes != null) { TypeNameBytes typeNameBytes = - new TypeNameBytes(classInfo.namespaceBytes.hashCode, classInfo.typeNameBytes.hashCode); - compositeClassNameBytes2ClassInfo.put(typeNameBytes, newClassInfo); + new TypeNameBytes(typeInfo.namespaceBytes.hashCode, typeInfo.typeNameBytes.hashCode); + compositeClassNameBytes2TypeInfo.put(typeNameBytes, newTypeInfo); } - return newClassInfo; + return newTypeInfo; } - return classInfo; + return typeInfo; } - private ClassInfo loadBytesToClassInfoWithTypeId( + private TypeInfo loadBytesToTypeInfoWithTypeId( int internalTypeId, MetaStringBytes packageBytes, MetaStringBytes simpleClassNameBytes) { TypeNameBytes typeNameBytes = new TypeNameBytes(packageBytes.hashCode, simpleClassNameBytes.hashCode); - ClassInfo classInfo = compositeClassNameBytes2ClassInfo.get(typeNameBytes); - if (classInfo == null) { - classInfo = - populateBytesToClassInfo( + TypeInfo typeInfo = compositeClassNameBytes2TypeInfo.get(typeNameBytes); + if (typeInfo == null) { + typeInfo = + populateBytesToTypeInfo( internalTypeId, typeNameBytes, packageBytes, simpleClassNameBytes); } - return classInfo; + return typeInfo; } - private ClassInfo populateBytesToClassInfo( + private TypeInfo populateBytesToTypeInfo( int typeId, TypeNameBytes typeNameBytes, MetaStringBytes packageBytes, @@ -1101,8 +1124,8 @@ private ClassInfo populateBytesToClassInfo( String namespace = packageBytes.decode(PACKAGE_DECODER); String typeName = simpleClassNameBytes.decode(TYPE_NAME_DECODER); String qualifiedName = qualifiedName(namespace, typeName); - ClassInfo classInfo = qualifiedType2ClassInfo.get(qualifiedName); - if (classInfo == null) { + TypeInfo typeInfo = qualifiedType2TypeInfo.get(qualifiedName); + if (typeInfo == null) { String msg = String.format("Class %s not registered", qualifiedName); Class type = null; if (config.deserializeNonexistentClass()) { @@ -1126,21 +1149,22 @@ private ClassInfo populateBytesToClassInfo( MetaStringBytes fullClassNameBytes = metaStringResolver.getOrCreateMetaStringBytes( PACKAGE_ENCODER.encode(qualifiedName, MetaString.Encoding.UTF_8)); - classInfo = - new ClassInfo( + typeInfo = + new TypeInfo( type, fullClassNameBytes, packageBytes, simpleClassNameBytes, false, null, - NOT_SUPPORT_XLANG); + NOT_SUPPORT_XLANG, + INVALID_USER_TYPE_ID); if (NonexistentClass.class.isAssignableFrom(TypeUtils.getComponentIfArray(type))) { - classInfo.serializer = NonexistentClassSerializers.getSerializer(fory, qualifiedName, type); + typeInfo.serializer = NonexistentClassSerializers.getSerializer(fory, qualifiedName, type); } } - compositeClassNameBytes2ClassInfo.put(typeNameBytes, classInfo); - return classInfo; + compositeClassNameBytes2TypeInfo.put(typeNameBytes, typeInfo); + return typeInfo; } @Override @@ -1170,7 +1194,7 @@ public DescriptorGrouper createDescriptorGrouper( private byte getInternalTypeId(Descriptor descriptor) { Class cls = descriptor.getRawType(); if (cls.isArray() && cls.getComponentType().isPrimitive()) { - return (byte) (Types.getDescriptorTypeId(fory, descriptor) & 0xff); + return (byte) Types.getDescriptorTypeId(fory, descriptor); } return getInternalTypeId(cls); } @@ -1192,7 +1216,7 @@ private byte getInternalTypeId(Class cls) { return Types.MAP; } if (isRegistered(cls)) { - return (byte) (getClassInfo(cls).getTypeId() & 0xff); + return (byte) getTypeInfo(cls).getTypeId(); } else { if (cls.isEnum()) { return Types.ENUM; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ArraySerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ArraySerializers.java index 68a0e654dc..2cf1e74690 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ArraySerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ArraySerializers.java @@ -26,10 +26,10 @@ import org.apache.fory.config.LongEncoding; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.Platform; -import org.apache.fory.resolver.ClassInfo; -import org.apache.fory.resolver.ClassInfoHolder; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.RefResolver; +import org.apache.fory.resolver.TypeInfo; +import org.apache.fory.resolver.TypeInfoHolder; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.collection.CollectionFlags; import org.apache.fory.serializer.collection.ForyArrayAsListSerializer; @@ -45,7 +45,7 @@ public class ArraySerializers { public static final class ObjectArraySerializer extends Serializer { private final Class innerType; private final Serializer componentTypeSerializer; - private final ClassInfoHolder classInfoHolder; + private final TypeInfoHolder classInfoHolder; private final int[] stubDims; private final GenericType componentGenericType; @@ -73,11 +73,11 @@ public ObjectArraySerializer(Fory fory, Class cls) { this.componentTypeSerializer = fory.getClassResolver().getSerializer(componentType); } } else { - // TODO add ClassInfo cache for non-final component type. + // TODO add TypeInfo cache for non-final component type. this.componentTypeSerializer = null; } this.stubDims = new int[dimension]; - classInfoHolder = fory.getClassResolver().nilClassInfoHolder(); + classInfoHolder = fory.getClassResolver().nilTypeInfoHolder(); } @Override @@ -96,16 +96,16 @@ public void write(MemoryBuffer buffer, T[] arr) { } else { Fory fory = this.fory; ClassResolver classResolver = fory.getClassResolver(); - ClassInfo classInfo = null; + TypeInfo typeInfo = null; Class elemClass = null; for (T t : arr) { if (!refResolver.writeRefOrNull(buffer, t)) { Class clz = t.getClass(); if (clz != elemClass) { elemClass = clz; - classInfo = classResolver.getClassInfo(clz); + typeInfo = classResolver.getTypeInfo(clz); } - fory.writeNonRef(buffer, t, classInfo); + fory.writeNonRef(buffer, t, typeInfo); } } } @@ -168,7 +168,7 @@ public T[] read(MemoryBuffer buffer) { } } else { Fory fory = this.fory; - ClassInfoHolder classInfoHolder = this.classInfoHolder; + TypeInfoHolder classInfoHolder = this.classInfoHolder; for (int i = 0; i < numElements; i++) { int nextReadRefId = refResolver.tryPreserveRefId(buffer); Object o; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/CodegenSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/CodegenSerializer.java index d500dcd0fc..841bec0466 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/CodegenSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/CodegenSerializer.java @@ -89,16 +89,16 @@ private Serializer getOrCreateGeneratedSerializer() { if (interpreterSerializer != null) { return interpreterSerializer; } - fory.getClassResolver().getClassInfo(type).setSerializer(null); + fory.getClassResolver().getTypeInfo(type).setSerializer(null); if (fory.getConfig().isAsyncCompilationEnabled()) { // jit not finished, avoid recursive call current serializer. Class sc = fory.getClassResolver().getSerializerClass(type, false); - fory.getClassResolver().getClassInfo(type).setSerializer(this); + fory.getClassResolver().getTypeInfo(type).setSerializer(this); return interpreterSerializer = Serializers.newSerializer(fory, type, sc); } else { Class sc = fory.getClassResolver().getSerializerClass(type); - fory.getClassResolver().getClassInfo(type).setSerializer(this); + fory.getClassResolver().getTypeInfo(type).setSerializer(this); checkArgument( Generated.GeneratedSerializer.class.isAssignableFrom(sc), "Expect jit serializer but got %s for class %s", diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java index ea20bdb83a..9726c0ea6e 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java @@ -29,9 +29,9 @@ import org.apache.fory.meta.TypeExtMeta; import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.reflect.TypeRef; -import org.apache.fory.resolver.ClassInfo; -import org.apache.fory.resolver.ClassInfoHolder; import org.apache.fory.resolver.RefMode; +import org.apache.fory.resolver.TypeInfo; +import org.apache.fory.resolver.TypeInfoHolder; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.converter.FieldConverter; import org.apache.fory.type.Descriptor; @@ -120,9 +120,10 @@ public static FieldGroups buildFieldInfos(Fory fory, DescriptorGrouper grouper) public static final class SerializationFieldInfo { public final Descriptor descriptor; + public final Class type; public final TypeRef typeRef; public final int dispatchId; - public final ClassInfo classInfo; + public final TypeInfo typeInfo; public final Serializer serializer; public final String qualifiedFieldName; public final FieldAccessor fieldAccessor; @@ -135,30 +136,31 @@ public static final class SerializationFieldInfo { public final boolean useDeclaredTypeInfo; public final GenericType genericType; - public final ClassInfoHolder classInfoHolder; + public final TypeInfoHolder classInfoHolder; public final boolean isArray; - public final ClassInfo containerClassInfo; + public final TypeInfo containerTypeInfo; SerializationFieldInfo(Fory fory, Descriptor d) { this.descriptor = d; + this.type = descriptor.getRawType(); this.typeRef = d.getTypeRef(); this.dispatchId = DispatchId.getDispatchId(fory, d); TypeResolver resolver = fory.getTypeResolver(); // invoke `copy` to avoid ObjectSerializer construct clear serializer by `clearSerializer`. if (resolver.isMonomorphic(descriptor)) { - classInfo = SerializationUtils.getClassInfo(fory, typeRef.getRawType()); + typeInfo = SerializationUtils.getTypeInfo(fory, typeRef.getRawType()); if (!fory.isShareMeta() && !fory.isCompatible() - && classInfo.getSerializer() instanceof ReplaceResolveSerializer) { + && typeInfo.getSerializer() instanceof ReplaceResolveSerializer) { // overwrite replace resolve serializer for final field - classInfo.setSerializer(new FinalFieldReplaceResolveSerializer(fory, classInfo.getCls())); + typeInfo.setSerializer(new FinalFieldReplaceResolveSerializer(fory, typeInfo.getCls())); } } else { - classInfo = null; + typeInfo = null; } - useDeclaredTypeInfo = classInfo != null && resolver.isMonomorphic(descriptor); - if (classInfo != null) { - serializer = classInfo.getSerializer(); + useDeclaredTypeInfo = typeInfo != null && resolver.isMonomorphic(descriptor); + if (typeInfo != null) { + serializer = typeInfo.getSerializer(); } else { serializer = null; } @@ -199,15 +201,15 @@ public static final class SerializationFieldInfo { if (field != null) { TypeUtils.applyRefTrackingOverride(t, field.getAnnotatedType(), fory.trackingRef()); } - classInfoHolder = resolver.nilClassInfoHolder(); + classInfoHolder = resolver.nilTypeInfoHolder(); isArray = cls.isArray(); if (!fory.isCrossLanguage()) { - containerClassInfo = null; + containerTypeInfo = null; } else { if (resolver.isMap(cls) || resolver.isCollection(cls) || resolver.isSet(cls)) { - containerClassInfo = resolver.getClassInfo(cls); + containerTypeInfo = resolver.getTypeInfo(cls); } else { - containerClassInfo = null; + containerTypeInfo = null; } } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ForwardSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ForwardSerializer.java index 2df75a6f03..11e3615d8d 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ForwardSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ForwardSerializer.java @@ -62,7 +62,7 @@ protected void register(T serializer, Class clz) { throw new UnsupportedOperationException(); } - protected void register(T serializer, Class clz, int id) { + protected void register(T serializer, Class clz, long id) { throw new UnsupportedOperationException(); } @@ -151,8 +151,8 @@ protected void register(LoaderBinding binding, Class clz) { } @Override - protected void register(LoaderBinding binding, Class clz, int id) { - binding.register(clz, (short) id); + protected void register(LoaderBinding binding, Class clz, long id) { + binding.register(clz, id); } @Override @@ -237,7 +237,7 @@ public synchronized void register(Class clz) { }); } - public synchronized void register(Class clz, int id) { + public synchronized void register(Class clz, long id) { serializerSet.forEach(serializer -> proxy.register(serializer, clz, id)); serializerCallback = serializerCallback.andThen( diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/LazySerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/LazySerializer.java index 4bea435abd..1f64c9b37a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/LazySerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/LazySerializer.java @@ -60,7 +60,7 @@ public void xwrite(MemoryBuffer buffer, Object value) { if (serializer == null) { serializer = serializerSupplier.get(); fory.getClassResolver().setSerializer(value.getClass(), serializer); - fory.getXtypeResolver().getClassInfo(value.getClass()).setSerializer(serializer); + fory.getXtypeResolver().getTypeInfo(value.getClass()).setSerializer(serializer); } serializer.xwrite(buffer, value); } @@ -74,7 +74,7 @@ public Object xread(MemoryBuffer buffer) { Object value = serializer.xread(buffer); if (unInit) { fory.getClassResolver().setSerializer(value.getClass(), serializer); - fory.getXtypeResolver().getClassInfo(value.getClass()).setSerializer(serializer); + fory.getXtypeResolver().getTypeInfo(value.getClass()).setSerializer(serializer); } return value; } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java index 4a463073bd..fc7abe0796 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java @@ -24,7 +24,7 @@ import org.apache.fory.collection.IdentityObjectIntMap; import org.apache.fory.collection.ObjectIntMap; import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.meta.ClassDef; +import org.apache.fory.meta.TypeDef; import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.resolver.MetaContext; import org.apache.fory.resolver.TypeResolver; @@ -48,7 +48,7 @@ */ @SuppressWarnings({"unchecked", "rawtypes"}) public class MetaSharedLayerSerializer extends MetaSharedLayerSerializerBase { - private final ClassDef layerClassDef; + private final TypeDef layerTypeDef; private final Class layerMarkerClass; private final SerializationFieldInfo[] buildInFields; private final SerializationFieldInfo[] otherFields; @@ -60,19 +60,19 @@ public class MetaSharedLayerSerializer extends MetaSharedLayerSerializerBase< * * @param fory the Fory instance * @param type the target class for this layer - * @param layerClassDef the ClassDef for this layer only (resolveParent=false) + * @param layerTypeDef the TypeDef for this layer only (resolveParent=false) * @param layerMarkerClass the generated marker class used as key in metaContext.classMap */ public MetaSharedLayerSerializer( - Fory fory, Class type, ClassDef layerClassDef, Class layerMarkerClass) { + Fory fory, Class type, TypeDef layerTypeDef, Class layerMarkerClass) { super(fory, type); - this.layerClassDef = layerClassDef; + this.layerTypeDef = layerTypeDef; this.layerMarkerClass = layerMarkerClass; TypeResolver typeResolver = fory.getTypeResolver(); this.binding = SerializationBinding.createBinding(fory); - // Build field infos from layerClassDef - Collection descriptors = layerClassDef.getDescriptors(typeResolver, type); + // Build field infos from layerTypeDef + Collection descriptors = layerTypeDef.getDescriptors(typeResolver, type); DescriptorGrouper descriptorGrouper = typeResolver.createDescriptorGrouper(descriptors, false); FieldGroups fieldGroups = FieldGroups.buildFieldInfos(fory, descriptorGrouper); this.buildInFields = fieldGroups.buildInFields; @@ -103,9 +103,9 @@ public void writeLayerClassMeta(MemoryBuffer buffer) { // Reference to previously written type: (index << 1) | 1, LSB=1 buffer.writeVarUint32((id << 1) | 1); } else { - // New type: index << 1, LSB=0, followed by ClassDef bytes inline + // New type: index << 1, LSB=0, followed by TypeDef bytes inline buffer.writeVarUint32(newId << 1); - buffer.writeBytes(layerClassDef.getEncoded()); + buffer.writeBytes(layerTypeDef.getEncoded()); } } @@ -244,7 +244,7 @@ public T readAndSetFields(MemoryBuffer buffer, T obj) { /** * Read fields only, without reading layer class meta. This is used when the caller has already * read and processed the layer class meta (e.g., for schema evolution where different senders may - * have different ClassDefs for the same layer). + * have different TypeDefs for the same layer). * * @param buffer the memory buffer to read from * @param obj the object to set field values on diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java index 9b62531003..4ecdcfda4a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java @@ -31,7 +31,7 @@ import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.Platform; -import org.apache.fory.meta.ClassDef; +import org.apache.fory.meta.TypeDef; import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.resolver.MapRefResolver; import org.apache.fory.resolver.RefResolver; @@ -50,8 +50,8 @@ import org.apache.fory.util.record.RecordUtils; /** - * A meta-shared compatible deserializer builder based on {@link ClassDef}. This serializer will - * compare fields between {@link ClassDef} and class fields, then create serializer to read and + * A meta-shared compatible deserializer builder based on {@link TypeDef}. This serializer will + * compare fields between {@link TypeDef} and class fields, then create serializer to read and * set/skip corresponding fields to support type forward/backward compatibility. Serializer are * forward to {@link ObjectSerializer} for now. We can consolidate fields between peers to create * better serializers to serialize common fields between peers for efficiency. @@ -77,7 +77,7 @@ public class MetaSharedSerializer extends AbstractObjectSerializer { private final boolean hasDefaultValues; private final DefaultValueUtils.DefaultValueField[] defaultValueFields; - public MetaSharedSerializer(Fory fory, Class type, ClassDef classDef) { + public MetaSharedSerializer(Fory fory, Class type, TypeDef typeDef) { super(fory, type); Preconditions.checkArgument( !fory.getConfig().checkClassVersion(), @@ -85,13 +85,13 @@ public MetaSharedSerializer(Fory fory, Class type, ClassDef classDef) { Preconditions.checkArgument( fory.getConfig().isMetaShareEnabled(), "Meta share must be enabled."); if (Utils.DEBUG_OUTPUT_ENABLED) { - LOG.info("========== MetaSharedSerializer ClassDef for {} ==========", type.getName()); - LOG.info("ClassDef fieldsInfo count: {}", classDef.getFieldCount()); - for (int i = 0; i < classDef.getFieldsInfo().size(); i++) { - LOG.info(" [{}] {}", i, classDef.getFieldsInfo().get(i)); + LOG.info("========== MetaSharedSerializer TypeDef for {} ==========", type.getName()); + LOG.info("TypeDef fieldsInfo count: {}", typeDef.getFieldCount()); + for (int i = 0; i < typeDef.getFieldsInfo().size(); i++) { + LOG.info(" [{}] {}", i, typeDef.getFieldsInfo().get(i)); } } - Collection descriptors = consolidateFields(fory.getTypeResolver(), type, classDef); + Collection descriptors = consolidateFields(fory.getTypeResolver(), type, typeDef); DescriptorGrouper descriptorGrouper = fory.getTypeResolver().createDescriptorGrouper(descriptors, false); if (Utils.DEBUG_OUTPUT_ENABLED) { @@ -202,6 +202,9 @@ public T read(MemoryBuffer buffer) { } // read order: primitive,boxed,final,other,collection,map for (SerializationFieldInfo fieldInfo : this.buildInFields) { + if (Utils.DEBUG_OUTPUT_VERBOSE) { + printFieldDebugInfo(fieldInfo, buffer); + } FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; if (fieldAccessor != null) { AbstractObjectSerializer.readBuildInFieldValue(binding, fieldInfo, buffer, targetObject); @@ -216,6 +219,9 @@ public T read(MemoryBuffer buffer) { } Generics generics = fory.getGenerics(); for (SerializationFieldInfo fieldInfo : containerFields) { + if (Utils.DEBUG_OUTPUT_VERBOSE) { + printFieldDebugInfo(fieldInfo, buffer); + } Object fieldValue = AbstractObjectSerializer.readContainerFieldValue(binding, generics, fieldInfo, buffer); FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; @@ -224,6 +230,9 @@ public T read(MemoryBuffer buffer) { } } for (SerializationFieldInfo fieldInfo : otherFields) { + if (Utils.DEBUG_OUTPUT_VERBOSE) { + printFieldDebugInfo(fieldInfo, buffer); + } Object fieldValue = binding.readField(fieldInfo, buffer); FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; if (fieldAccessor != null) { @@ -244,6 +253,9 @@ private void readFields(MemoryBuffer buffer, Object[] fields) { SerializationBinding binding = this.binding; // read order: primitive,boxed,final,other,collection,map for (SerializationFieldInfo fieldInfo : this.buildInFields) { + if (Utils.DEBUG_OUTPUT_ENABLED) { + printFieldDebugInfo(fieldInfo, buffer); + } if (fieldInfo.fieldAccessor != null) { fields[counter++] = AbstractObjectSerializer.readBuildInFieldValue(binding, fieldInfo, buffer); @@ -258,18 +270,32 @@ private void readFields(MemoryBuffer buffer, Object[] fields) { } Generics generics = fory.getGenerics(); for (SerializationFieldInfo fieldInfo : containerFields) { + if (Utils.DEBUG_OUTPUT_ENABLED) { + printFieldDebugInfo(fieldInfo, buffer); + } Object fieldValue = AbstractObjectSerializer.readContainerFieldValue(binding, generics, fieldInfo, buffer); fields[counter++] = fieldValue; } for (SerializationFieldInfo fieldInfo : otherFields) { + if (Utils.DEBUG_OUTPUT_ENABLED) { + printFieldDebugInfo(fieldInfo, buffer); + } Object fieldValue = binding.readField(fieldInfo, buffer); fields[counter++] = fieldValue; } } + private void printFieldDebugInfo(SerializationFieldInfo fieldInfo, MemoryBuffer buffer) { + LOG.info( + "[Java] read field {} of type {}, reader index {}", + fieldInfo.descriptor.getName(), + fieldInfo.typeRef, + buffer.readerIndex()); + } + public static Collection consolidateFields( - TypeResolver resolver, Class cls, ClassDef classDef) { - return classDef.getDescriptors(resolver, cls); + TypeResolver resolver, Class cls, TypeDef typeDef) { + return typeDef.getDescriptors(resolver, cls); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClass.java b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClass.java index 10ad0d54e2..6cf332b7f4 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClass.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClass.java @@ -22,7 +22,7 @@ import org.apache.fory.collection.LazyMap; import org.apache.fory.config.CompatibleMode; import org.apache.fory.config.Config; -import org.apache.fory.meta.ClassDef; +import org.apache.fory.meta.TypeDef; import org.apache.fory.type.TypeUtils; /** @@ -106,10 +106,10 @@ enum NonexistentEnum implements NonexistentClass { class NonexistentSkip implements NonexistentClass {} class NonexistentMetaShared extends LazyMap implements NonexistentClass { - final ClassDef classDef; + final TypeDef typeDef; - public NonexistentMetaShared(ClassDef classDef) { - this.classDef = classDef; + public NonexistentMetaShared(TypeDef typeDef) { + this.typeDef = typeDef; } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java index aabf35dfa3..6ec9d2f9f2 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java @@ -31,7 +31,7 @@ import org.apache.fory.logging.Logger; import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.meta.ClassDef; +import org.apache.fory.meta.TypeDef; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.MetaContext; import org.apache.fory.resolver.MetaStringResolver; @@ -68,22 +68,21 @@ private ClassFieldsInfo(FieldGroups fieldGroups, int classVersionHash) { public static final class NonexistentClassSerializer extends Serializer { private static final int NONEXISTENT_META_SHARED_ID_SIZE = computeVarUint32Size(ClassResolver.NONEXISTENT_META_SHARED_ID); - private final ClassDef classDef; + private final TypeDef typeDef; private final LongMap fieldsInfoMap; private final SerializationBinding binding; - public NonexistentClassSerializer(Fory fory, ClassDef classDef) { + public NonexistentClassSerializer(Fory fory, TypeDef typeDef) { super(fory, NonexistentClass.NonexistentMetaShared.class); - this.classDef = classDef; + this.typeDef = typeDef; fieldsInfoMap = new LongMap<>(); binding = SerializationBinding.createBinding(fory); Preconditions.checkArgument(fory.getConfig().isMetaShareEnabled()); - if (Utils.DEBUG_OUTPUT_ENABLED && classDef != null) { - LOG.info( - "========== NonexistentClassSerializer ClassDef for {} ==========", type.getName()); - LOG.info("ClassDef fieldsInfo count: {}", classDef.getFieldCount()); - for (int i = 0; i < classDef.getFieldsInfo().size(); i++) { - LOG.info(" [{}] {}", i, classDef.getFieldsInfo().get(i)); + if (Utils.DEBUG_OUTPUT_ENABLED && typeDef != null) { + LOG.info("========== NonexistentClassSerializer TypeDef for {} ==========", type.getName()); + LOG.info("TypeDef fieldsInfo count: {}", typeDef.getFieldCount()); + for (int i = 0; i < typeDef.getFieldsInfo().size(); i++) { + LOG.info(" [{}] {}", i, typeDef.getFieldsInfo().get(i)); } } } @@ -91,25 +90,25 @@ public NonexistentClassSerializer(Fory fory, ClassDef classDef) { /** * Multiple un existed class will correspond to this `NonexistentMetaSharedClass`. When querying * classinfo by `class`, it may dispatch to same `NonexistentClassSerializer`, so we can't use - * `classDef` in this serializer, but use `classDef` in `NonexistentMetaSharedClass` instead. + * `typeDef` in this serializer, but use `typeDef` in `NonexistentMetaSharedClass` instead. * *

    NonexistentMetaShared is registered with a fixed internal typeId for dispatch. This * serializer rewinds that placeholder typeId and writes the original class's typeId, then - * writes the shared ClassDef inline using the stream meta protocol. + * writes the shared TypeDef inline using the stream meta protocol. */ - private void writeClassDef(MemoryBuffer buffer, NonexistentClass.NonexistentMetaShared value) { + private void writeTypeDef(MemoryBuffer buffer, NonexistentClass.NonexistentMetaShared value) { MetaContext metaContext = fory.getSerializationContext().getMetaContext(); IdentityObjectIntMap classMap = metaContext.classMap; int newId = classMap.size; // class not exist, use class def id for identity. - int id = classMap.putOrGet(value.classDef.getId(), newId); + int id = classMap.putOrGet(value.typeDef.getId(), newId); if (id >= 0) { // Reference to previously written type: (index << 1) | 1, LSB=1 buffer.writeVarUint32((id << 1) | 1); } else { - // New type: index << 1, LSB=0, followed by ClassDef bytes inline + // New type: index << 1, LSB=0, followed by TypeDef bytes inline buffer.writeVarUint32(newId << 1); - buffer.writeBytes(value.classDef.getEncoded()); + buffer.writeBytes(value.typeDef.getEncoded()); } } @@ -129,41 +128,56 @@ private static int computeVarUint32Size(int value) { return 5; } - private int resolveTypeId(ClassDef classDef) { - if (classDef.getClassSpec().isEnum) { - if (classDef.isNamed()) { + private int resolveTypeId(TypeDef typeDef) { + if (typeDef.getClassSpec().isEnum) { + if (typeDef.isNamed()) { return Types.NAMED_ENUM; } - return (classDef.getUserTypeId() << 8) | Types.ENUM; + return Types.ENUM; } - if (classDef.isNamed()) { - return classDef.isCompatible() ? Types.NAMED_COMPATIBLE_STRUCT : Types.NAMED_STRUCT; + if (typeDef.isNamed()) { + return typeDef.isCompatible() ? Types.NAMED_COMPATIBLE_STRUCT : Types.NAMED_STRUCT; } - int internalTypeId = classDef.isCompatible() ? Types.COMPATIBLE_STRUCT : Types.STRUCT; - return (classDef.getUserTypeId() << 8) | internalTypeId; + return typeDef.isCompatible() ? Types.COMPATIBLE_STRUCT : Types.STRUCT; } @Override public void write(MemoryBuffer buffer, Object v) { NonexistentClass.NonexistentMetaShared value = (NonexistentClass.NonexistentMetaShared) v; - int typeId = resolveTypeId(value.classDef); - int typeIdSize = computeVarUint32Size(typeId); - if (typeIdSize == NONEXISTENT_META_SHARED_ID_SIZE) { - buffer.increaseWriterIndex(-NONEXISTENT_META_SHARED_ID_SIZE); - buffer.writeVarUint32Small7(typeId); + int typeId = resolveTypeId(value.typeDef); + int userTypeId = value.typeDef.isNamed() ? -1 : value.typeDef.getUserTypeId(); + int typeIdSize = 1; + int userTypeIdSize = userTypeId != -1 ? computeVarUint32Size(userTypeId) : 0; + if (fory.isCrossLanguage()) { + buffer.writeUint8(typeId); + if (userTypeIdSize > 0) { + buffer.writeVarUint32(userTypeId); + } } else { - int originalWriterIndex = buffer.writerIndex(); - int placeholderStart = originalWriterIndex - NONEXISTENT_META_SHARED_ID_SIZE; - int payloadStart = placeholderStart + NONEXISTENT_META_SHARED_ID_SIZE; - int payloadLength = originalWriterIndex - payloadStart; - byte[] payload = buffer.getBytes(payloadStart, payloadLength); - buffer.writerIndex(placeholderStart); - buffer.writeVarUint32Small7(typeId); - buffer.writeBytes(payload); + int totalSize = typeIdSize + userTypeIdSize; + if (totalSize == NONEXISTENT_META_SHARED_ID_SIZE) { + buffer.increaseWriterIndex(-NONEXISTENT_META_SHARED_ID_SIZE); + buffer.writeUint8(typeId); + if (userTypeIdSize > 0) { + buffer.writeVarUint32(userTypeId); + } + } else { + int originalWriterIndex = buffer.writerIndex(); + int placeholderStart = originalWriterIndex - NONEXISTENT_META_SHARED_ID_SIZE; + int payloadStart = placeholderStart + NONEXISTENT_META_SHARED_ID_SIZE; + int payloadLength = originalWriterIndex - payloadStart; + byte[] payload = buffer.getBytes(payloadStart, payloadLength); + buffer.writerIndex(placeholderStart); + buffer.writeUint8(typeId); + if (userTypeIdSize > 0) { + buffer.writeVarUint32(userTypeId); + } + buffer.writeBytes(payload); + } } - writeClassDef(buffer, value); - ClassDef classDef = value.classDef; - ClassFieldsInfo fieldsInfo = getClassFieldsInfo(classDef); + writeTypeDef(buffer, value); + TypeDef typeDef = value.typeDef; + ClassFieldsInfo fieldsInfo = getClassFieldsInfo(typeDef); Fory fory = this.fory; RefResolver refResolver = fory.getRefResolver(); if (fory.checkClassVersion()) { @@ -186,14 +200,14 @@ public void write(MemoryBuffer buffer, Object v) { } } - private ClassFieldsInfo getClassFieldsInfo(ClassDef classDef) { - ClassFieldsInfo fieldsInfo = fieldsInfoMap.get(classDef.getId()); + private ClassFieldsInfo getClassFieldsInfo(TypeDef typeDef) { + ClassFieldsInfo fieldsInfo = fieldsInfoMap.get(typeDef.getId()); TypeResolver resolver = getTypeResolver(fory); if (fieldsInfo == null) { // Use `NonexistentSkipClass` since it doesn't have any field. Collection descriptors = MetaSharedSerializer.consolidateFields( - resolver, NonexistentClass.NonexistentSkip.class, classDef); + resolver, NonexistentClass.NonexistentSkip.class, typeDef); DescriptorGrouper grouper = fory.getClassResolver().createDescriptorGrouper(descriptors, false); FieldGroups fieldGroups = FieldGroups.buildFieldInfos(fory, grouper); @@ -202,7 +216,7 @@ private ClassFieldsInfo getClassFieldsInfo(ClassDef classDef) { classVersionHash = ObjectSerializer.computeStructHash(fory, grouper); } fieldsInfo = new ClassFieldsInfo(fieldGroups, classVersionHash); - fieldsInfoMap.put(classDef.getId(), fieldsInfo); + fieldsInfoMap.put(typeDef.getId(), fieldsInfo); } return fieldsInfo; } @@ -210,13 +224,13 @@ private ClassFieldsInfo getClassFieldsInfo(ClassDef classDef) { @Override public Object read(MemoryBuffer buffer) { NonexistentClass.NonexistentMetaShared obj = - new NonexistentClass.NonexistentMetaShared(classDef); + new NonexistentClass.NonexistentMetaShared(typeDef); Fory fory = this.fory; RefResolver refResolver = fory.getRefResolver(); refResolver.reference(obj); List entries = new ArrayList<>(); // read order: primitive,boxed,final,other,collection,map - ClassFieldsInfo allFieldsInfo = getClassFieldsInfo(classDef); + ClassFieldsInfo allFieldsInfo = getClassFieldsInfo(typeDef); for (SerializationFieldInfo fieldInfo : allFieldsInfo.buildInFields) { Object fieldValue = AbstractObjectSerializer.readBuildInFieldValue(binding, fieldInfo, buffer); @@ -289,7 +303,7 @@ public static Serializer getSerializer(Fory fory, String className, Class cls if (fory.getConfig().isMetaShareEnabled()) { throw new IllegalStateException( String.format( - "Serializer of class %s should be set in ClassResolver#getMetaSharedClassInfo", + "Serializer of class %s should be set in ClassResolver#getMetaSharedTypeInfo", className)); } else { return new ObjectSerializer(fory, cls); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java index d935bca915..c6cdc5b008 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java @@ -31,7 +31,7 @@ import org.apache.fory.logging.Logger; import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.meta.ClassDef; +import org.apache.fory.meta.TypeDef; import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.resolver.RefResolver; import org.apache.fory.resolver.TypeResolver; @@ -89,15 +89,15 @@ public ObjectSerializer(Fory fory, Class cls, boolean resolveParent) { Collection descriptors; boolean shareMeta = fory.getConfig().isMetaShareEnabled(); if (shareMeta) { - ClassDef classDef = typeResolver.getTypeDef(cls, resolveParent); + TypeDef typeDef = typeResolver.getTypeDef(cls, resolveParent); if (Utils.DEBUG_OUTPUT_ENABLED) { - LOG.info("========== ObjectSerializer ClassDef for {} ==========", cls.getName()); - LOG.info("ClassDef fieldsInfo count: {}", classDef.getFieldsInfo().size()); - for (int i = 0; i < classDef.getFieldsInfo().size(); i++) { - LOG.info(" [{}] {}", i, classDef.getFieldsInfo().get(i)); + LOG.info("========== ObjectSerializer TypeDef for {} ==========", cls.getName()); + LOG.info("TypeDef fieldsInfo count: {}", typeDef.getFieldsInfo().size()); + for (int i = 0; i < typeDef.getFieldsInfo().size(); i++) { + LOG.info(" [{}] {}", i, typeDef.getFieldsInfo().get(i)); } } - descriptors = classDef.getDescriptors(typeResolver, cls); + descriptors = typeDef.getDescriptors(typeResolver, cls); } else { descriptors = typeResolver.getFieldDescriptors(cls, resolveParent); } @@ -153,14 +153,26 @@ public void write(MemoryBuffer buffer, T value) { writeOtherFields(buffer, value); } + private void printWriteFieldDebugInfo(SerializationFieldInfo fieldInfo, MemoryBuffer buffer) { + LOG.info( + "[Java] write field {} of type {}, writer index {}", + fieldInfo.descriptor.getName(), + fieldInfo.typeRef, + buffer.writerIndex()); + } + + private void printReadFieldDebugInfo(SerializationFieldInfo fieldInfo, MemoryBuffer buffer) { + LOG.info( + "[Java] read field {} of type {}, reader index {}", + fieldInfo.descriptor.getName(), + fieldInfo.typeRef, + buffer.readerIndex()); + } + private void writeOtherFields(MemoryBuffer buffer, T value) { for (SerializationFieldInfo fieldInfo : otherFields) { - if (Utils.DEBUG_OUTPUT_ENABLED) { - LOG.info( - "[Java] write field {} of type {}, writer index {}", - fieldInfo.descriptor.getName(), - fieldInfo.typeRef, - buffer.writerIndex()); + if (Utils.DEBUG_OUTPUT_VERBOSE) { + printWriteFieldDebugInfo(fieldInfo, buffer); } FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; Object fieldValue = fieldAccessor.getObject(value); @@ -170,12 +182,8 @@ private void writeOtherFields(MemoryBuffer buffer, T value) { private void writeBuildInFields(MemoryBuffer buffer, T value, Fory fory) { for (SerializationFieldInfo fieldInfo : this.buildInFields) { - if (Utils.DEBUG_OUTPUT_ENABLED) { - LOG.info( - "[Java] write field {} of type {}, writer index {}", - fieldInfo.descriptor.getName(), - fieldInfo.typeRef, - buffer.writerIndex()); + if (Utils.DEBUG_OUTPUT_VERBOSE) { + printWriteFieldDebugInfo(fieldInfo, buffer); } AbstractObjectSerializer.writeBuildInField(binding, fieldInfo, buffer, value); } @@ -185,12 +193,8 @@ private void writeContainerFields( MemoryBuffer buffer, T value, Fory fory, RefResolver refResolver) { Generics generics = fory.getGenerics(); for (SerializationFieldInfo fieldInfo : containerFields) { - if (Utils.DEBUG_OUTPUT_ENABLED) { - LOG.info( - "[Java] write field {} of type {}, writer index {}", - fieldInfo.descriptor.getName(), - fieldInfo.typeRef, - buffer.writerIndex()); + if (Utils.DEBUG_OUTPUT_VERBOSE) { + printWriteFieldDebugInfo(fieldInfo, buffer); } FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; Object fieldValue = fieldAccessor.getObject(value); @@ -228,14 +232,23 @@ public Object[] readFields(MemoryBuffer buffer) { int counter = 0; // read order: primitive,boxed,final,other,collection,map for (SerializationFieldInfo fieldInfo : this.buildInFields) { + if (Utils.DEBUG_OUTPUT_VERBOSE) { + printReadFieldDebugInfo(fieldInfo, buffer); + } fieldValues[counter++] = readBuildInFieldValue(binding, fieldInfo, buffer); } Generics generics = fory.getGenerics(); for (SerializationFieldInfo fieldInfo : containerFields) { + if (Utils.DEBUG_OUTPUT_VERBOSE) { + printReadFieldDebugInfo(fieldInfo, buffer); + } Object fieldValue = readContainerFieldValue(binding, generics, fieldInfo, buffer); fieldValues[counter++] = fieldValue; } for (SerializationFieldInfo fieldInfo : otherFields) { + if (Utils.DEBUG_OUTPUT_VERBOSE) { + printReadFieldDebugInfo(fieldInfo, buffer); + } Object fieldValue = binding.readField(fieldInfo, buffer); fieldValues[counter++] = fieldValue; } @@ -250,17 +263,25 @@ public T readAndSetFields(MemoryBuffer buffer, T obj) { } // read order: primitive,boxed,final,other,collection,map for (SerializationFieldInfo fieldInfo : this.buildInFields) { - FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; + if (Utils.DEBUG_OUTPUT_VERBOSE) { + printReadFieldDebugInfo(fieldInfo, buffer); + } // a numeric type can have only three kinds: primitive, not_null_boxed, nullable_boxed readBuildInFieldValue(binding, fieldInfo, buffer, obj); } Generics generics = fory.getGenerics(); for (SerializationFieldInfo fieldInfo : containerFields) { + if (Utils.DEBUG_OUTPUT_VERBOSE) { + printReadFieldDebugInfo(fieldInfo, buffer); + } Object fieldValue = readContainerFieldValue(binding, generics, fieldInfo, buffer); FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; fieldAccessor.putObject(obj, fieldValue); } for (SerializationFieldInfo fieldInfo : otherFields) { + if (Utils.DEBUG_OUTPUT_VERBOSE) { + printReadFieldDebugInfo(fieldInfo, buffer); + } Object fieldValue = binding.readField(fieldInfo, buffer); FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; fieldAccessor.putObject(obj, fieldValue); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java index bab3e8acd1..0716a3b8ef 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java @@ -52,15 +52,15 @@ import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.Platform; -import org.apache.fory.meta.ClassDef; -import org.apache.fory.meta.ClassDefEncoder; import org.apache.fory.meta.FieldInfo; import org.apache.fory.meta.FieldTypes; +import org.apache.fory.meta.NativeTypeDefEncoder; +import org.apache.fory.meta.TypeDef; import org.apache.fory.reflect.ObjectCreator; import org.apache.fory.reflect.ObjectCreators; import org.apache.fory.reflect.ReflectionUtils; -import org.apache.fory.resolver.ClassInfo; import org.apache.fory.resolver.MetaContext; +import org.apache.fory.resolver.TypeInfo; import org.apache.fory.type.Descriptor; import org.apache.fory.type.TypeUtils; import org.apache.fory.type.Types; @@ -86,8 +86,8 @@ public class ObjectStreamSerializer extends AbstractObjectSerializer { private static final Logger LOG = LoggerFactory.getLogger(ObjectStreamSerializer.class); private final SlotInfo[] slotsInfos; - // Instance-level cache: ClassDef ID -> ClassInfo (shared across all slots) - private final LongMap classDefIdToClassInfo = new LongMap<>(4, 0.4f); + // Instance-level cache: TypeDef ID -> TypeInfo (shared across all slots) + private final LongMap typeDefIdToTypeInfo = new LongMap<>(4, 0.4f); /** * Interface for slot information used in ObjectStreamSerializer. This allows both full SlotsInfo @@ -96,19 +96,19 @@ public class ObjectStreamSerializer extends AbstractObjectSerializer { private interface SlotInfo { Class getCls(); - StreamClassInfo getStreamClassInfo(); + StreamTypeInfo getStreamTypeInfo(); MetaSharedLayerSerializerBase getSlotsSerializer(); /** - * Read the layer ClassDef from buffer (if meta share enabled) and return the appropriate - * serializer. When meta share is enabled, reads the ClassDef from buffer, caches the serializer - * by ClassDef ID, and returns it. When meta share is disabled, returns the default slots + * Read the layer TypeDef from buffer (if meta share enabled) and return the appropriate + * serializer. When meta share is enabled, reads the TypeDef from buffer, caches the serializer + * by TypeDef ID, and returns it. When meta share is disabled, returns the default slots * serializer. Also stores the returned serializer for later retrieval via {@link * #getCurrentReadSerializer()}. * * @param fory the Fory instance - * @param buffer the memory buffer to read ClassDef from + * @param buffer the memory buffer to read TypeDef from * @return the serializer to use for reading */ MetaSharedLayerSerializerBase getReadSerializer(Fory fory, MemoryBuffer buffer); @@ -218,8 +218,8 @@ public void write(MemoryBuffer buffer, Object value) { if (fory.getConfig().isMetaShareEnabled()) { serializer.writeLayerClassMeta(buffer); } - StreamClassInfo streamClassInfo = slotsInfo.getStreamClassInfo(); - Method writeObjectMethod = streamClassInfo.writeObjectMethod; + StreamTypeInfo streamTypeInfo = slotsInfo.getStreamTypeInfo(); + Method writeObjectMethod = streamTypeInfo.writeObjectMethod; if (writeObjectMethod == null) { // No custom writeObject - write fields directly serializer.writeFieldsOnly(buffer, value); @@ -234,8 +234,8 @@ public void write(MemoryBuffer buffer, Object value) { objectOutputStream.buffer = buffer; objectOutputStream.curPut = null; objectOutputStream.fieldsWritten = false; - if (streamClassInfo.writeObjectFunc != null) { - streamClassInfo.writeObjectFunc.accept(value, objectOutputStream); + if (streamTypeInfo.writeObjectFunc != null) { + streamTypeInfo.writeObjectFunc.accept(value, objectOutputStream); } else { writeObjectMethod.invoke(value, objectOutputStream); } @@ -280,11 +280,11 @@ public Object read(MemoryBuffer buffer) { break; } else { // Receiver has an extra layer that sender doesn't have - call readObjectNoData - StreamClassInfo streamClassInfo = candidateSlot.getStreamClassInfo(); - Method readObjectNoData = streamClassInfo.readObjectNoData; + StreamTypeInfo streamTypeInfo = candidateSlot.getStreamTypeInfo(); + Method readObjectNoData = streamTypeInfo.readObjectNoData; if (readObjectNoData != null) { - if (streamClassInfo.readObjectNoDataFunc != null) { - streamClassInfo.readObjectNoDataFunc.accept(obj); + if (streamTypeInfo.readObjectNoDataFunc != null) { + streamTypeInfo.readObjectNoDataFunc.accept(obj); } else { readObjectNoData.invoke(obj); } @@ -294,17 +294,17 @@ public Object read(MemoryBuffer buffer) { } if (matchedSlot == null) { - // Sender has a layer that receiver doesn't have - read ClassDef and skip the data + // Sender has a layer that receiver doesn't have - read TypeDef and skip the data skipUnknownLayerData(buffer, currentClass); continue; } - // Read data for the matched layer - getReadSerializer reads ClassDef from buffer - // This must be called exactly once per layer to read the ClassDef + // Read data for the matched layer - getReadSerializer reads TypeDef from buffer + // This must be called exactly once per layer to read the TypeDef matchedSlot.getReadSerializer(fory, buffer); - StreamClassInfo streamClassInfo = matchedSlot.getStreamClassInfo(); - Method readObjectMethod = streamClassInfo.readObjectMethod; + StreamTypeInfo streamTypeInfo = matchedSlot.getStreamTypeInfo(); + Method readObjectMethod = streamTypeInfo.readObjectMethod; if (readObjectMethod == null) { // For standard field serialization - use getCurrentReadSerializer() @@ -327,8 +327,8 @@ public Object read(MemoryBuffer buffer) { objectInputStream.targetObject = obj; objectInputStream.getField = getField; objectInputStream.callbacks = callbacks; - if (streamClassInfo.readObjectFunc != null) { - streamClassInfo.readObjectFunc.accept(obj, objectInputStream); + if (streamTypeInfo.readObjectFunc != null) { + streamTypeInfo.readObjectFunc.accept(obj, objectInputStream); } else { readObjectMethod.invoke(obj, objectInputStream); } @@ -347,11 +347,11 @@ public Object read(MemoryBuffer buffer) { // Handle any remaining receiver-only layers at the end while (slotIndex < slotsInfos.length) { SlotInfo remainingSlot = slotsInfos[slotIndex++]; - StreamClassInfo streamClassInfo = remainingSlot.getStreamClassInfo(); - Method readObjectNoData = streamClassInfo.readObjectNoData; + StreamTypeInfo streamTypeInfo = remainingSlot.getStreamTypeInfo(); + Method readObjectNoData = streamTypeInfo.readObjectNoData; if (readObjectNoData != null) { - if (streamClassInfo.readObjectNoDataFunc != null) { - streamClassInfo.readObjectNoDataFunc.accept(obj); + if (streamTypeInfo.readObjectNoDataFunc != null) { + streamTypeInfo.readObjectNoDataFunc.accept(obj); } else { readObjectNoData.invoke(obj); } @@ -376,7 +376,7 @@ public Object read(MemoryBuffer buffer) { */ private void skipUnknownLayerData(MemoryBuffer buffer, Class senderClass) { // For layers without custom writeObject, we can skip using a serializer created from the - // ClassDef. Note: For layers with custom writeObject, the sender would have that class + // TypeDef. Note: For layers with custom writeObject, the sender would have that class // locally, and we'd have a matching slot. This method is only called when sender has a // layer the receiver doesn't have. @@ -387,7 +387,7 @@ private void skipUnknownLayerData(MemoryBuffer buffer, Class senderClass) { + ". Schema evolution with removed parent classes requires meta share."); } - // Read ClassInfo from buffer (maintains index alignment with readClassInfos) + // Read TypeInfo from buffer (maintains index alignment with readTypeInfos) MetaContext metaContext = fory.getSerializationContext().getMetaContext(); if (metaContext == null) { throw new IllegalStateException("MetaContext is null but meta share is enabled"); @@ -395,35 +395,34 @@ private void skipUnknownLayerData(MemoryBuffer buffer, Class senderClass) { int indexMarker = buffer.readVarUint32Small14(); boolean isRef = (indexMarker & 1) == 1; int index = indexMarker >>> 1; - ClassInfo classInfo; + TypeInfo typeInfo; if (isRef) { - // Reference to previously read ClassInfo - classInfo = metaContext.readClassInfos.get(index); + // Reference to previously read TypeInfo + typeInfo = metaContext.readTypeInfos.get(index); } else { - // New ClassDef in stream - read ID first to check cache - long classDefId = buffer.readInt64(); - classInfo = classDefIdToClassInfo.get(classDefId); - if (classInfo != null) { - // Already cached - skip the ClassDef bytes, reuse existing ClassInfo - ClassDef.skipClassDef(buffer, classDefId); + // New TypeDef in stream - read ID first to check cache + long typeDefId = buffer.readInt64(); + typeInfo = typeDefIdToTypeInfo.get(typeDefId); + if (typeInfo != null) { + // Already cached - skip the TypeDef bytes, reuse existing TypeInfo + TypeDef.skipTypeDef(buffer, typeDefId); } else { - // Not cached - read full ClassDef and create ClassInfo - ClassDef classDef = ClassDef.readClassDef(fory, buffer, classDefId); - classInfo = new ClassInfo(senderClass, classDef); - classDefIdToClassInfo.put(classDefId, classInfo); + // Not cached - read full TypeDef and create TypeInfo + TypeDef typeDef = TypeDef.readTypeDef(fory, buffer, typeDefId); + typeInfo = new TypeInfo(senderClass, typeDef); + typeDefIdToTypeInfo.put(typeDefId, typeInfo); } - metaContext.readClassInfos.add(classInfo); + metaContext.readTypeInfos.add(typeInfo); } - // Get or create serializer from ClassInfo to skip the fields + // Get or create serializer from TypeInfo to skip the fields MetaSharedLayerSerializerBase skipSerializer = - (MetaSharedLayerSerializerBase) classInfo.getSerializer(); + (MetaSharedLayerSerializerBase) typeInfo.getSerializer(); if (skipSerializer == null) { Class layerMarkerClass = LayerMarkerClassGenerator.getOrCreate(fory, senderClass, 0); MetaSharedLayerSerializer newSerializer = - new MetaSharedLayerSerializer( - fory, senderClass, classInfo.getClassDef(), layerMarkerClass); - classInfo.setSerializer(newSerializer); + new MetaSharedLayerSerializer(fory, senderClass, typeInfo.getTypeDef(), layerMarkerClass); + typeInfo.setSerializer(newSerializer); skipSerializer = newSerializer; } SerializationBinding binding = SerializationBinding.createBinding(fory); @@ -455,13 +454,12 @@ private static void throwSerializationException(Class type, Exception e) { * images, it needs to reuse the serializers that were generated at build time. * * @param fory the Fory instance - * @param layerClassDef the ClassDef for this layer + * @param layerTypeDef the TypeDef for this layer * @param type the target class type */ private static void ensureFieldSerializersGenerated( - Fory fory, ClassDef layerClassDef, Class type) { - Collection descriptors = - layerClassDef.getDescriptors(fory.getClassResolver(), type); + Fory fory, TypeDef layerTypeDef, Class type) { + Collection descriptors = layerTypeDef.getDescriptors(fory.getClassResolver(), type); for (Descriptor descriptor : descriptors) { Class fieldType = descriptor.getRawType(); if (fieldType != null && !fieldType.isPrimitive()) { @@ -512,25 +510,26 @@ private static FieldTypes.FieldType buildFieldTypeFromClass(Fory fory, Class // For primitives, use RegisteredFieldType if (fieldType.isPrimitive()) { int typeId = Types.getTypeId(fory, fieldType); - return new FieldTypes.RegisteredFieldType(false, false, typeId); + return new FieldTypes.RegisteredFieldType(false, false, typeId, -1); } // For boxed primitives Class unwrapped = TypeUtils.unwrap(fieldType); if (unwrapped.isPrimitive()) { int typeId = Types.getTypeId(fory, unwrapped); - return new FieldTypes.RegisteredFieldType(true, true, typeId); + return new FieldTypes.RegisteredFieldType(true, true, typeId, -1); } // For registered types if (fory.getClassResolver().isRegisteredById(fieldType)) { - int typeId = fory.getClassResolver().getTypeIdForClassDef(fieldType); - return new FieldTypes.RegisteredFieldType(true, true, typeId); + int typeId = fory.getClassResolver().getTypeIdForTypeDef(fieldType); + int userTypeId = fory.getClassResolver().getUserTypeIdForTypeDef(fieldType); + return new FieldTypes.RegisteredFieldType(true, true, typeId, userTypeId); } // For enums if (fieldType.isEnum()) { - return new FieldTypes.EnumFieldType(true, -1); + return new FieldTypes.EnumFieldType(true, -1, -1); } // For arrays, collections, maps - use ObjectFieldType as a fallback @@ -542,7 +541,7 @@ private static FieldTypes.FieldType buildFieldTypeFromClass(Fory fory, Class * Information about a class's stream methods (writeObject, readObject, readObjectNoData) and * their optimized MethodHandle equivalents for fast invocation. */ - private static class StreamClassInfo { + private static class StreamTypeInfo { private final Method writeObjectMethod; private final Method readObjectMethod; private final Method readObjectNoData; @@ -550,7 +549,7 @@ private static class StreamClassInfo { private final BiConsumer readObjectFunc; private final Consumer readObjectNoDataFunc; - private StreamClassInfo(Class type) { + private StreamTypeInfo(Class type) { // ObjectStreamClass.lookup has cache inside, invocation cost won't be big. ObjectStreamClass objectStreamClass = safeObjectStreamClassLookup(type); // In JDK17, set private jdk method accessible will fail by default, use ObjectStreamClass @@ -593,11 +592,11 @@ private StreamClassInfo(Class type) { } } - private static final ClassValue STREAM_CLASS_INFO_CACHE = - new ClassValue() { + private static final ClassValue STREAM_CLASS_INFO_CACHE = + new ClassValue() { @Override - protected StreamClassInfo computeValue(Class type) { - return new StreamClassInfo(type); + protected StreamTypeInfo computeValue(Class type) { + return new StreamTypeInfo(type); } }; @@ -608,7 +607,7 @@ protected StreamClassInfo computeValue(Class type) { */ private class SlotsInfo implements SlotInfo { private final Class cls; - private final StreamClassInfo streamClassInfo; + private final StreamTypeInfo streamTypeInfo; // mark non-final for async-jit to update it to jit-serializer. private MetaSharedLayerSerializerBase slotsSerializer; private final ObjectIntMap fieldIndexMap; @@ -623,21 +622,21 @@ private class SlotsInfo implements SlotInfo { public SlotsInfo(Fory fory, Class type) { this.cls = type; ObjectStreamClass objectStreamClass = safeObjectStreamClassLookup(type); - streamClassInfo = STREAM_CLASS_INFO_CACHE.get(type); + streamTypeInfo = STREAM_CLASS_INFO_CACHE.get(type); - // Build ClassDef from ObjectStreamClass fields (handles serialPersistentFields). + // Build TypeDef from ObjectStreamClass fields (handles serialPersistentFields). // This ensures the serializer uses the same field layout as defined by // serialPersistentFields (if present) rather than actual class fields. - ClassDef layerClassDef; + TypeDef layerTypeDef; if (objectStreamClass != null) { List fieldInfos = buildFieldInfoFromObjectStreamClass(fory, objectStreamClass, type); - layerClassDef = - ClassDefEncoder.buildClassDefWithFieldInfos( + layerTypeDef = + NativeTypeDefEncoder.buildTypeDefWithFieldInfos( fory.getClassResolver(), type, fieldInfos, true); } else { // Fallback when ObjectStreamClass is not available (e.g., GraalVM native image) - layerClassDef = fory.getClassResolver().getTypeDef(type, false); + layerTypeDef = fory.getClassResolver().getTypeDef(type, false); } // Generate marker class for this layer. Use 0 as layer index since each class // has its own SlotsInfo, and the (class, 0) pair is unique for each class. @@ -645,7 +644,7 @@ public SlotsInfo(Fory fory, Class type) { // Create interpreter-mode serializer first this.slotsSerializer = - new MetaSharedLayerSerializer(fory, type, layerClassDef, layerMarkerClass); + new MetaSharedLayerSerializer(fory, type, layerTypeDef, layerMarkerClass); // Register JIT callback to replace with JIT serializer when ready if (fory.getConfig().isCodeGenEnabled()) { @@ -655,7 +654,7 @@ public SlotsInfo(Fory fory, Class type) { () -> MetaSharedLayerSerializer.class, () -> CodecUtils.loadOrGenMetaSharedLayerCodecClass( - type, fory, layerClassDef, layerMarkerClass), + type, fory, layerTypeDef, layerMarkerClass), c -> thisInfo.slotsSerializer = (MetaSharedLayerSerializerBase) @@ -665,16 +664,16 @@ public SlotsInfo(Fory fory, Class type) { // In GraalVM, ensure serializers are generated for all field types at build time // so they're available when new Fory instances are created at runtime if (GraalvmSupport.isGraalBuildtime()) { - ensureFieldSerializersGenerated(fory, layerClassDef, type); + ensureFieldSerializersGenerated(fory, layerTypeDef, type); } // Build fieldIndexMap and putFieldTypes from serializer's field order. // This ensures putFields/writeFields API uses the same order as the serializer // (buildIn, container, other groups), not ObjectStreamClass order. fieldIndexMap = new ObjectIntMap<>(4, 0.4f); - if (streamClassInfo != null - && (streamClassInfo.writeObjectMethod != null - || streamClassInfo.readObjectMethod != null)) { + if (streamTypeInfo != null + && (streamTypeInfo.writeObjectMethod != null + || streamTypeInfo.readObjectMethod != null)) { this.numPutFields = slotsSerializer.getNumFields(); this.putFieldTypes = new Class[numPutFields]; slotsSerializer.populateFieldInfo(fieldIndexMap, putFieldTypes); @@ -683,7 +682,7 @@ public SlotsInfo(Fory fory, Class type) { this.putFieldTypes = null; } - if (streamClassInfo != null && streamClassInfo.writeObjectMethod != null) { + if (streamTypeInfo != null && streamTypeInfo.writeObjectMethod != null) { try { objectOutputStream = new ForyObjectOutputStream(this); } catch (IOException e) { @@ -693,7 +692,7 @@ public SlotsInfo(Fory fory, Class type) { } else { objectOutputStream = null; } - if (streamClassInfo != null && streamClassInfo.readObjectMethod != null) { + if (streamTypeInfo != null && streamTypeInfo.readObjectMethod != null) { try { objectInputStream = new ForyObjectInputStream(this); } catch (IOException e) { @@ -712,8 +711,8 @@ public Class getCls() { } @Override - public StreamClassInfo getStreamClassInfo() { - return streamClassInfo; + public StreamTypeInfo getStreamTypeInfo() { + return streamTypeInfo; } @Override @@ -759,21 +758,21 @@ public MetaSharedLayerSerializerBase getReadSerializer(Fory fory, MemoryBuffer b // Meta share not enabled - use the default slots serializer result = slotsSerializer; } else { - // Read ClassInfo from buffer (creates new or returns existing) - ClassInfo classInfo = readLayerClassInfo(fory, buffer); - if (classInfo == null) { + // Read TypeInfo from buffer (creates new or returns existing) + TypeInfo typeInfo = readLayerTypeInfo(fory, buffer); + if (typeInfo == null) { result = slotsSerializer; } else { - // Get or create serializer from ClassInfo - Serializer serializer = classInfo.getSerializer(); + // Get or create serializer from TypeInfo + Serializer serializer = typeInfo.getSerializer(); if (serializer != null) { result = (MetaSharedLayerSerializerBase) serializer; } else { - // Create a new serializer based on the ClassDef from stream + // Create a new serializer based on the TypeDef from stream Class layerMarkerClass = LayerMarkerClassGenerator.getOrCreate(fory, cls, 0); MetaSharedLayerSerializer newSerializer = - new MetaSharedLayerSerializer(fory, cls, classInfo.getClassDef(), layerMarkerClass); - classInfo.setSerializer(newSerializer); + new MetaSharedLayerSerializer(fory, cls, typeInfo.getTypeDef(), layerMarkerClass); + typeInfo.setSerializer(newSerializer); result = newSerializer; } } @@ -788,7 +787,7 @@ public MetaSharedLayerSerializerBase getCurrentReadSerializer() { return currentReadSerializer; } - private ClassInfo readLayerClassInfo(Fory fory, MemoryBuffer buffer) { + private TypeInfo readLayerTypeInfo(Fory fory, MemoryBuffer buffer) { MetaContext metaContext = fory.getSerializationContext().getMetaContext(); if (metaContext == null) { return null; @@ -797,23 +796,23 @@ private ClassInfo readLayerClassInfo(Fory fory, MemoryBuffer buffer) { boolean isRef = (indexMarker & 1) == 1; int index = indexMarker >>> 1; if (isRef) { - // Reference to previously read ClassInfo - return metaContext.readClassInfos.get(index); + // Reference to previously read TypeInfo + return metaContext.readTypeInfos.get(index); } else { - // New ClassDef in stream - read ID first to check cache - long classDefId = buffer.readInt64(); - ClassInfo classInfo = classDefIdToClassInfo.get(classDefId); - if (classInfo != null) { - // Already cached - skip the ClassDef bytes, reuse existing ClassInfo - ClassDef.skipClassDef(buffer, classDefId); + // New TypeDef in stream - read ID first to check cache + long typeDefId = buffer.readInt64(); + TypeInfo typeInfo = typeDefIdToTypeInfo.get(typeDefId); + if (typeInfo != null) { + // Already cached - skip the TypeDef bytes, reuse existing TypeInfo + TypeDef.skipTypeDef(buffer, typeDefId); } else { - // Not cached - read full ClassDef and create ClassInfo - ClassDef classDef = ClassDef.readClassDef(fory, buffer, classDefId); - classInfo = new ClassInfo(cls, classDef); - classDefIdToClassInfo.put(classDefId, classInfo); + // Not cached - read full TypeDef and create TypeInfo + TypeDef typeDef = TypeDef.readTypeDef(fory, buffer, typeDefId); + typeInfo = new TypeInfo(cls, typeDef); + typeDefIdToTypeInfo.put(typeDefId, typeInfo); } - metaContext.readClassInfos.add(classInfo); - return classInfo; + metaContext.readTypeInfos.add(typeInfo); + return typeInfo; } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java index 951024df2e..e9b6f9f437 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java @@ -33,9 +33,9 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.Platform; import org.apache.fory.reflect.ReflectionUtils; -import org.apache.fory.resolver.ClassInfo; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.RefResolver; +import org.apache.fory.resolver.TypeInfo; import org.apache.fory.util.Preconditions; import org.apache.fory.util.unsafe._JDKAccess; @@ -213,8 +213,8 @@ private static Serializer createDataSerializer( protected final RefResolver refResolver; protected final ClassResolver classResolver; protected final MethodInfoCache jdkMethodInfoWriteCache; - protected final ClassInfo writeClassInfo; - protected final Map, MethodInfoCache> classClassInfoHolderMap = new HashMap<>(); + protected final TypeInfo writeTypeInfo; + protected final Map, MethodInfoCache> classTypeInfoHolderMap = new HashMap<>(); public ReplaceResolveSerializer(Fory fory, Class type) { this(fory, type, false, true); @@ -235,17 +235,17 @@ public ReplaceResolveSerializer( } if (type != ReplaceStub.class) { jdkMethodInfoWriteCache = newJDKMethodInfoCache(type, fory); - classClassInfoHolderMap.put(type, jdkMethodInfoWriteCache); + classTypeInfoHolderMap.put(type, jdkMethodInfoWriteCache); if (isFinalField) { - writeClassInfo = null; + writeTypeInfo = null; } else { // FIXME new classinfo may miss serializer update in async compilation mode. - int typeId = classResolver.getTypeIdForClassDef(type); - writeClassInfo = classResolver.newClassInfo(type, this, typeId); + int typeId = classResolver.getTypeIdForTypeDef(type); + writeTypeInfo = classResolver.newTypeInfo(type, this, typeId); } } else { jdkMethodInfoWriteCache = null; - writeClassInfo = null; + writeTypeInfo = null; } } @@ -295,7 +295,7 @@ public void write(MemoryBuffer buffer, Object value) { protected void writeObject( MemoryBuffer buffer, Object value, MethodInfoCache jdkMethodInfoCache) { - classResolver.writeClassInternal(buffer, writeClassInfo); + classResolver.writeClassInternal(buffer, writeTypeInfo); jdkMethodInfoCache.objectSerializer.write(buffer, value); } @@ -308,7 +308,7 @@ public Object read(MemoryBuffer buffer) { int nextReadRefId = refResolver.tryPreserveRefId(buffer); if (nextReadRefId >= Fory.NOT_NULL_VALUE_FLAG) { // ref value or not-null value - Object o = fory.readData(buffer, classResolver.readClassInfo(buffer)); + Object o = fory.readData(buffer, classResolver.readTypeInfo(buffer)); refResolver.setReadObject(nextReadRefId, o); refResolver.setReadObject(outerRefId, o); return o; @@ -365,10 +365,10 @@ public Object copy(Object originObj) { } protected MethodInfoCache getMethodInfoCache(Class cls) { - MethodInfoCache jdkMethodInfoCache = classClassInfoHolderMap.get(cls); + MethodInfoCache jdkMethodInfoCache = classTypeInfoHolderMap.get(cls); if (jdkMethodInfoCache == null) { jdkMethodInfoCache = newJDKMethodInfoCache(cls, fory); - classClassInfoHolderMap.put(cls, jdkMethodInfoCache); + classTypeInfoHolderMap.put(cls, jdkMethodInfoCache); } return jdkMethodInfoCache; } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java index c621fbecae..2dae7ecad9 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java @@ -25,11 +25,11 @@ import org.apache.fory.logging.Logger; import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.resolver.ClassInfo; -import org.apache.fory.resolver.ClassInfoHolder; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.RefMode; import org.apache.fory.resolver.RefResolver; +import org.apache.fory.resolver.TypeInfo; +import org.apache.fory.resolver.TypeInfoHolder; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.resolver.XtypeResolver; import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo; @@ -55,26 +55,26 @@ abstract class SerializationBinding { abstract void writeRef(MemoryBuffer buffer, T obj, Serializer serializer); - abstract void writeRef(MemoryBuffer buffer, Object obj, ClassInfoHolder classInfoHolder); + abstract void writeRef(MemoryBuffer buffer, Object obj, TypeInfoHolder classInfoHolder); - abstract void writeRef(MemoryBuffer buffer, Object obj, ClassInfo classInfo); + abstract void writeRef(MemoryBuffer buffer, Object obj, TypeInfo typeInfo); abstract void writeNonRef(MemoryBuffer buffer, Object obj); abstract void writeNonRef(MemoryBuffer buffer, Object obj, Serializer serializer); - abstract void writeNonRef(MemoryBuffer buffer, Object obj, ClassInfoHolder classInfoHolder); + abstract void writeNonRef(MemoryBuffer buffer, Object obj, TypeInfoHolder classInfoHolder); abstract void writeNullable(MemoryBuffer buffer, Object obj); abstract void writeNullable(MemoryBuffer buffer, Object obj, Serializer serializer); - abstract void writeNullable(MemoryBuffer buffer, Object obj, ClassInfoHolder classInfoHolder); + abstract void writeNullable(MemoryBuffer buffer, Object obj, TypeInfoHolder classInfoHolder); - abstract void writeNullable(MemoryBuffer buffer, Object obj, ClassInfo classInfo); + abstract void writeNullable(MemoryBuffer buffer, Object obj, TypeInfo typeInfo); abstract void writeNullable( - MemoryBuffer buffer, Object obj, ClassInfoHolder classInfoHolder, boolean nullable); + MemoryBuffer buffer, Object obj, TypeInfoHolder classInfoHolder, boolean nullable); abstract void writeNullable( MemoryBuffer buffer, Object obj, Serializer serializer, boolean nullable); @@ -104,13 +104,13 @@ public final Object readField(SerializationFieldInfo fieldInfo, MemoryBuffer buf abstract Object readRef(MemoryBuffer buffer, SerializationFieldInfo field); - abstract Object readRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder); + abstract Object readRef(MemoryBuffer buffer, TypeInfoHolder classInfoHolder); abstract Object readRef(MemoryBuffer buffer); abstract Object readNonRef(MemoryBuffer buffer); - abstract Object readNonRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder); + abstract Object readNonRef(MemoryBuffer buffer, TypeInfoHolder classInfoHolder); abstract Object readNonRef(MemoryBuffer buffer, SerializationFieldInfo field); @@ -168,13 +168,13 @@ public void writeRef(MemoryBuffer buffer, T obj, Serializer serializer) { } @Override - public void writeRef(MemoryBuffer buffer, Object obj, ClassInfoHolder classInfoHolder) { + public void writeRef(MemoryBuffer buffer, Object obj, TypeInfoHolder classInfoHolder) { fory.writeRef(buffer, obj, classInfoHolder); } @Override - public void writeRef(MemoryBuffer buffer, Object obj, ClassInfo classInfo) { - fory.writeRef(buffer, obj, classInfo); + public void writeRef(MemoryBuffer buffer, Object obj, TypeInfo typeInfo) { + fory.writeRef(buffer, obj, typeInfo); } @Override @@ -185,13 +185,13 @@ public T readRef(MemoryBuffer buffer, Serializer serializer) { @Override public Object readRef(MemoryBuffer buffer, SerializationFieldInfo field) { if (field.useDeclaredTypeInfo) { - return fory.readRef(buffer, field.classInfo.getSerializer()); + return fory.readRef(buffer, field.typeInfo.getSerializer()); } return fory.readRef(buffer, field.classInfoHolder); } @Override - public Object readRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { + public Object readRef(MemoryBuffer buffer, TypeInfoHolder classInfoHolder) { return fory.readRef(buffer, classInfoHolder); } @@ -206,14 +206,14 @@ public Object readNonRef(MemoryBuffer buffer) { } @Override - public Object readNonRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { + public Object readNonRef(MemoryBuffer buffer, TypeInfoHolder classInfoHolder) { return fory.readNonRef(buffer, classInfoHolder); } @Override public Object readNonRef(MemoryBuffer buffer, SerializationFieldInfo field) { if (field.useDeclaredTypeInfo) { - return fory.readNonRef(buffer, field.classInfo); + return fory.readNonRef(buffer, field.typeInfo); } return fory.readNonRef(buffer, field.classInfoHolder); } @@ -221,7 +221,7 @@ public Object readNonRef(MemoryBuffer buffer, SerializationFieldInfo field) { @Override public Object readNullable(MemoryBuffer buffer, SerializationFieldInfo field) { if (field.useDeclaredTypeInfo) { - return fory.readNullable(buffer, field.classInfo.getSerializer()); + return fory.readNullable(buffer, field.typeInfo.getSerializer()); } return fory.readNullable(buffer, field.classInfoHolder); } @@ -253,7 +253,7 @@ public Object readContainerFieldValueRef( int nextReadRefId = refResolver.tryPreserveRefId(buffer); if (nextReadRefId >= NOT_NULL_VALUE_FLAG) { // ref value or not-null value - Object o = fory.readData(buffer, classResolver.readClassInfo(buffer)); + Object o = fory.readData(buffer, classResolver.readTypeInfo(buffer)); refResolver.setReadObject(nextReadRefId, o); return o; } else { @@ -270,26 +270,32 @@ public void write(MemoryBuffer buffer, Serializer serializer, Object value) { Object readField(SerializationFieldInfo fieldInfo, RefMode refMode, MemoryBuffer buffer) { if (fieldInfo.useDeclaredTypeInfo) { if (refMode == RefMode.TRACKING) { - return fory.readRef(buffer, fieldInfo.classInfo); + return fory.readRef(buffer, fieldInfo.typeInfo); } else { if (refMode != RefMode.NULL_ONLY || buffer.readByte() != Fory.NULL_FLAG) { // Preserve a dummy ref ID so ObjectSerializer.read() can pop it. // This is needed when global ref tracking is enabled but field ref tracking is // disabled. refResolver.preserveRefId(-1); - return fory.readNonRef(buffer, fieldInfo.classInfo); + return fory.readNonRef(buffer, fieldInfo.typeInfo); } } } else { if (refMode == RefMode.TRACKING) { - return fory.readRef(buffer, fieldInfo.classInfoHolder); + int nextReadRefId = refResolver.tryPreserveRefId(buffer); + if (nextReadRefId >= NOT_NULL_VALUE_FLAG) { + // ref value or not-null value + Object o = + classResolver.readTypeInfo(buffer, fieldInfo.type).getSerializer().read(buffer); + refResolver.setReadObject(nextReadRefId, o); + return o; + } else { + return refResolver.getReadObject(); + } } else { if (refMode != RefMode.NULL_ONLY || buffer.readByte() != Fory.NULL_FLAG) { - // Preserve a dummy ref ID so ObjectSerializer.read() can pop it. - // This is needed when global ref tracking is enabled but field ref tracking is - // disabled. - refResolver.preserveRefId(-1); - return fory.readNonRef(buffer, fieldInfo.classInfoHolder); + TypeInfo typeInfo = classResolver.readTypeInfo(buffer, fieldInfo.type); + return typeInfo.getSerializer().read(buffer, RefMode.NONE); } } } @@ -316,8 +322,8 @@ public void writeNonRef(MemoryBuffer buffer, Object obj, Serializer serializer) } @Override - public void writeNonRef(MemoryBuffer buffer, Object obj, ClassInfoHolder classInfoHolder) { - fory.writeNonRef(buffer, obj, classResolver.getClassInfo(obj.getClass(), classInfoHolder)); + public void writeNonRef(MemoryBuffer buffer, Object obj, TypeInfoHolder classInfoHolder) { + fory.writeNonRef(buffer, obj, classResolver.getTypeInfo(obj.getClass(), classInfoHolder)); } @Override @@ -341,28 +347,28 @@ public void writeNullable(MemoryBuffer buffer, Object obj, Serializer serializer } @Override - public void writeNullable(MemoryBuffer buffer, Object obj, ClassInfoHolder classInfoHolder) { + public void writeNullable(MemoryBuffer buffer, Object obj, TypeInfoHolder classInfoHolder) { if (obj == null) { buffer.writeByte(Fory.NULL_FLAG); } else { buffer.writeByte(NOT_NULL_VALUE_FLAG); - fory.writeNonRef(buffer, obj, classResolver.getClassInfo(obj.getClass(), classInfoHolder)); + fory.writeNonRef(buffer, obj, classResolver.getTypeInfo(obj.getClass(), classInfoHolder)); } } @Override - public void writeNullable(MemoryBuffer buffer, Object obj, ClassInfo classInfo) { + public void writeNullable(MemoryBuffer buffer, Object obj, TypeInfo typeInfo) { if (obj == null) { buffer.writeByte(Fory.NULL_FLAG); } else { buffer.writeByte(NOT_NULL_VALUE_FLAG); - fory.writeNonRef(buffer, obj, classInfo); + fory.writeNonRef(buffer, obj, typeInfo); } } @Override public void writeNullable( - MemoryBuffer buffer, Object obj, ClassInfoHolder classInfoHolder, boolean nullable) { + MemoryBuffer buffer, Object obj, TypeInfoHolder classInfoHolder, boolean nullable) { if (nullable) { writeNullable(buffer, obj, classInfoHolder); } else { @@ -384,9 +390,9 @@ public void writeNullable( public void writeContainerFieldValue( SerializationFieldInfo fieldInfo, MemoryBuffer buffer, Object fieldValue) { if (fieldInfo.useDeclaredTypeInfo) { - ClassInfo classInfo = - typeResolver.getClassInfo(fieldValue.getClass(), fieldInfo.classInfoHolder); - fory.writeNonRef(buffer, fieldValue, classInfo); + TypeInfo typeInfo = + typeResolver.getTypeInfo(fieldValue.getClass(), fieldInfo.classInfoHolder); + fory.writeNonRef(buffer, fieldValue, typeInfo); } else { fory.writeNonRef(buffer, fieldValue, fieldInfo.classInfoHolder); } @@ -396,7 +402,7 @@ public void writeContainerFieldValue( void writeField( SerializationFieldInfo fieldInfo, RefMode refMode, MemoryBuffer buffer, Object fieldValue) { if (fieldInfo.useDeclaredTypeInfo) { - Serializer serializer = fieldInfo.classInfo.getSerializer(); + Serializer serializer = fieldInfo.typeInfo.getSerializer(); if (refMode == RefMode.TRACKING) { if (!refResolver.writeRefOrNull(buffer, fieldValue)) { serializer.write(buffer, fieldValue); @@ -442,7 +448,7 @@ static final class XlangSerializationBinding extends SerializationBinding { void writeField( SerializationFieldInfo fieldInfo, RefMode refMode, MemoryBuffer buffer, Object fieldValue) { if (fieldInfo.useDeclaredTypeInfo) { - Serializer serializer = fieldInfo.classInfo.getSerializer(); + Serializer serializer = fieldInfo.typeInfo.getSerializer(); if (refMode == RefMode.TRACKING) { if (!refResolver.writeRefOrNull(buffer, fieldValue)) { serializer.xwrite(buffer, fieldValue); @@ -486,13 +492,13 @@ public void writeRef(MemoryBuffer buffer, T obj, Serializer serializer) { } @Override - public void writeRef(MemoryBuffer buffer, Object obj, ClassInfoHolder classInfoHolder) { + public void writeRef(MemoryBuffer buffer, Object obj, TypeInfoHolder classInfoHolder) { fory.xwriteRef(buffer, obj, classInfoHolder); } @Override - public void writeRef(MemoryBuffer buffer, Object obj, ClassInfo classInfo) { - fory.xwriteRef(buffer, obj, classInfo); + public void writeRef(MemoryBuffer buffer, Object obj, TypeInfo typeInfo) { + fory.xwriteRef(buffer, obj, typeInfo); } @Override @@ -522,7 +528,7 @@ public Object readRef(MemoryBuffer buffer, SerializationFieldInfo field) { } @Override - public Object readRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { + public Object readRef(MemoryBuffer buffer, TypeInfoHolder classInfoHolder) { return fory.xreadRef(buffer, classInfoHolder); } @@ -537,8 +543,8 @@ public Object readNonRef(MemoryBuffer buffer) { } @Override - public Object readNonRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { - return fory.xreadNonRef(buffer, xtypeResolver.readClassInfo(buffer, classInfoHolder)); + public Object readNonRef(MemoryBuffer buffer, TypeInfoHolder classInfoHolder) { + return fory.xreadNonRef(buffer, xtypeResolver.readTypeInfo(buffer, classInfoHolder)); } @Override @@ -565,7 +571,7 @@ public Object readNonRef(MemoryBuffer buffer, SerializationFieldInfo field) { @Override public Object readNullable(MemoryBuffer buffer, SerializationFieldInfo field) { if (field.useDeclaredTypeInfo) { - return fory.xreadNullable(buffer, field.classInfo.getSerializer()); + return fory.xreadNullable(buffer, field.typeInfo.getSerializer()); } return fory.xreadNullable(buffer, field.classInfoHolder); } @@ -587,14 +593,14 @@ public Object readNullable( @Override public Object readContainerFieldValue(MemoryBuffer buffer, SerializationFieldInfo field) { - return fory.xreadNonRef(buffer, field.containerClassInfo); + return fory.xreadNonRef(buffer, field.containerTypeInfo); } @Override public Object readContainerFieldValueRef(MemoryBuffer buffer, SerializationFieldInfo field) { int nextReadRefId = refResolver.tryPreserveRefId(buffer); if (nextReadRefId >= NOT_NULL_VALUE_FLAG) { - Object o = fory.xreadNonRef(buffer, field.containerClassInfo); + Object o = fory.xreadNonRef(buffer, field.containerTypeInfo); refResolver.setReadObject(nextReadRefId, o); return o; } else { @@ -610,15 +616,24 @@ public void write(MemoryBuffer buffer, Serializer serializer, Object value) { @Override Object readField(SerializationFieldInfo fieldInfo, RefMode refMode, MemoryBuffer buffer) { if (fieldInfo.useDeclaredTypeInfo) { - return fieldInfo.classInfo.getSerializer().xread(buffer, refMode); + return fieldInfo.typeInfo.getSerializer().xread(buffer, refMode); } else { if (refMode == RefMode.TRACKING) { - return fory.xreadRef(buffer, fieldInfo.classInfoHolder); + int nextReadRefId = refResolver.tryPreserveRefId(buffer); + if (nextReadRefId >= NOT_NULL_VALUE_FLAG) { + // ref value or not-null value + Object o = + xtypeResolver.readTypeInfo(buffer, fieldInfo.type).getSerializer().xread(buffer); + refResolver.setReadObject(nextReadRefId, o); + return o; + } else { + return refResolver.getReadObject(); + } } else { if (refMode != RefMode.NULL_ONLY || buffer.readByte() != Fory.NULL_FLAG) { - ClassInfo classInfo = xtypeResolver.readClassInfo(buffer, fieldInfo.classInfoHolder); - Serializer serializer = classInfo.getSerializer(); - return serializer.xread(buffer, refMode); + TypeInfo typeInfo = xtypeResolver.readTypeInfo(buffer, fieldInfo.type); + Serializer serializer = typeInfo.getSerializer(); + return serializer.xread(buffer, RefMode.NONE); } } } @@ -645,8 +660,8 @@ public void writeNonRef(MemoryBuffer buffer, Object obj, Serializer serializer) } @Override - public void writeNonRef(MemoryBuffer buffer, Object obj, ClassInfoHolder classInfoHolder) { - fory.xwriteNonRef(buffer, obj, xtypeResolver.getClassInfo(obj.getClass(), classInfoHolder)); + public void writeNonRef(MemoryBuffer buffer, Object obj, TypeInfoHolder classInfoHolder) { + fory.xwriteNonRef(buffer, obj, xtypeResolver.getTypeInfo(obj.getClass(), classInfoHolder)); } @Override @@ -670,28 +685,28 @@ public void writeNullable(MemoryBuffer buffer, Object obj, Serializer serializer } @Override - public void writeNullable(MemoryBuffer buffer, Object obj, ClassInfoHolder classInfoHolder) { + public void writeNullable(MemoryBuffer buffer, Object obj, TypeInfoHolder classInfoHolder) { if (obj == null) { buffer.writeByte(Fory.NULL_FLAG); } else { buffer.writeByte(NOT_NULL_VALUE_FLAG); - fory.xwriteNonRef(buffer, obj, xtypeResolver.getClassInfo(obj.getClass(), classInfoHolder)); + fory.xwriteNonRef(buffer, obj, xtypeResolver.getTypeInfo(obj.getClass(), classInfoHolder)); } } @Override - public void writeNullable(MemoryBuffer buffer, Object obj, ClassInfo classInfo) { + public void writeNullable(MemoryBuffer buffer, Object obj, TypeInfo typeInfo) { if (obj == null) { buffer.writeByte(Fory.NULL_FLAG); } else { buffer.writeByte(NOT_NULL_VALUE_FLAG); - fory.xwriteNonRef(buffer, obj, classInfo); + fory.xwriteNonRef(buffer, obj, typeInfo); } } @Override public void writeNullable( - MemoryBuffer buffer, Object obj, ClassInfoHolder classInfoHolder, boolean nullable) { + MemoryBuffer buffer, Object obj, TypeInfoHolder classInfoHolder, boolean nullable) { if (nullable) { writeNullable(buffer, obj, classInfoHolder); } else { @@ -713,9 +728,9 @@ public void writeNullable( public void writeContainerFieldValue( SerializationFieldInfo fieldInfo, MemoryBuffer buffer, Object fieldValue) { assert fieldInfo.useDeclaredTypeInfo; - ClassInfo classInfo = - typeResolver.getClassInfo(fieldValue.getClass(), fieldInfo.classInfoHolder); - fory.xwriteData(buffer, classInfo, fieldValue); + TypeInfo typeInfo = + typeResolver.getTypeInfo(fieldValue.getClass(), fieldInfo.classInfoHolder); + fory.xwriteData(buffer, typeInfo, fieldValue); } } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationUtils.java b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationUtils.java index 60f1ae47fb..ec2bad055a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationUtils.java @@ -23,7 +23,7 @@ import java.util.Map; import org.apache.fory.Fory; import org.apache.fory.annotation.Internal; -import org.apache.fory.resolver.ClassInfo; +import org.apache.fory.resolver.TypeInfo; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.collection.CollectionSerializer; import org.apache.fory.serializer.collection.MapSerializer; @@ -34,8 +34,8 @@ public static TypeResolver getTypeResolver(Fory fory) { return fory.isCrossLanguage() ? fory.getXtypeResolver() : fory.getClassResolver(); } - public static ClassInfo getClassInfo(Fory fory, Class cls) { - return getTypeResolver(fory).getClassInfo(cls); + public static TypeInfo getTypeInfo(Fory fory, Class cls) { + return getTypeResolver(fory).getTypeInfo(cls); } public static void validateSerializer( diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java index 48654a80df..8ade12701d 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java @@ -47,7 +47,7 @@ import org.apache.fory.collection.Tuple2; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.Platform; -import org.apache.fory.meta.ClassDef; +import org.apache.fory.meta.TypeDef; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.util.ExceptionUtils; @@ -87,8 +87,8 @@ public static Serializer newSerializer( return new ObjectSerializer(fory, type); } if (serializerClass == MetaSharedSerializer.class) { - ClassDef classDef = fory.getClassResolver().getTypeDef(type, true); - return new MetaSharedSerializer(fory, type, classDef); + TypeDef typeDef = fory.getClassResolver().getTypeDef(type, true); + return new MetaSharedSerializer(fory, type, typeDef); } Tuple2 ctrInfo = CTR_MAP.getIfPresent(serializerClass); if (ctrInfo != null) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/UnionSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/UnionSerializer.java index c8987e104c..d3c3fe385d 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/UnionSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/UnionSerializer.java @@ -32,8 +32,8 @@ import org.apache.fory.logging.Logger; import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.resolver.ClassInfo; import org.apache.fory.resolver.RefResolver; +import org.apache.fory.resolver.TypeInfo; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.type.Types; import org.apache.fory.type.union.Union; @@ -76,7 +76,7 @@ public class UnionSerializer extends Serializer { private final BiFunction factory; private final Map> caseValueTypes; - private final LongMap finalCaseClassInfo; + private final LongMap finalCaseTypeInfo; private boolean finalCaseSerializersResolved; private final TypeResolver resolver; @@ -88,7 +88,7 @@ public UnionSerializer(Fory fory, Class cls) { } else { this.factory = createFactory(cls); } - finalCaseClassInfo = new LongMap<>(); + finalCaseTypeInfo = new LongMap<>(); this.caseValueTypes = resolveCaseValueTypes(cls); resolver = fory.getTypeResolver(); } @@ -172,12 +172,12 @@ public Union xread(MemoryBuffer buffer) { int nextReadRefId = refResolver.tryPreserveRefId(buffer); if (nextReadRefId >= Fory.NOT_NULL_VALUE_FLAG) { // ref value or not-null value - ClassInfo declared = getFinalCaseClassInfo(index); - ClassInfo readClassInfo = resolver.readClassInfo(buffer, declared); + TypeInfo declared = getFinalCaseTypeInfo(index); + TypeInfo readTypeInfo = resolver.readTypeInfo(buffer, declared); if (declared != null) { caseValue = Serializers.read(buffer, declared.getSerializer()); } else { - caseValue = Serializers.read(buffer, readClassInfo.getSerializer()); + caseValue = Serializers.read(buffer, readTypeInfo.getSerializer()); } refResolver.setReadObject(nextReadRefId, caseValue); } else { @@ -197,25 +197,25 @@ public Union copy(Union union) { } private void writeCaseValue(MemoryBuffer buffer, Object value, int typeId, int caseId) { - byte internalTypeId = (byte) (typeId & 0xff); + byte internalTypeId = (byte) typeId; boolean primitiveArray = Types.isPrimitiveArray(internalTypeId); Serializer serializer; - ClassInfo classInfo; + TypeInfo typeInfo; if (value == null) { buffer.writeByte(Fory.NULL_FLAG); return; } - classInfo = getFinalCaseClassInfo(caseId); - if (classInfo == null) { + typeInfo = getFinalCaseTypeInfo(caseId); + if (typeInfo == null) { Preconditions.checkArgument(!primitiveArray); if (!Types.isUserDefinedType(internalTypeId)) { - classInfo = resolver.getClassInfoByTypeId(internalTypeId); + typeInfo = resolver.getTypeInfoByTypeId(internalTypeId); } else { - classInfo = resolver.getClassInfo(value.getClass()); + typeInfo = resolver.getTypeInfo(value.getClass()); } } - Preconditions.checkArgument(classInfo != null); - serializer = classInfo.getSerializer(); + Preconditions.checkArgument(typeInfo != null); + serializer = typeInfo.getSerializer(); RefResolver refResolver = fory.getRefResolver(); if (serializer != null && serializer.needToWriteRef()) { if (refResolver.writeRefOrNull(buffer, value)) { @@ -225,15 +225,15 @@ private void writeCaseValue(MemoryBuffer buffer, Object value, int typeId, int c buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); } if (!Types.isUserDefinedType(internalTypeId)) { - buffer.writeVarUint32Small7(typeId); + buffer.writeUint8(typeId); } else { - resolver.writeClassInfo(buffer, classInfo); + resolver.writeTypeInfo(buffer, typeInfo); } writeValue(buffer, value, typeId, serializer); } private void writeValue(MemoryBuffer buffer, Object value, int typeId, Serializer serializer) { - int internalTypeId = typeId & 0xff; + int internalTypeId = typeId; switch (internalTypeId) { case Types.BOOL: buffer.writeBoolean((Boolean) value); @@ -294,15 +294,15 @@ private void writeValue(MemoryBuffer buffer, Object value, int typeId, Serialize throw new IllegalStateException("Missing serializer for union type id " + typeId); } - private ClassInfo getFinalCaseClassInfo(int caseId) { + private TypeInfo getFinalCaseTypeInfo(int caseId) { if (!finalCaseSerializersResolved) { - resolveFinalCaseClassInfo(); + resolveFinalCaseTypeInfo(); finalCaseSerializersResolved = true; } - return finalCaseClassInfo.get(caseId); + return finalCaseTypeInfo.get(caseId); } - private void resolveFinalCaseClassInfo() { + private void resolveFinalCaseTypeInfo() { for (Map.Entry> entry : caseValueTypes.entrySet()) { Class expectedType = entry.getValue(); if (!isFinalCaseType(expectedType)) { @@ -311,8 +311,8 @@ private void resolveFinalCaseClassInfo() { if (expectedType.isPrimitive()) { continue; } - ClassInfo classInfo = fory.getTypeResolver().getClassInfo(expectedType); - finalCaseClassInfo.put(entry.getKey(), classInfo); + TypeInfo typeInfo = fory.getTypeResolver().getTypeInfo(expectedType); + finalCaseTypeInfo.put(entry.getKey(), typeInfo); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java index 0841155757..3854ba7fd4 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java @@ -39,7 +39,7 @@ import org.apache.fory.builder.LayerMarkerClassGenerator; import org.apache.fory.config.CompatibleMode; import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.meta.ClassDef; +import org.apache.fory.meta.TypeDef; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.MetaContext; @@ -238,11 +238,11 @@ private static Serializer[] buildSlotsSerializers( while (!superClasses.contains(cls)) { Serializer slotsSerializer; if (fory.getConfig().getCompatibleMode() == CompatibleMode.COMPATIBLE) { - ClassDef layerClassDef = fory.getClassResolver().getTypeDef(cls, false); + TypeDef layerTypeDef = fory.getClassResolver().getTypeDef(cls, false); // Use layer index within class hierarchy (not global counter) // This ensures unique marker classes for each layer Class layerMarkerClass = LayerMarkerClassGenerator.getOrCreate(fory, cls, layerIndex); - slotsSerializer = new MetaSharedLayerSerializer(fory, cls, layerClassDef, layerMarkerClass); + slotsSerializer = new MetaSharedLayerSerializer(fory, cls, layerTypeDef, layerMarkerClass); } else { slotsSerializer = new ObjectSerializer<>(fory, cls, false); } @@ -289,8 +289,8 @@ private static void readAndSkipLayerClassMeta(Fory fory, MemoryBuffer buffer) { // Reference to previously read type - nothing more to read return; } - // New type - need to read and skip the ClassDef bytes + // New type - need to read and skip the TypeDef bytes long id = buffer.readInt64(); - ClassDef.skipClassDef(buffer, id); + TypeDef.skipTypeDef(buffer, id); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionLikeSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionLikeSerializer.java index 319bb5f0d2..d1f054534e 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionLikeSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionLikeSerializer.java @@ -25,10 +25,10 @@ import org.apache.fory.annotation.CodegenInvoke; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.reflect.ReflectionUtils; -import org.apache.fory.resolver.ClassInfo; -import org.apache.fory.resolver.ClassInfoHolder; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.RefResolver; +import org.apache.fory.resolver.TypeInfo; +import org.apache.fory.resolver.TypeInfoHolder; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.Serializer; import org.apache.fory.type.GenericType; @@ -42,7 +42,7 @@ public abstract class CollectionLikeSerializer extends Serializer { private MethodHandle constructor; private int numElements; protected final boolean supportCodegenHook; - protected final ClassInfoHolder elementClassInfoHolder; + protected final TypeInfoHolder elementTypeInfoHolder; private final TypeResolver typeResolver; protected final SerializationBinding binding; @@ -62,7 +62,7 @@ public CollectionLikeSerializer(Fory fory, Class cls) { public CollectionLikeSerializer(Fory fory, Class cls, boolean supportCodegenHook) { super(fory, cls); this.supportCodegenHook = supportCodegenHook; - elementClassInfoHolder = fory.getClassResolver().nilClassInfoHolder(); + elementTypeInfoHolder = fory.getClassResolver().nilTypeInfoHolder(); this.typeResolver = fory.isCrossLanguage() ? fory.getXtypeResolver() : fory.getClassResolver(); binding = SerializationBinding.createBinding(fory); } @@ -71,7 +71,7 @@ public CollectionLikeSerializer( Fory fory, Class cls, boolean supportCodegenHook, boolean immutable) { super(fory, cls, immutable); this.supportCodegenHook = supportCodegenHook; - elementClassInfoHolder = fory.getClassResolver().nilClassInfoHolder(); + elementTypeInfoHolder = fory.getClassResolver().nilTypeInfoHolder(); this.typeResolver = fory.isCrossLanguage() ? fory.getXtypeResolver() : fory.getClassResolver(); binding = SerializationBinding.createBinding(fory); } @@ -131,17 +131,17 @@ protected final int writeElementsHeader(MemoryBuffer buffer, Collection value) { } } else { if (trackingRef) { - return writeTypeHeader(buffer, value, elemGenericType.getCls(), elementClassInfoHolder); + return writeTypeHeader(buffer, value, elemGenericType.getCls(), elementTypeInfoHolder); } else { return writeTypeNullabilityHeader( - buffer, value, elemGenericType.getCls(), elementClassInfoHolder); + buffer, value, elemGenericType.getCls(), elementTypeInfoHolder); } } } else { if (fory.trackingRef()) { - return writeTypeHeader(buffer, value, elementClassInfoHolder); + return writeTypeHeader(buffer, value, elementTypeInfoHolder); } else { - return writeTypeNullabilityHeader(buffer, value, null, elementClassInfoHolder); + return writeTypeNullabilityHeader(buffer, value, null, elementTypeInfoHolder); } } } @@ -165,7 +165,7 @@ public int writeNullabilityHeader(MemoryBuffer buffer, Collection value) { */ @CodegenInvoke public int writeTypeHeader( - MemoryBuffer buffer, Collection value, Class declareElementType, ClassInfoHolder cache) { + MemoryBuffer buffer, Collection value, Class declareElementType, TypeInfoHolder cache) { int bitmap = CollectionFlags.TRACKING_REF; boolean hasDifferentClass = false; Class elemClass = null; @@ -196,7 +196,7 @@ public int writeTypeHeader( buffer.writeByte(bitmap); // Update classinfo, the caller will use it. TypeResolver typeResolver = this.typeResolver; - typeResolver.writeClassInfo(buffer, typeResolver.getClassInfo(elemClass, cache)); + typeResolver.writeTypeInfo(buffer, typeResolver.getTypeInfo(elemClass, cache)); } } return bitmap; @@ -204,7 +204,7 @@ public int writeTypeHeader( /** Maybe track elements ref, or write elements nullability. */ @CodegenInvoke - public int writeTypeHeader(MemoryBuffer buffer, Collection value, ClassInfoHolder cache) { + public int writeTypeHeader(MemoryBuffer buffer, Collection value, TypeInfoHolder cache) { int bitmap = 0; boolean hasDifferentClass = false; Class elemClass = null; @@ -234,12 +234,12 @@ public int writeTypeHeader(MemoryBuffer buffer, Collection value, ClassInfoHolde elemClass = void.class; } bitmap |= CollectionFlags.IS_SAME_TYPE; - ClassInfo classInfo = typeResolver.getClassInfo(elemClass, cache); - if (classInfo.getSerializer().needToWriteRef()) { + TypeInfo typeInfo = typeResolver.getTypeInfo(elemClass, cache); + if (typeInfo.getSerializer().needToWriteRef()) { bitmap |= CollectionFlags.TRACKING_REF; } buffer.writeByte(bitmap); - typeResolver.writeClassInfo(buffer, classInfo); + typeResolver.writeTypeInfo(buffer, typeInfo); } return bitmap; } @@ -250,7 +250,7 @@ public int writeTypeHeader(MemoryBuffer buffer, Collection value, ClassInfoHolde */ @CodegenInvoke public int writeTypeNullabilityHeader( - MemoryBuffer buffer, Collection value, Class declareElementType, ClassInfoHolder cache) { + MemoryBuffer buffer, Collection value, Class declareElementType, TypeInfoHolder cache) { int bitmap = 0; boolean containsNull = false; boolean hasDifferentClass = false; @@ -285,8 +285,8 @@ public int writeTypeNullabilityHeader( } else { buffer.writeByte(bitmap); TypeResolver typeResolver = this.typeResolver; - ClassInfo classInfo = typeResolver.getClassInfo(elemClass, cache); - typeResolver.writeClassInfo(buffer, classInfo); + TypeInfo typeInfo = typeResolver.getTypeInfo(elemClass, cache); + typeResolver.writeTypeInfo(buffer, typeInfo); } } return bitmap; @@ -347,7 +347,7 @@ private void generalJavaWrite( Preconditions.checkNotNull(elemGenericType); serializer = elemGenericType.getSerializer(typeResolver); } else { - serializer = elementClassInfoHolder.getSerializer(); + serializer = elementTypeInfoHolder.getSerializer(); } writeSameTypeElements(fory, buffer, serializer, flags, collection); } else { @@ -474,9 +474,9 @@ public Collection newCollection(Collection collection) { public void copyElements(Collection originCollection, Collection newCollection) { for (Object element : originCollection) { if (element != null) { - ClassInfo classInfo = typeResolver.getClassInfo(element.getClass(), elementClassInfoHolder); - if (!classInfo.getSerializer().isImmutable()) { - element = fory.copyObject(element, classInfo.getTypeId()); + TypeInfo typeInfo = typeResolver.getTypeInfo(element.getClass(), elementTypeInfoHolder); + if (!typeInfo.getSerializer().isImmutable()) { + element = fory.copyObject(element, typeInfo.getTypeId()); } } newCollection.add(element); @@ -487,9 +487,9 @@ public void copyElements(Collection originCollection, Object[] elements) { int index = 0; for (Object element : originCollection) { if (element != null) { - ClassInfo classInfo = typeResolver.getClassInfo(element.getClass(), elementClassInfoHolder); - if (!classInfo.getSerializer().isImmutable()) { - element = fory.copyObject(element, classInfo.getSerializer()); + TypeInfo typeInfo = typeResolver.getTypeInfo(element.getClass(), elementTypeInfoHolder); + if (!typeInfo.getSerializer().isImmutable()) { + element = fory.copyObject(element, typeInfo.getSerializer()); } } elements[index++] = element; @@ -562,7 +562,7 @@ private void generalJavaRead( Serializer serializer; TypeResolver typeResolver = this.typeResolver; if ((flags & CollectionFlags.IS_DECL_ELEMENT_TYPE) != CollectionFlags.IS_DECL_ELEMENT_TYPE) { - serializer = typeResolver.readClassInfo(buffer, elementClassInfoHolder).getSerializer(); + serializer = typeResolver.readTypeInfo(buffer, elementTypeInfoHolder).getSerializer(); } else { serializer = elemGenericType.getSerializer(typeResolver); } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java index cd5b7b6dd8..6b5c368dcd 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java @@ -54,10 +54,10 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.Platform; import org.apache.fory.reflect.ReflectionUtils; -import org.apache.fory.resolver.ClassInfo; -import org.apache.fory.resolver.ClassInfoHolder; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.RefResolver; +import org.apache.fory.resolver.TypeInfo; +import org.apache.fory.resolver.TypeInfoHolder; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.ReplaceResolveSerializer; import org.apache.fory.serializer.Serializer; @@ -515,8 +515,8 @@ public SetFromMapSerializer(Fory fory, Class> type) { @Override public Collection newCollection(MemoryBuffer buffer) { - final ClassInfo mapClassInfo = fory.getClassResolver().readClassInfo(buffer); - final MapLikeSerializer mapSerializer = (MapLikeSerializer) mapClassInfo.getSerializer(); + final TypeInfo mapTypeInfo = fory.getClassResolver().readTypeInfo(buffer); + final MapLikeSerializer mapSerializer = (MapLikeSerializer) mapTypeInfo.getSerializer(); RefResolver refResolver = fory.getRefResolver(); // It's possible that elements or nested fields has circular ref to set. int refId = refResolver.lastPreservedRefId(); @@ -554,9 +554,9 @@ public Collection newCollection(Collection originCollection) { @Override public Collection onCollectionWrite(MemoryBuffer buffer, Set value) { final Map map = (Map) Platform.getObject(value, MAP_FIELD_OFFSET); - final ClassInfo classInfo = fory.getClassResolver().getClassInfo(map.getClass()); - MapLikeSerializer mapSerializer = (MapLikeSerializer) classInfo.getSerializer(); - fory.getClassResolver().writeClassInfo(buffer, classInfo); + final TypeInfo typeInfo = fory.getClassResolver().getTypeInfo(map.getClass()); + MapLikeSerializer mapSerializer = (MapLikeSerializer) typeInfo.getSerializer(); + fory.getClassResolver().writeTypeInfo(buffer, typeInfo); if (mapSerializer.supportCodegenHook) { buffer.writeBoolean(true); mapSerializer.onMapWrite(buffer, map); @@ -571,26 +571,26 @@ public Collection onCollectionWrite(MemoryBuffer buffer, Set value) { public static final class ConcurrentHashMapKeySetViewSerializer extends CollectionSerializer { - private final ClassInfoHolder mapClassInfoHolder; - private final ClassInfoHolder valueClassInfoHolder; + private final TypeInfoHolder mapTypeInfoHolder; + private final TypeInfoHolder valueTypeInfoHolder; public ConcurrentHashMapKeySetViewSerializer( Fory fory, Class type) { super(fory, type, false); - mapClassInfoHolder = fory.getClassResolver().nilClassInfoHolder(); - valueClassInfoHolder = fory.getClassResolver().nilClassInfoHolder(); + mapTypeInfoHolder = fory.getClassResolver().nilTypeInfoHolder(); + valueTypeInfoHolder = fory.getClassResolver().nilTypeInfoHolder(); } @Override public void write(MemoryBuffer buffer, ConcurrentHashMap.KeySetView value) { - fory.writeRef(buffer, value.getMap(), mapClassInfoHolder); - fory.writeRef(buffer, value.getMappedValue(), valueClassInfoHolder); + fory.writeRef(buffer, value.getMap(), mapTypeInfoHolder); + fory.writeRef(buffer, value.getMappedValue(), valueTypeInfoHolder); } @Override public ConcurrentHashMap.KeySetView read(MemoryBuffer buffer) { - ConcurrentHashMap map = (ConcurrentHashMap) fory.readRef(buffer, mapClassInfoHolder); - Object value = fory.readRef(buffer, valueClassInfoHolder); + ConcurrentHashMap map = (ConcurrentHashMap) fory.readRef(buffer, mapTypeInfoHolder); + Object value = fory.readRef(buffer, valueTypeInfoHolder); return map.keySet(value); } @@ -668,7 +668,7 @@ public void write(MemoryBuffer buffer, EnumSet object) { @Override public EnumSet read(MemoryBuffer buffer) { - Class elemClass = fory.getClassResolver().readClassInfo(buffer).getCls(); + Class elemClass = fory.getClassResolver().readTypeInfo(buffer).getCls(); EnumSet object = EnumSet.noneOf(elemClass); Serializer elemSerializer = fory.getClassResolver().getSerializer(elemClass); int length = buffer.readVarUint32Small7(); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapLikeSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapLikeSerializer.java index 573aaa030d..0abd12a4c1 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapLikeSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapLikeSerializer.java @@ -44,9 +44,9 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.reflect.TypeRef; -import org.apache.fory.resolver.ClassInfo; -import org.apache.fory.resolver.ClassInfoHolder; import org.apache.fory.resolver.RefResolver; +import org.apache.fory.resolver.TypeInfo; +import org.apache.fory.resolver.TypeInfoHolder; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.Serializer; import org.apache.fory.type.GenericType; @@ -61,10 +61,10 @@ public abstract class MapLikeSerializer extends Serializer { protected MethodHandle constructor; protected final boolean supportCodegenHook; - protected final ClassInfoHolder keyClassInfoWriteCache; - protected final ClassInfoHolder keyClassInfoReadCache; - protected final ClassInfoHolder valueClassInfoWriteCache; - protected final ClassInfoHolder valueClassInfoReadCache; + protected final TypeInfoHolder keyTypeInfoWriteCache; + protected final TypeInfoHolder keyTypeInfoReadCache; + protected final TypeInfoHolder valueTypeInfoWriteCache; + protected final TypeInfoHolder valueTypeInfoReadCache; // support map subclass whose key or value generics only are available, // or one of types is already instantiated in subclass, ex: `Subclass implements Map` @@ -96,10 +96,10 @@ public MapLikeSerializer(Fory fory, Class cls, boolean supportCodegenHook, bo this.typeResolver = fory.isCrossLanguage() ? fory.getXtypeResolver() : fory.getClassResolver(); trackRef = fory.trackingRef(); this.supportCodegenHook = supportCodegenHook; - keyClassInfoWriteCache = typeResolver.nilClassInfoHolder(); - keyClassInfoReadCache = typeResolver.nilClassInfoHolder(); - valueClassInfoWriteCache = typeResolver.nilClassInfoHolder(); - valueClassInfoReadCache = typeResolver.nilClassInfoHolder(); + keyTypeInfoWriteCache = typeResolver.nilTypeInfoHolder(); + keyTypeInfoReadCache = typeResolver.nilTypeInfoHolder(); + valueTypeInfoWriteCache = typeResolver.nilTypeInfoHolder(); + valueTypeInfoReadCache = typeResolver.nilTypeInfoHolder(); partialGenericKVTypeMap = new IdentityMap<>(); objType = typeResolver.buildGenericType(Object.class); binding = SerializationBinding.createBinding(fory); @@ -181,7 +181,7 @@ private void writeNullValueChunk(MemoryBuffer buffer, Serializer keySerializer, } } else { buffer.writeByte(VALUE_HAS_NULL | TRACKING_KEY_REF); - binding.writeRef(buffer, key, keyClassInfoWriteCache); + binding.writeRef(buffer, key, keyTypeInfoWriteCache); } } @@ -203,7 +203,7 @@ private void writeNullKeyChunk(MemoryBuffer buffer, Serializer valueSerializer, } } else { buffer.writeByte(KEY_HAS_NULL | TRACKING_VALUE_REF); - binding.writeRef(buffer, value, valueClassInfoWriteCache); + binding.writeRef(buffer, value, valueTypeInfoWriteCache); } } else { buffer.writeByte(KV_NULL); @@ -268,7 +268,7 @@ private void writeKeyForNullValueChunkGeneric( if (!keyType.isMonomorphic()) { if (trackingRef) { buffer.writeByte(VALUE_HAS_NULL | TRACKING_KEY_REF); - binding.writeRef(buffer, key, keyClassInfoWriteCache); + binding.writeRef(buffer, key, keyTypeInfoWriteCache); } else { buffer.writeByte(VALUE_HAS_NULL); binding.writeNonRef(buffer, key); @@ -299,7 +299,7 @@ private void writeValueForNullKeyChunkGeneric( if (!valueType.isMonomorphic()) { if (trackingRef) { buffer.writeByte(KEY_HAS_NULL | TRACKING_VALUE_REF); - binding.writeRef(buffer, value, valueClassInfoWriteCache); + binding.writeRef(buffer, value, valueTypeInfoWriteCache); } else { buffer.writeByte(KEY_HAS_NULL); binding.writeNonRef(buffer, value); @@ -343,12 +343,12 @@ private Entry writeJavaChunk( if (keySerializer != null) { chunkHeader |= KEY_DECL_TYPE; } else { - keySerializer = writeKeyClassInfo(classResolver, keyType, buffer); + keySerializer = writeKeyTypeInfo(classResolver, keyType, buffer); } if (valueSerializer != null) { chunkHeader |= VALUE_DECL_TYPE; } else { - valueSerializer = writeValueClassInfo(classResolver, valueType, buffer); + valueSerializer = writeValueTypeInfo(classResolver, valueType, buffer); } // noinspection Duplicates boolean keyWriteRef = keySerializer.needToWriteRef(); @@ -394,18 +394,18 @@ private Entry writeJavaChunk( return entry; } - private Serializer writeKeyClassInfo( + private Serializer writeKeyTypeInfo( TypeResolver classResolver, Class keyType, MemoryBuffer buffer) { - ClassInfo classInfo = classResolver.getClassInfo(keyType, keyClassInfoWriteCache); - classResolver.writeClassInfo(buffer, classInfo); - return classInfo.getSerializer(); + TypeInfo typeInfo = classResolver.getTypeInfo(keyType, keyTypeInfoWriteCache); + classResolver.writeTypeInfo(buffer, typeInfo); + return typeInfo.getSerializer(); } - private Serializer writeValueClassInfo( + private Serializer writeValueTypeInfo( TypeResolver classResolver, Class valueType, MemoryBuffer buffer) { - ClassInfo classInfo = classResolver.getClassInfo(valueType, valueClassInfoWriteCache); - classResolver.writeClassInfo(buffer, classInfo); - return classInfo.getSerializer(); + TypeInfo typeInfo = classResolver.getTypeInfo(valueType, valueTypeInfoWriteCache); + classResolver.writeTypeInfo(buffer, typeInfo); + return typeInfo.getSerializer(); } @CodegenInvoke @@ -456,13 +456,13 @@ public Entry writeJavaChunkGeneric( chunkHeader |= KEY_DECL_TYPE; keySerializer = keyGenericType.getSerializer(classResolver); } else { - keySerializer = writeKeyClassInfo(classResolver, keyType, buffer); + keySerializer = writeKeyTypeInfo(classResolver, keyType, buffer); } if (valueGenericTypeFinal) { chunkHeader |= VALUE_DECL_TYPE; valueSerializer = valueGenericType.getSerializer(classResolver); } else { - valueSerializer = writeValueClassInfo(classResolver, valueType, buffer); + valueSerializer = writeValueTypeInfo(classResolver, valueType, buffer); } boolean trackingKeyRef = keyGenericType.trackingRef(typeResolver); boolean trackingValueRef = valueGenericType.trackingRef(typeResolver); @@ -543,17 +543,16 @@ protected void copyEntry(Map originMap, Map newMap) { for (Map.Entry entry : originMap.entrySet()) { K key = entry.getKey(); if (key != null) { - ClassInfo classInfo = classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache); - if (!classInfo.getSerializer().isImmutable()) { - key = fory.copyObject(key, classInfo.getTypeId()); + TypeInfo typeInfo = classResolver.getTypeInfo(key.getClass(), keyTypeInfoWriteCache); + if (!typeInfo.getSerializer().isImmutable()) { + key = fory.copyObject(key, typeInfo.getTypeId()); } } V value = entry.getValue(); if (value != null) { - ClassInfo classInfo = - classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache); - if (!classInfo.getSerializer().isImmutable()) { - value = fory.copyObject(value, classInfo.getTypeId()); + TypeInfo typeInfo = classResolver.getTypeInfo(value.getClass(), valueTypeInfoWriteCache); + if (!typeInfo.getSerializer().isImmutable()) { + value = fory.copyObject(value, typeInfo.getTypeId()); } } newMap.put(key, value); @@ -565,17 +564,16 @@ protected void copyEntry(Map originMap, Builder builder) { for (Entry entry : originMap.entrySet()) { K key = entry.getKey(); if (key != null) { - ClassInfo classInfo = classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache); - if (!classInfo.getSerializer().isImmutable()) { - key = fory.copyObject(key, classInfo.getTypeId()); + TypeInfo typeInfo = classResolver.getTypeInfo(key.getClass(), keyTypeInfoWriteCache); + if (!typeInfo.getSerializer().isImmutable()) { + key = fory.copyObject(key, typeInfo.getTypeId()); } } V value = entry.getValue(); if (value != null) { - ClassInfo classInfo = - classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache); - if (!classInfo.getSerializer().isImmutable()) { - value = fory.copyObject(value, classInfo.getTypeId()); + TypeInfo typeInfo = classResolver.getTypeInfo(value.getClass(), valueTypeInfoWriteCache); + if (!typeInfo.getSerializer().isImmutable()) { + value = fory.copyObject(value, typeInfo.getTypeId()); } } builder.put(key, value); @@ -588,17 +586,16 @@ protected void copyEntry(Map originMap, Object[] elements) { for (Entry entry : originMap.entrySet()) { K key = entry.getKey(); if (key != null) { - ClassInfo classInfo = classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache); - if (!classInfo.getSerializer().isImmutable()) { - key = fory.copyObject(key, classInfo.getTypeId()); + TypeInfo typeInfo = classResolver.getTypeInfo(key.getClass(), keyTypeInfoWriteCache); + if (!typeInfo.getSerializer().isImmutable()) { + key = fory.copyObject(key, typeInfo.getTypeId()); } } V value = entry.getValue(); if (value != null) { - ClassInfo classInfo = - classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache); - if (!classInfo.getSerializer().isImmutable()) { - value = fory.copyObject(value, classInfo.getTypeId()); + TypeInfo typeInfo = classResolver.getTypeInfo(value.getClass(), valueTypeInfoWriteCache); + if (!typeInfo.getSerializer().isImmutable()) { + value = fory.copyObject(value, typeInfo.getTypeId()); } } elements[index++] = key; @@ -669,9 +666,9 @@ public long readJavaNullChunk( } } else { if (trackKeyRef) { - key = binding.readRef(buffer, keyClassInfoReadCache); + key = binding.readRef(buffer, keyTypeInfoReadCache); } else { - key = binding.readNonRef(buffer, keyClassInfoReadCache); + key = binding.readNonRef(buffer, keyTypeInfoReadCache); } } map.put(key, null); @@ -715,9 +712,9 @@ private void readNullKeyChunk( } } else { if (trackValueRef) { - value = binding.readRef(buffer, valueClassInfoReadCache); + value = binding.readRef(buffer, valueTypeInfoReadCache); } else { - value = binding.readNonRef(buffer, valueClassInfoReadCache); + value = binding.readNonRef(buffer, valueTypeInfoReadCache); } } map.put(null, value); @@ -798,10 +795,10 @@ public long readJavaChunk( boolean valueIsDeclaredType = (chunkHeader & VALUE_DECL_TYPE) != 0; int chunkSize = buffer.readUnsignedByte(); if (!keyIsDeclaredType) { - keySerializer = typeResolver.readClassInfo(buffer, keyClassInfoReadCache).getSerializer(); + keySerializer = typeResolver.readTypeInfo(buffer, keyTypeInfoReadCache).getSerializer(); } if (!valueIsDeclaredType) { - valueSerializer = typeResolver.readClassInfo(buffer, valueClassInfoReadCache).getSerializer(); + valueSerializer = typeResolver.readTypeInfo(buffer, valueTypeInfoReadCache).getSerializer(); } fory.incReadDepth(); for (int i = 0; i < chunkSize; i++) { @@ -847,12 +844,12 @@ private long readJavaChunkGeneric( int chunkSize = buffer.readUnsignedByte(); Serializer keySerializer, valueSerializer; if (!keyIsDeclaredType) { - keySerializer = typeResolver.readClassInfo(buffer, keyClassInfoReadCache).getSerializer(); + keySerializer = typeResolver.readTypeInfo(buffer, keyTypeInfoReadCache).getSerializer(); } else { keySerializer = keyGenericType.getSerializer(typeResolver); } if (!valueIsDeclaredType) { - valueSerializer = typeResolver.readClassInfo(buffer, valueClassInfoReadCache).getSerializer(); + valueSerializer = typeResolver.readTypeInfo(buffer, valueTypeInfoReadCache).getSerializer(); } else { valueSerializer = valueGenericType.getSerializer(typeResolver); } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapSerializers.java index c17b4a2c2f..90d8469b3c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapSerializers.java @@ -37,8 +37,8 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.Platform; import org.apache.fory.reflect.ReflectionUtils; -import org.apache.fory.resolver.ClassInfo; import org.apache.fory.resolver.ClassResolver; +import org.apache.fory.resolver.TypeInfo; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.ReplaceResolveSerializer; import org.apache.fory.serializer.Serializer; @@ -327,7 +327,7 @@ public Map onMapWrite(MemoryBuffer buffer, EnumMap value) { @Override public EnumMap newMap(MemoryBuffer buffer) { setNumElements(buffer.readVarUint32Small7()); - Class keyType = fory.getClassResolver().readClassInfo(buffer).getCls(); + Class keyType = fory.getClassResolver().readTypeInfo(buffer).getCls(); return new EnumMap(keyType); } @@ -349,10 +349,9 @@ protected void copyEntry(Map originMap, Map newMap) { for (Entry entry : originMap.entrySet()) { V value = entry.getValue(); if (value != null) { - ClassInfo classInfo = - classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache); - if (!classInfo.getSerializer().isImmutable()) { - value = fory.copyObject(value, classInfo.getTypeId()); + TypeInfo typeInfo = classResolver.getTypeInfo(value.getClass(), valueTypeInfoWriteCache); + if (!typeInfo.getSerializer().isImmutable()) { + value = fory.copyObject(value, typeInfo.getTypeId()); } } newMap.put(entry.getKey(), value); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SerializationBinding.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SerializationBinding.java index de9eb917d6..3577e26184 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SerializationBinding.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SerializationBinding.java @@ -21,8 +21,8 @@ import org.apache.fory.Fory; import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.resolver.ClassInfoHolder; import org.apache.fory.resolver.RefMode; +import org.apache.fory.resolver.TypeInfoHolder; import org.apache.fory.serializer.Serializer; // This polymorphic interface has cost, do not expose it as a public class @@ -34,7 +34,7 @@ interface SerializationBinding { void writeRef(MemoryBuffer buffer, T obj, Serializer serializer); - void writeRef(MemoryBuffer buffer, Object obj, ClassInfoHolder classInfoHolder); + void writeRef(MemoryBuffer buffer, Object obj, TypeInfoHolder classInfoHolder); void writeNonRef(MemoryBuffer buffer, Object elem); @@ -44,13 +44,13 @@ interface SerializationBinding { T readRef(MemoryBuffer buffer, Serializer serializer); - Object readRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder); + Object readRef(MemoryBuffer buffer, TypeInfoHolder classInfoHolder); Object readRef(MemoryBuffer buffer); Object readNonRef(MemoryBuffer buffer); - Object readNonRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder); + Object readNonRef(MemoryBuffer buffer, TypeInfoHolder classInfoHolder); static SerializationBinding createBinding(Fory fory) { if (fory.isCrossLanguage()) { @@ -78,7 +78,7 @@ public void writeRef(MemoryBuffer buffer, T obj, Serializer serializer) { } @Override - public void writeRef(MemoryBuffer buffer, Object obj, ClassInfoHolder classInfoHolder) { + public void writeRef(MemoryBuffer buffer, Object obj, TypeInfoHolder classInfoHolder) { fory.writeRef(buffer, obj, classInfoHolder); } @@ -90,7 +90,7 @@ public T readRef(MemoryBuffer buffer, Serializer serializer) { } @Override - public Object readRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { + public Object readRef(MemoryBuffer buffer, TypeInfoHolder classInfoHolder) { return fory.readRef(buffer, classInfoHolder); } @@ -105,7 +105,7 @@ public Object readNonRef(MemoryBuffer buffer) { } @Override - public Object readNonRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { + public Object readNonRef(MemoryBuffer buffer, TypeInfoHolder classInfoHolder) { return fory.readNonRef(buffer, classInfoHolder); } @@ -146,7 +146,7 @@ public void writeRef(MemoryBuffer buffer, T obj, Serializer serializer) { } @Override - public void writeRef(MemoryBuffer buffer, Object obj, ClassInfoHolder classInfoHolder) { + public void writeRef(MemoryBuffer buffer, Object obj, TypeInfoHolder classInfoHolder) { fory.xwriteRef(buffer, obj); } @@ -158,7 +158,7 @@ public T readRef(MemoryBuffer buffer, Serializer serializer) { } @Override - public Object readRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { + public Object readRef(MemoryBuffer buffer, TypeInfoHolder classInfoHolder) { return fory.xreadRef(buffer); } @@ -173,7 +173,7 @@ public Object readNonRef(MemoryBuffer buffer) { } @Override - public Object readNonRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { + public Object readNonRef(MemoryBuffer buffer, TypeInfoHolder classInfoHolder) { return fory.xreadNonRef(buffer, classInfoHolder); } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/struct/Fingerprint.java b/java/fory-core/src/main/java/org/apache/fory/serializer/struct/Fingerprint.java index d31f622951..19776c7777 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/struct/Fingerprint.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/struct/Fingerprint.java @@ -37,7 +37,7 @@ import org.apache.fory.logging.Logger; import org.apache.fory.logging.LoggerFactory; import org.apache.fory.reflect.ReflectionUtils; -import org.apache.fory.resolver.ClassInfo; +import org.apache.fory.resolver.TypeInfo; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.type.Descriptor; import org.apache.fory.type.TypeUtils; @@ -170,15 +170,14 @@ private static int getTypeId(Fory fory, Descriptor descriptor) { if (ReflectionUtils.isAbstract(cls) || cls.isInterface() || cls.isEnum()) { return Types.UNKNOWN; } - ClassInfo classInfo = resolver.getClassInfo(cls, false); - if (classInfo == null) { + TypeInfo typeInfo = resolver.getTypeInfo(cls, false); + if (typeInfo == null) { return Types.UNKNOWN; } int typeId = Types.getDescriptorTypeId(fory, descriptor); - int internalTypeId = typeId & 0xff; // union must also be set to `UNKNOWN`, we can't know a type is union at compile-time for some // languages. - if (Types.isUserDefinedType((byte) internalTypeId)) { + if (Types.isUserDefinedType((byte) typeId)) { return Types.UNKNOWN; } return typeId; diff --git a/java/fory-core/src/main/java/org/apache/fory/type/Generics.java b/java/fory-core/src/main/java/org/apache/fory/type/Generics.java index 4dbf13616b..231f13a7dc 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/Generics.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/Generics.java @@ -21,7 +21,7 @@ import org.apache.fory.Fory; import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.resolver.ClassInfo; +import org.apache.fory.resolver.TypeInfo; // Derived from // https://github.com/EsotericSoftware/kryo/blob/135df69526615bb3f6b34846e58ba3fec3b631c3/src/com/esotericsoftware/kryo/util/DefaultGenerics.java. @@ -50,7 +50,7 @@ public Generics(Fory fory) { * {@link #nextGenericType}. Fory serialization depth should be increased after this call and * before {@link #nextGenericType}. * - * @see Fory#writeRef(MemoryBuffer, Object, ClassInfo) + * @see Fory#writeRef(MemoryBuffer, Object, TypeInfo) */ public void pushGenericType(GenericType fieldType) { int size = genericTypesSize++; @@ -78,7 +78,7 @@ private GenericType[] allocateGenericTypes(GenericType[] genericTypes, int size) * #pushGenericType(GenericType)} was not called. Fory serialization depth should be decreased * before this call and after {@link #nextGenericType}. * - * @see Fory#writeRef(MemoryBuffer, Object, ClassInfo) + * @see Fory#writeRef(MemoryBuffer, Object, TypeInfo) */ public void popGenericType() { int size = genericTypesSize; diff --git a/java/fory-core/src/main/java/org/apache/fory/type/Types.java b/java/fory-core/src/main/java/org/apache/fory/type/Types.java index 568ee5d86f..21620cd59c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/Types.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/Types.java @@ -24,8 +24,8 @@ import org.apache.fory.Fory; import org.apache.fory.meta.TypeExtMeta; import org.apache.fory.reflect.TypeRef; -import org.apache.fory.resolver.ClassInfo; import org.apache.fory.resolver.ClassResolver; +import org.apache.fory.resolver.TypeInfo; import org.apache.fory.util.Preconditions; public class Types { @@ -211,9 +211,10 @@ public class Types { /** Bound value for range checks (types with id >= BOUND are not internal types). */ public static final int BOUND = 64; + public static final int INVALID_USER_TYPE_ID = -1; + // Helper methods public static boolean isNamedType(int value) { - assert value < 0xff; switch (value) { case NAMED_STRUCT: case NAMED_COMPATIBLE_STRUCT: @@ -226,8 +227,21 @@ public static boolean isNamedType(int value) { } } + /** Return true if type is user type and registered by id. */ + public static boolean isUserTypeRegisteredById(int typeId) { + switch (typeId) { + case Types.ENUM: + case Types.STRUCT: + case Types.COMPATIBLE_STRUCT: + case Types.EXT: + case Types.TYPED_UNION: + return true; + default: + return false; + } + } + public static boolean isStructType(int value) { - assert value < 0xff; return value == STRUCT || value == COMPATIBLE_STRUCT || value == NAMED_STRUCT @@ -249,7 +263,7 @@ public static boolean isUnionType(int value) { return value == UNION || value == TYPED_UNION || value == NAMED_UNION; } - public static boolean isUserDefinedType(byte typeId) { + public static boolean isUserDefinedType(int typeId) { return isStructType(typeId) || isExtType(typeId) || isEnumType(typeId) @@ -352,12 +366,12 @@ public static int getDescriptorTypeId(Fory fory, Descriptor d) { } private static int getUnionDescriptorTypeId(Fory fory, Class rawType) { - ClassInfo classInfo = fory.getTypeResolver().getClassInfo(rawType, false); - if (classInfo == null) { + TypeInfo typeInfo = fory.getTypeResolver().getTypeInfo(rawType, false); + if (typeInfo == null) { return -1; } - int internalTypeId = classInfo.getTypeId() & 0xff; - if (Types.isUnionType(internalTypeId)) { + int typeId = typeInfo.getTypeId(); + if (Types.isUnionType(typeId)) { return Types.UNION; } return -1; @@ -386,9 +400,9 @@ public static int getTypeId(Fory fory, Class clz) { return Types.FLOAT64; } } - ClassInfo classInfo = fory.getTypeResolver().getClassInfo(clz, false); - if (classInfo != null) { - return classInfo.getTypeId(); + TypeInfo typeInfo = fory.getTypeResolver().getTypeInfo(clz, false); + if (typeInfo != null) { + return typeInfo.getTypeId(); } return Types.UNKNOWN; } diff --git a/java/fory-core/src/main/java/org/apache/fory/util/LoaderBinding.java b/java/fory-core/src/main/java/org/apache/fory/util/LoaderBinding.java index 7f60e1f27d..a67fa65c22 100644 --- a/java/fory-core/src/main/java/org/apache/fory/util/LoaderBinding.java +++ b/java/fory-core/src/main/java/org/apache/fory/util/LoaderBinding.java @@ -175,10 +175,14 @@ public void register(Class clz) { bindingCallback = bindingCallback.andThen(fory -> fory.register(clz)); } - public void register(Class clz, int id) { - Preconditions.checkArgument(id < Short.MAX_VALUE); - foryMap.values().forEach(fory -> fory.register(clz, (short) id)); - bindingCallback = bindingCallback.andThen(fory -> fory.register(clz, (short) id)); + public void register(Class clz, long id) { + long unsignedId = id < 0 ? id & 0xffff_ffffL : id; + Preconditions.checkArgument( + unsignedId >= 0 && unsignedId <= 0xffff_fffEL, + "User type id must be in range [0, 0xfffffffe]"); + int checkedId = (int) unsignedId; + foryMap.values().forEach(fory -> fory.register(clz, checkedId)); + bindingCallback = bindingCallback.andThen(fory -> fory.register(clz, checkedId)); } public void setBindingCallback(Consumer bindingCallback) { diff --git a/java/fory-core/src/main/java/org/apache/fory/util/Utils.java b/java/fory-core/src/main/java/org/apache/fory/util/Utils.java index 17d910c15a..f40172dada 100644 --- a/java/fory-core/src/main/java/org/apache/fory/util/Utils.java +++ b/java/fory-core/src/main/java/org/apache/fory/util/Utils.java @@ -22,8 +22,10 @@ /** Misc common utils. */ public class Utils { public static final boolean DEBUG_OUTPUT_ENABLED; + public static final boolean DEBUG_OUTPUT_VERBOSE; static { DEBUG_OUTPUT_ENABLED = "1".equals(System.getenv("ENABLE_FORY_DEBUG_OUTPUT")); + DEBUG_OUTPUT_VERBOSE = "1".equals(System.getenv("ENABLE_FORY_DEBUG_OUTPUT_VERBOSE")); } } diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties index cf6e873456..e024cb2211 100644 --- a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties @@ -245,10 +245,10 @@ Args=--initialize-at-build-time=org.apache.fory.memory.MemoryBuffer,\ org.apache.fory.meta.FieldTypes$MapFieldType,\ org.apache.fory.meta.FieldTypes$ObjectFieldType,\ org.apache.fory.meta.FieldTypes$RegisteredFieldType,\ - org.apache.fory.meta.ClassDef,\ + org.apache.fory.meta.TypeDef,\ org.apache.fory.meta.FieldInfo,\ - org.apache.fory.meta.ClassDefDecoder,\ - org.apache.fory.meta.ClassDefEncoder,\ + org.apache.fory.meta.NativeTypeDefDecoder,\ + org.apache.fory.meta.NativeTypeDefEncoder,\ org.apache.fory.meta.ClassSpec,\ org.apache.fory.meta.TypeExtMeta,\ org.apache.fory.meta.DeflaterMetaCompressor,\ @@ -280,8 +280,8 @@ Args=--initialize-at-build-time=org.apache.fory.memory.MemoryBuffer,\ org.apache.fory.reflect.ObjectCreators$RecordObjectCreator,\ org.apache.fory.resolver.ClassChecker,\ org.apache.fory.resolver.TypeChecker,\ - org.apache.fory.resolver.ClassInfo,\ - org.apache.fory.resolver.ClassInfoHolder,\ + org.apache.fory.resolver.TypeInfo,\ + org.apache.fory.resolver.TypeInfoHolder,\ org.apache.fory.resolver.ClassResolver$2,\ org.apache.fory.resolver.TypeResolver$ExtRegistry,\ org.apache.fory.resolver.TypeResolver$GraalvmClassRegistry,\ @@ -504,7 +504,7 @@ Args=--initialize-at-build-time=org.apache.fory.memory.MemoryBuffer,\ org.apache.fory.serializer.ObjectStreamSerializer$1,\ org.apache.fory.serializer.ObjectStreamSerializer$ForyObjectInputStream,\ org.apache.fory.serializer.ObjectStreamSerializer$ForyObjectOutputStream,\ - org.apache.fory.serializer.ObjectStreamSerializer$StreamClassInfo,\ + org.apache.fory.serializer.ObjectStreamSerializer$StreamTypeInfo,\ org.apache.fory.serializer.ObjectStreamSerializer$SlotsInfo,\ org.apache.fory.serializer.collection.ChildContainerSerializers$ChildCollectionSerializer,\ org.apache.fory.serializer.collection.ChildContainerSerializers$ChildMapSerializer,\ diff --git a/java/fory-core/src/test/java/org/apache/fory/CyclicTest.java b/java/fory-core/src/test/java/org/apache/fory/CyclicTest.java index be478904b1..b6ce96bf80 100644 --- a/java/fory-core/src/test/java/org/apache/fory/CyclicTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/CyclicTest.java @@ -27,8 +27,6 @@ import java.util.ArrayList; import java.util.List; import java.util.zip.GZIPOutputStream; -import org.apache.fory.config.CompatibleMode; -import org.apache.fory.config.ForyBuilder; import org.apache.fory.config.Language; import org.apache.fory.test.bean.Cyclic; import org.apache.fory.test.bean.FinalCyclic; @@ -46,33 +44,28 @@ public static Object[][] beans() { } @DataProvider - public static Object[][] fory() { + public static Object[][] config() { return Sets.cartesianProduct( ImmutableSet.of(true, false), // enableCodegen ImmutableSet.of(true, false), // async compilation - ImmutableSet.of(true, false), // scoped meta share - ImmutableSet.of( - CompatibleMode.SCHEMA_CONSISTENT, CompatibleMode.COMPATIBLE) // structFieldsRepeat + ImmutableSet.of(true, false) // compatible ) .stream() .map(List::toArray) - .map( - c -> - new Object[] { - Fory.builder() - .withLanguage(Language.JAVA) - .withCodegen((Boolean) c[0]) - .withAsyncCompilation((Boolean) c[1]) - .withScopedMetaShare((Boolean) c[2]) - .withCompatibleMode((CompatibleMode) c[3]) - .requireClassRegistration(false) - }) .toArray(Object[][]::new); } - @Test(dataProvider = "fory") - public void testBean(ForyBuilder builder) { - Fory fory = builder.withMetaShare(false).withRefTracking(true).build(); + @Test(dataProvider = "config") + public void testBean(boolean enableCodegen, boolean asyncCompilation, boolean compatible) { + Fory fory = + Fory.builder() + .withLanguage(Language.JAVA) + .requireClassRegistration(false) + .withRefTracking(true) + .withCodegen(enableCodegen) + .withAsyncCompilation(asyncCompilation) + .withCompatible(compatible) + .build(); for (Object[] objects : beans()) { Object notCyclic = objects[0]; Object cyclic = objects[1]; diff --git a/java/fory-core/src/test/java/org/apache/fory/ForyTestBase.java b/java/fory-core/src/test/java/org/apache/fory/ForyTestBase.java index 4c190c9c23..5c6eb71d08 100644 --- a/java/fory-core/src/test/java/org/apache/fory/ForyTestBase.java +++ b/java/fory-core/src/test/java/org/apache/fory/ForyTestBase.java @@ -251,9 +251,7 @@ public static Object[][] basicMultiConfigFory() { return Sets.cartesianProduct( ImmutableSet.of(true, false), // trackingRef ImmutableSet.of(true, false), // codeGen - ImmutableSet.of(true, false), // scoped meta share - ImmutableSet.of( - CompatibleMode.COMPATIBLE, CompatibleMode.SCHEMA_CONSISTENT)) // CompatibleMode + ImmutableSet.of(true, false)) // compatible .stream() .map(List::toArray) .toArray(Object[][]::new); diff --git a/java/fory-core/src/test/java/org/apache/fory/TestUtils.java b/java/fory-core/src/test/java/org/apache/fory/TestUtils.java index c516b95c49..f6bf4eefa5 100644 --- a/java/fory-core/src/test/java/org/apache/fory/TestUtils.java +++ b/java/fory-core/src/test/java/org/apache/fory/TestUtils.java @@ -31,7 +31,7 @@ import java.util.stream.Collectors; import org.apache.fory.collection.Tuple3; import org.apache.fory.memory.Platform; -import org.apache.fory.meta.ClassDef; +import org.apache.fory.meta.TypeDef; import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.type.Descriptor; @@ -344,8 +344,8 @@ public static Tuple3, Map, Map> getCom */ public static Map objectToMap(Fory fory, Object obj) { Class cls = obj.getClass(); - ClassDef classDef = fory.getClassResolver().getTypeDef(cls, true); - List descriptors = classDef.getDescriptors(fory.getTypeResolver(), cls); + TypeDef typeDef = fory.getClassResolver().getTypeDef(cls, true); + List descriptors = typeDef.getDescriptors(fory.getTypeResolver(), cls); Map result = new LinkedHashMap<>(); for (Descriptor descriptor : descriptors) { Field field = descriptor.getField(); diff --git a/java/fory-core/src/test/java/org/apache/fory/annotation/ForyAnnotationTest.java b/java/fory-core/src/test/java/org/apache/fory/annotation/ForyAnnotationTest.java index 300ae1a876..7255149fdd 100644 --- a/java/fory-core/src/test/java/org/apache/fory/annotation/ForyAnnotationTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/annotation/ForyAnnotationTest.java @@ -26,7 +26,6 @@ import lombok.Data; import org.apache.fory.Fory; import org.apache.fory.ForyTestBase; -import org.apache.fory.config.CompatibleMode; import org.apache.fory.config.Language; import org.testng.Assert; import org.testng.annotations.Test; @@ -141,19 +140,14 @@ public static class BeanM1 { } @Test(dataProvider = "basicMultiConfigFory") - public void testForyFieldAnnotation( - boolean trackingRef, - boolean codeGen, - boolean scopedMetaShare, - CompatibleMode compatibleMode) { + public void testForyFieldAnnotation(boolean trackingRef, boolean codeGen, boolean compatible) { Fory fory = Fory.builder() .withLanguage(Language.JAVA) .withRefTracking(trackingRef) .requireClassRegistration(false) .withCodegen(codeGen) - .withCompatibleMode(compatibleMode) - .withScopedMetaShare(scopedMetaShare) + .withCompatible(compatible) .build(); BeanM o = new BeanM(); byte[] bytes = fory.serialize(o); diff --git a/java/fory-core/src/test/java/org/apache/fory/annotation/ForyFieldTagIdTest.java b/java/fory-core/src/test/java/org/apache/fory/annotation/ForyFieldTagIdTest.java index d31856f6a6..1f5e1c0e04 100644 --- a/java/fory-core/src/test/java/org/apache/fory/annotation/ForyFieldTagIdTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/annotation/ForyFieldTagIdTest.java @@ -30,8 +30,8 @@ import org.apache.fory.Fory; import org.apache.fory.ForyTestBase; import org.apache.fory.config.Language; -import org.apache.fory.meta.ClassDef; import org.apache.fory.meta.FieldInfo; +import org.apache.fory.meta.TypeDef; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -65,8 +65,8 @@ public void testFieldInfoCreationWithTagIds( fory.register(TestClass.class, "test.TestClass"); } - ClassDef classDef = ClassDef.buildClassDef(fory, TestClass.class); - List fieldsInfo = classDef.getFieldsInfo(); + TypeDef typeDef = TypeDef.buildTypeDef(fory, TestClass.class); + List fieldsInfo = typeDef.getFieldsInfo(); // Should have 4 fields assertEquals(fieldsInfo.size(), 4); diff --git a/java/fory-core/src/test/java/org/apache/fory/builder/JITContextTest.java b/java/fory-core/src/test/java/org/apache/fory/builder/JITContextTest.java index 415d4ade4b..8201f0ad96 100644 --- a/java/fory-core/src/test/java/org/apache/fory/builder/JITContextTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/builder/JITContextTest.java @@ -52,23 +52,31 @@ public class JITContextTest extends ForyTestBase { public static Object[][] config1() { return Sets.cartesianProduct( ImmutableSet.of(true, false), // referenceTracking - ImmutableSet.of(true, false), // scoped meta share - ImmutableSet.of(CompatibleMode.COMPATIBLE, CompatibleMode.SCHEMA_CONSISTENT)) + ImmutableSet.of(true, false)) // compatible + .stream() + .map(List::toArray) + .toArray(Object[][]::new); + } + + @DataProvider + public static Object[][] config2() { + return Sets.cartesianProduct( + ImmutableSet.of(true, false), // referenceTracking + ImmutableSet.of(true, false), // compatible + ImmutableSet.of(true, false)) // scopedMetaShare .stream() .map(List::toArray) .toArray(Object[][]::new); } @Test(dataProvider = "config1", timeOut = 60_000) - public void testAsyncCompilation( - boolean referenceTracking, boolean scopedMetaShare, CompatibleMode compatibleMode) + public void testAsyncCompilation(boolean referenceTracking, boolean compatible) throws InterruptedException { Fory fory = Fory.builder() .withLanguage(Language.JAVA) .withRefTracking(referenceTracking) - .withCompatibleMode(compatibleMode) - .withScopedMetaShare(scopedMetaShare) + .withCompatible(compatible) .requireClassRegistration(false) .withAsyncCompilation(true) .build(); @@ -101,15 +109,15 @@ private Serializer getSerializer(Fory fory, Class cls) { } } - @Test(dataProvider = "config1", timeOut = 60_000) + @Test(dataProvider = "config2", timeOut = 60_000) public void testAsyncCompilationMetaShared( - boolean referenceTracking, boolean scopedMetaShare, CompatibleMode compatibleMode) + boolean referenceTracking, boolean compatible, boolean scopedMetaShare) throws InterruptedException { Fory fory = Fory.builder() .withLanguage(Language.JAVA) .withRefTracking(referenceTracking) - .withCompatibleMode(compatibleMode) + .withCompatible(compatible) .withScopedMetaShare(scopedMetaShare) .requireClassRegistration(false) .withAsyncCompilation(true) diff --git a/java/fory-core/src/test/java/org/apache/fory/meta/ClassDefTest.java b/java/fory-core/src/test/java/org/apache/fory/meta/ClassDefTest.java deleted file mode 100644 index 502d7d3e90..0000000000 --- a/java/fory-core/src/test/java/org/apache/fory/meta/ClassDefTest.java +++ /dev/null @@ -1,503 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.fory.meta; - -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertTrue; - -import com.google.common.collect.ImmutableList; -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.TreeSet; -import org.apache.fory.Fory; -import org.apache.fory.ForyTestBase; -import org.apache.fory.annotation.ForyField; -import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.reflect.ReflectionUtils; -import org.apache.fory.reflect.TypeRef; -import org.apache.fory.resolver.ClassResolver; -import org.apache.fory.test.bean.Foo; -import org.apache.fory.type.Descriptor; -import org.apache.fory.type.Types; -import org.testng.Assert; -import org.testng.annotations.Test; - -public class ClassDefTest extends ForyTestBase { - static class TestFieldsOrderClass1 { - private int intField2; - private boolean booleanField; - private Object objField; - private long longField; - } - - static class TestFieldsOrderClass2 extends TestFieldsOrderClass1 { - private int intField1; - private boolean booleanField; - private int childIntField2; - private boolean childBoolField1; - private byte childByteField; - private short childShortField; - private long childLongField; - } - - static class DuplicateFieldClass extends TestFieldsOrderClass1 { - private int intField1; - private boolean booleanField; - private Object objField; - private long longField; - } - - static class ContainerClass extends TestFieldsOrderClass1 { - private int intField1; - private long longField; - private Collection collection; - private List list1; - private List list2; - private List list3; - private Map map1; - private Map map2; - private Map map3; - } - - @Test - public void testFieldsOrder() { - List fieldList = new ArrayList<>(); - Collections.addAll(fieldList, TestFieldsOrderClass1.class.getDeclaredFields()); - Collections.addAll(fieldList, TestFieldsOrderClass2.class.getDeclaredFields()); - TreeSet sorted = new TreeSet<>(ClassDef.FIELD_COMPARATOR); - sorted.addAll(fieldList); - assertEquals(fieldList.size(), sorted.size()); - fieldList.sort(ClassDef.FIELD_COMPARATOR); - } - - @Test - public void testClassDefSerialization() throws NoSuchFieldException { - Fory fory = Fory.builder().withMetaShare(true).build(); - { - ClassDef classDef = - ClassDef.buildClassDef( - fory.getClassResolver(), - TestFieldsOrderClass1.class, - ImmutableList.of(TestFieldsOrderClass1.class.getDeclaredField("longField"))); - MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(32); - classDef.writeClassDef(buffer); - ClassDef classDef1 = ClassDef.readClassDef(fory, buffer); - assertEquals(classDef1.getClassName(), classDef.getClassName()); - assertEquals(classDef1, classDef); - } - { - ClassDef classDef = - ClassDef.buildClassDef( - fory.getClassResolver(), - TestFieldsOrderClass1.class, - ReflectionUtils.getFields(TestFieldsOrderClass1.class, true)); - assertEquals(classDef.getClassName(), TestFieldsOrderClass1.class.getName()); - assertEquals( - classDef.getFieldsInfo().size(), - ReflectionUtils.getFields(TestFieldsOrderClass1.class, true).size()); - MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(32); - classDef.writeClassDef(buffer); - ClassDef classDef1 = ClassDef.readClassDef(fory, buffer); - assertEquals(classDef1.getClassName(), classDef.getClassName()); - assertEquals(classDef1, classDef); - } - { - ClassDef classDef = - ClassDef.buildClassDef( - fory.getClassResolver(), - TestFieldsOrderClass2.class, - ReflectionUtils.getFields(TestFieldsOrderClass2.class, true)); - assertEquals(classDef.getClassName(), TestFieldsOrderClass2.class.getName()); - assertEquals( - classDef.getFieldsInfo().size(), - ReflectionUtils.getFields(TestFieldsOrderClass2.class, true).size()); - MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(32); - classDef.writeClassDef(buffer); - ClassDef classDef1 = ClassDef.readClassDef(fory, buffer); - assertEquals(classDef1.getClassName(), classDef.getClassName()); - assertEquals(classDef1, classDef); - } - } - - @Test - public void testDuplicateFieldsClass() { - Fory fory = Fory.builder().withMetaShare(true).build(); - { - ClassDef classDef = - ClassDef.buildClassDef( - fory.getClassResolver(), - DuplicateFieldClass.class, - ReflectionUtils.getFields(DuplicateFieldClass.class, true)); - assertEquals(classDef.getClassName(), DuplicateFieldClass.class.getName()); - assertEquals( - classDef.getFieldsInfo().size(), - ReflectionUtils.getFields(DuplicateFieldClass.class, true).size()); - MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(32); - classDef.writeClassDef(buffer); - ClassDef classDef1 = ClassDef.readClassDef(fory, buffer); - assertEquals(classDef1.getClassName(), classDef.getClassName()); - assertEquals(classDef1, classDef); - } - } - - @Test - public void testContainerClass() { - Fory fory = Fory.builder().withMetaShare(true).build(); - List fields = ReflectionUtils.getFields(ContainerClass.class, true); - ClassDef classDef = - ClassDef.buildClassDef(fory.getClassResolver(), ContainerClass.class, fields); - assertEquals(classDef.getClassName(), ContainerClass.class.getName()); - assertEquals(classDef.getFieldsInfo().size(), fields.size()); - MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(32); - classDef.writeClassDef(buffer); - ClassDef classDef1 = ClassDef.readClassDef(fory, buffer); - assertEquals(classDef1.getClassName(), classDef.getClassName()); - assertEquals(classDef1, classDef); - } - - @Test - public void testInterface() { - Fory fory = Fory.builder().withMetaShare(true).build(); - ClassDef classDef = ClassDef.buildClassDef(fory, Map.class); - assertTrue(classDef.getFieldsInfo().isEmpty()); - assertTrue(classDef.hasFieldsMeta()); - } - - @Test - public void testTypeExtInfo() { - Fory fory = Fory.builder().withRefTracking(true).withMetaShare(true).build(); - ClassResolver classResolver = fory.getClassResolver(); - assertTrue( - classResolver.needToWriteRef( - TypeRef.of(Foo.class, new TypeExtMeta(Types.STRUCT, true, true)))); - assertFalse( - classResolver.needToWriteRef( - TypeRef.of(Foo.class, new TypeExtMeta(Types.STRUCT, true, false)))); - } - - // Test classes for duplicate tag ID validation - static class ClassWithDuplicateTagIds { - @ForyField(id = 1) - private String field1; - - @ForyField(id = 1) - private String field2; - - @ForyField(id = 2) - private int field3; - } - - static class ClassWithDuplicateTagIdsMultiple { - @ForyField(id = 5) - private String field1; - - @ForyField(id = 5) - private String field2; - - @ForyField(id = 5) - private int field3; - } - - static class ClassWithValidTagIds { - @ForyField(id = 1) - private String field1; - - @ForyField(id = 2) - private String field2; - - @ForyField(id = 3) - private int field3; - } - - @Test - public void testDuplicateTagIdsThrowsException() { - Fory fory = Fory.builder().withMetaShare(true).build(); - List fields = ReflectionUtils.getFields(ClassWithDuplicateTagIds.class, true); - - Assert.assertThrows( - IllegalArgumentException.class, - () -> - ClassDef.buildClassDef( - fory.getClassResolver(), ClassWithDuplicateTagIds.class, fields)); - } - - @Test - public void testDuplicateTagIdsMultipleThrowsException() { - Fory fory = Fory.builder().withMetaShare(true).build(); - List fields = ReflectionUtils.getFields(ClassWithDuplicateTagIdsMultiple.class, true); - - Assert.assertThrows( - IllegalArgumentException.class, - () -> - ClassDef.buildClassDef( - fory.getClassResolver(), ClassWithDuplicateTagIdsMultiple.class, fields)); - } - - @Test - public void testValidTagIdsSucceeds() { - Fory fory = Fory.builder().withMetaShare(true).build(); - List fields = ReflectionUtils.getFields(ClassWithValidTagIds.class, true); - - // Should not throw any exception - ClassDef classDef = - ClassDef.buildClassDef(fory.getClassResolver(), ClassWithValidTagIds.class, fields); - assertEquals(classDef.getClassName(), ClassWithValidTagIds.class.getName()); - assertEquals(classDef.getFieldsInfo().size(), fields.size()); - } - - // Test classes for getDescriptors method - static class TargetClassWithDuplicateTagIds { - @ForyField(id = 100) - private String field1; - - @ForyField(id = 100) // Duplicate tag ID - private String field2; - - @ForyField(id = 200) - private int field3; - } - - static class TargetClassWithValidTags { - @ForyField(id = 10) - private String taggedField1; - - @ForyField(id = 20) - private int taggedField2; - - private String normalField; - } - - static class TargetClassWithMixedTags { - @ForyField(id = 50) - private String field1; - - private String field2; - - @ForyField(id = 60) - private int field3; - } - - @Test - public void testGetDescriptorsWithDuplicateTagIds() { - Fory fory = Fory.builder().withMetaShare(true).build(); - - // Build a ClassDef with valid fields (no duplicates in ClassDef itself) - List sourceFields = ReflectionUtils.getFields(ClassWithValidTagIds.class, true); - ClassDef classDef = - ClassDef.buildClassDef(fory.getClassResolver(), ClassWithValidTagIds.class, sourceFields); - - // Try to get descriptors for a class that has duplicate tag IDs - Assert.assertThrows( - IllegalArgumentException.class, - () -> - classDef.getDescriptors(fory.getClassResolver(), TargetClassWithDuplicateTagIds.class)); - } - - @Test - public void testGetDescriptorsWithValidTags() { - Fory fory = Fory.builder().withMetaShare(true).build(); - - // Build a ClassDef with tagged fields - List sourceFields = ReflectionUtils.getFields(TargetClassWithValidTags.class, true); - ClassDef classDef = - ClassDef.buildClassDef( - fory.getClassResolver(), TargetClassWithValidTags.class, sourceFields); - - // Get descriptors should succeed - List descriptors = - classDef.getDescriptors(fory.getClassResolver(), TargetClassWithValidTags.class); - - assertEquals(descriptors.size(), 3); - } - - @Test - public void testGetDescriptorsWithMixedTags() { - Fory fory = Fory.builder().withMetaShare(true).build(); - - // Build a ClassDef with mixed tagged and non-tagged fields - List sourceFields = ReflectionUtils.getFields(TargetClassWithMixedTags.class, true); - ClassDef classDef = - ClassDef.buildClassDef( - fory.getClassResolver(), TargetClassWithMixedTags.class, sourceFields); - - // Get descriptors should succeed - List descriptors = - classDef.getDescriptors(fory.getClassResolver(), TargetClassWithMixedTags.class); - - assertEquals(descriptors.size(), 3); - - // Verify that tagged fields are matched by tag, not by name - boolean foundField1 = false; - boolean foundField2 = false; - boolean foundField3 = false; - - for (Descriptor desc : descriptors) { - if (desc.getName().equals("field1")) { - foundField1 = true; - } else if (desc.getName().equals("field2")) { - foundField2 = true; - } else if (desc.getName().equals("field3")) { - foundField3 = true; - } - } - - assertTrue(foundField1); - assertTrue(foundField2); - assertTrue(foundField3); - } - - static class SourceClassWithTags { - @ForyField(id = 100) - private String renamedField; // This will be matched by tag ID - - @ForyField(id = 200) - private int anotherField; - } - - static class TargetClassWithDifferentNames { - @ForyField(id = 100) - private String differentName; // Same tag ID as renamedField - - @ForyField(id = 200) - private int alsoRenamed; // Same tag ID as anotherField - } - - @Test - public void testGetDescriptorsMatchesByTagNotName() { - Fory fory = Fory.builder().withMetaShare(true).build(); - - // Build a ClassDef from source class with specific tag IDs - List sourceFields = ReflectionUtils.getFields(SourceClassWithTags.class, true); - ClassDef classDef = - ClassDef.buildClassDef(fory.getClassResolver(), SourceClassWithTags.class, sourceFields); - - // Get descriptors for target class with different field names but same tag IDs - List descriptors = - classDef.getDescriptors(fory.getClassResolver(), TargetClassWithDifferentNames.class); - - // Should match fields by tag ID, not by name - assertEquals(descriptors.size(), 2); - - // Verify the descriptors were matched correctly (by tag, not name) - // When matched by tag, descriptors will have the target class field information - for (Descriptor desc : descriptors) { - // The descriptor should have the field from the target class since it was matched by tag - assertTrue( - desc.getName().equals("differentName") || desc.getName().equals("alsoRenamed"), - "Descriptor name should match target class field names when matched by tag ID"); - } - } - - static class TargetClassWithZeroTagId { - @ForyField(id = 0) - private String field1; - - @ForyField(id = 0) // Duplicate tag ID 0 - private String field2; - } - - @Test - public void testGetDescriptorsWithDuplicateZeroTagIds() { - Fory fory = Fory.builder().withMetaShare(true).build(); - - // Build a ClassDef with some fields - List sourceFields = ReflectionUtils.getFields(ClassWithValidTagIds.class, true); - ClassDef classDef = - ClassDef.buildClassDef(fory.getClassResolver(), ClassWithValidTagIds.class, sourceFields); - - // Try to get descriptors for a class that has duplicate tag ID 0 - Assert.assertThrows( - IllegalArgumentException.class, - () -> classDef.getDescriptors(fory.getClassResolver(), TargetClassWithZeroTagId.class)); - } - - static class EmptyClass { - // No fields - } - - @Test - public void testGetDescriptorsWithEmptyClass() { - Fory fory = Fory.builder().withMetaShare(true).build(); - - // Build a ClassDef with no fields - List sourceFields = ReflectionUtils.getFields(EmptyClass.class, true); - ClassDef classDef = - ClassDef.buildClassDef(fory.getClassResolver(), EmptyClass.class, sourceFields); - - // Get descriptors should succeed and return empty list - List descriptors = - classDef.getDescriptors(fory.getClassResolver(), EmptyClass.class); - - assertEquals(descriptors.size(), 0); - } - - static class InheritedBaseClass { - @ForyField(id = 10) - private String baseField; - } - - static class InheritedChildClass extends InheritedBaseClass { - @ForyField(id = 20) - private String childField; - } - - static class InheritedChildWithDuplicateTag extends InheritedBaseClass { - @ForyField(id = 10) // Duplicate with baseField - private String childField; - } - - @Test - public void testGetDescriptorsWithInheritance() { - Fory fory = Fory.builder().withMetaShare(true).build(); - - // Build a ClassDef with inherited fields - List sourceFields = ReflectionUtils.getFields(InheritedChildClass.class, true); - ClassDef classDef = - ClassDef.buildClassDef(fory.getClassResolver(), InheritedChildClass.class, sourceFields); - - // Get descriptors should succeed - List descriptors = - classDef.getDescriptors(fory.getClassResolver(), InheritedChildClass.class); - - // Should have both base and child fields - assertEquals(descriptors.size(), 2); - } - - @Test - public void testGetDescriptorsWithInheritedDuplicateTag() { - Fory fory = Fory.builder().withMetaShare(true).build(); - - // Build a ClassDef with some fields - List sourceFields = ReflectionUtils.getFields(InheritedBaseClass.class, true); - ClassDef classDef = - ClassDef.buildClassDef(fory.getClassResolver(), InheritedBaseClass.class, sourceFields); - - // Try to get descriptors for a class that has duplicate tag ID across inheritance - Assert.assertThrows( - IllegalArgumentException.class, - () -> - classDef.getDescriptors(fory.getClassResolver(), InheritedChildWithDuplicateTag.class)); - } -} diff --git a/java/fory-core/src/test/java/org/apache/fory/meta/ClassDefEncoderTest.java b/java/fory-core/src/test/java/org/apache/fory/meta/NativeTypeDefEncoderTest.java similarity index 74% rename from java/fory-core/src/test/java/org/apache/fory/meta/ClassDefEncoderTest.java rename to java/fory-core/src/test/java/org/apache/fory/meta/NativeTypeDefEncoderTest.java index 0b33afbfb7..dfae1a8f49 100644 --- a/java/fory-core/src/test/java/org/apache/fory/meta/ClassDefEncoderTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/meta/NativeTypeDefEncoderTest.java @@ -19,8 +19,8 @@ package org.apache.fory.meta; -import static org.apache.fory.meta.ClassDefEncoder.buildFieldsInfo; -import static org.apache.fory.meta.ClassDefEncoder.getClassFields; +import static org.apache.fory.meta.NativeTypeDefEncoder.buildFieldsInfo; +import static org.apache.fory.meta.NativeTypeDefEncoder.getClassFields; import java.io.Serializable; import java.util.List; @@ -36,20 +36,20 @@ import org.testng.Assert; import org.testng.annotations.Test; -public class ClassDefEncoderTest { +public class NativeTypeDefEncoderTest { @Test - public void testBasicClassDef() { + public void testBasicTypeDef() { Fory fory = Fory.builder().withMetaShare(true).build(); - Class type = ClassDefTest.TestFieldsOrderClass1.class; + Class type = TypeDefTest.TestFieldsOrderClass1.class; List fieldsInfo = buildFieldsInfo(fory.getClassResolver(), type); MemoryBuffer buffer = - ClassDefEncoder.encodeClassDef( + NativeTypeDefEncoder.encodeTypeDef( fory.getClassResolver(), type, getClassFields(type, fieldsInfo), true); - ClassDef classDef = ClassDef.readClassDef(fory, buffer); - Assert.assertEquals(classDef.getClassName(), type.getName()); - Assert.assertEquals(classDef.getFieldsInfo().size(), type.getDeclaredFields().length); - Assert.assertEquals(classDef.getFieldsInfo(), fieldsInfo); + TypeDef typeDef = TypeDef.readTypeDef(fory, buffer); + Assert.assertEquals(typeDef.getClassName(), type.getName()); + Assert.assertEquals(typeDef.getFieldsInfo().size(), type.getDeclaredFields().length); + Assert.assertEquals(typeDef.getFieldsInfo(), fieldsInfo); } @Test @@ -59,10 +59,10 @@ public void testBigMetaEncoding() { MapFields.class, BeanA.class, Struct.createStructClass("TestBigMetaEncoding", 5) }) { Fory fory = Fory.builder().withMetaShare(true).build(); - ClassDef classDef = ClassDef.buildClassDef(fory, type); - ClassDef classDef1 = - ClassDef.readClassDef(fory, MemoryBuffer.fromByteArray(classDef.getEncoded())); - Assert.assertEquals(classDef1, classDef); + TypeDef typeDef = TypeDef.buildTypeDef(fory, type); + TypeDef typeDef1 = + TypeDef.readTypeDef(fory, MemoryBuffer.fromByteArray(typeDef.getEncoded())); + Assert.assertEquals(typeDef1, typeDef); } } @@ -76,23 +76,21 @@ public static class Foo2 extends Foo1 {} @Test public void testEmptySubClassSerializer() { Fory fory = Fory.builder().withLanguage(Language.JAVA).requireClassRegistration(true).build(); - ClassDef classDef = ClassDef.buildClassDef(fory, Foo2.class); - ClassDef classDef1 = - ClassDef.readClassDef(fory, MemoryBuffer.fromByteArray(classDef.getEncoded())); - Assert.assertEquals(classDef, classDef1); + TypeDef typeDef = TypeDef.buildTypeDef(fory, Foo2.class); + TypeDef typeDef1 = TypeDef.readTypeDef(fory, MemoryBuffer.fromByteArray(typeDef.getEncoded())); + Assert.assertEquals(typeDef, typeDef1); } @Test public void testBigClassNameObject() { Fory fory = Fory.builder().withMetaShare(true).build(); - ClassDef classDef = - ClassDef.buildClassDef( + TypeDef typeDef = + TypeDef.buildTypeDef( fory, TestClassLengthTestClassLengthTestClassLengthTestClassLengthTestClassLengthTestClassLengthTestClassLength .InnerClassTestLengthInnerClassTestLengthInnerClassTestLength.class); - ClassDef classDef1 = - ClassDef.readClassDef(fory, MemoryBuffer.fromByteArray(classDef.getEncoded())); - Assert.assertEquals(classDef1, classDef); + TypeDef typeDef1 = TypeDef.readTypeDef(fory, MemoryBuffer.fromByteArray(typeDef.getEncoded())); + Assert.assertEquals(typeDef1, typeDef); } @Data @@ -112,14 +110,14 @@ public static class InnerClassTestLengthInnerClassTestLengthInnerClassTestLength @Test public void testPrependHeader() { - MemoryBuffer inputBuffer = MemoryBuffer.newHeapBuffer(ClassDef.META_SIZE_MASKS + 1); - inputBuffer.writerIndex(ClassDef.META_SIZE_MASKS + 1); - MemoryBuffer outputBuffer = ClassDefEncoder.prependHeader(inputBuffer, true, false); + MemoryBuffer inputBuffer = MemoryBuffer.newHeapBuffer(TypeDef.META_SIZE_MASKS + 1); + inputBuffer.writerIndex(TypeDef.META_SIZE_MASKS + 1); + MemoryBuffer outputBuffer = NativeTypeDefEncoder.prependHeader(inputBuffer, true, false); long header = outputBuffer.readInt64(); - Assert.assertEquals(header & ClassDef.META_SIZE_MASKS, ClassDef.META_SIZE_MASKS); - Assert.assertEquals(header & ClassDef.COMPRESS_META_FLAG, ClassDef.COMPRESS_META_FLAG); - Assert.assertEquals(header & ClassDef.HAS_FIELDS_META_FLAG, 0); + Assert.assertEquals(header & TypeDef.META_SIZE_MASKS, TypeDef.META_SIZE_MASKS); + Assert.assertEquals(header & TypeDef.COMPRESS_META_FLAG, TypeDef.COMPRESS_META_FLAG); + Assert.assertEquals(header & TypeDef.HAS_FIELDS_META_FLAG, 0); } @Test @@ -140,10 +138,10 @@ public void testAbstractParentClass() { fory1.register(BaseAbstractClass.class); fory1.register(ChildClass.class); for (Fory fory : new Fory[] {fory0, fory1}) { - ClassDef classDef = ClassDef.buildClassDef(fory, ChildClass.class); - ClassDef classDef1 = - ClassDef.readClassDef(fory, MemoryBuffer.fromByteArray(classDef.getEncoded())); - Assert.assertEquals(classDef, classDef1); + TypeDef typeDef = TypeDef.buildTypeDef(fory, ChildClass.class); + TypeDef typeDef1 = + TypeDef.readTypeDef(fory, MemoryBuffer.fromByteArray(typeDef.getEncoded())); + Assert.assertEquals(typeDef, typeDef1); ChildClass c = new ChildClass(); c.setId("123"); c.setName("test"); @@ -172,7 +170,7 @@ public static class ChildClass extends BaseAbstractClass { private String name; } - // Test classes for duplicate tag ID validation in ClassDefEncoder + // Test classes for duplicate tag ID validation in TypeDefEncoder @Data public static class ClassWithDuplicateTagIds { @ForyField(id = 10) diff --git a/java/fory-core/src/test/java/org/apache/fory/meta/TypeDefTest.java b/java/fory-core/src/test/java/org/apache/fory/meta/TypeDefTest.java index 81c4e88450..623c4be083 100644 --- a/java/fory-core/src/test/java/org/apache/fory/meta/TypeDefTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/meta/TypeDefTest.java @@ -20,58 +20,521 @@ package org.apache.fory.meta; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; +import com.google.common.collect.ImmutableList; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.TreeSet; import org.apache.fory.Fory; import org.apache.fory.ForyTestBase; +import org.apache.fory.annotation.ForyField; import org.apache.fory.config.Language; import org.apache.fory.data.AllUnsignedFields; import org.apache.fory.data.UnsignedArrayFields; import org.apache.fory.data.UnsignedScalarFields; import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.meta.ClassDefTest.TestFieldsOrderClass1; -import org.apache.fory.meta.ClassDefTest.TestFieldsOrderClass2; import org.apache.fory.reflect.ReflectionUtils; +import org.apache.fory.reflect.TypeRef; +import org.apache.fory.resolver.ClassResolver; +import org.apache.fory.test.bean.Foo; +import org.apache.fory.type.Descriptor; import org.apache.fory.type.Types; +import org.testng.Assert; import org.testng.annotations.Test; public class TypeDefTest extends ForyTestBase { + static class TestFieldsOrderClass1 { + private int intField2; + private boolean booleanField; + private Object objField; + private long longField; + } + + static class TestFieldsOrderClass2 extends TestFieldsOrderClass1 { + private int intField1; + private boolean booleanField; + private int childIntField2; + private boolean childBoolField1; + private byte childByteField; + private short childShortField; + private long childLongField; + } + + static class DuplicateFieldClass extends TestFieldsOrderClass1 { + private int intField1; + private boolean booleanField; + private Object objField; + private long longField; + } + + static class ContainerClass extends TestFieldsOrderClass1 { + private int intField1; + private long longField; + private Collection collection; + private List list1; + private List list2; + private List list3; + private Map map1; + private Map map2; + private Map map3; + } + + @Test + public void testFieldsOrder() { + List fieldList = new ArrayList<>(); + Collections.addAll(fieldList, TestFieldsOrderClass1.class.getDeclaredFields()); + Collections.addAll(fieldList, TestFieldsOrderClass2.class.getDeclaredFields()); + TreeSet sorted = new TreeSet<>(TypeDef.FIELD_COMPARATOR); + sorted.addAll(fieldList); + assertEquals(fieldList.size(), sorted.size()); + fieldList.sort(TypeDef.FIELD_COMPARATOR); + } + + @Test + public void testTypeDefSerialization() throws NoSuchFieldException { + Fory fory = Fory.builder().withMetaShare(true).build(); + { + TypeDef typeDef = + TypeDef.buildTypeDef( + fory.getClassResolver(), + TestFieldsOrderClass1.class, + ImmutableList.of(TestFieldsOrderClass1.class.getDeclaredField("longField"))); + MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(32); + typeDef.writeTypeDef(buffer); + TypeDef typeDef1 = TypeDef.readTypeDef(fory, buffer); + assertEquals(typeDef1.getClassName(), typeDef.getClassName()); + assertEquals(typeDef1, typeDef); + } + { + TypeDef typeDef = + TypeDef.buildTypeDef( + fory.getClassResolver(), + TestFieldsOrderClass1.class, + ReflectionUtils.getFields(TestFieldsOrderClass1.class, true)); + assertEquals(typeDef.getClassName(), TestFieldsOrderClass1.class.getName()); + assertEquals( + typeDef.getFieldsInfo().size(), + ReflectionUtils.getFields(TestFieldsOrderClass1.class, true).size()); + MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(32); + typeDef.writeTypeDef(buffer); + TypeDef typeDef1 = TypeDef.readTypeDef(fory, buffer); + assertEquals(typeDef1.getClassName(), typeDef.getClassName()); + assertEquals(typeDef1, typeDef); + } + { + TypeDef typeDef = + TypeDef.buildTypeDef( + fory.getClassResolver(), + TestFieldsOrderClass2.class, + ReflectionUtils.getFields(TestFieldsOrderClass2.class, true)); + assertEquals(typeDef.getClassName(), TestFieldsOrderClass2.class.getName()); + assertEquals( + typeDef.getFieldsInfo().size(), + ReflectionUtils.getFields(TestFieldsOrderClass2.class, true).size()); + MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(32); + typeDef.writeTypeDef(buffer); + TypeDef typeDef1 = TypeDef.readTypeDef(fory, buffer); + assertEquals(typeDef1.getClassName(), typeDef.getClassName()); + assertEquals(typeDef1, typeDef); + } + } + + @Test + public void testDuplicateFieldsClass() { + Fory fory = Fory.builder().withMetaShare(true).build(); + { + TypeDef typeDef = + TypeDef.buildTypeDef( + fory.getClassResolver(), + DuplicateFieldClass.class, + ReflectionUtils.getFields(DuplicateFieldClass.class, true)); + assertEquals(typeDef.getClassName(), DuplicateFieldClass.class.getName()); + assertEquals( + typeDef.getFieldsInfo().size(), + ReflectionUtils.getFields(DuplicateFieldClass.class, true).size()); + MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(32); + typeDef.writeTypeDef(buffer); + TypeDef typeDef1 = TypeDef.readTypeDef(fory, buffer); + assertEquals(typeDef1.getClassName(), typeDef.getClassName()); + assertEquals(typeDef1, typeDef); + } + } + + @Test + public void testContainerClass() { + Fory fory = Fory.builder().withMetaShare(true).build(); + List fields = ReflectionUtils.getFields(ContainerClass.class, true); + TypeDef typeDef = TypeDef.buildTypeDef(fory.getClassResolver(), ContainerClass.class, fields); + assertEquals(typeDef.getClassName(), ContainerClass.class.getName()); + assertEquals(typeDef.getFieldsInfo().size(), fields.size()); + MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(32); + typeDef.writeTypeDef(buffer); + TypeDef typeDef1 = TypeDef.readTypeDef(fory, buffer); + assertEquals(typeDef1.getClassName(), typeDef.getClassName()); + assertEquals(typeDef1, typeDef); + } + + @Test + public void testInterface() { + Fory fory = Fory.builder().withMetaShare(true).build(); + TypeDef typeDef = TypeDef.buildTypeDef(fory, Map.class); + assertTrue(typeDef.getFieldsInfo().isEmpty()); + assertTrue(typeDef.hasFieldsMeta()); + } + + @Test + public void testTypeExtInfo() { + Fory fory = Fory.builder().withRefTracking(true).withMetaShare(true).build(); + ClassResolver classResolver = fory.getClassResolver(); + assertTrue( + classResolver.needToWriteRef( + TypeRef.of(Foo.class, new TypeExtMeta(Types.STRUCT, true, true)))); + assertFalse( + classResolver.needToWriteRef( + TypeRef.of(Foo.class, new TypeExtMeta(Types.STRUCT, true, false)))); + } + + // Test classes for duplicate tag ID validation + static class ClassWithDuplicateTagIds { + @ForyField(id = 1) + private String field1; + + @ForyField(id = 1) + private String field2; + + @ForyField(id = 2) + private int field3; + } + + static class ClassWithDuplicateTagIdsMultiple { + @ForyField(id = 5) + private String field1; + + @ForyField(id = 5) + private String field2; + + @ForyField(id = 5) + private int field3; + } + + static class ClassWithValidTagIds { + @ForyField(id = 1) + private String field1; + + @ForyField(id = 2) + private String field2; + + @ForyField(id = 3) + private int field3; + } + + @Test + public void testDuplicateTagIdsThrowsException() { + Fory fory = Fory.builder().withMetaShare(true).build(); + List fields = ReflectionUtils.getFields(ClassWithDuplicateTagIds.class, true); + + Assert.assertThrows( + IllegalArgumentException.class, + () -> + TypeDef.buildTypeDef(fory.getClassResolver(), ClassWithDuplicateTagIds.class, fields)); + } + + @Test + public void testDuplicateTagIdsMultipleThrowsException() { + Fory fory = Fory.builder().withMetaShare(true).build(); + List fields = ReflectionUtils.getFields(ClassWithDuplicateTagIdsMultiple.class, true); + + Assert.assertThrows( + IllegalArgumentException.class, + () -> + TypeDef.buildTypeDef( + fory.getClassResolver(), ClassWithDuplicateTagIdsMultiple.class, fields)); + } + + @Test + public void testValidTagIdsSucceeds() { + Fory fory = Fory.builder().withMetaShare(true).build(); + List fields = ReflectionUtils.getFields(ClassWithValidTagIds.class, true); + + // Should not throw any exception + TypeDef typeDef = + TypeDef.buildTypeDef(fory.getClassResolver(), ClassWithValidTagIds.class, fields); + assertEquals(typeDef.getClassName(), ClassWithValidTagIds.class.getName()); + assertEquals(typeDef.getFieldsInfo().size(), fields.size()); + } + + // Test classes for getDescriptors method + static class TargetClassWithDuplicateTagIds { + @ForyField(id = 100) + private String field1; + + @ForyField(id = 100) // Duplicate tag ID + private String field2; + + @ForyField(id = 200) + private int field3; + } + + static class TargetClassWithValidTags { + @ForyField(id = 10) + private String taggedField1; + + @ForyField(id = 20) + private int taggedField2; + + private String normalField; + } + + static class TargetClassWithMixedTags { + @ForyField(id = 50) + private String field1; + + private String field2; + + @ForyField(id = 60) + private int field3; + } + + @Test + public void testGetDescriptorsWithDuplicateTagIds() { + Fory fory = Fory.builder().withMetaShare(true).build(); + + // Build a TypeDef with valid fields (no duplicates in TypeDef itself) + List sourceFields = ReflectionUtils.getFields(ClassWithValidTagIds.class, true); + TypeDef typeDef = + TypeDef.buildTypeDef(fory.getClassResolver(), ClassWithValidTagIds.class, sourceFields); + + // Try to get descriptors for a class that has duplicate tag IDs + Assert.assertThrows( + IllegalArgumentException.class, + () -> + typeDef.getDescriptors(fory.getClassResolver(), TargetClassWithDuplicateTagIds.class)); + } + + @Test + public void testGetDescriptorsWithValidTags() { + Fory fory = Fory.builder().withMetaShare(true).build(); + + // Build a TypeDef with tagged fields + List sourceFields = ReflectionUtils.getFields(TargetClassWithValidTags.class, true); + TypeDef typeDef = + TypeDef.buildTypeDef(fory.getClassResolver(), TargetClassWithValidTags.class, sourceFields); + + // Get descriptors should succeed + List descriptors = + typeDef.getDescriptors(fory.getClassResolver(), TargetClassWithValidTags.class); + + assertEquals(descriptors.size(), 3); + } + + @Test + public void testGetDescriptorsWithMixedTags() { + Fory fory = Fory.builder().withMetaShare(true).build(); + + // Build a TypeDef with mixed tagged and non-tagged fields + List sourceFields = ReflectionUtils.getFields(TargetClassWithMixedTags.class, true); + TypeDef typeDef = + TypeDef.buildTypeDef(fory.getClassResolver(), TargetClassWithMixedTags.class, sourceFields); + + // Get descriptors should succeed + List descriptors = + typeDef.getDescriptors(fory.getClassResolver(), TargetClassWithMixedTags.class); + + assertEquals(descriptors.size(), 3); + + // Verify that tagged fields are matched by tag, not by name + boolean foundField1 = false; + boolean foundField2 = false; + boolean foundField3 = false; + + for (Descriptor desc : descriptors) { + if (desc.getName().equals("field1")) { + foundField1 = true; + } else if (desc.getName().equals("field2")) { + foundField2 = true; + } else if (desc.getName().equals("field3")) { + foundField3 = true; + } + } + + assertTrue(foundField1); + assertTrue(foundField2); + assertTrue(foundField3); + } + + static class SourceClassWithTags { + @ForyField(id = 100) + private String renamedField; // This will be matched by tag ID + + @ForyField(id = 200) + private int anotherField; + } + + static class TargetClassWithDifferentNames { + @ForyField(id = 100) + private String differentName; // Same tag ID as renamedField + + @ForyField(id = 200) + private int alsoRenamed; // Same tag ID as anotherField + } + + @Test + public void testGetDescriptorsMatchesByTagNotName() { + Fory fory = Fory.builder().withMetaShare(true).build(); + + // Build a TypeDef from source class with specific tag IDs + List sourceFields = ReflectionUtils.getFields(SourceClassWithTags.class, true); + TypeDef typeDef = + TypeDef.buildTypeDef(fory.getClassResolver(), SourceClassWithTags.class, sourceFields); + + // Get descriptors for target class with different field names but same tag IDs + List descriptors = + typeDef.getDescriptors(fory.getClassResolver(), TargetClassWithDifferentNames.class); + + // Should match fields by tag ID, not by name + assertEquals(descriptors.size(), 2); + + // Verify the descriptors were matched correctly (by tag, not name) + // When matched by tag, descriptors will have the target class field information + for (Descriptor desc : descriptors) { + // The descriptor should have the field from the target class since it was matched by tag + assertTrue( + desc.getName().equals("differentName") || desc.getName().equals("alsoRenamed"), + "Descriptor name should match target class field names when matched by tag ID"); + } + } + + static class TargetClassWithZeroTagId { + @ForyField(id = 0) + private String field1; + + @ForyField(id = 0) // Duplicate tag ID 0 + private String field2; + } + + @Test + public void testGetDescriptorsWithDuplicateZeroTagIds() { + Fory fory = Fory.builder().withMetaShare(true).build(); + + // Build a TypeDef with some fields + List sourceFields = ReflectionUtils.getFields(ClassWithValidTagIds.class, true); + TypeDef typeDef = + TypeDef.buildTypeDef(fory.getClassResolver(), ClassWithValidTagIds.class, sourceFields); + + // Try to get descriptors for a class that has duplicate tag ID 0 + Assert.assertThrows( + IllegalArgumentException.class, + () -> typeDef.getDescriptors(fory.getClassResolver(), TargetClassWithZeroTagId.class)); + } + + static class EmptyClass { + // No fields + } + + @Test + public void testGetDescriptorsWithEmptyClass() { + Fory fory = Fory.builder().withMetaShare(true).build(); + + // Build a TypeDef with no fields + List sourceFields = ReflectionUtils.getFields(EmptyClass.class, true); + TypeDef typeDef = TypeDef.buildTypeDef(fory.getClassResolver(), EmptyClass.class, sourceFields); + + // Get descriptors should succeed and return empty list + List descriptors = + typeDef.getDescriptors(fory.getClassResolver(), EmptyClass.class); + + assertEquals(descriptors.size(), 0); + } + + static class InheritedBaseClass { + @ForyField(id = 10) + private String baseField; + } + + static class InheritedChildClass extends InheritedBaseClass { + @ForyField(id = 20) + private String childField; + } + + static class InheritedChildWithDuplicateTag extends InheritedBaseClass { + @ForyField(id = 10) // Duplicate with baseField + private String childField; + } + + @Test + public void testGetDescriptorsWithInheritance() { + Fory fory = Fory.builder().withMetaShare(true).build(); + + // Build a TypeDef with inherited fields + List sourceFields = ReflectionUtils.getFields(InheritedChildClass.class, true); + TypeDef typeDef = + TypeDef.buildTypeDef(fory.getClassResolver(), InheritedChildClass.class, sourceFields); + + // Get descriptors should succeed + List descriptors = + typeDef.getDescriptors(fory.getClassResolver(), InheritedChildClass.class); + + // Should have both base and child fields + assertEquals(descriptors.size(), 2); + } + + @Test + public void testGetDescriptorsWithInheritedDuplicateTag() { + Fory fory = Fory.builder().withMetaShare(true).build(); + + // Build a TypeDef with some fields + List sourceFields = ReflectionUtils.getFields(InheritedBaseClass.class, true); + TypeDef typeDef = + TypeDef.buildTypeDef(fory.getClassResolver(), InheritedBaseClass.class, sourceFields); + + // Try to get descriptors for a class that has duplicate tag ID across inheritance + Assert.assertThrows( + IllegalArgumentException.class, + () -> + typeDef.getDescriptors(fory.getClassResolver(), InheritedChildWithDuplicateTag.class)); + } @Test - public void testClassDefSerialization() { + public void testTypeDefSerializationBasic() { Fory fory = builder().withLanguage(Language.XLANG).withMetaShare(true).build(); fory.register(TestFieldsOrderClass1.class, "demo.Class1"); - ClassDef classDef = ClassDef.buildClassDef(fory, TestFieldsOrderClass1.class, true); + TypeDef typeDef = TypeDef.buildTypeDef(fory, TestFieldsOrderClass1.class, true); MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(32); - classDef.writeClassDef(buffer); - ClassDef classDef1 = ClassDef.readClassDef(fory, buffer); - assertEquals(classDef1.getClassName(), classDef.getClassName()); - assertEquals(classDef1, classDef); + typeDef.writeTypeDef(buffer); + TypeDef typeDef1 = TypeDef.readTypeDef(fory, buffer); + assertEquals(typeDef1.getClassName(), typeDef.getClassName()); + assertEquals(typeDef1, typeDef); } @Test - public void testClassDefInheritanceDuplicatedFields() { + public void testTypeDefInheritanceDuplicatedFields() { Fory fory = builder().withLanguage(Language.XLANG).withMetaShare(true).build(); fory.register(TestFieldsOrderClass2.class, "demo.Class2"); - ClassDef classDef = ClassDef.buildClassDef(fory, TestFieldsOrderClass2.class); - assertEquals(classDef.getClassName(), TestFieldsOrderClass2.class.getName()); + TypeDef typeDef = TypeDef.buildTypeDef(fory, TestFieldsOrderClass2.class); + assertEquals(typeDef.getClassName(), TestFieldsOrderClass2.class.getName()); // xtype ignore duplicate fields from parent class. assertEquals( - classDef.getFieldsInfo().size(), + typeDef.getFieldsInfo().size(), ReflectionUtils.getFields(TestFieldsOrderClass2.class, true).size() - 1); MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(32); - classDef.writeClassDef(buffer); - ClassDef classDef1 = ClassDef.readClassDef(fory, buffer); - assertEquals(classDef1.getClassName(), classDef.getClassName()); - assertEquals(classDef1, classDef); + typeDef.writeTypeDef(buffer); + TypeDef typeDef1 = TypeDef.readTypeDef(fory, buffer); + assertEquals(typeDef1.getClassName(), typeDef.getClassName()); + assertEquals(typeDef1, typeDef); } @Test public void testUnsignedScalarFieldsTypeIds() { Fory fory = builder().withLanguage(Language.XLANG).withMetaShare(true).build(); fory.register(UnsignedScalarFields.class, "test.UnsignedScalarFields"); - ClassDef classDef = ClassDef.buildClassDef(fory, UnsignedScalarFields.class); + TypeDef typeDef = TypeDef.buildTypeDef(fory, UnsignedScalarFields.class); // Build expected type IDs map: field name -> type id Map expectedTypeIds = new HashMap<>(); @@ -83,7 +546,7 @@ public void testUnsignedScalarFieldsTypeIds() { expectedTypeIds.put("u64Var", Types.VAR_UINT64); expectedTypeIds.put("u64Tagged", Types.TAGGED_UINT64); - for (FieldInfo fieldInfo : classDef.getFieldsInfo()) { + for (FieldInfo fieldInfo : typeDef.getFieldsInfo()) { String fieldName = fieldInfo.getFieldName(); int actualTypeId = fieldInfo.getFieldType().typeId; Integer expectedTypeId = expectedTypeIds.get(fieldName); @@ -98,7 +561,7 @@ public void testUnsignedScalarFieldsTypeIds() { public void testUnsignedArrayFieldsTypeIds() { Fory fory = builder().withLanguage(Language.XLANG).withMetaShare(true).build(); fory.register(UnsignedArrayFields.class, "test.UnsignedArrayFields"); - ClassDef classDef = ClassDef.buildClassDef(fory, UnsignedArrayFields.class); + TypeDef typeDef = TypeDef.buildTypeDef(fory, UnsignedArrayFields.class); // Build expected type IDs map: field name -> type id Map expectedTypeIds = new HashMap<>(); @@ -107,7 +570,7 @@ public void testUnsignedArrayFieldsTypeIds() { expectedTypeIds.put("u32Array", Types.UINT32_ARRAY); expectedTypeIds.put("u64Array", Types.UINT64_ARRAY); - for (FieldInfo fieldInfo : classDef.getFieldsInfo()) { + for (FieldInfo fieldInfo : typeDef.getFieldsInfo()) { String fieldName = fieldInfo.getFieldName(); int actualTypeId = fieldInfo.getFieldType().typeId; Integer expectedTypeId = expectedTypeIds.get(fieldName); @@ -122,7 +585,7 @@ public void testUnsignedArrayFieldsTypeIds() { public void testAllUnsignedFieldsTypeIds() { Fory fory = builder().withLanguage(Language.XLANG).withMetaShare(true).build(); fory.register(AllUnsignedFields.class, "test.AllUnsignedFields"); - ClassDef classDef = ClassDef.buildClassDef(fory, AllUnsignedFields.class); + TypeDef typeDef = TypeDef.buildTypeDef(fory, AllUnsignedFields.class); // Build expected type IDs map: field name -> type id Map expectedTypeIds = new HashMap<>(); @@ -137,7 +600,7 @@ public void testAllUnsignedFieldsTypeIds() { expectedTypeIds.put("u32Array", Types.UINT32_ARRAY); expectedTypeIds.put("u64Array", Types.UINT64_ARRAY); - for (FieldInfo fieldInfo : classDef.getFieldsInfo()) { + for (FieldInfo fieldInfo : typeDef.getFieldsInfo()) { String fieldName = fieldInfo.getFieldName(); int actualTypeId = fieldInfo.getFieldType().typeId; Integer expectedTypeId = expectedTypeIds.get(fieldName); diff --git a/java/fory-core/src/test/java/org/apache/fory/resolver/ClassResolverTest.java b/java/fory-core/src/test/java/org/apache/fory/resolver/ClassResolverTest.java index aacd2c1f0a..7c0581c19c 100644 --- a/java/fory-core/src/test/java/org/apache/fory/resolver/ClassResolverTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/resolver/ClassResolverTest.java @@ -104,10 +104,12 @@ public void testRegisterClassByName() { classResolver.register(C2.class, "", "C2"); classResolver.register(Foo.class, "ns", "Foo"); - Assert.assertTrue(fory.serialize(C1.class).length < 12); + byte[] serialized = fory.serialize(C1.class); + Assert.assertTrue(serialized.length < 13, "Serialize length " + serialized.length); serDeCheck(fory, C1.class); - Assert.assertTrue(fory.serialize(C2.class).length < 12); + serialized = fory.serialize(C2.class); + Assert.assertTrue(serialized.length < 13, "Serialize length " + serialized.length); serDeCheck(fory, C2.class); Foo foo = new Foo(); @@ -122,7 +124,7 @@ public void testRegisterClass() { @Test public void testRegisterClassWithUserIds() { - // Test that user IDs 0 and 1 work correctly with unified type IDs. + // Test that user IDs 0 and 1 work correctly with separated user type IDs. Fory fory = Fory.builder().withLanguage(Language.JAVA).requireClassRegistration(true).build(); ClassResolver classResolver = fory.getClassResolver(); @@ -131,17 +133,17 @@ public void testRegisterClassWithUserIds() { // Register with user ID 1 classResolver.register(Bar.class, 1); - // Verify registered IDs are user IDs and type IDs encode user ID and internal type. + // Verify registered IDs are user IDs and type IDs are internal types only. assertEquals(classResolver.getRegisteredClassId(Foo.class).shortValue(), (short) 0); assertEquals(classResolver.getRegisteredClassId(Bar.class).shortValue(), (short) 1); - int fooTypeId = classResolver.getClassInfo(Foo.class).getTypeId(); - int barTypeId = classResolver.getClassInfo(Bar.class).getTypeId(); - assertEquals(fooTypeId >>> 8, 0); - assertEquals(barTypeId >>> 8, 1); - int fooInternalType = fooTypeId & 0xff; - int barInternalType = barTypeId & 0xff; - assertTrue(fooInternalType == Types.STRUCT || fooInternalType == Types.COMPATIBLE_STRUCT); - assertTrue(barInternalType == Types.STRUCT || barInternalType == Types.COMPATIBLE_STRUCT); + TypeInfo fooTypeInfo = classResolver.getTypeInfo(Foo.class); + TypeInfo barTypeInfo = classResolver.getTypeInfo(Bar.class); + assertEquals(fooTypeInfo.getUserTypeId(), 0); + assertEquals(barTypeInfo.getUserTypeId(), 1); + int fooTypeId = fooTypeInfo.getTypeId(); + int barTypeId = barTypeInfo.getTypeId(); + assertTrue(fooTypeId == Types.STRUCT || fooTypeId == Types.COMPATIBLE_STRUCT); + assertTrue(barTypeId == Types.STRUCT || barTypeId == Types.COMPATIBLE_STRUCT); // Verify serialization/deserialization works Foo foo = new Foo(); @@ -277,8 +279,8 @@ public void testWriteClassName() { MemoryBuffer buffer = MemoryUtils.buffer(32); classResolver.writeClassAndUpdateCache(buffer, getClass()); classResolver.writeClassAndUpdateCache(buffer, getClass()); - Assert.assertSame(classResolver.readClassInfo(buffer).getCls(), getClass()); - Assert.assertSame(classResolver.readClassInfo(buffer).getCls(), getClass()); + Assert.assertSame(classResolver.readTypeInfo(buffer).getCls(), getClass()); + Assert.assertSame(classResolver.readTypeInfo(buffer).getCls(), getClass()); classResolver.reset(); buffer.writerIndex(0); buffer.readerIndex(0); @@ -336,7 +338,7 @@ public void testNeedToWriteReference() { ClassResolver classResolver = fory.getClassResolver(); Assert.assertFalse( classResolver.needToWriteRef(TypeRef.of(TestNeedToWriteReferenceClass.class))); - assertNull(classResolver.getClassInfo(TestNeedToWriteReferenceClass.class, false)); + assertNull(classResolver.getTypeInfo(TestNeedToWriteReferenceClass.class, false)); } @Test @@ -350,23 +352,23 @@ public void testSetSerializer() { ClassResolver classResolver = fory.getClassResolver(); { classResolver.setSerializer(Foo.class, new ObjectSerializer<>(fory, Foo.class)); - ClassInfo classInfo = classResolver.getClassInfo(Foo.class); - assertSame(classInfo.getSerializer().getClass(), ObjectSerializer.class); + TypeInfo typeInfo = classResolver.getTypeInfo(Foo.class); + assertSame(typeInfo.getSerializer().getClass(), ObjectSerializer.class); // Create another ObjectSerializer to test setSerializer updates the existing classInfo classResolver.setSerializer(Foo.class, new ObjectSerializer<>(fory, Foo.class, true)); - Assert.assertSame(classResolver.getClassInfo(Foo.class), classInfo); - assertSame(classInfo.getSerializer().getClass(), ObjectSerializer.class); + Assert.assertSame(classResolver.getTypeInfo(Foo.class), typeInfo); + assertSame(typeInfo.getSerializer().getClass(), ObjectSerializer.class); } { classResolver.registerInternal(Bar.class); - ClassInfo classInfo = classResolver.getClassInfo(Bar.class); + TypeInfo typeInfo = classResolver.getTypeInfo(Bar.class); classResolver.setSerializer(Bar.class, new ObjectSerializer<>(fory, Bar.class)); - Assert.assertSame(classResolver.getClassInfo(Bar.class), classInfo); - assertSame(classInfo.getSerializer().getClass(), ObjectSerializer.class); + Assert.assertSame(classResolver.getTypeInfo(Bar.class), typeInfo); + assertSame(typeInfo.getSerializer().getClass(), ObjectSerializer.class); // Create another ObjectSerializer to test setSerializer updates the existing classInfo classResolver.setSerializer(Bar.class, new ObjectSerializer<>(fory, Bar.class, true)); - Assert.assertSame(classResolver.getClassInfo(Bar.class), classInfo); - assertSame(classInfo.getSerializer().getClass(), ObjectSerializer.class); + Assert.assertSame(classResolver.getTypeInfo(Bar.class), typeInfo); + assertSame(typeInfo.getSerializer().getClass(), ObjectSerializer.class); } } diff --git a/java/fory-core/src/test/java/org/apache/fory/resolver/MetaContextTest.java b/java/fory-core/src/test/java/org/apache/fory/resolver/MetaContextTest.java index 054d66f843..c54b411a3b 100644 --- a/java/fory-core/src/test/java/org/apache/fory/resolver/MetaContextTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/resolver/MetaContextTest.java @@ -50,7 +50,7 @@ public void testShareClassName() { } @Test(dataProvider = "enableCodegen") - public void testShareClassDefCompatible(boolean enableCodegen) { + public void testShareTypeDefCompatible(boolean enableCodegen) { Fory fory = Fory.builder() .withLanguage(Language.JAVA) diff --git a/java/fory-core/src/test/java/org/apache/fory/resolver/ClassInfoTest.java b/java/fory-core/src/test/java/org/apache/fory/resolver/TypeInfoTest.java similarity index 90% rename from java/fory-core/src/test/java/org/apache/fory/resolver/ClassInfoTest.java rename to java/fory-core/src/test/java/org/apache/fory/resolver/TypeInfoTest.java index b526a3236d..f5dcd3d4b6 100644 --- a/java/fory-core/src/test/java/org/apache/fory/resolver/ClassInfoTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/resolver/TypeInfoTest.java @@ -25,11 +25,11 @@ import org.apache.fory.config.Language; import org.testng.annotations.Test; -public class ClassInfoTest { +public class TypeInfoTest { @Test public void testEncodePackageNameAndTypeName() { Fory fory1 = Fory.builder().withLanguage(Language.JAVA).requireClassRegistration(false).build(); - ClassInfo info1 = fory1.getClassResolver().getClassInfo(org.apache.fory.test.bean.Foo.class); + TypeInfo info1 = fory1.getClassResolver().getTypeInfo(org.apache.fory.test.bean.Foo.class); assertNotNull(info1.namespaceBytes); assertNotNull(info1.typeNameBytes); } diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/CodegenSerializerTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/CodegenSerializerTest.java index cc85de8564..a4025b45a5 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/CodegenSerializerTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/CodegenSerializerTest.java @@ -175,7 +175,7 @@ static class NonFinalPackage extends NonFinalPublic {} @EqualsAndHashCode(callSuper = true) private static class NonFinalPrivate extends NonFinalPublic {} - public static class TestCacheNonFinalClassInfo { + public static class TestCacheNonFinalTypeInfo { public String str; public NonFinalPublic nonFinalPublic; public List finalList; @@ -185,9 +185,9 @@ public static class TestCacheNonFinalClassInfo { } @Test - public void testCacheNonFinalClassInfo() { + public void testCacheNonFinalTypeInfo() { Fory fory = builder().withLanguage(Language.JAVA).requireClassRegistration(false).build(); - TestCacheNonFinalClassInfo obj = new TestCacheNonFinalClassInfo(); + TestCacheNonFinalTypeInfo obj = new TestCacheNonFinalTypeInfo(); obj.finalList = new ArrayList<>(ImmutableList.of("a", "b")); obj.nonFinalPublicList = new ArrayList<>( diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/CompatibleSerializerTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/CompatibleSerializerTest.java index 39c305019f..cdf267071e 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/CompatibleSerializerTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/CompatibleSerializerTest.java @@ -345,7 +345,7 @@ public void testSerializeJavaObject() { } @Test - public void testSerializeJavaObjectWithClassInfo() { + public void testSerializeJavaObjectWithTypeInfo() { Fory fory = Fory.builder() .withLanguage(Language.JAVA) diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/FinalFieldReplaceResolveSerializerTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/FinalFieldReplaceResolveSerializerTest.java index 6030b713ef..9b69a1dcea 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/FinalFieldReplaceResolveSerializerTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/FinalFieldReplaceResolveSerializerTest.java @@ -340,11 +340,11 @@ public void testMultipleFinalFieldsWithReplaceCopy(Fory fory) { } /** - * Verify that the writeClassInfo field is null for FieldReplaceResolveSerializer. This is what + * Verify that the writeTypeInfo field is null for FieldReplaceResolveSerializer. This is what * prevents class names from being written. */ @Test - public void testWriteClassInfoIsNull() throws Exception { + public void testWriteTypeInfoIsNull() throws Exception { Fory fory = Fory.builder() .withLanguage(Language.JAVA) @@ -361,26 +361,26 @@ public void testWriteClassInfoIsNull() throws Exception { FinalFieldReplaceResolveSerializer finalFieldSerializer = new FinalFieldReplaceResolveSerializer(fory, listClass); - // Use reflection to check that writeClassInfo is null - java.lang.reflect.Field writeClassInfoField = - ReplaceResolveSerializer.class.getDeclaredField("writeClassInfo"); - writeClassInfoField.setAccessible(true); - Object writeClassInfo = writeClassInfoField.get(finalFieldSerializer); + // Use reflection to check that writeTypeInfo is null + java.lang.reflect.Field writeTypeInfoField = + ReplaceResolveSerializer.class.getDeclaredField("writeTypeInfo"); + writeTypeInfoField.setAccessible(true); + Object writeTypeInfo = writeTypeInfoField.get(finalFieldSerializer); - // For FieldReplaceResolveSerializer, writeClassInfo should be null + // For FieldReplaceResolveSerializer, writeTypeInfo should be null assertNull( - writeClassInfo, - "FieldReplaceResolveSerializer should have writeClassInfo=null to avoid writing class names"); + writeTypeInfo, + "FieldReplaceResolveSerializer should have writeTypeInfo=null to avoid writing class names"); // Compare with ReplaceResolveSerializer (non-final) ReplaceResolveSerializer nonFinalFieldSerializer = new ReplaceResolveSerializer(fory, listClass, false, true); - Object writeClassInfoNonFinal = writeClassInfoField.get(nonFinalFieldSerializer); + Object writeTypeInfoNonFinal = writeTypeInfoField.get(nonFinalFieldSerializer); - // For ReplaceResolveSerializer (non-final), writeClassInfo should NOT be null + // For ReplaceResolveSerializer (non-final), writeTypeInfo should NOT be null assertNotNull( - writeClassInfoNonFinal, - "ReplaceResolveSerializer (non-final) should have writeClassInfo set to write class names"); + writeTypeInfoNonFinal, + "ReplaceResolveSerializer (non-final) should have writeTypeInfo set to write class names"); } @Test(dataProvider = "enableCodegen") @@ -447,7 +447,7 @@ public void testWriteObjectSkipsClassNameWrite() { // The key point: FieldReplaceResolveSerializer.writeObject() directly calls // jdkMethodInfoCache.objectSerializer.write(buffer, value) - // without calling classResolver.writeClassInternal(buffer, writeClassInfo) + // without calling classResolver.writeClassInternal(buffer, writeTypeInfo) } // TODO fix: bug with CompatibleMode and final field replace/resolve on main branch diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/MetaSharedCompatibleTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/MetaSharedCompatibleTest.java index 94715db221..db32ae5b6f 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/MetaSharedCompatibleTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/MetaSharedCompatibleTest.java @@ -38,7 +38,7 @@ import org.apache.fory.config.CompatibleMode; import org.apache.fory.config.ForyBuilder; import org.apache.fory.config.Language; -import org.apache.fory.meta.ClassDefEncoderTest; +import org.apache.fory.meta.NativeTypeDefEncoderTest; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.resolver.MetaContext; import org.apache.fory.serializer.collection.UnmodifiableSerializersTest; @@ -670,7 +670,7 @@ public void testBigClassNameObject() { .withScopedMetaShare(false) .build(); Object o = - new ClassDefEncoderTest + new NativeTypeDefEncoderTest .TestClassLengthTestClassLengthTestClassLengthTestClassLengthTestClassLengthTestClassLengthTestClassLength .InnerClassTestLengthInnerClassTestLengthInnerClassTestLength(); serDeMetaSharedCheck(fory, o); diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/ProtocolInteroperabilityTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/ProtocolInteroperabilityTest.java index 83c7b0c76a..b3bf10f9a7 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/ProtocolInteroperabilityTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/ProtocolInteroperabilityTest.java @@ -61,8 +61,7 @@ public static Object[][] fory() { return Sets.cartesianProduct( ImmutableSet.of(true, false), // referenceTracking ImmutableSet.of(true, false), // compressNumber - ImmutableSet.of(true, false), // scopedMetaShare - ImmutableSet.of(CompatibleMode.SCHEMA_CONSISTENT, CompatibleMode.COMPATIBLE)) + ImmutableSet.of(true, false)) // compatible .stream() .map(List::toArray) .map( @@ -73,8 +72,7 @@ public static Object[][] fory() { .withRefTracking((Boolean) c[0]) .withNumberCompressed((Boolean) c[1]) .withCodegen(false) - .withScopedMetaShare((Boolean) c[2]) - .withCompatibleMode((CompatibleMode) c[3]) + .withCompatible((Boolean) c[2]) .requireClassRegistration(false) .build(), builder() @@ -82,8 +80,7 @@ public static Object[][] fory() { .withRefTracking((Boolean) c[0]) .withNumberCompressed((Boolean) c[1]) .withCodegen(true) - .withScopedMetaShare((Boolean) c[2]) - .withCompatibleMode((CompatibleMode) c[3]) + .withCompatible((Boolean) c[2]) .requireClassRegistration(false) .build() }) diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/RegisterTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/RegisterTest.java index 941092dbc2..bf1ac16b1d 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/RegisterTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/RegisterTest.java @@ -49,7 +49,9 @@ public void testRegisterForCompatible(boolean enableCodegen) { A a1 = fory1.deserialize(fory2.serialize(a), A.class); Assert.assertNotNull(a1); - Assert.assertNull(a1.b); + Object b = a1.b; + Assert.assertNotNull(b); + Assert.assertEquals(b.getClass(), B.class); Fory fory3 = builder.requireClassRegistration(false).build(); fory3.register(A.class, (short) 1000); diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/collection/MapSerializersTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/collection/MapSerializersTest.java index cada3c93a4..e27b53b73e 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/collection/MapSerializersTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/collection/MapSerializersTest.java @@ -67,18 +67,14 @@ public class MapSerializersTest extends ForyTestBase { @Test(dataProvider = "basicMultiConfigFory") public void basicTestCaseWithMultiConfig( - boolean trackingRef, - boolean codeGen, - boolean scopedMetaShare, - CompatibleMode compatibleMode) { + boolean trackingRef, boolean codeGen, boolean compatible) { Fory fory = builder() .withLanguage(Language.JAVA) .withRefTracking(trackingRef) .requireClassRegistration(false) .withCodegen(codeGen) - .withCompatibleMode(compatibleMode) - .withScopedMetaShare(scopedMetaShare) + .withCompatible(compatible) .build(); // testBasicMap diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java b/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java index bba05fd3af..712715f553 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java @@ -1241,7 +1241,7 @@ static class AnimalListHolder { @Data static class AnimalMapHolder { - // Using snake_case field name to test fallback lookup in ClassDef.getDescriptors() + // Using snake_case field name to test fallback lookup in TypeDef.getDescriptors() Map animal_map; } diff --git a/java/fory-extensions/src/test/java/org/apache/fory/extension/meta/ClassDefEncoderTest.java b/java/fory-extensions/src/test/java/org/apache/fory/extension/meta/TypeDefEncoderTest.java similarity index 70% rename from java/fory-extensions/src/test/java/org/apache/fory/extension/meta/ClassDefEncoderTest.java rename to java/fory-extensions/src/test/java/org/apache/fory/extension/meta/TypeDefEncoderTest.java index f295d71251..aea91d4570 100644 --- a/java/fory-extensions/src/test/java/org/apache/fory/extension/meta/ClassDefEncoderTest.java +++ b/java/fory-extensions/src/test/java/org/apache/fory/extension/meta/TypeDefEncoderTest.java @@ -19,19 +19,19 @@ package org.apache.fory.extension.meta; -import static org.apache.fory.meta.ClassDefEncoder.buildFieldsInfo; -import static org.apache.fory.meta.ClassDefEncoder.getClassFields; +import static org.apache.fory.meta.NativeTypeDefEncoder.buildFieldsInfo; +import static org.apache.fory.meta.NativeTypeDefEncoder.getClassFields; import java.util.List; import org.apache.fory.Fory; import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.meta.ClassDef; -import org.apache.fory.meta.ClassDefEncoder; import org.apache.fory.meta.FieldInfo; +import org.apache.fory.meta.NativeTypeDefEncoder; +import org.apache.fory.meta.TypeDef; import org.testng.Assert; import org.testng.annotations.Test; -public class ClassDefEncoderTest { +public class TypeDefEncoderTest { static class TestFieldsOrderClass1 { private int intField2; @@ -41,17 +41,17 @@ static class TestFieldsOrderClass1 { } @Test - public void testBasicClassDef_zstdMetaCompressor() throws Exception { + public void testBasicTypeDefZstdMetaCompressor() throws Exception { Fory fory = Fory.builder().withMetaShare(true).withMetaCompressor(new ZstdMetaCompressor()).build(); Class type = TestFieldsOrderClass1.class; List fieldsInfo = buildFieldsInfo(fory.getClassResolver(), type); MemoryBuffer buffer = - ClassDefEncoder.encodeClassDef( + NativeTypeDefEncoder.encodeTypeDef( fory.getClassResolver(), type, getClassFields(type, fieldsInfo), true); - ClassDef classDef = ClassDef.readClassDef(fory, buffer); - Assert.assertEquals(classDef.getClassName(), type.getName()); - Assert.assertEquals(classDef.getFieldsInfo().size(), type.getDeclaredFields().length); - Assert.assertEquals(classDef.getFieldsInfo(), fieldsInfo); + TypeDef typeDef = TypeDef.readTypeDef(fory, buffer); + Assert.assertEquals(typeDef.getClassName(), type.getName()); + Assert.assertEquals(typeDef.getFieldsInfo().size(), type.getDeclaredFields().length); + Assert.assertEquals(typeDef.getFieldsInfo(), fieldsInfo); } } diff --git a/javascript/packages/fory/lib/fory.ts b/javascript/packages/fory/lib/fory.ts index 9714463449..f5d9e1c6d4 100644 --- a/javascript/packages/fory/lib/fory.ts +++ b/javascript/packages/fory/lib/fory.ts @@ -17,7 +17,7 @@ * under the License. */ -import ClassResolver from "./classResolver"; +import TypeResolver from "./typeResolver"; import { BinaryWriter } from "./writer"; import { BinaryReader } from "./reader"; import { ReferenceResolver } from "./referenceResolver"; @@ -33,7 +33,7 @@ import { MetaStringResolver } from "./metaStringResolver"; export default class { binaryReader: BinaryReader; binaryWriter: BinaryWriter; - classResolver: ClassResolver; + typeResolver: TypeResolver; typeMetaResolver: TypeMetaResolver; metaStringResolver: MetaStringResolver; referenceResolver: ReferenceResolver; @@ -47,10 +47,10 @@ export default class { this.binaryWriter = new BinaryWriter(this.config); this.referenceResolver = new ReferenceResolver(this.binaryReader); this.typeMetaResolver = new TypeMetaResolver(this); - this.classResolver = new ClassResolver(this); + this.typeResolver = new TypeResolver(this); this.metaStringResolver = new MetaStringResolver(this); - this.classResolver.init(); - this.anySerializer = this.classResolver.getSerializerById(TypeId.UNKNOWN); + this.typeResolver.init(); + this.anySerializer = this.typeResolver.getSerializerById(TypeId.UNKNOWN); } private initConfig(config: Partial | undefined) { @@ -90,11 +90,11 @@ export default class { if (constructor.prototype?.[ForyTypeInfoSymbol]) { const typeInfo: TypeInfo = ((constructor.prototype[ForyTypeInfoSymbol])).structTypeInfo; serializer = new Gen(this, { constructor }).generateSerializer(typeInfo); - this.classResolver.registerSerializer(typeInfo, serializer); + this.typeResolver.registerSerializer(typeInfo, serializer); } else { const typeInfo = constructor; serializer = new Gen(this).generateSerializer(typeInfo); - this.classResolver.registerSerializer(typeInfo, serializer); + this.typeResolver.registerSerializer(typeInfo, serializer); } TypeInfo.detach(); return { @@ -117,12 +117,12 @@ export default class { replaceSerializerReader(typeInfo: TypeInfo) { TypeInfo.attach(this); const serializer = new Gen(this, { constroctor: (typeInfo as StructTypeInfo).options.constructor }).reGenerateSerializer(typeInfo); - const result = this.classResolver.registerSerializer(typeInfo, { + const result = this.typeResolver.registerSerializer(typeInfo, { getHash: serializer.getHash, read: serializer.read, readNoRef: serializer.readNoRef, readRef: serializer.readRef, - readClassInfo: serializer.readClassInfo, + readTypeInfo: serializer.readTypeInfo, } as any)!; TypeInfo.detach(); return result; diff --git a/javascript/packages/fory/lib/gen/any.ts b/javascript/packages/fory/lib/gen/any.ts index 96da6d2199..d2330f4db0 100644 --- a/javascript/packages/fory/lib/gen/any.ts +++ b/javascript/packages/fory/lib/gen/any.ts @@ -21,28 +21,31 @@ import { TypeInfo } from "../typeInfo"; import { CodecBuilder } from "./builder"; import { BaseSerializerGenerator } from "./serializer"; import { CodegenRegistry } from "./router"; -import { Mode, RefFlags, Serializer, TypeId } from "../type"; +import { Mode, Serializer, TypeId } from "../type"; import { Scope } from "./scope"; import Fory from "../fory"; export class AnyHelper { static detectSerializer(fory: Fory) { - const typeId = fory.binaryReader.readVarUint32Small7(); + const typeId = fory.binaryReader.uint8(); + let userTypeId = -1; + if (TypeId.needsUserTypeId(typeId) && typeId !== TypeId.COMPATIBLE_STRUCT) { + userTypeId = fory.binaryReader.readVarUint32Small7(); + } let serializer: Serializer | undefined; - const internalTypeId = typeId & 0xff; - - switch (internalTypeId) { + switch (typeId) { + case TypeId.COMPATIBLE_STRUCT: case TypeId.NAMED_ENUM: case TypeId.NAMED_STRUCT: case TypeId.NAMED_EXT: case TypeId.NAMED_UNION: case TypeId.NAMED_COMPATIBLE_STRUCT: - if (fory.config.mode === Mode.Compatible) { + if (fory.config.mode === Mode.Compatible || typeId === TypeId.COMPATIBLE_STRUCT) { const typeMeta = fory.typeMetaResolver.readTypeMeta(fory.binaryReader); const ns = typeMeta.getNs(); const typeName = typeMeta.getTypeName(); const named = `${ns}$${typeName}`; - serializer = fory.classResolver.getSerializerByName(named); + serializer = fory.typeResolver.getSerializerByName(named); if (!serializer) { throw new Error(`can't find implements of typeId: ${typeId}`); } @@ -53,26 +56,11 @@ export class AnyHelper { } else { const ns = fory.metaStringResolver.readNamespace(fory.binaryReader); const typeName = fory.metaStringResolver.readTypeName(fory.binaryReader); - serializer = fory.classResolver.getSerializerByName(`${ns}$${typeName}`); - } - break; - case TypeId.COMPATIBLE_STRUCT: - if (fory.config.mode === Mode.Compatible) { - const typeMeta = fory.typeMetaResolver.readTypeMeta(fory.binaryReader); - serializer = fory.classResolver.getSerializerById(typeId); - if (!serializer) { - throw new Error(`can't find implements of typeId: ${typeId}`); - } - const hash = serializer.getHash(); - if (hash !== typeMeta.getHash()) { - serializer = fory.typeMetaResolver.genSerializerByTypeMetaRuntime(typeMeta); - } - } else { - serializer = fory.classResolver.getSerializerById(typeId); + serializer = fory.typeResolver.getSerializerByName(`${ns}$${typeName}`); } break; default: - serializer = fory.classResolver.getSerializerById(typeId); + serializer = fory.typeResolver.getSerializerById(typeId, userTypeId); break; } if (!serializer) { @@ -86,7 +74,7 @@ export class AnyHelper { throw new Error("can not guess the type of null or undefined"); } - const serializer = fory.classResolver.getSerializerByData(v); + const serializer = fory.typeResolver.getSerializerByData(v); if (!serializer) { throw new Error(`Failed to detect the Fory serializer from JavaScript type: ${typeof v}`); } @@ -112,14 +100,14 @@ class AnySerializerGenerator extends BaseSerializerGenerator { `; } - writeClassInfo(accessor: string): string { + writeTypeInfo(accessor: string): string { return ` ${this.writerSerializer} = ${this.builder.getExternal(AnyHelper.name)}.getSerializer(${this.builder.getForyName()}, ${accessor}); - ${this.writerSerializer}.writeClassInfo(); + ${this.writerSerializer}.writeTypeInfo(); `; } - readClassInfo(): string { + readTypeInfo(): string { return ` ${this.detectedSerializer} = ${this.builder.getExternal(AnyHelper.name)}.detectSerializer(${this.builder.getForyName()}); `; diff --git a/javascript/packages/fory/lib/gen/builder.ts b/javascript/packages/fory/lib/gen/builder.ts index 73f0fa9e2a..972f5df23c 100644 --- a/javascript/packages/fory/lib/gen/builder.ts +++ b/javascript/packages/fory/lib/gen/builder.ts @@ -292,7 +292,7 @@ class ReferenceResolverBuilder { } } -class ClassResolverBuilder { +class TypeResolverBuilder { constructor(private holder: string) { } @@ -301,8 +301,11 @@ class ClassResolverBuilder { return this.holder; } - getSerializerById(id: string | number) { - return `${this.holder}.getSerializerById(${id})`; + getSerializerById(id: string | number, userTypeId?: string | number) { + if (userTypeId === undefined) { + return `${this.holder}.getSerializerById(${id})`; + } + return `${this.holder}.getSerializerById(${id}, ${userTypeId})`; } getSerializerByName(name: string) { @@ -375,18 +378,18 @@ export class CodecBuilder { writer: BinaryWriterBuilder; typeMeta: TypeMetaBuilder; // Use the TypeMetaWrapper referenceResolver: ReferenceResolverBuilder; - classResolver: ClassResolverBuilder; + typeResolver: TypeResolverBuilder; typeMetaResolver: TypeMetaResolverBuilder; metaStringResolver: MetaStringResolverBuilder; constructor(scope: Scope, public fory: Fory) { const br = scope.declareByName("br", "fory.binaryReader"); const bw = scope.declareByName("bw", "fory.binaryWriter"); - const cr = scope.declareByName("cr", "fory.classResolver"); + const cr = scope.declareByName("cr", "fory.typeResolver"); const rr = scope.declareByName("rr", "fory.referenceResolver"); this.reader = new BinaryReaderBuilder(br); this.writer = new BinaryWriterBuilder(bw); - this.classResolver = new ClassResolverBuilder(cr); + this.typeResolver = new TypeResolverBuilder(cr); this.referenceResolver = new ReferenceResolverBuilder(rr); this.typeMeta = new TypeMetaBuilder("fory"); // Initialize the TypeMetaWrapper this.typeMetaResolver = new TypeMetaResolverBuilder("fory.typeMetaResolver"); diff --git a/javascript/packages/fory/lib/gen/collection.ts b/javascript/packages/fory/lib/gen/collection.ts index 34ddd828ab..74c18946b3 100644 --- a/javascript/packages/fory/lib/gen/collection.ts +++ b/javascript/packages/fory/lib/gen/collection.ts @@ -56,7 +56,7 @@ class CollectionAnySerializer { if ((item === undefined || item === null) && !includeNone) { includeNone = true; } - const current = this.fory.classResolver.getSerializerByData(item); + const current = this.fory.typeResolver.getSerializerByData(item); if (!current) { throw new Error("can't detect the type of item in list"); } @@ -95,7 +95,7 @@ class CollectionAnySerializer { this.fory.binaryWriter.writeVarUint32Small7(size); const { serializer, isSame, includeNone, trackingRef } = this.writeElementsHeader(value); if (isSame) { - serializer!.writeClassInfo(value); + serializer!.writeTypeInfo(value); if (trackingRef) { for (const item of value) { if (!serializer!.writeRefOrNull(item)) { @@ -119,7 +119,7 @@ class CollectionAnySerializer { } else { if (trackingRef) { for (const item of value) { - const serializer = this.fory.classResolver.getSerializerByData(item); + const serializer = this.fory.typeResolver.getSerializerByData(item); serializer?.writeRef(item); } } else if (includeNone) { @@ -127,7 +127,7 @@ class CollectionAnySerializer { if (item === null || item === undefined) { this.fory.binaryWriter.uint8(RefFlags.NullFlag); } else { - const serializer = this.fory.classResolver.getSerializerByData(item); + const serializer = this.fory.typeResolver.getSerializerByData(item); this.fory.binaryWriter.uint8(RefFlags.NotNullValueFlag); serializer!.write(item); } @@ -141,6 +141,7 @@ class CollectionAnySerializer { } read(accessor: (result: any, index: number, v: any) => void, createCollection: (len: number) => any, fromRef: boolean): any { + void fromRef; const len = this.fory.binaryReader.readVarUint32Small7(); const flags = this.fory.binaryReader.uint8(); const isSame = flags & CollectionFlags.SAME_TYPE; diff --git a/javascript/packages/fory/lib/gen/enum.ts b/javascript/packages/fory/lib/gen/enum.ts index 2767e20d91..65e289f217 100644 --- a/javascript/packages/fory/lib/gen/enum.ts +++ b/javascript/packages/fory/lib/gen/enum.ts @@ -57,16 +57,16 @@ class EnumSerializerGenerator extends BaseSerializerGenerator { `; } - readClassInfo(): string { - return ` - ${ - // skip the typeId - this.builder.reader.readVarUint32Small7() - } - - ${ - TypeId.isNamedType(this.getTypeId()) - ? ` + readTypeInfo(): string { + const internalTypeId = this.getInternalTypeId(); + let readUserTypeIdStmt = ""; + let namesStmt = ""; + switch (internalTypeId) { + case TypeId.ENUM: + readUserTypeIdStmt = `${this.builder.reader.readVarUint32Small7()};`; + break; + case TypeId.NAMED_ENUM: + namesStmt = ` ${ // skip the namespace this.builder.metaStringResolver.readNamespace(this.builder.reader.ownName()) @@ -76,15 +76,29 @@ class EnumSerializerGenerator extends BaseSerializerGenerator { // skip the namespace this.builder.metaStringResolver.readTypeName(this.builder.reader.ownName()) } - ` - : ""} + `; + break; + default: + break; + } + return ` + ${ + // skip the typeId + this.builder.reader.uint8() + } + ${readUserTypeIdStmt} + ${namesStmt} `; } - writeClassInfo(): string { + writeTypeInfo(): string { const internalTypeId = this.getInternalTypeId(); let typeMeta = ""; + let writeUserTypeIdStmt = ""; switch (internalTypeId) { + case TypeId.ENUM: + writeUserTypeIdStmt = this.builder.writer.writeVarUint32Small7(this.typeInfo.userTypeId); + break; case TypeId.NAMED_ENUM: { const typeInfo = this.typeInfo.castToStruct(); @@ -100,7 +114,8 @@ class EnumSerializerGenerator extends BaseSerializerGenerator { break; } return ` - ${this.builder.writer.writeVarUint32Small7(this.getTypeId())}; + ${this.builder.writer.uint8(this.getTypeId())}; + ${writeUserTypeIdStmt} ${typeMeta} `; } diff --git a/javascript/packages/fory/lib/gen/index.ts b/javascript/packages/fory/lib/gen/index.ts index fbaabe245d..1234148ae5 100644 --- a/javascript/packages/fory/lib/gen/index.ts +++ b/javascript/packages/fory/lib/gen/index.ts @@ -63,11 +63,11 @@ export class Gen { } private register(typeInfo: StructTypeInfo, serializer?: Serializer) { - this.fory.classResolver.registerSerializer(typeInfo, serializer); + this.fory.typeResolver.registerSerializer(typeInfo, serializer); } private isRegistered(typeInfo: TypeInfo) { - return !!this.fory.classResolver.typeInfoExists(typeInfo); + return !!this.fory.typeResolver.typeInfoExists(typeInfo); } private traversalContainer(typeInfo: TypeInfo) { @@ -106,7 +106,7 @@ export class Gen { this.traversalContainer(typeInfo); const exists = this.isRegistered(typeInfo); if (exists) { - return this.fory.classResolver.getSerializerByTypeInfo(typeInfo); + return this.fory.typeResolver.getSerializerByTypeInfo(typeInfo); } return this.reGenerateSerializer(typeInfo); } diff --git a/javascript/packages/fory/lib/gen/map.ts b/javascript/packages/fory/lib/gen/map.ts index 8632b005e2..7cb32e1304 100644 --- a/javascript/packages/fory/lib/gen/map.ts +++ b/javascript/packages/fory/lib/gen/map.ts @@ -97,10 +97,10 @@ class MapChunkWriter { // chunkSize default 0 | KV header this.fory.binaryWriter.uint16(header); if (this.keySerializer) { - this.keySerializer.writeClassInfo(null); + this.keySerializer.writeTypeInfo(null); } if (this.valueSerializer) { - this.valueSerializer.writeClassInfo(null); + this.valueSerializer.writeTypeInfo(null); } return header; } @@ -137,10 +137,10 @@ class MapAnySerializer { constructor(private fory: Fory, keySerializerId: null | number, valueSerializerId: null | number) { if (keySerializerId !== null) { - fory.classResolver.getSerializerById(keySerializerId); + fory.typeResolver.getSerializerById(keySerializerId); } if (valueSerializerId !== null) { - fory.classResolver.getSerializerById(valueSerializerId); + fory.typeResolver.getSerializerById(valueSerializerId); } } @@ -175,8 +175,8 @@ class MapAnySerializer { const mapChunkWriter = new MapChunkWriter(this.fory, this.keySerializer, this.valueSerializer); this.fory.binaryWriter.writeVarUint32Small7(value.size); for (const [k, v] of value.entries()) { - const keySerializer = this.keySerializer !== null ? this.keySerializer : this.fory.classResolver.getSerializerByData(k); - const valueSerializer = this.valueSerializer !== null ? this.valueSerializer : this.fory.classResolver.getSerializerByData(v); + const keySerializer = this.keySerializer !== null ? this.keySerializer : this.fory.typeResolver.getSerializerByData(k); + const valueSerializer = this.valueSerializer !== null ? this.valueSerializer : this.fory.typeResolver.getSerializerByData(v); const header = mapChunkWriter.next( MapHeadUtil.elementInfo(keySerializer!.getTypeId()!, k == null ? 1 : 0, keySerializer!.needToWriteRef() ? 1 : 0), diff --git a/javascript/packages/fory/lib/gen/router.ts b/javascript/packages/fory/lib/gen/router.ts index 38226de51d..a934eb618b 100644 --- a/javascript/packages/fory/lib/gen/router.ts +++ b/javascript/packages/fory/lib/gen/router.ts @@ -52,7 +52,7 @@ export class CodegenRegistry { } static get(typeId: number) { - return this.map.get(typeId & 0xff); + return this.map.get(typeId); } static getExternal() { diff --git a/javascript/packages/fory/lib/gen/serializer.ts b/javascript/packages/fory/lib/gen/serializer.ts index 4b76eb72ff..986f2a35fd 100644 --- a/javascript/packages/fory/lib/gen/serializer.ts +++ b/javascript/packages/fory/lib/gen/serializer.ts @@ -18,7 +18,7 @@ */ import { CodecBuilder } from "./builder"; -import { RefFlags } from "../type"; +import { RefFlags, TypeId } from "../type"; import { Scope } from "./scope"; import { TypeInfo } from "../typeInfo"; import { refTrackingAbleTypeId } from "../meta/TypeMeta"; @@ -27,7 +27,7 @@ import { BinaryWriter } from "../writer"; export const makeHead = (flag: RefFlags, typeId: number) => { const writer = new BinaryWriter(); writer.uint8(flag); - writer.writeVarUint32Small7(typeId); + writer.uint8(typeId); const buffer = writer.dump(); return buffer; }; @@ -36,7 +36,7 @@ export interface SerializerGenerator { writeRef(accessor: string): string; writeNoRef(accessor: string): string; writeRefOrNull(assignStmt: (v: string) => string, accessor: string): string; - writeClassInfo(accessor: string): string; + writeTypeInfo(accessor: string): string; write(accessor: string): string; writeEmbed(): any; @@ -44,9 +44,9 @@ export interface SerializerGenerator { getFixedSize(): number; needToWriteRef(): boolean; - readRef(assignStmt: (v: string) => string, withoutClassInfo?: boolean): string; + readRef(assignStmt: (v: string) => string, withoutTypeInfo?: boolean): string; readNoRef(assignStmt: (v: string) => string, refState: string): string; - readClassInfo(): string; + readTypeInfo(): string; read(assignStmt: (v: string) => string, refState: string): string; readEmbed(): any; getHash(): string; @@ -115,7 +115,7 @@ export abstract class BaseSerializerGenerator implements SerializerGenerator { writeNoRef(accessor: string) { return ` - ${this.writeClassInfo(accessor)}; + ${this.writeTypeInfo(accessor)}; ${this.write(accessor)}; `; } @@ -148,9 +148,16 @@ export abstract class BaseSerializerGenerator implements SerializerGenerator { `; } - writeClassInfo(accessor: string) { + writeTypeInfo(accessor: string) { + void accessor; + const typeId = this.getTypeId(); + const userTypeId = this.typeInfo.userTypeId; + const userTypeStmt = TypeId.needsUserTypeId(typeId) && typeId !== TypeId.COMPATIBLE_STRUCT + ? this.builder.writer.writeVarUint32Small7(userTypeId) + : ""; return ` - ${this.builder.writer.writeVarUint32Small7(this.getTypeId())}; + ${this.builder.writer.uint8(typeId)}; + ${userTypeStmt} `; } @@ -162,36 +169,42 @@ export abstract class BaseSerializerGenerator implements SerializerGenerator { return this.typeInfo.typeId; } + getUserTypeId() { + return this.typeInfo.userTypeId; + } + getInternalTypeId() { - if (this.getTypeId() <= 0xff) { - return this.getTypeId(); - } - return this.getTypeId() & 0xff; + return this.getTypeId(); } abstract read(assignStmt: (v: string) => string, refState: string): string; - readClassInfo(): string { + readTypeInfo(): string { + const typeId = this.getTypeId(); + const readUserTypeStmt = TypeId.needsUserTypeId(typeId) && typeId !== TypeId.COMPATIBLE_STRUCT + ? `${this.builder.reader.readVarUint32Small7()};` + : ""; return ` - ${this.builder.reader.readVarUint32Small7()}; + ${this.builder.reader.uint8()}; + ${readUserTypeStmt} `; } readNoRef(assignStmt: (v: string) => string, refState: string): string { return ` - ${this.readClassInfo()} + ${this.readTypeInfo()} ${this.read(assignStmt, refState)}; `; } - readRef(assignStmt: (v: string) => string, withoutClassInfo = false): string { + readRef(assignStmt: (v: string) => string, withoutTypeInfo = false): string { const refFlag = this.scope.uniqueName("refFlag"); return ` const ${refFlag} = ${this.builder.reader.int8()}; switch (${refFlag}) { case ${RefFlags.NotNullValueFlag}: case ${RefFlags.RefValueFlag}: - ${!withoutClassInfo ? this.readNoRef(assignStmt, `${refFlag} === ${RefFlags.RefValueFlag}`) : this.read(assignStmt, `${refFlag} === ${RefFlags.RefValueFlag}`)} + ${!withoutTypeInfo ? this.readNoRef(assignStmt, `${refFlag} === ${RefFlags.RefValueFlag}`) : this.read(assignStmt, `${refFlag} === ${RefFlags.RefValueFlag}`)} break; case ${RefFlags.RefFlag}: ${assignStmt(this.builder.referenceResolver.getReadObject(this.builder.reader.varUInt32()))} @@ -247,8 +260,8 @@ export abstract class BaseSerializerGenerator implements SerializerGenerator { const writeRefOrNull = (v) => { ${this.writeRefOrNull(expr => `return ${expr};`, "v")} }; - const writeClassInfo = (v) => { - ${this.writeClassInfo("v")} + const writeTypeInfo = (v) => { + ${this.writeTypeInfo("v")} }; const read = (fromRef) => { ${this.read(assignStmt => `return ${assignStmt}`, "fromRef")} @@ -259,8 +272,8 @@ export abstract class BaseSerializerGenerator implements SerializerGenerator { const readNoRef = (fromRef) => { ${this.readNoRef(assignStmt => `return ${assignStmt}`, "fromRef")} }; - const readClassInfo = () => { - ${this.readClassInfo()} + const readTypeInfo = () => { + ${this.readTypeInfo()} }; `; return ` @@ -271,18 +284,19 @@ export abstract class BaseSerializerGenerator implements SerializerGenerator { fixedSize: ${this.getFixedSize()}, needToWriteRef: () => ${this.needToWriteRef()}, getTypeId: () => ${this.getTypeId()}, + getUserTypeId: () => ${this.getUserTypeId()}, getHash, write, writeRef, writeNoRef, writeRefOrNull, - writeClassInfo, + writeTypeInfo, read, readRef, readNoRef, - readClassInfo, + readTypeInfo, }; } `; diff --git a/javascript/packages/fory/lib/gen/struct.ts b/javascript/packages/fory/lib/gen/struct.ts index 592445a4dc..b18173deab 100644 --- a/javascript/packages/fory/lib/gen/struct.ts +++ b/javascript/packages/fory/lib/gen/struct.ts @@ -204,7 +204,7 @@ class StructSerializerGenerator extends BaseSerializerGenerator { readNoRef(assignStmt: (v: string) => string, refState: string): string { return ` - ${this.readClassInfo()} + ${this.readTypeInfo()} if (${this.metaChangedSerializer} !== null) { ${assignStmt(`${this.metaChangedSerializer}.read(${refState})`)} } else { @@ -213,32 +213,60 @@ class StructSerializerGenerator extends BaseSerializerGenerator { `; } - readClassInfo(): string { + readTypeInfo(): string { const typeMeta = this.scope.uniqueName("typeMeta"); + const internalTypeId = this.getInternalTypeId(); let namesStmt = ""; - if (!this.builder.fory.isCompatible() && TypeId.isNamedType(this.getTypeId())) { - namesStmt = ` - ${ - this.builder.metaStringResolver.readNamespace(this.builder.reader.ownName()) - }; - ${ - this.builder.metaStringResolver.readTypeName(this.builder.reader.ownName()) - }; - `; - } let typeMetaStmt = ""; - if (this.builder.fory.isCompatible()) { - typeMetaStmt = ` - const ${typeMeta} = ${this.builder.typeMetaResolver.readTypeMeta(this.builder.reader.ownName())}; - if (getHash() !== ${typeMeta}.getHash()) { - ${this.metaChangedSerializer} = ${this.builder.typeMetaResolver.genSerializerByTypeMetaRuntime(typeMeta)} - } - `; + let readUserTypeIdStmt = ""; + switch (internalTypeId) { + case TypeId.STRUCT: + readUserTypeIdStmt = `${this.builder.reader.readVarUint32Small7()};`; + break; + case TypeId.NAMED_COMPATIBLE_STRUCT: + case TypeId.COMPATIBLE_STRUCT: + typeMetaStmt = ` + const ${typeMeta} = ${this.builder.typeMetaResolver.readTypeMeta(this.builder.reader.ownName())}; + if (getHash() !== ${typeMeta}.getHash()) { + ${this.metaChangedSerializer} = ${this.builder.typeMetaResolver.genSerializerByTypeMetaRuntime(typeMeta)} + } + `; + break; + case TypeId.NAMED_STRUCT: + if (!this.builder.fory.isCompatible()) { + namesStmt = ` + ${ + this.builder.metaStringResolver.readNamespace(this.builder.reader.ownName()) + }; + ${ + this.builder.metaStringResolver.readTypeName(this.builder.reader.ownName()) + }; + `; + } else { + typeMetaStmt = ` + const ${typeMeta} = ${this.builder.typeMetaResolver.readTypeMeta(this.builder.reader.ownName())}; + if (getHash() !== ${typeMeta}.getHash()) { + ${this.metaChangedSerializer} = ${this.builder.typeMetaResolver.genSerializerByTypeMetaRuntime(typeMeta)} + } + `; + } + break; + default: + if (this.builder.fory.isCompatible()) { + typeMetaStmt = ` + const ${typeMeta} = ${this.builder.typeMetaResolver.readTypeMeta(this.builder.reader.ownName())}; + if (getHash() !== ${typeMeta}.getHash()) { + ${this.metaChangedSerializer} = ${this.builder.typeMetaResolver.genSerializerByTypeMetaRuntime(typeMeta)} + } + `; + } + break; } return ` ${ - this.builder.reader.readVarUint32Small7() + this.builder.reader.uint8() }; + ${readUserTypeIdStmt} ${ namesStmt } @@ -255,8 +283,8 @@ class StructSerializerGenerator extends BaseSerializerGenerator { const name = this.scope.declare( "tag_ser", TypeId.isNamedType(this.typeInfo.typeId) - ? this.builder.classResolver.getSerializerByName(CodecBuilder.replaceBackslashAndQuote(this.typeInfo.named!)) - : this.builder.classResolver.getSerializerById(this.typeInfo.typeId) + ? this.builder.typeResolver.getSerializerByName(CodecBuilder.replaceBackslashAndQuote(this.typeInfo.named!)) + : this.builder.typeResolver.getSerializerById(this.typeInfo.typeId, this.typeInfo.userTypeId) ); return accessor(`${name}.${prop}(${args.join(",")})`); }; @@ -271,8 +299,8 @@ class StructSerializerGenerator extends BaseSerializerGenerator { const name = this.scope.declare( "tag_ser", TypeId.isNamedType(this.typeInfo.typeId) - ? this.builder.classResolver.getSerializerByName(CodecBuilder.replaceBackslashAndQuote(this.typeInfo.named!)) - : this.builder.classResolver.getSerializerById(this.typeInfo.typeId) + ? this.builder.typeResolver.getSerializerByName(CodecBuilder.replaceBackslashAndQuote(this.typeInfo.named!)) + : this.builder.typeResolver.getSerializerById(this.typeInfo.typeId, this.typeInfo.userTypeId) ); return `${name}.${prop}(${accessor})`; }; @@ -280,12 +308,22 @@ class StructSerializerGenerator extends BaseSerializerGenerator { }); } - writeClassInfo(): string { + writeTypeInfo(): string { const internalTypeId = this.getInternalTypeId(); let typeMeta = ""; + let writeUserTypeIdStmt = ""; switch (internalTypeId) { - case TypeId.NAMED_STRUCT: + case TypeId.STRUCT: + writeUserTypeIdStmt = this.builder.writer.writeVarUint32Small7(this.typeInfo.userTypeId); + break; case TypeId.NAMED_COMPATIBLE_STRUCT: + case TypeId.COMPATIBLE_STRUCT: + { + const bytes = this.scope.declare("typeInfoBytes", `new Uint8Array([${TypeMeta.fromTypeInfo( this.typeInfo).toBytes().join(",")}])`); + typeMeta = this.builder.typeMetaResolver.writeTypeMeta(this.builder.getTypeInfo(), this.builder.writer.ownName(), bytes); + } + break; + case TypeId.NAMED_STRUCT: if (this.builder.fory.config.mode !== Mode.Compatible) { const typeInfo = this.typeInfo.castToStruct(); const nsBytes = this.scope.declare("nsBytes", this.builder.metaStringResolver.encodeNamespace(CodecBuilder.replaceBackslashAndQuote(typeInfo.namespace))); @@ -299,17 +337,12 @@ class StructSerializerGenerator extends BaseSerializerGenerator { typeMeta = this.builder.typeMetaResolver.writeTypeMeta(this.builder.getTypeInfo(), this.builder.writer.ownName(), bytes); } break; - case TypeId.COMPATIBLE_STRUCT: - { - const bytes = this.scope.declare("typeInfoBytes", `new Uint8Array([${TypeMeta.fromTypeInfo( this.typeInfo).toBytes().join(",")}])`); - typeMeta = this.builder.typeMetaResolver.writeTypeMeta(this.builder.getTypeInfo(), this.builder.writer.ownName(), bytes); - } - break; default: break; } return ` - ${this.builder.writer.writeVarUint32Small7(this.getTypeId())}; + ${this.builder.writer.uint8(this.getTypeId())}; + ${writeUserTypeIdStmt} ${typeMeta} `; } @@ -324,7 +357,7 @@ class StructSerializerGenerator extends BaseSerializerGenerator { fixedSize += propGenerator.getFixedSize(); }); } else { - fixedSize += this.builder.fory.classResolver.getSerializerByName(typeInfo.named!)!.fixedSize; + fixedSize += this.builder.fory.typeResolver.getSerializerByName(typeInfo.named!)!.fixedSize; } return fixedSize; } diff --git a/javascript/packages/fory/lib/meta/TypeMeta.ts b/javascript/packages/fory/lib/meta/TypeMeta.ts index 89b8cd320f..06d36dc913 100644 --- a/javascript/packages/fory/lib/meta/TypeMeta.ts +++ b/javascript/packages/fory/lib/meta/TypeMeta.ts @@ -121,6 +121,7 @@ function getPrimitiveTypeSize(typeId: number) { type InnerFieldInfoOptions = { key?: InnerFieldInfo; value?: InnerFieldInfo; inner?: InnerFieldInfo }; interface InnerFieldInfo { typeId: number; + userTypeId: number; trackingRef: boolean; nullable: boolean; options?: InnerFieldInfoOptions; @@ -129,6 +130,7 @@ class FieldInfo { constructor( public fieldName: string, public typeId: number, + public userTypeId = -1, public trackingRef = false, public nullable = false, public options: InnerFieldInfoOptions = {}, @@ -153,6 +155,11 @@ class FieldInfo { static writeTypeId(writer: BinaryWriter, typeInfo: InnerFieldInfo, writeFlags = false) { let { typeId } = typeInfo; + if (typeId === TypeId.NAMED_ENUM) { + typeId = TypeId.ENUM; + } else if (typeId === TypeId.NAMED_UNION || typeId === TypeId.TYPED_UNION) { + typeId = TypeId.UNION; + } const { trackingRef, nullable } = typeInfo; if (writeFlags) { typeId = (typeId << 2); @@ -162,9 +169,11 @@ class FieldInfo { if (trackingRef) { typeId |= 0b1; } + writer.writeVarUint32Small7(typeId); + } else { + writer.uint8(typeId); } - writer.writeVarUint32Small7(typeId); - switch (typeInfo.typeId & 0xff) { + switch (typeInfo.typeId) { case TypeId.LIST: FieldInfo.writeTypeId(writer, typeInfo.options!.inner!, true); break; @@ -204,6 +213,7 @@ export class TypeMeta { typeId: number; typeName: string; namespace: string; + userTypeId: number; }) { } @@ -213,13 +223,27 @@ export class TypeMeta { static fromTypeInfo(typeInfo: StructTypeInfo) { let fieldInfo = Object.entries(typeInfo.options.props!).map(([fieldName, typeInfo]) => { - return new FieldInfo(fieldName, typeInfo.typeId, false, false, typeInfo.options); + let fieldTypeId = typeInfo.typeId; + if (fieldTypeId === TypeId.NAMED_ENUM) { + fieldTypeId = TypeId.ENUM; + } else if (fieldTypeId === TypeId.NAMED_UNION || fieldTypeId === TypeId.TYPED_UNION) { + fieldTypeId = TypeId.UNION; + } + return new FieldInfo( + fieldName, + fieldTypeId, + -1, + false, + false, + typeInfo.options, + ); }); fieldInfo = TypeMeta.groupFieldsByType(fieldInfo); return new TypeMeta(fieldInfo, { typeId: typeInfo.typeId, namespace: typeInfo.namespace, typeName: typeInfo.typeName, + userTypeId: typeInfo.userTypeId ?? -1, }); } @@ -244,6 +268,7 @@ export class TypeMeta { } let typeId: number; + let userTypeId = -1; let namespace = ""; let typeName = ""; @@ -253,7 +278,8 @@ export class TypeMeta { typeName = this.readTypeName(reader); typeId = TypeId.NAMED_STRUCT; // Default for named types } else { - typeId = reader.varUInt32(); + typeId = reader.uint8(); + userTypeId = reader.varUInt32(); } // Read fields @@ -268,6 +294,7 @@ export class TypeMeta { typeId, namespace, typeName, + userTypeId, }; return new TypeMeta(fields, typeInfo); @@ -284,7 +311,7 @@ export class TypeMeta { } // Read type ID - const { typeId, trackingRef, nullable, options } = this.readTypeId(reader); + const { typeId, userTypeId, trackingRef, nullable, options } = this.readTypeId(reader); let fieldName: string; if (encodingFlags === 3) { @@ -296,29 +323,42 @@ export class TypeMeta { fieldName = fieldDecoder.decode(reader, size + 1, encoding || Encoding.UTF_8); } - return new FieldInfo(fieldName, typeId, trackingRef, nullable, options); + return new FieldInfo(fieldName, typeId, userTypeId, trackingRef, nullable, options); } private static readTypeId(reader: BinaryReader, readFlag = false): InnerFieldInfo { const options: InnerFieldInfoOptions = {}; - let typeId = reader.readVarUint32Small7(); let nullable = false; let trackingRef = false; if (readFlag) { + let typeId = reader.readVarUint32Small7(); nullable = Boolean(typeId & 0b10); trackingRef = Boolean(typeId & 0b1); typeId = typeId >> 2; + if (typeId === TypeId.NAMED_ENUM) { + typeId = TypeId.ENUM; + } else if (typeId === TypeId.NAMED_UNION || typeId === TypeId.TYPED_UNION) { + typeId = TypeId.UNION; + } + this.readNestedTypeInfo(reader, typeId, options); + return { typeId, userTypeId: -1, nullable, trackingRef, options }; } + let typeId = reader.uint8(); + if (typeId === TypeId.NAMED_ENUM) { + typeId = TypeId.ENUM; + } else if (typeId === TypeId.NAMED_UNION || typeId === TypeId.TYPED_UNION) { + typeId = TypeId.UNION; + } + this.readNestedTypeInfo(reader, typeId, options); + return { typeId, userTypeId: -1, nullable, trackingRef, options }; + } - const baseTypeId = typeId & 0xff; - - // Handle nested type IDs for collections - switch (baseTypeId) { + private static readNestedTypeInfo(reader: BinaryReader, typeId: number, options: InnerFieldInfoOptions) { + switch (typeId) { case TypeId.LIST: options.inner = this.readTypeId(reader, true); break; case TypeId.SET: - // Read inner type options.key = this.readTypeId(reader, true); break; case TypeId.MAP: @@ -328,8 +368,6 @@ export class TypeMeta { default: break; } - - return { typeId, nullable, trackingRef, options }; } private static readPkgName(reader: BinaryReader): string { @@ -365,6 +403,10 @@ export class TypeMeta { return this.type.typeName; } + getUserTypeId(): number { + return this.type.userTypeId; + } + getFieldInfo(): FieldInfo[] { return this.fields; } @@ -380,7 +422,11 @@ export class TypeMeta { } if (!TypeId.isNamedType(this.type.typeId)) { - writer.varUInt32(this.type.typeId); + writer.uint8(this.type.typeId); + if (this.type.userTypeId === undefined || this.type.userTypeId === -1) { + throw new Error(`userTypeId required for typeId ${this.type.typeId}`); + } + writer.varUInt32(this.type.userTypeId); } else { currentClassHeader |= REGISTER_BY_NAME_FLAG; const ns = this.type.namespace; diff --git a/javascript/packages/fory/lib/metaStringResolver.ts b/javascript/packages/fory/lib/metaStringResolver.ts index 1efac43bc4..85244f274e 100644 --- a/javascript/packages/fory/lib/metaStringResolver.ts +++ b/javascript/packages/fory/lib/metaStringResolver.ts @@ -51,8 +51,11 @@ export class MetaStringResolver { bytes.dynamicWriteStringId = this.dynamicNameId; this.dynamicNameId += 1; this.disposeMetaStringBytes.push(bytes); - writer.varUInt32(bytes.bytes.getBytes().byteLength << 1); - writer.uint8(bytes.bytes.getEncoding()); + const len = bytes.bytes.getBytes().byteLength; + writer.varUInt32(len << 1); + if (len !== 0) { + writer.uint8(bytes.bytes.getEncoding()); + } writer.buffer(bytes.bytes.getBytes()); } } @@ -70,7 +73,11 @@ export class MetaStringResolver { if (idOrLen & 1) { return this.names[idOrLen >> 1]; } else { - const len = idOrLen >> 1; // not used + const len = idOrLen >> 1; + if (len === 0) { + this.names.push(""); + return ""; + } const encoding = reader.uint8(); const name = this.typenameDecoder.decode(reader, len, encoding); this.names.push(name); @@ -83,7 +90,11 @@ export class MetaStringResolver { if (idOrLen & 1) { return this.names[idOrLen >> 1]; } else { - const len = idOrLen >> 1; // not used + const len = idOrLen >> 1; + if (len === 0) { + this.names.push(""); + return ""; + } const encoding = reader.uint8(); const name = this.namespaceDecoder.decode(reader, len, encoding); this.names.push(name); diff --git a/javascript/packages/fory/lib/type.ts b/javascript/packages/fory/lib/type.ts index 8be2fd9532..448814b66e 100644 --- a/javascript/packages/fory/lib/type.ts +++ b/javascript/packages/fory/lib/type.ts @@ -137,31 +137,38 @@ export const TypeId = { TypeId.NAMED_EXT, TypeId.NAMED_STRUCT, TypeId.NAMED_UNION, - ].includes((id & 0xff) as any); + ].includes(id as any); }, polymorphicType(id: number) { - return [TypeId.STRUCT, TypeId.NAMED_STRUCT, TypeId.COMPATIBLE_STRUCT, TypeId.NAMED_COMPATIBLE_STRUCT, TypeId.EXT, TypeId.NAMED_EXT].includes((id & 0xff) as any); + return [TypeId.STRUCT, TypeId.NAMED_STRUCT, TypeId.COMPATIBLE_STRUCT, TypeId.NAMED_COMPATIBLE_STRUCT, TypeId.EXT, TypeId.NAMED_EXT].includes(id as any); }, structType(id: number) { - return [TypeId.STRUCT, TypeId.NAMED_STRUCT, TypeId.COMPATIBLE_STRUCT, TypeId.NAMED_COMPATIBLE_STRUCT].includes((id & 0xff) as any); + return [TypeId.STRUCT, TypeId.NAMED_STRUCT, TypeId.COMPATIBLE_STRUCT, TypeId.NAMED_COMPATIBLE_STRUCT].includes(id as any); }, extType(id: number) { - return [TypeId.EXT, TypeId.NAMED_EXT].includes((id & 0xff) as any); + return [TypeId.EXT, TypeId.NAMED_EXT].includes(id as any); }, enumType(id: number) { - return [TypeId.ENUM, TypeId.NAMED_ENUM].includes((id & 0xff) as any); + return [TypeId.ENUM, TypeId.NAMED_ENUM].includes(id as any); }, userDefinedType(id: number) { - const internalId = id & 0xff; return this.structType(id) || this.extType(id) || this.enumType(id) - || internalId == TypeId.TYPED_UNION - || internalId == TypeId.NAMED_UNION; + || id == TypeId.TYPED_UNION + || id == TypeId.NAMED_UNION; }, isBuiltin(id: number) { - const internalId = id & 0xff; - return !this.userDefinedType(id) && internalId !== TypeId.UNKNOWN; + return !this.userDefinedType(id) && id !== TypeId.UNKNOWN; + }, + needsUserTypeId(id: number) { + return [ + TypeId.ENUM, + TypeId.STRUCT, + TypeId.COMPATIBLE_STRUCT, + TypeId.EXT, + TypeId.TYPED_UNION, + ].includes(id as any); }, } as const; @@ -172,23 +179,24 @@ export enum ConfigFlags { } // read, write -export type Serializer = { +export type Serializer = { fixedSize: number; needToWriteRef: () => boolean; getTypeId: () => number; + getUserTypeId: () => number; getHash: () => number; // for writing - write: (v: T2) => void; - writeRef: (v: T2) => void; - writeNoRef: (v: T2) => void; - writeRefOrNull: (v: T2) => boolean; - writeClassInfo: (v: T2) => void; - - read: (fromRef: boolean) => T2; - readRef: () => T2; - readNoRef: (fromRef: boolean) => T2; - readClassInfo: () => void; + write: (v: T) => void; + writeRef: (v: T) => void; + writeNoRef: (v: T) => void; + writeRefOrNull: (v: T) => boolean; + writeTypeInfo: (v: T) => void; + + read: (fromRef: boolean) => T; + readRef: () => T; + readNoRef: (fromRef: boolean) => T; + readTypeInfo: () => void; }; export enum RefFlags { diff --git a/javascript/packages/fory/lib/typeInfo.ts b/javascript/packages/fory/lib/typeInfo.ts index 23721e6be7..41a2c3c09e 100644 --- a/javascript/packages/fory/lib/typeInfo.ts +++ b/javascript/packages/fory/lib/typeInfo.ts @@ -71,6 +71,8 @@ export class TypeInfo extends ExtensibleFunction { named = ""; namespace = ""; typeName = ""; + // Stored as unsigned 32-bit; -1 (0xffffffff) means "unset". + userTypeId = -1; options?: any; dynamic: "TRUE" | "FALSE" | "AUTO" = "AUTO"; static fory: WeakRef | null = null; @@ -83,7 +85,7 @@ export class TypeInfo extends ExtensibleFunction { TypeInfo.fory = null; } - private constructor(private _typeId: number) { + private constructor(private _typeId: number, userTypeId = -1) { super(function (target: any, key?: string | { name?: string }) { if (key === undefined) { initMeta(target, that as unknown as StructTypeInfo); @@ -97,10 +99,16 @@ export class TypeInfo extends ExtensibleFunction { }); // eslint-disable-next-line const that = this; + if (userTypeId !== -1) { + if (!Number.isInteger(userTypeId) || userTypeId < 0 || userTypeId > 0xFFFFFFFE) { + throw new Error("userTypeId must be in range [0, 0xfffffffe]"); + } + } + this.userTypeId = userTypeId; } computeTypeId(fory?: Fory) { - const internalTypeId = this._typeId & 0xff; + const internalTypeId = this._typeId; if (internalTypeId !== TypeId.STRUCT && internalTypeId !== TypeId.NAMED_STRUCT) { return this._typeId; } @@ -108,10 +116,10 @@ export class TypeInfo extends ExtensibleFunction { throw new Error("fory is not attached") } if (internalTypeId === TypeId.NAMED_STRUCT && fory.config.mode === Mode.Compatible) { - return ((this._typeId >> 8) << 8) | TypeId.NAMED_COMPATIBLE_STRUCT + return TypeId.NAMED_COMPATIBLE_STRUCT; } if (internalTypeId === TypeId.STRUCT && fory.config.mode === Mode.Compatible) { - return ((this._typeId >> 8) << 8) | TypeId.COMPATIBLE_STRUCT + return TypeId.COMPATIBLE_STRUCT; } return this._typeId; } @@ -133,7 +141,7 @@ export class TypeInfo extends ExtensibleFunction { if (TypeId.enumType(this._typeId)) { return true; } - const internalTypeId = this._typeId & 0xff; + const internalTypeId = this._typeId; const fory = TypeInfo.fory?.deref(); if (!fory) { throw new Error("fory is not attached") @@ -194,12 +202,14 @@ export class TypeInfo extends ExtensibleFunction { } } let finalTypeId = 0; + let userTypeId = -1; if (typeId !== undefined) { - finalTypeId = (typeId << 8) | TypeId.STRUCT; + finalTypeId = TypeId.STRUCT; + userTypeId = typeId; } else { - finalTypeId = TypeId.NAMED_STRUCT; + finalTypeId = TypeId.NAMED_STRUCT; } - const typeInfo = new TypeInfo(finalTypeId).cast(); + const typeInfo = new TypeInfo(finalTypeId, userTypeId).cast(); typeInfo.options = { props: props || {}, withConstructor, @@ -252,8 +262,9 @@ export class TypeInfo extends ExtensibleFunction { typeName = splits.slice(1).join("."); } } - const finalTypeId = typeId !== undefined ? ((typeId << 8) | TypeId.ENUM) : TypeId.NAMED_ENUM; - const typeInfo = new TypeInfo(finalTypeId); + const finalTypeId = typeId !== undefined ? TypeId.ENUM : TypeId.NAMED_ENUM; + const userTypeId = typeId !== undefined ? typeId : -1; + const typeInfo = new TypeInfo(finalTypeId, userTypeId); typeInfo.cast().options = { inner: props, }; diff --git a/javascript/packages/fory/lib/typeMetaResolver.ts b/javascript/packages/fory/lib/typeMetaResolver.ts index 2d8b660ca5..00ede32f42 100644 --- a/javascript/packages/fory/lib/typeMetaResolver.ts +++ b/javascript/packages/fory/lib/typeMetaResolver.ts @@ -17,7 +17,7 @@ * under the License. */ -import { StructTypeInfo, Type, TypeInfo } from "./typeInfo"; +import { StructTypeInfo, TypeInfo } from "./typeInfo"; import fory from "./fory"; import { TypeMeta } from "./meta/TypeMeta"; import { BinaryReader } from "./reader"; @@ -37,7 +37,8 @@ export class TypeMetaResolver { typeInfo.options.props = Object.fromEntries(typeMeta.getFieldInfo().map((x) => { const typeId = x.getTypeId(); const fieldName = x.getFieldName(); - const fieldTypeInfo = this.fory.classResolver.getTypeInfo(typeId); + const declared = typeInfo.options.props?.[fieldName]; + const fieldTypeInfo = declared ?? this.fory.typeResolver.getTypeInfo(typeId); if (!fieldTypeInfo) { throw new Error(`typeid: ${typeId} in prop ${fieldName} not registered`); } @@ -57,11 +58,12 @@ export class TypeMetaResolver { const typeName = typeMeta.getTypeName(); const ns = typeMeta.getNs(); const typeId = typeMeta.getTypeId(); + const userTypeId = typeMeta.getUserTypeId(); let typeInfo; if (!TypeId.isNamedType(typeId)) { - typeInfo = this.fory.classResolver.getTypeInfo(typeId); + typeInfo = this.fory.typeResolver.getTypeInfo(typeId, userTypeId); } else { - typeInfo = this.fory.classResolver.getTypeInfo(`${ns}$${typeName}`); + typeInfo = this.fory.typeResolver.getTypeInfo(`${ns}$${typeName}`); } if (!typeInfo) { throw new Error(`${typeId} not registered`); // todo diff --git a/javascript/packages/fory/lib/classResolver.ts b/javascript/packages/fory/lib/typeResolver.ts similarity index 82% rename from javascript/packages/fory/lib/classResolver.ts rename to javascript/packages/fory/lib/typeResolver.ts index c0724675a9..e7af28b488 100644 --- a/javascript/packages/fory/lib/classResolver.ts +++ b/javascript/packages/fory/lib/typeResolver.ts @@ -28,6 +28,9 @@ const uninitSerialize = { getTypeId: () => { throw new Error("uninitSerialize"); }, + getUserTypeId: () => { + throw new Error("uninitSerialize"); + }, needToWriteRef: () => { throw new Error("uninitSerialize"); }, @@ -35,35 +38,42 @@ const uninitSerialize = { throw new Error("uninitSerialize"); }, write: (v: any) => { + void v; throw new Error("uninitSerialize"); }, writeRef: (v: any) => { + void v; throw new Error("uninitSerialize"); }, writeNoRef: (v: any) => { + void v; throw new Error("uninitSerialize"); }, writeRefOrNull: (v: any) => { + void v; throw new Error("uninitSerialize"); }, - writeClassInfo: (v: any) => { + writeTypeInfo: (v: any) => { + void v; throw new Error("uninitSerialize"); }, read: (fromRef: boolean) => { + void fromRef; throw new Error("uninitSerialize"); }, readRef: () => { throw new Error("uninitSerialize"); }, readNoRef: (fromRef: boolean) => { + void fromRef; throw new Error("uninitSerialize"); }, - readClassInfo: () => { + readTypeInfo: () => { throw new Error("uninitSerialize"); }, }; -export default class ClassResolver { +export default class TypeResolver { private internalSerializer: Serializer[] = new Array(300); private customSerializer: Map = new Map(); private typeInfoMap: Map = new Map(); @@ -137,17 +147,35 @@ export default class ClassResolver { constructor(private fory: Fory) { } + private makeUserTypeKey(userTypeId: number) { + return `u:${userTypeId}`; + } + init() { this.initInternalSerializer(); } - getTypeInfo(typeIdOrName: number | string) { + getTypeInfo(typeIdOrName: number | string, userTypeId?: number) { + if (typeof typeIdOrName === "number" && userTypeId !== undefined && TypeId.needsUserTypeId(typeIdOrName)) { + return this.typeInfoMap.get(this.makeUserTypeKey(userTypeId)); + } return this.typeInfoMap.get(typeIdOrName); } registerSerializer(typeInfo: TypeInfo, serializer: Serializer = uninitSerialize) { - if (!TypeId.isNamedType(typeInfo.typeId)) { - const id = typeInfo.typeId; + const typeId = typeInfo.typeId; + if (!TypeId.isNamedType(typeId)) { + if (TypeId.needsUserTypeId(typeId) && typeInfo.userTypeId !== -1) { + const key = this.makeUserTypeKey(typeInfo.userTypeId); + this.typeInfoMap.set(key, typeInfo); + if (this.customSerializer.has(key)) { + Object.assign(this.customSerializer.get(key)!, serializer); + } else { + this.customSerializer.set(key, { ...serializer }); + } + return this.customSerializer.get(key); + } + const id = typeId; this.typeInfoMap.set(id, typeInfo); if (id <= 0xFF) { if (this.internalSerializer[id]) { @@ -156,14 +184,13 @@ export default class ClassResolver { this.internalSerializer[id] = { ...serializer }; } return this.internalSerializer[id]; + } + if (this.customSerializer.has(id)) { + Object.assign(this.customSerializer.get(id)!, serializer); } else { - if (this.customSerializer.has(id)) { - Object.assign(this.customSerializer.get(id)!, serializer); - } else { - this.customSerializer.set(id, { ...serializer }); - } - return this.customSerializer.get(id); + this.customSerializer.set(id, { ...serializer }); } + return this.customSerializer.get(id); } else { const namedTypeInfo = typeInfo.castToStruct(); const name = namedTypeInfo.named!; @@ -181,6 +208,9 @@ export default class ClassResolver { if (typeInfo.isNamedType()) { return this.typeInfoMap.has((typeInfo.castToStruct()).named!); } + if (TypeId.needsUserTypeId(typeInfo.typeId) && typeInfo.userTypeId !== -1) { + return this.typeInfoMap.has(this.makeUserTypeKey(typeInfo.userTypeId)); + } return this.typeInfoMap.has(typeInfo.typeId); } @@ -189,10 +219,13 @@ export default class ClassResolver { if (TypeId.isNamedType(typeId)) { return this.customSerializer.get((typeInfo.castToStruct()).named!); } - return this.getSerializerById(typeId); + return this.getSerializerById(typeId, typeInfo.userTypeId); } - getSerializerById(id: number) { + getSerializerById(id: number, userTypeId?: number) { + if (TypeId.needsUserTypeId(id) && userTypeId !== undefined && userTypeId !== -1) { + return this.customSerializer.get(this.makeUserTypeKey(userTypeId))!; + } if (id <= 0xff) { return this.internalSerializer[id]!; } else { diff --git a/licenserc.toml b/licenserc.toml index 2af7f88572..e4982ff2e8 100644 --- a/licenserc.toml +++ b/licenserc.toml @@ -25,6 +25,7 @@ excludes = [ # Generated "benchmarks/java_benchmark/src/main/java/org/apache/fory/benchmark/state/generated/**", + "integration_tests/idl_tests/**/auto_id.*", # Derived "ci/format.sh", diff --git a/python/pyfory/_fory.py b/python/pyfory/_fory.py index 1ff7181419..8b58320636 100644 --- a/python/pyfory/_fory.py +++ b/python/pyfory/_fory.py @@ -47,6 +47,8 @@ USE_TYPE_ID = 1 # preserve 0 as flag for type id not set in TypeInfo` NO_TYPE_ID = 0 +# 0xffffffff means "unset" for user type id. +NO_USER_TYPE_ID = 0xFFFFFFFF INT64_TYPE_ID = TypeId.VARINT64 FLOAT64_TYPE_ID = TypeId.FLOAT64 BOOL_TYPE_ID = TypeId.BOOL @@ -523,27 +525,27 @@ def write_ref(self, buffer, obj, typeinfo=None): if self.ref_resolver.write_ref_or_null(buffer, obj): return if typeinfo is None: - typeinfo = self.type_resolver.get_typeinfo(cls) - self.type_resolver.write_typeinfo(buffer, typeinfo) + typeinfo = self.type_resolver.get_type_info(cls) + self.type_resolver.write_type_info(buffer, typeinfo) typeinfo.serializer.write(buffer, obj) def write_no_ref(self, buffer, obj): cls = type(obj) if cls is str: - buffer.write_var_uint32(STRING_TYPE_ID) + buffer.write_uint8(STRING_TYPE_ID) buffer.write_string(obj) return elif cls is int: - buffer.write_var_uint32(INT64_TYPE_ID) + buffer.write_uint8(INT64_TYPE_ID) buffer.write_varint64(obj) return elif cls is bool: - buffer.write_var_uint32(BOOL_TYPE_ID) + buffer.write_uint8(BOOL_TYPE_ID) buffer.write_bool(obj) return else: - typeinfo = self.type_resolver.get_typeinfo(cls) - self.type_resolver.write_typeinfo(buffer, typeinfo) + typeinfo = self.type_resolver.get_type_info(cls) + self.type_resolver.write_type_info(buffer, typeinfo) typeinfo.serializer.write(buffer, obj) def xwrite_ref(self, buffer, obj, serializer=None): @@ -562,8 +564,8 @@ def xwrite_no_ref(self, buffer, obj, serializer=None): serializer.xwrite(buffer, obj) return cls = type(obj) - typeinfo = self.type_resolver.get_typeinfo(cls) - self.type_resolver.write_typeinfo(buffer, typeinfo) + typeinfo = self.type_resolver.get_type_info(cls) + self.type_resolver.write_type_info(buffer, typeinfo) typeinfo.serializer.xwrite(buffer, obj) def deserialize( @@ -641,7 +643,7 @@ def read_ref(self, buffer): ref_id = ref_resolver.try_preserve_ref_id(buffer) # indicates that the object is first read. if ref_id >= NOT_NULL_VALUE_FLAG: - typeinfo = self.type_resolver.read_typeinfo(buffer) + typeinfo = self.type_resolver.read_type_info(buffer) self.inc_depth() o = typeinfo.serializer.read(buffer) self.dec_depth() @@ -652,7 +654,7 @@ def read_ref(self, buffer): def read_no_ref(self, buffer): """Deserialize not-null and non-reference object from buffer.""" - typeinfo = self.type_resolver.read_typeinfo(buffer) + typeinfo = self.type_resolver.read_type_info(buffer) self.inc_depth() o = typeinfo.serializer.read(buffer) self.dec_depth() @@ -677,7 +679,7 @@ def xread_ref(self, buffer, serializer=None): def xread_no_ref(self, buffer, serializer=None): if serializer is None: - serializer = self.type_resolver.read_typeinfo(buffer).serializer + serializer = self.type_resolver.read_type_info(buffer).serializer # Push -1 to read_ref_ids so reference() can pop it and skip reference tracking # This handles the case where xread_no_ref is called directly without xread_ref if self.ref_tracking: @@ -687,7 +689,7 @@ def xread_no_ref(self, buffer, serializer=None): def _xread_no_ref_internal(self, buffer, serializer): """Internal method to read without pushing to read_ref_ids.""" if serializer is None: - serializer = self.type_resolver.read_typeinfo(buffer).serializer + serializer = self.type_resolver.read_type_info(buffer).serializer self.inc_depth() o = serializer.xread(buffer) self.dec_depth() @@ -746,8 +748,8 @@ def write_ref_pyobject(self, buffer, value, typeinfo=None): if self.ref_resolver.write_ref_or_null(buffer, value): return if typeinfo is None: - typeinfo = self.type_resolver.get_typeinfo(type(value)) - self.type_resolver.write_typeinfo(buffer, typeinfo) + typeinfo = self.type_resolver.get_type_info(type(value)) + self.type_resolver.write_type_info(buffer, typeinfo) typeinfo.serializer.write(buffer, value) def read_ref_pyobject(self, buffer): diff --git a/python/pyfory/collection.pxi b/python/pyfory/collection.pxi index 3142c511b7..aa62736c32 100644 --- a/python/pyfory/collection.pxi +++ b/python/pyfory/collection.pxi @@ -40,7 +40,7 @@ cdef class CollectionSerializer(Serializer): cdef c_bool is_py cdef int8_t elem_tracking_ref cdef elem_type - cdef TypeInfo elem_typeinfo + cdef TypeInfo elem_type_info def __init__(self, fory, type_, elem_serializer=None, elem_tracking_ref=None): super().__init__(fory, type_) @@ -49,11 +49,11 @@ cdef class CollectionSerializer(Serializer): self.elem_serializer = elem_serializer if elem_serializer is None: self.elem_type = None - self.elem_typeinfo = self.type_resolver.get_typeinfo(None) + self.elem_type_info = self.type_resolver.get_type_info(None) self.elem_tracking_ref = -1 else: self.elem_type = elem_serializer.type_ - self.elem_typeinfo = fory.type_resolver.get_typeinfo(self.elem_type) + self.elem_type_info = fory.type_resolver.get_type_info(self.elem_type) self.elem_tracking_ref = (elem_serializer.need_to_write_ref) if elem_tracking_ref is not None: self.elem_tracking_ref = (1 if elem_tracking_ref else 0) @@ -62,7 +62,7 @@ cdef class CollectionSerializer(Serializer): cdef inline pair[int8_t, int64_t] write_header(self, Buffer buffer, value): cdef int8_t collect_flag = COLL_DEFAULT_FLAG elem_type = self.elem_type - cdef TypeInfo elem_typeinfo = self.elem_typeinfo + cdef TypeInfo elem_type_info = self.elem_type_info cdef c_bool has_null = False cdef c_bool has_same_type = True if elem_type is None: @@ -76,7 +76,7 @@ cdef class CollectionSerializer(Serializer): has_same_type = False if has_same_type: collect_flag |= COLL_IS_SAME_TYPE - elem_typeinfo = self.type_resolver.get_typeinfo(elem_type) + elem_type_info = self.type_resolver.get_type_info(elem_type) else: collect_flag |= COLL_IS_DECL_ELEMENT_TYPE | COLL_IS_SAME_TYPE for s in value: @@ -89,14 +89,14 @@ cdef class CollectionSerializer(Serializer): if self.elem_tracking_ref == 1: collect_flag |= COLL_TRACKING_REF elif self.elem_tracking_ref == -1: - if not has_same_type or elem_typeinfo.serializer.need_to_write_ref: + if not has_same_type or elem_type_info.serializer.need_to_write_ref: collect_flag |= COLL_TRACKING_REF buffer.write_var_uint32(len(value)) buffer.write_int8(collect_flag) if (has_same_type and collect_flag & COLL_IS_DECL_ELEMENT_TYPE == 0): - self.type_resolver.write_typeinfo(buffer, elem_typeinfo) - return pair[int8_t, int64_t](collect_flag, obj2int(elem_typeinfo)) + self.type_resolver.write_type_info(buffer, elem_type_info) + return pair[int8_t, int64_t](collect_flag, obj2int(elem_type_info)) cpdef write(self, Buffer buffer, value): if len(value) == 0: @@ -104,13 +104,13 @@ cdef class CollectionSerializer(Serializer): return cdef pair[int8_t, int64_t] header_pair = self.write_header(buffer, value) cdef int8_t collect_flag = header_pair.first - cdef int64_t elem_typeinfo_ptr = header_pair.second - cdef TypeInfo elem_typeinfo = int2obj(elem_typeinfo_ptr) - cdef elem_type = elem_typeinfo.cls + cdef int64_t elem_type_info_ptr = header_pair.second + cdef TypeInfo elem_type_info = int2obj(elem_type_info_ptr) + cdef elem_type = elem_type_info.cls cdef MapRefResolver ref_resolver = self.ref_resolver cdef TypeResolver type_resolver = self.type_resolver cdef c_bool is_py = self.is_py - cdef serializer = type(elem_typeinfo.serializer) + cdef serializer = type(elem_type_info.serializer) cdef c_bool tracking_ref cdef c_bool has_null if (collect_flag & COLL_IS_SAME_TYPE) != 0: @@ -124,13 +124,13 @@ cdef class CollectionSerializer(Serializer): elif serializer is Float64Serializer: self._write_float(buffer, value) elif (collect_flag & COLL_TRACKING_REF) == 0: - self._write_same_type_no_ref(buffer, value, elem_typeinfo) + self._write_same_type_no_ref(buffer, value, elem_type_info) else: - self._write_same_type_ref(buffer, value, elem_typeinfo) + self._write_same_type_ref(buffer, value, elem_type_info) elif (collect_flag & COLL_TRACKING_REF) != 0: - self._write_same_type_ref(buffer, value, elem_typeinfo) + self._write_same_type_ref(buffer, value, elem_type_info) else: - self._write_same_type_has_null(buffer, value, elem_typeinfo) + self._write_same_type_has_null(buffer, value, elem_type_info) else: # Check tracking_ref and has_null flags for different types writing tracking_ref = (collect_flag & COLL_TRACKING_REF) != 0 @@ -153,8 +153,8 @@ cdef class CollectionSerializer(Serializer): buffer.write_double(s) else: if not ref_resolver.write_ref_or_null(buffer, s): - typeinfo = type_resolver.get_typeinfo(cls) - type_resolver.write_typeinfo(buffer, typeinfo) + typeinfo = type_resolver.get_type_info(cls) + type_resolver.write_type_info(buffer, typeinfo) if is_py: typeinfo.serializer.write(buffer, s) else: @@ -163,8 +163,8 @@ cdef class CollectionSerializer(Serializer): # When ref tracking is disabled and no nulls, write type info directly for s in value: cls = type(s) - typeinfo = type_resolver.get_typeinfo(cls) - type_resolver.write_typeinfo(buffer, typeinfo) + typeinfo = type_resolver.get_type_info(cls) + type_resolver.write_type_info(buffer, typeinfo) if is_py: typeinfo.serializer.write(buffer, s) else: @@ -177,8 +177,8 @@ cdef class CollectionSerializer(Serializer): else: buffer.write_int8(NOT_NULL_VALUE_FLAG) cls = type(s) - typeinfo = type_resolver.get_typeinfo(cls) - type_resolver.write_typeinfo(buffer, typeinfo) + typeinfo = type_resolver.get_type_info(cls) + type_resolver.write_type_info(buffer, typeinfo) if is_py: typeinfo.serializer.write(buffer, s) else: @@ -345,27 +345,27 @@ cdef class ListSerializer(CollectionSerializer): ref_resolver.reference(list_) cdef c_bool is_py = self.is_py cdef TypeInfo typeinfo - cdef int32_t type_id = -1 + cdef uint8_t type_id = 0 cdef c_bool tracking_ref cdef c_bool has_null cdef int8_t head_flag if (collect_flag & COLL_IS_SAME_TYPE) != 0: if collect_flag & COLL_IS_DECL_ELEMENT_TYPE == 0: - typeinfo = self.type_resolver.read_typeinfo(buffer) + typeinfo = self.type_resolver.read_type_info(buffer) else: - typeinfo = self.elem_typeinfo + typeinfo = self.elem_type_info if (collect_flag & COLL_HAS_NULL) == 0: type_id = typeinfo.type_id - if type_id == TypeId.STRING: + if type_id == TypeId.STRING: self._read_string(buffer, len_, list_) return list_ - elif type_id == TypeId.VARINT64: + elif type_id == TypeId.VARINT64: self._read_int(buffer, len_, list_) return list_ - elif type_id == TypeId.BOOL: + elif type_id == TypeId.BOOL: self._read_bool(buffer, len_, list_) return list_ - elif type_id == TypeId.FLOAT64: + elif type_id == TypeId.FLOAT64: self._read_float(buffer, len_, list_) return list_ elif (collect_flag & COLL_TRACKING_REF) == 0: @@ -390,7 +390,7 @@ cdef class ListSerializer(CollectionSerializer): elif not has_null: # When ref tracking is disabled and no nulls, read type info directly for i in range(len_): - typeinfo = type_resolver.read_typeinfo(buffer) + typeinfo = type_resolver.read_type_info(buffer) if is_py: elem = typeinfo.serializer.read(buffer) else: @@ -404,7 +404,7 @@ cdef class ListSerializer(CollectionSerializer): if head_flag == NULL_FLAG: elem = None else: - typeinfo = type_resolver.read_typeinfo(buffer) + typeinfo = type_resolver.read_type_info(buffer) if is_py: elem = typeinfo.serializer.read(buffer) else: @@ -433,18 +433,18 @@ cdef inline get_next_element( if ref_id < NOT_NULL_VALUE_FLAG: return ref_resolver.get_read_object() # indicates that the object is first read. - typeinfo = type_resolver.read_typeinfo(buffer) - cdef int32_t type_id = typeinfo.type_id + typeinfo = type_resolver.read_type_info(buffer) + cdef uint8_t type_id = typeinfo.type_id # Note that all read operations in fast paths of list/tuple/set/dict/sub_dict # must match corresponding writing operations. Otherwise, ref tracking will # error. - if type_id == TypeId.STRING: + if type_id == TypeId.STRING: return buffer.read_string() - elif type_id == TypeId.VARINT32: + elif type_id == TypeId.VARINT32: return buffer.read_varint64() - elif type_id == TypeId.BOOL: + elif type_id == TypeId.BOOL: return buffer.read_bool() - elif type_id == TypeId.FLOAT64: + elif type_id == TypeId.FLOAT64: return buffer.read_double() else: if is_py: @@ -467,27 +467,27 @@ cdef class TupleSerializer(CollectionSerializer): cdef int8_t collect_flag = buffer.read_int8() cdef c_bool is_py = self.is_py cdef TypeInfo typeinfo - cdef int32_t type_id = -1 + cdef uint8_t type_id = 0 cdef c_bool tracking_ref cdef c_bool has_null cdef int8_t head_flag if (collect_flag & COLL_IS_SAME_TYPE) != 0: if collect_flag & COLL_IS_DECL_ELEMENT_TYPE == 0: - typeinfo = self.type_resolver.read_typeinfo(buffer) + typeinfo = self.type_resolver.read_type_info(buffer) else: - typeinfo = self.elem_typeinfo + typeinfo = self.elem_type_info if (collect_flag & COLL_HAS_NULL) == 0: type_id = typeinfo.type_id - if type_id == TypeId.STRING: + if type_id == TypeId.STRING: self._read_string(buffer, len_, tuple_) return tuple_ - if type_id == TypeId.VARINT64: + if type_id == TypeId.VARINT64: self._read_int(buffer, len_, tuple_) return tuple_ - if type_id == TypeId.BOOL: + if type_id == TypeId.BOOL: self._read_bool(buffer, len_, tuple_) return tuple_ - if type_id == TypeId.FLOAT64: + if type_id == TypeId.FLOAT64: self._read_float(buffer, len_, tuple_) return tuple_ elif (collect_flag & COLL_TRACKING_REF) == 0: @@ -512,7 +512,7 @@ cdef class TupleSerializer(CollectionSerializer): elif not has_null: # When ref tracking is disabled and no nulls, read type info directly for i in range(len_): - typeinfo = type_resolver.read_typeinfo(buffer) + typeinfo = type_resolver.read_type_info(buffer) if is_py: elem = typeinfo.serializer.read(buffer) else: @@ -526,7 +526,7 @@ cdef class TupleSerializer(CollectionSerializer): if head_flag == NULL_FLAG: elem = None else: - typeinfo = type_resolver.read_typeinfo(buffer) + typeinfo = type_resolver.read_type_info(buffer) if is_py: elem = typeinfo.serializer.read(buffer) else: @@ -563,28 +563,28 @@ cdef class SetSerializer(CollectionSerializer): cdef int8_t collect_flag = buffer.read_int8() cdef int32_t ref_id cdef TypeInfo typeinfo - cdef int32_t type_id = -1 + cdef uint8_t type_id = 0 cdef c_bool is_py = self.is_py cdef c_bool tracking_ref cdef c_bool has_null cdef int8_t head_flag if (collect_flag & COLL_IS_SAME_TYPE) != 0: if collect_flag & COLL_IS_DECL_ELEMENT_TYPE == 0: - typeinfo = self.type_resolver.read_typeinfo(buffer) + typeinfo = self.type_resolver.read_type_info(buffer) else: - typeinfo = self.elem_typeinfo + typeinfo = self.elem_type_info if (collect_flag & COLL_HAS_NULL) == 0: type_id = typeinfo.type_id - if type_id == TypeId.STRING: + if type_id == TypeId.STRING: self._read_string(buffer, len_, instance) return instance - if type_id == TypeId.VARINT64: + if type_id == TypeId.VARINT64: self._read_int(buffer, len_, instance) return instance - if type_id == TypeId.BOOL: + if type_id == TypeId.BOOL: self._read_bool(buffer, len_, instance) return instance - if type_id == TypeId.FLOAT64: + if type_id == TypeId.FLOAT64: self._read_float(buffer, len_, instance) return instance elif (collect_flag & COLL_TRACKING_REF) == 0: @@ -608,15 +608,15 @@ cdef class SetSerializer(CollectionSerializer): instance.add(ref_resolver.get_read_object()) continue # indicates that the object is first read. - typeinfo = type_resolver.read_typeinfo(buffer) + typeinfo = type_resolver.read_type_info(buffer) type_id = typeinfo.type_id - if type_id == TypeId.STRING: + if type_id == TypeId.STRING: instance.add(buffer.read_string()) - elif type_id == TypeId.VARINT64: + elif type_id == TypeId.VARINT64: instance.add(buffer.read_varint64()) - elif type_id == TypeId.BOOL: + elif type_id == TypeId.BOOL: instance.add(buffer.read_bool()) - elif type_id == TypeId.FLOAT64: + elif type_id == TypeId.FLOAT64: instance.add(buffer.read_double()) else: if is_py: @@ -628,15 +628,15 @@ cdef class SetSerializer(CollectionSerializer): elif not has_null: # When ref tracking is disabled and no nulls, read type info directly for i in range(len_): - typeinfo = type_resolver.read_typeinfo(buffer) + typeinfo = type_resolver.read_type_info(buffer) type_id = typeinfo.type_id - if type_id == TypeId.STRING: + if type_id == TypeId.STRING: instance.add(buffer.read_string()) - elif type_id == TypeId.VARINT64: + elif type_id == TypeId.VARINT64: instance.add(buffer.read_varint64()) - elif type_id == TypeId.BOOL: + elif type_id == TypeId.BOOL: instance.add(buffer.read_bool()) - elif type_id == TypeId.FLOAT64: + elif type_id == TypeId.FLOAT64: instance.add(buffer.read_double()) else: if is_py: @@ -650,15 +650,15 @@ cdef class SetSerializer(CollectionSerializer): if head_flag == NULL_FLAG: instance.add(None) else: - typeinfo = type_resolver.read_typeinfo(buffer) + typeinfo = type_resolver.read_type_info(buffer) type_id = typeinfo.type_id - if type_id == TypeId.STRING: + if type_id == TypeId.STRING: instance.add(buffer.read_string()) - elif type_id == TypeId.VARINT64: + elif type_id == TypeId.VARINT64: instance.add(buffer.read_varint64()) - elif type_id == TypeId.BOOL: + elif type_id == TypeId.BOOL: instance.add(buffer.read_bool()) - elif type_id == TypeId.FLOAT64: + elif type_id == TypeId.FLOAT64: instance.add(buffer.read_double()) else: if is_py: @@ -752,7 +752,7 @@ cdef class MapSerializer(Serializer): cdef Serializer key_serializer = self.key_serializer cdef Serializer value_serializer = self.value_serializer cdef type key_cls, value_cls, key_serializer_type, value_serializer_type - cdef TypeInfo key_typeinfo, value_typeinfo + cdef TypeInfo key_type_info, value_type_info cdef int32_t chunk_size_offset, chunk_header, chunk_size cdef c_bool key_write_ref, value_write_ref cdef int has_next = PyDict_Next(obj, &pos, &key_addr, &value_addr) @@ -832,15 +832,15 @@ cdef class MapSerializer(Serializer): if key_serializer is not None: chunk_header |= KEY_DECL_TYPE else: - key_typeinfo = self.type_resolver.get_typeinfo(key_cls) - type_resolver.write_typeinfo(buffer, key_typeinfo) - key_serializer = key_typeinfo.serializer + key_type_info = self.type_resolver.get_type_info(key_cls) + type_resolver.write_type_info(buffer, key_type_info) + key_serializer = key_type_info.serializer if value_serializer is not None: chunk_header |= VALUE_DECL_TYPE else: - value_typeinfo = self.type_resolver.get_typeinfo(value_cls) - type_resolver.write_typeinfo(buffer, value_typeinfo) - value_serializer = value_typeinfo.serializer + value_type_info = self.type_resolver.get_type_info(value_cls) + type_resolver.write_type_info(buffer, value_type_info) + value_serializer = value_type_info.serializer if self.key_serializer is not None: key_write_ref = self.key_tracking_ref == 1 else: @@ -917,7 +917,7 @@ cdef class MapSerializer(Serializer): cdef dict map_ = _PyDict_NewPresized(size) ref_resolver.reference(map_) cdef int32_t ref_id - cdef TypeInfo key_typeinfo, value_typeinfo + cdef TypeInfo key_type_info, value_type_info cdef int32_t chunk_header = 0 if size != 0: chunk_header = buffer.read_uint8() @@ -999,9 +999,9 @@ cdef class MapSerializer(Serializer): value_is_declared_type = (chunk_header & VALUE_DECL_TYPE) != 0 chunk_size = buffer.read_uint8() if not key_is_declared_type: - key_serializer = type_resolver.read_typeinfo(buffer).serializer + key_serializer = type_resolver.read_type_info(buffer).serializer if not value_is_declared_type: - value_serializer = type_resolver.read_typeinfo(buffer).serializer + value_serializer = type_resolver.read_type_info(buffer).serializer key_serializer_type = type(key_serializer) value_serializer_type = type(value_serializer) for i in range(chunk_size): diff --git a/python/pyfory/collection.py b/python/pyfory/collection.py index 8dd30f8bda..1fcef05c99 100644 --- a/python/pyfory/collection.py +++ b/python/pyfory/collection.py @@ -50,7 +50,7 @@ class CollectionSerializer(Serializer): "is_py", "elem_tracking_ref", "elem_type", - "elem_typeinfo", + "elem_type_info", ) def __init__(self, fory, type_, elem_serializer=None, elem_tracking_ref=None): @@ -60,11 +60,11 @@ def __init__(self, fory, type_, elem_serializer=None, elem_tracking_ref=None): self.elem_serializer = elem_serializer if elem_serializer is None: self.elem_type = None - self.elem_typeinfo = self.type_resolver.get_typeinfo(None) + self.elem_type_info = self.type_resolver.get_type_info(None) self.elem_tracking_ref = -1 else: self.elem_type = elem_serializer.type_ - self.elem_typeinfo = fory.type_resolver.get_typeinfo(self.elem_type) + self.elem_type_info = fory.type_resolver.get_type_info(self.elem_type) self.elem_tracking_ref = int(elem_serializer.need_to_write_ref) if elem_tracking_ref is not None: self.elem_tracking_ref = 1 if elem_tracking_ref else 0 @@ -73,7 +73,7 @@ def __init__(self, fory, type_, elem_serializer=None, elem_tracking_ref=None): def write_header(self, buffer, value): collect_flag = COLL_DEFAULT_FLAG elem_type = self.elem_type - elem_typeinfo = self.elem_typeinfo + elem_type_info = self.elem_type_info has_null = False has_same_type = True if elem_type is None: @@ -88,7 +88,7 @@ def write_header(self, buffer, value): if has_same_type: collect_flag |= COLL_IS_SAME_TYPE if elem_type is not None: - elem_typeinfo = self.type_resolver.get_typeinfo(elem_type) + elem_type_info = self.type_resolver.get_type_info(elem_type) else: collect_flag |= COLL_IS_DECL_ELEMENT_TYPE | COLL_IS_SAME_TYPE for s in value: @@ -102,13 +102,13 @@ def write_header(self, buffer, value): if self.elem_tracking_ref == 1: collect_flag |= COLL_TRACKING_REF elif self.elem_tracking_ref == -1: - if not has_same_type or elem_typeinfo.serializer.need_to_write_ref: + if not has_same_type or elem_type_info.serializer.need_to_write_ref: collect_flag |= COLL_TRACKING_REF buffer.write_var_uint32(len(value)) buffer.write_int8(collect_flag) if has_same_type and (collect_flag & COLL_IS_DECL_ELEMENT_TYPE) == 0: - self.type_resolver.write_typeinfo(buffer, elem_typeinfo) - return collect_flag, elem_typeinfo + self.type_resolver.write_type_info(buffer, elem_type_info) + return collect_flag, elem_type_info def write(self, buffer, value): if len(value) == 0: @@ -166,8 +166,8 @@ def _write_different_types(self, buffer, value, collect_flag=0): # When ref tracking is enabled, write with ref handling for s in value: if not self.ref_resolver.write_ref_or_null(buffer, s): - typeinfo = self.type_resolver.get_typeinfo(type(s)) - self.type_resolver.write_typeinfo(buffer, typeinfo) + typeinfo = self.type_resolver.get_type_info(type(s)) + self.type_resolver.write_type_info(buffer, typeinfo) if self.is_py: typeinfo.serializer.write(buffer, s) else: @@ -175,8 +175,8 @@ def _write_different_types(self, buffer, value, collect_flag=0): elif not has_null: # When ref tracking is disabled and no nulls, write type info directly for s in value: - typeinfo = self.type_resolver.get_typeinfo(type(s)) - self.type_resolver.write_typeinfo(buffer, typeinfo) + typeinfo = self.type_resolver.get_type_info(type(s)) + self.type_resolver.write_type_info(buffer, typeinfo) if self.is_py: typeinfo.serializer.write(buffer, s) else: @@ -188,8 +188,8 @@ def _write_different_types(self, buffer, value, collect_flag=0): buffer.write_int8(NULL_FLAG) else: buffer.write_int8(NOT_NULL_VALUE_FLAG) - typeinfo = self.type_resolver.get_typeinfo(type(s)) - self.type_resolver.write_typeinfo(buffer, typeinfo) + typeinfo = self.type_resolver.get_type_info(type(s)) + self.type_resolver.write_type_info(buffer, typeinfo) if self.is_py: typeinfo.serializer.write(buffer, s) else: @@ -203,9 +203,9 @@ def read(self, buffer): collect_flag = buffer.read_int8() if (collect_flag & COLL_IS_SAME_TYPE) != 0: if collect_flag & COLL_IS_DECL_ELEMENT_TYPE == 0: - typeinfo = self.type_resolver.read_typeinfo(buffer) + typeinfo = self.type_resolver.read_type_info(buffer) else: - typeinfo = self.elem_typeinfo + typeinfo = self.elem_type_info if (collect_flag & COLL_TRACKING_REF) != 0: self._read_same_type_ref(buffer, len_, collection_, typeinfo) elif (collect_flag & COLL_HAS_NULL) == 0: @@ -281,7 +281,7 @@ def _read_different_types(self, buffer, len_, collection_, collect_flag): elif not has_null: # When ref tracking is disabled and no nulls, read type info directly for i in range(len_): - typeinfo = self.type_resolver.read_typeinfo(buffer) + typeinfo = self.type_resolver.read_type_info(buffer) if typeinfo is None: elem = None elif self.is_py: @@ -296,7 +296,7 @@ def _read_different_types(self, buffer, len_, collection_, collect_flag): if head_flag == NULL_FLAG: elem = None else: - typeinfo = self.type_resolver.read_typeinfo(buffer) + typeinfo = self.type_resolver.read_type_info(buffer) if typeinfo is None: elem = None elif self.is_py: @@ -353,7 +353,7 @@ def get_next_element(buffer, ref_resolver, type_resolver, is_py): ref_id = ref_resolver.try_preserve_ref_id(buffer) if ref_id < NOT_NULL_VALUE_FLAG: return ref_resolver.get_read_object() - typeinfo = type_resolver.read_typeinfo(buffer) + typeinfo = type_resolver.read_type_info(buffer) if is_py: obj = typeinfo.serializer.read(buffer) else: @@ -484,16 +484,16 @@ def write(self, buffer, o): if key_serializer is not None: chunk_header |= KEY_DECL_TYPE else: - key_typeinfo = self.type_resolver.get_typeinfo(key_cls) - type_resolver.write_typeinfo(buffer, key_typeinfo) - key_serializer = key_typeinfo.serializer + key_type_info = self.type_resolver.get_type_info(key_cls) + type_resolver.write_type_info(buffer, key_type_info) + key_serializer = key_type_info.serializer if value_serializer is not None: chunk_header |= VALUE_DECL_TYPE else: - value_typeinfo = self.type_resolver.get_typeinfo(value_cls) - type_resolver.write_typeinfo(buffer, value_typeinfo) - value_serializer = value_typeinfo.serializer + value_type_info = self.type_resolver.get_type_info(value_cls) + type_resolver.write_type_info(buffer, value_type_info) + value_serializer = value_type_info.serializer if self.key_serializer is not None: key_write_ref = self.key_tracking_ref @@ -596,9 +596,9 @@ def read(self, buffer): value_is_declared_type = (chunk_header & VALUE_DECL_TYPE) != 0 chunk_size = buffer.read_uint8() if not key_is_declared_type: - key_serializer = type_resolver.read_typeinfo(buffer).serializer + key_serializer = type_resolver.read_type_info(buffer).serializer if not value_is_declared_type: - value_serializer = type_resolver.read_typeinfo(buffer).serializer + value_serializer = type_resolver.read_type_info(buffer).serializer for i in range(chunk_size): if track_key_ref: ref_id = ref_resolver.try_preserve_ref_id(buffer) diff --git a/python/pyfory/includes/libserialization.pxd b/python/pyfory/includes/libserialization.pxd index 31928c5823..2c0a6ed5db 100644 --- a/python/pyfory/includes/libserialization.pxd +++ b/python/pyfory/includes/libserialization.pxd @@ -15,14 +15,14 @@ # specific language governing permissions and limitations # under the License. -from libc.stdint cimport int32_t +from libc.stdint cimport int32_t, uint8_t from libcpp cimport bool as c_bool from pyfory.includes.libutil cimport CBuffer cdef extern from "fory/type/type.h" namespace "fory" nogil: # Declare the C++ TypeId enum - cdef enum class TypeId(int32_t): + cdef enum class TypeId(uint8_t): UNKNOWN = 0 BOOL = 1 INT8 = 2 @@ -78,8 +78,14 @@ cdef extern from "fory/type/type.h" namespace "fory" nogil: FLOAT64_ARRAY = 52 BOUND = 64 - cdef c_bool is_namespaced_type(int32_t type_id) - cdef c_bool is_type_share_meta(int32_t type_id) + cdef enum class TypeRegistrationKind(int32_t): + INTERNAL = 0 + BY_ID = 1 + BY_NAME = 2 + + cdef TypeRegistrationKind get_type_registration_kind(TypeId type_id) + cdef c_bool is_namespaced_type(TypeId type_id) + cdef c_bool is_type_share_meta(TypeId type_id) cdef extern from "fory/python/pyfory.h" namespace "fory": int Fory_PyBooleanSequenceWriteToBuffer(object collection, CBuffer *buffer, Py_ssize_t start_index) diff --git a/python/pyfory/meta/typedef.py b/python/pyfory/meta/typedef.py index 811116ae91..ee822392e7 100644 --- a/python/pyfory/meta/typedef.py +++ b/python/pyfory/meta/typedef.py @@ -19,6 +19,7 @@ import typing from typing import List from pyfory.types import TypeId, is_primitive_type, is_polymorphic_type, is_union_type +from pyfory._fory import NO_USER_TYPE_ID from pyfory.buffer import Buffer from pyfory.type_util import infer_field from pyfory.meta.metastring import Encoding @@ -51,12 +52,21 @@ class TypeDef: def __init__( - self, namespace: str, typename: str, cls: type, type_id: int, fields: List["FieldInfo"], encoded: bytes = None, is_compressed: bool = False + self, + namespace: str, + typename: str, + cls: type, + type_id: int, + fields: List["FieldInfo"], + encoded: bytes = None, + is_compressed: bool = False, + user_type_id: int = NO_USER_TYPE_ID, ): self.namespace = namespace self.typename = typename self.cls = cls self.type_id = type_id + self.user_type_id = user_type_id self.fields = fields self.encoded = encoded self.is_compressed = is_compressed @@ -132,17 +142,17 @@ def _resolve_field_names_from_tag_ids(self): return resolved_names def create_serializer(self, resolver): - if self.type_id & 0xFF == TypeId.NAMED_EXT: - return resolver.get_typeinfo_by_name(self.namespace, self.typename).serializer - if self.type_id & 0xFF == TypeId.NAMED_ENUM: + if self.type_id == TypeId.NAMED_EXT: + return resolver.get_type_info_by_name(self.namespace, self.typename).serializer + if self.type_id == TypeId.NAMED_ENUM: try: - return resolver.get_typeinfo_by_name(self.namespace, self.typename).serializer + return resolver.get_type_info_by_name(self.namespace, self.typename).serializer except Exception: from pyfory.serializer import NonExistEnumSerializer return NonExistEnumSerializer(resolver.fory) - if self.type_id & 0xFF == TypeId.NAMED_UNION: - return resolver.get_typeinfo_by_name(self.namespace, self.typename).serializer + if self.type_id == TypeId.NAMED_UNION: + return resolver.get_type_info_by_name(self.namespace, self.typename).serializer from pyfory.struct import DataClassSerializer @@ -157,6 +167,13 @@ def create_serializer(self, resolver): resolved_name = field_names[i] nullable_fields[resolved_name] = field_info.field_type.is_nullable + dynamic_fields = {} + for i, field_info in enumerate(self.fields): + resolved_name = field_names[i] + type_id = field_info.field_type.type_id + if is_polymorphic_type(type_id): + dynamic_fields[resolved_name] = True + return DataClassSerializer( fory, self.cls, @@ -164,10 +181,15 @@ def create_serializer(self, resolver): field_names=field_names, serializers=self.create_fields_serializer(resolver, field_names), nullable_fields=nullable_fields, + dynamic_fields=dynamic_fields, ) def __repr__(self): - return f"TypeDef(namespace={self.namespace}, typename={self.typename}, cls={self.cls}, type_id={self.type_id}, fields={self.fields}, is_compressed={self.is_compressed})" + return ( + f"TypeDef(namespace={self.namespace}, typename={self.typename}, cls={self.cls}, " + f"type_id={self.type_id}, user_type_id={self.user_type_id}, " + f"fields={self.fields}, is_compressed={self.is_compressed})" + ) def _snake_to_camel(s: str) -> str: @@ -213,8 +235,16 @@ def __repr__(self): class FieldType: - def __init__(self, type_id: int, is_monomorphic: bool, is_nullable: bool, is_tracking_ref: bool): + def __init__( + self, + type_id: int, + is_monomorphic: bool, + is_nullable: bool, + is_tracking_ref: bool, + user_type_id: int = NO_USER_TYPE_ID, + ): self.type_id = type_id + self.user_type_id = user_type_id self.is_monomorphic = is_monomorphic self.is_nullable = is_nullable self.is_tracking_ref = is_tracking_ref @@ -228,7 +258,9 @@ def xwrite(self, buffer: Buffer, write_flags: bool = True): xtype_id |= 0b10 if self.is_tracking_ref: xtype_id |= 0b1 - buffer.write_var_uint32(xtype_id) + buffer.write_var_uint32(xtype_id) + else: + buffer.write_uint8(xtype_id) # Handle nested types if self.type_id in [TypeId.LIST, TypeId.SET]: self.element_type.xwrite(buffer, True) @@ -246,6 +278,7 @@ def xread(cls, buffer: Buffer, resolver): @classmethod def xread_with_type(cls, buffer: Buffer, resolver, xtype_id: int, is_nullable: bool, is_tracking_ref: bool): + user_type_id = NO_USER_TYPE_ID if xtype_id in [TypeId.LIST, TypeId.SET]: element_type = cls.xread(buffer, resolver) return CollectionFieldType(xtype_id, True, is_nullable, is_tracking_ref, element_type) @@ -254,11 +287,11 @@ def xread_with_type(cls, buffer: Buffer, resolver, xtype_id: int, is_nullable: b value_type = cls.xread(buffer, resolver) return MapFieldType(xtype_id, True, is_nullable, is_tracking_ref, key_type, value_type) elif xtype_id == TypeId.UNKNOWN: - return DynamicFieldType(xtype_id, False, is_nullable, is_tracking_ref) + return DynamicFieldType(xtype_id, False, is_nullable, is_tracking_ref, user_type_id=user_type_id) else: # For primitive types, determine if they are monomorphic based on the type is_monomorphic = not is_polymorphic_type(xtype_id) - return FieldType(xtype_id, is_monomorphic, is_nullable, is_tracking_ref) + return FieldType(xtype_id, is_monomorphic, is_nullable, is_tracking_ref, user_type_id=user_type_id) def create_serializer(self, resolver, type_): # Handle list wrapper @@ -268,12 +301,12 @@ def create_serializer(self, resolver, type_): if type_ is None: return None try: - return resolver.get_typeinfo(cls=type_).serializer + return resolver.get_type_info(cls=type_).serializer except Exception: return None # Types that need to be handled dynamically during deserialization # For these types, we don't know the concrete type at compile time - if self.type_id & 0xFF in [ + if self.type_id in [ TypeId.EXT, TypeId.NAMED_EXT, TypeId.STRUCT, @@ -282,26 +315,29 @@ def create_serializer(self, resolver, type_): TypeId.NAMED_COMPATIBLE_STRUCT, TypeId.UNKNOWN, ]: - return None - if self.type_id & 0xFF in [TypeId.ENUM, TypeId.NAMED_ENUM]: + if type_ is None: + return None + try: + return resolver.get_type_info(cls=type_).serializer + except Exception: + return None + if self.type_id in [TypeId.ENUM]: try: if issubclass(type_, enum.Enum): - return resolver.get_typeinfo(cls=type_).serializer + return resolver.get_type_info(cls=type_).serializer except Exception: pass from pyfory.serializer import NonExistEnumSerializer return NonExistEnumSerializer(resolver.fory) - typeinfo = resolver.get_typeinfo_by_id(self.type_id) + typeinfo = resolver.get_type_info_by_id(self.type_id) return typeinfo.serializer def __repr__(self): - type_id = self.type_id - if type_id > 128: - type_id = f"{type_id}, fory_id={type_id & 0xFF}, user_id={type_id >> 8}" return ( - f"FieldType(type_id={type_id}, is_monomorphic={self.is_monomorphic}, " - f"is_nullable={self.is_nullable}, is_tracking_ref={self.is_tracking_ref})" + f"FieldType(type_id={self.type_id}, user_type_id={self.user_type_id}, " + f"is_monomorphic={self.is_monomorphic}, is_nullable={self.is_nullable}, " + f"is_tracking_ref={self.is_tracking_ref})" ) @@ -374,8 +410,15 @@ def __repr__(self): class DynamicFieldType(FieldType): - def __init__(self, type_id: int, is_monomorphic: bool, is_nullable: bool, is_tracking_ref: bool): - super().__init__(type_id, is_monomorphic, is_nullable, is_tracking_ref) + def __init__( + self, + type_id: int, + is_monomorphic: bool, + is_nullable: bool, + is_tracking_ref: bool, + user_type_id: int = NO_USER_TYPE_ID, + ): + super().__init__(type_id, is_monomorphic, is_nullable, is_tracking_ref, user_type_id=user_type_id) def create_serializer(self, resolver, type_): # For dynamic field types (UNKNOWN, STRUCT, etc.), default to None so @@ -516,8 +559,11 @@ def build_field_type_from_type_ids_with_ref( type_id = type_ids[0] if type_id is None: type_id = TypeId.UNKNOWN + if type_id == TypeId.NAMED_ENUM: + type_id = TypeId.ENUM + if type_id in (TypeId.NAMED_UNION, TypeId.TYPED_UNION): + type_id = TypeId.UNION assert type_id >= 0, f"Unknown type: {type_id} for field: {field_name}" - type_id = type_id & 0xFF morphic = not is_polymorphic_type(type_id) if type_id in [TypeId.SET, TypeId.LIST]: elem_hint = None @@ -592,30 +638,33 @@ def build_field_type_from_type_ids_with_ref( TypeId.COMPATIBLE_STRUCT, TypeId.NAMED_COMPATIBLE_STRUCT, ]: - return DynamicFieldType(type_id, False, is_nullable, is_tracking_ref) + return DynamicFieldType(type_id, False, is_nullable, is_tracking_ref, user_type_id=NO_USER_TYPE_ID) else: if type_id <= 0 or type_id >= TypeId.BOUND: raise TypeError(f"Unknown type: {type_id} for field: {field_name}") # union/enum go here too - return FieldType(type_id, morphic, is_nullable, is_tracking_ref) + return FieldType(type_id, morphic, is_nullable, is_tracking_ref, user_type_id=NO_USER_TYPE_ID) def build_field_type(type_resolver, field_name: str, type_hint, visitor, is_nullable=False): """Build field type from type hint.""" type_ids = infer_field(field_name, type_hint, visitor) try: - return build_field_type_from_type_ids(type_resolver, field_name, type_ids, visitor, is_nullable) + return build_field_type_from_type_ids(type_resolver, field_name, type_ids, visitor, is_nullable, type_hint=type_hint) except Exception as e: raise TypeError(f"Error building field type for field: {field_name} with type hint: {type_hint} in class: {visitor.cls}") from e -def build_field_type_from_type_ids(type_resolver, field_name: str, type_ids, visitor, is_nullable=False): +def build_field_type_from_type_ids(type_resolver, field_name: str, type_ids, visitor, is_nullable=False, type_hint=None): tracking_ref = type_resolver.fory.ref_tracking type_id = type_ids[0] if type_id is None: type_id = TypeId.UNKNOWN + if type_id == TypeId.NAMED_ENUM: + type_id = TypeId.ENUM + if type_id in (TypeId.NAMED_UNION, TypeId.TYPED_UNION): + type_id = TypeId.UNION assert type_id >= 0, f"Unknown type: {type_id} for field: {field_name}" - type_id = type_id & 0xFF morphic = not is_polymorphic_type(type_id) if type_id in [TypeId.SET, TypeId.LIST]: elem_type = build_field_type_from_type_ids(type_resolver, field_name, type_ids[1], visitor, is_nullable=False) @@ -631,9 +680,9 @@ def build_field_type_from_type_ids(type_resolver, field_name: str, type_ids, vis TypeId.NAMED_STRUCT, TypeId.COMPATIBLE_STRUCT, TypeId.NAMED_COMPATIBLE_STRUCT, - ] or is_union_type(type_id): - return DynamicFieldType(type_id, False, is_nullable, tracking_ref) + ]: + return DynamicFieldType(type_id, False, is_nullable, tracking_ref, user_type_id=NO_USER_TYPE_ID) else: if type_id <= 0 or type_id >= TypeId.BOUND: raise TypeError(f"Unknown type: {type_id} for field: {field_name}") - return FieldType(type_id, morphic, is_nullable, tracking_ref) + return FieldType(type_id, morphic, is_nullable, tracking_ref, user_type_id=NO_USER_TYPE_ID) diff --git a/python/pyfory/meta/typedef_decoder.py b/python/pyfory/meta/typedef_decoder.py index 56f69e6d99..59880ed08d 100644 --- a/python/pyfory/meta/typedef_decoder.py +++ b/python/pyfory/meta/typedef_decoder.py @@ -40,6 +40,7 @@ TAG_ID_SIZE_THRESHOLD, ) from pyfory.types import TypeId +from pyfory._fory import NO_USER_TYPE_ID from pyfory.meta.metastring import MetaStringDecoder, Encoding @@ -121,12 +122,13 @@ def decode_typedef(buffer: Buffer, resolver, header=None) -> TypeDef: is_registered_by_name = (meta_header & REGISTER_BY_NAME_FLAG) != 0 type_cls = None + user_type_id = NO_USER_TYPE_ID # Read type info if is_registered_by_name: namespace = read_namespace(meta_buffer) typename = read_typename(meta_buffer) # Look up the type_id from namespace and typename - type_info = resolver.get_typeinfo_by_name(namespace, typename) + type_info = resolver.get_type_info_by_name(namespace, typename) if type_info: type_id = type_info.type_id type_cls = type_info.cls @@ -134,15 +136,16 @@ def decode_typedef(buffer: Buffer, resolver, header=None) -> TypeDef: # Fallback to COMPATIBLE_STRUCT if not found type_id = TypeId.COMPATIBLE_STRUCT else: - type_id = meta_buffer.read_var_uint32() - if resolver.is_registered_by_id(type_id=type_id): - type_info = resolver.get_typeinfo_by_id(type_id) + type_id = meta_buffer.read_uint8() + user_type_id = meta_buffer.read_var_uint32() + if resolver.is_registered_by_id(type_id=type_id, user_type_id=user_type_id): + type_info = resolver.get_type_info_by_id(type_id, user_type_id=user_type_id) type_cls = type_info.cls namespace = type_info.decode_namespace() typename = type_info.decode_typename() else: namespace = "fory" - typename = f"Nonexistent{type_id}" + typename = f"Nonexistent{user_type_id if user_type_id != NO_USER_TYPE_ID else type_id}" name = namespace + "." + typename if namespace else typename # Read fields info if present field_infos = [] @@ -163,7 +166,16 @@ def decode_typedef(buffer: Buffer, resolver, header=None) -> TypeDef: type_cls = make_dataclass(class_name, field_definitions) # Create TypeDef object - type_def = TypeDef(namespace, typename, type_cls, type_id, field_infos, meta_data, is_compressed) + type_def = TypeDef( + namespace, + typename, + type_cls, + type_id, + field_infos, + meta_data, + is_compressed, + user_type_id=user_type_id, + ) return type_def @@ -248,7 +260,7 @@ def read_field_info(buffer: Buffer, resolver, defined_class: str) -> FieldInfo: tag_id = size_or_tag # Read field type info (no field name to read for TAG_ID) - xtype_id = buffer.read_var_uint32() + xtype_id = buffer.read_uint8() field_type = FieldType.xread_with_type(buffer, resolver, xtype_id, is_nullable, is_tracking_ref) # For TAG_ID encoding, use tag_id as field name placeholder @@ -263,7 +275,7 @@ def read_field_info(buffer: Buffer, resolver, defined_class: str) -> FieldInfo: encoding = FIELD_NAME_ENCODINGS[encoding_type] # Read field type info BEFORE field name (matching Java TypeDefDecoder order) - xtype_id = buffer.read_var_uint32() + xtype_id = buffer.read_uint8() field_type = FieldType.xread_with_type(buffer, resolver, xtype_id, is_nullable, is_tracking_ref) # Read field name meta string diff --git a/python/pyfory/meta/typedef_encoder.py b/python/pyfory/meta/typedef_encoder.py index fc2d7fdb01..158d6eb97e 100644 --- a/python/pyfory/meta/typedef_encoder.py +++ b/python/pyfory/meta/typedef_encoder.py @@ -36,6 +36,7 @@ TAG_ID_SIZE_THRESHOLD, ) from pyfory.meta.metastring import MetaStringEncoder +from pyfory._fory import NO_USER_TYPE_ID from pyfory.buffer import Buffer from pyfory.lib.mmh3 import hash_buffer @@ -85,16 +86,17 @@ def encode_typedef(type_resolver, cls, include_fields: bool = True): buffer.write_uint8(header) # Write type info + type_id, user_type_id = type_resolver.get_registered_type_ids(cls) if type_resolver.is_registered_by_name(cls): namespace, typename = type_resolver.get_registered_name(cls) write_namespace(buffer, namespace) write_typename(buffer, typename) - # Use the actual type_id from the resolver, not a generic one - type_id = type_resolver.get_registered_id(cls) else: assert type_resolver.is_registered_by_id(cls=cls), "Class must be registered by name or id" - type_id = type_resolver.get_registered_id(cls) - buffer.write_var_uint32(type_id) + buffer.write_uint8(type_id) + if user_type_id in {None, NO_USER_TYPE_ID}: + raise ValueError(f"user_type_id required for type_id {type_id}") + buffer.write_var_uint32(user_type_id) # Write fields info write_fields_info(type_resolver, buffer, field_infos) @@ -118,7 +120,7 @@ def encode_typedef(type_resolver, cls, include_fields: bool = True): splits.insert(0, "") namespace, typename = splits - result = TypeDef(namespace, typename, cls, type_id, field_infos, binary, is_compressed) + result = TypeDef(namespace, typename, cls, type_id, field_infos, binary, is_compressed, user_type_id=user_type_id) return result @@ -147,7 +149,7 @@ def prepend_header(buffer: bytes, is_compressed: bool, has_fields_meta: bool): def write_namespace(buffer: Buffer, namespace: str): """Write namespace using meta string encoding.""" # - Package name encoding(omitted when class is registered): - # - encoding algorithm: `UTF8/ALL_TO_LOWER_SPECIAL/LOWER_UPPER_DIGIT_SPECIAL` + # - encoding algorithm: `UTF_8/ALL_TO_LOWER_SPECIAL/LOWER_UPPER_DIGIT_SPECIAL` # - Header: `6 bits size | 2 bits encoding flags`. # The `6 bits size: 0~63` will be used to indicate size `0~62`, # the value `63` the size need more byte to read, the encoding will encode `size - 62` as a varint next. @@ -159,7 +161,7 @@ def write_typename(buffer: Buffer, typename: str): """Write typename using meta string encoding.""" # - Class name encoding(omitted when class is registered): # - encoding algorithm: - # `UTF8/LOWER_UPPER_DIGIT_SPECIAL/FIRST_TO_LOWER_SPECIAL/ALL_TO_LOWER_SPECIAL` + # `UTF_8/LOWER_UPPER_DIGIT_SPECIAL/FIRST_TO_LOWER_SPECIAL/ALL_TO_LOWER_SPECIAL` # - header: `6 bits size | 2 bits encoding flags`. # The `6 bits size: 0~63` will be used to indicate size `1~64`, # the value `63` the size need more byte to read, the encoding will encode `size - 63` as a varint next. diff --git a/python/pyfory/registry.py b/python/pyfory/registry.py index d6dbc71aa4..0f6c70ff9e 100644 --- a/python/pyfory/registry.py +++ b/python/pyfory/registry.py @@ -100,6 +100,7 @@ float32, float64, is_struct_type, + needs_user_type_id, ) from pyfory.type_util import ( load_class, @@ -109,6 +110,7 @@ DYNAMIC_TYPE_ID, # preserve 0 as flag for type id not set in TypeInfo` NO_TYPE_ID, + NO_USER_TYPE_ID, ) from pyfory.meta.typedef import TypeDef from pyfory.meta.typedef_decoder import decode_typedef, skip_typedef @@ -131,6 +133,7 @@ class TypeInfo: __slots__ = ( "cls", "type_id", + "user_type_id", "serializer", "namespace_bytes", "typename_bytes", @@ -142,6 +145,7 @@ def __init__( self, cls: type = None, type_id: int = NO_TYPE_ID, + user_type_id: int = NO_USER_TYPE_ID, serializer: Serializer = None, namespace_bytes=None, typename_bytes=None, @@ -150,6 +154,7 @@ def __init__( ): self.cls = cls self.type_id = type_id + self.user_type_id = user_type_id self.serializer = serializer self.namespace_bytes = namespace_bytes self.typename_bytes = typename_bytes @@ -157,7 +162,7 @@ def __init__( self.type_def = type_def def __repr__(self): - return f"TypeInfo(cls={self.cls}, type_id={self.type_id}, serializer={self.serializer})" + return f"TypeInfo(cls={self.cls}, type_id={self.type_id}, user_type_id={self.user_type_id}, serializer={self.serializer})" def decode_namespace(self) -> str: if self.namespace_bytes is None: @@ -178,13 +183,13 @@ class TypeResolver: "_types_info", "_hash_to_metastring", "_metastr_to_type", - "_hash_to_typeinfo", - "_dynamic_id_to_typeinfo_list", + "_hash_to_type_info", + "_dynamic_id_to_type_info_list", "_dynamic_id_to_metastr_list", "_dynamic_write_string_id", "_dynamic_written_metastr", - "_ns_type_to_typeinfo", - "_named_type_to_typeinfo", + "_ns_type_to_type_info", + "_named_type_to_type_info", "namespace_encoder", "namespace_decoder", "typename_encoder", @@ -193,8 +198,10 @@ class TypeResolver: "require_registration", "metastring_resolver", "language", - "_type_id_to_typeinfo", - "_meta_shared_typeinfo", + "_type_id_to_type_info", + "_user_type_id_to_type_info", + "_used_user_type_ids", + "_meta_shared_type_info", "meta_share", "serialization_context", "_internal_py_serializer_map", @@ -208,20 +215,22 @@ def __init__(self, fory, meta_share=False, meta_compressor=None): self._metastr_to_str = dict() self._metastr_to_type = dict() self._hash_to_metastring = dict() - self._hash_to_typeinfo = dict() + self._hash_to_type_info = dict() self._dynamic_written_metastr = [] - self._type_id_to_typeinfo = dict() + self._type_id_to_type_info = dict() + self._user_type_id_to_type_info = dict() + self._used_user_type_ids = set() self._type_id_counter = 64 self._dynamic_write_string_id = 0 # hold objects to avoid gc, since `flat_hash_map/vector` doesn't # hold python reference. self._types_info = dict() - self._ns_type_to_typeinfo = dict() - self._named_type_to_typeinfo = dict() + self._ns_type_to_type_info = dict() + self._named_type_to_type_info = dict() self.namespace_encoder = MetaStringEncoder(".", "_") self.namespace_decoder = MetaStringDecoder(".", "_") # Cache for TypeDef and TypeInfo tuples (similar to Java's classIdToDef) - self._meta_shared_typeinfo = {} + self._meta_shared_type_info = {} self.typename_encoder = MetaStringEncoder("$", "_") self.typename_decoder = MetaStringDecoder("$", "_") self.meta_compressor = meta_compressor if meta_compressor is not None else DeflaterMetaCompressor() @@ -238,7 +247,6 @@ def initialize(self): def _initialize_py(self): register = functools.partial(self._register_type, internal=True) - register(type(None), serializer=NoneSerializer) register(tuple, serializer=TupleSerializer) register(slice, serializer=SliceSerializer) if np is not None: @@ -280,7 +288,7 @@ def _initialize_xlang(self): def _initialize_common(self): register = functools.partial(self._register_type, internal=True) register(type(None), type_id=TypeId.NONE, serializer=NoneSerializer) - # Also register None value to map to type(None) for get_typeinfo(None) calls + # Also register None value to map to type(None) for get_type_info(None) calls self._types_info[None] = self._types_info[type(None)] register(bool, type_id=TypeId.BOOL, serializer=BooleanSerializer) # Signed integers @@ -340,7 +348,7 @@ def _initialize_common(self): type_id=typeid, serializer=Numpy1DArraySerializer(self.fory, ftype, dtype), ) - self._type_id_to_typeinfo[typeid] = typeinfo + self._type_id_to_type_info[typeid] = typeinfo register(list, type_id=TypeId.LIST, serializer=ListSerializer) register(set, type_id=TypeId.SET, serializer=SetSerializer) register(dict, type_id=TypeId.MAP, serializer=MapSerializer) @@ -386,12 +394,15 @@ def register_union( if typename is None and type_id is None: type_id = self._next_type_id() if type_id not in {0, None}: - actual_type_id = (type_id << 8) + TypeId.TYPED_UNION + user_type_id = type_id + type_id = TypeId.TYPED_UNION else: - actual_type_id = TypeId.NAMED_UNION + user_type_id = NO_USER_TYPE_ID + type_id = TypeId.NAMED_UNION return self.__register_type( cls, - type_id=actual_type_id, + type_id=type_id, + user_type_id=user_type_id, namespace=namespace, typename=typename, serializer=serializer, @@ -403,6 +414,7 @@ def _register_type( cls: Union[type, TypeVar], *, type_id: int = None, + user_type_id: int = NO_USER_TYPE_ID, namespace: str = None, typename: str = None, serializer=None, @@ -410,6 +422,12 @@ def _register_type( ): """Register type with given type id or typename. If typename is not None, it will be used for cross-language serialization.""" + if internal: + if type_id is not None and type_id >= 0 and type_id > 0xFF: + raise ValueError(f"Internal type id overflow: {type_id}") + else: + if user_type_id not in {None, NO_USER_TYPE_ID} and (user_type_id < 0 or user_type_id > 0xFFFFFFFE): + raise ValueError(f"user_type_id must be in range [0, 0xfffffffe], got {user_type_id}") if serializer is not None and not isinstance(serializer, Serializer): try: serializer = serializer(self.fory, cls) @@ -418,21 +436,27 @@ def _register_type( serializer = serializer(self.fory) except BaseException: serializer = serializer() + if ( + cls in self._types_info + and type_id is None + and typename is None + and namespace is None + and serializer is None + and user_type_id in {None, NO_USER_TYPE_ID} + ): + return self._types_info[cls] n_params = len({typename, type_id, None}) - 1 if n_params == 0 and typename is None: type_id = self._next_type_id() if n_params == 2: raise TypeError(f"type name {typename} and id {type_id} should not be set at the same time") - if type_id not in {0, None}: - # multiple type can have same tpe id - if type_id in self._type_id_to_typeinfo and cls in self._types_info: - raise TypeError(f"{cls} registered already") - elif cls in self._types_info: + if cls in self._types_info: raise TypeError(f"{cls} registered already") register_type = self._register_xtype if self.fory.language == Language.XLANG else self._register_pytype return register_type( cls, type_id=type_id, + user_type_id=user_type_id, namespace=namespace, typename=typename, serializer=serializer, @@ -444,6 +468,7 @@ def _register_xtype( cls: Union[type, TypeVar], *, type_id: int = None, + user_type_id: int = NO_USER_TYPE_ID, namespace: str = None, typename: str = None, serializer=None, @@ -452,19 +477,40 @@ def _register_xtype( if serializer is None: if issubclass(cls, enum.Enum): serializer = EnumSerializer(self.fory, cls) - type_id = TypeId.NAMED_ENUM if type_id is None else ((type_id << 8) + TypeId.ENUM) + if type_id is None: + type_id = TypeId.NAMED_ENUM + user_type_id = NO_USER_TYPE_ID + else: + user_type_id = type_id + type_id = TypeId.ENUM else: serializer = None if self.meta_share: - type_id = TypeId.NAMED_COMPATIBLE_STRUCT if type_id is None else ((type_id << 8) + TypeId.COMPATIBLE_STRUCT) + if type_id is None: + type_id = TypeId.NAMED_COMPATIBLE_STRUCT + user_type_id = NO_USER_TYPE_ID + else: + user_type_id = type_id + type_id = TypeId.COMPATIBLE_STRUCT else: - type_id = TypeId.NAMED_STRUCT if type_id is None else ((type_id << 8) + TypeId.STRUCT) + if type_id is None: + type_id = TypeId.NAMED_STRUCT + user_type_id = NO_USER_TYPE_ID + else: + user_type_id = type_id + type_id = TypeId.STRUCT elif not internal: - type_id = TypeId.NAMED_EXT if type_id is None else ((type_id << 8) + TypeId.EXT) + if type_id is None: + type_id = TypeId.NAMED_EXT + user_type_id = NO_USER_TYPE_ID + else: + user_type_id = type_id + type_id = TypeId.EXT return self.__register_type( cls, type_id=type_id, + user_type_id=user_type_id, serializer=serializer, namespace=namespace, typename=typename, @@ -476,6 +522,7 @@ def _register_pytype( cls: Union[type, TypeVar], *, type_id: int = None, + user_type_id: int = NO_USER_TYPE_ID, namespace: str = None, typename: str = None, serializer: Serializer = None, @@ -487,6 +534,7 @@ def _register_pytype( return self.__register_type( cls, type_id=type_id, + user_type_id=user_type_id, namespace=namespace, typename=typename, serializer=serializer, @@ -498,6 +546,7 @@ def __register_type( cls: Union[type, TypeVar], *, type_id: int = None, + user_type_id: int = NO_USER_TYPE_ID, namespace: str = None, typename: str = None, serializer: Serializer = None, @@ -505,17 +554,15 @@ def __register_type( ): dynamic_type = type_id is not None and type_id < 0 # In metashare mode, for struct types, we want to keep serializer=None - # so that _set_typeinfo will be called to create the TypeDef-based serializer + # so that _set_type_info will be called to create the TypeDef-based serializer # This applies to both types registered by name and by ID - should_create_serializer = ( - not internal and serializer is None and not (self.meta_share and type_id is not None and is_struct_type(type_id & 0xFF)) - ) + should_create_serializer = not internal and serializer is None and not (self.meta_share and type_id is not None and is_struct_type(type_id)) if should_create_serializer: serializer = self._create_serializer(cls) if typename is None: - typeinfo = TypeInfo(cls, type_id, serializer, None, None, dynamic_type) + typeinfo = TypeInfo(cls, type_id, user_type_id, serializer, None, None, dynamic_type) else: if namespace is None: splits = typename.rsplit(".", 1) @@ -527,26 +574,34 @@ def __register_type( ns_meta_bytes = self.metastring_resolver.get_metastr_bytes(ns_metastr) type_metastr = self.typename_encoder.encode(typename) type_meta_bytes = self.metastring_resolver.get_metastr_bytes(type_metastr) - typeinfo = TypeInfo(cls, type_id, serializer, ns_meta_bytes, type_meta_bytes, dynamic_type) - self._named_type_to_typeinfo[(namespace, typename)] = typeinfo - self._ns_type_to_typeinfo[(ns_meta_bytes, type_meta_bytes)] = typeinfo + typeinfo = TypeInfo(cls, type_id, user_type_id, serializer, ns_meta_bytes, type_meta_bytes, dynamic_type) + self._named_type_to_type_info[(namespace, typename)] = typeinfo + self._ns_type_to_type_info[(ns_meta_bytes, type_meta_bytes)] = typeinfo self._types_info[cls] = typeinfo - if type_id is not None and type_id != 0 and (self.language == Language.PYTHON or not TypeId.is_namespaced_type(type_id)): - if type_id not in self._type_id_to_typeinfo or not internal: - self._type_id_to_typeinfo[type_id] = typeinfo + if type_id is not None and type_id != 0: + if needs_user_type_id(type_id) and user_type_id not in {None, NO_USER_TYPE_ID}: + existing = self._user_type_id_to_type_info.get(user_type_id) + if existing is not None and existing.cls is not cls: + raise TypeError(f"user_type_id {user_type_id} already registered for {existing.cls}") + if needs_user_type_id(type_id) and user_type_id not in {None, NO_USER_TYPE_ID}: + if user_type_id not in self._user_type_id_to_type_info or not internal: + self._user_type_id_to_type_info[user_type_id] = typeinfo + self._used_user_type_ids.add(user_type_id) + elif self.language == Language.PYTHON or not TypeId.is_namespaced_type(type_id): + if type_id not in self._type_id_to_type_info or not internal: + self._type_id_to_type_info[type_id] = typeinfo self._types_info[cls] = typeinfo # Create TypeDef for named non-struct types when meta_share is enabled if self.meta_share and type_id is not None: - base_type_id = type_id & 0xFF - if base_type_id in (TypeId.NAMED_ENUM, TypeId.NAMED_EXT, TypeId.NAMED_UNION): - type_def = encode_typedef(self, cls, include_fields=is_struct_type(base_type_id)) + if type_id in (TypeId.NAMED_ENUM, TypeId.NAMED_EXT, TypeId.NAMED_UNION): + type_def = encode_typedef(self, cls, include_fields=is_struct_type(type_id)) if type_def is not None: typeinfo.type_def = type_def return typeinfo def _next_type_id(self): type_id = self._type_id_counter = self._type_id_counter + 1 - while type_id in self._type_id_to_typeinfo: + while type_id in self._used_user_type_ids: type_id = self._type_id_counter = self._type_id_counter + 1 return type_id @@ -558,14 +613,22 @@ def register_serializer(self, cls: Union[type, TypeVar], serializer): if self.fory.language == Language.PYTHON: typeinfo.serializer = serializer return - type_id = prev_type_id = typeinfo.type_id - self._type_id_to_typeinfo.pop(prev_type_id) + prev_type_id = typeinfo.type_id + prev_user_type_id = typeinfo.user_type_id + if needs_user_type_id(prev_type_id) and prev_user_type_id not in {None, NO_USER_TYPE_ID}: + self._user_type_id_to_type_info.pop(prev_user_type_id, None) + else: + self._type_id_to_type_info.pop(prev_type_id, None) if typeinfo.serializer is not serializer: if typeinfo.typename_bytes is not None: - type_id = typeinfo.type_id & 0xFFFFFF00 | TypeId.NAMED_EXT + typeinfo.type_id = TypeId.NAMED_EXT + typeinfo.user_type_id = NO_USER_TYPE_ID else: - type_id = typeinfo.type_id & 0xFFFFFF00 | TypeId.EXT - self._type_id_to_typeinfo[type_id] = typeinfo + typeinfo.type_id = TypeId.EXT + if needs_user_type_id(typeinfo.type_id) and typeinfo.user_type_id not in {None, NO_USER_TYPE_ID}: + self._user_type_id_to_type_info[typeinfo.user_type_id] = typeinfo + else: + self._type_id_to_type_info[typeinfo.type_id] = typeinfo def get_serializer(self, cls: type): """ @@ -573,18 +636,18 @@ def get_serializer(self, cls: type): ------- Returns or create serializer for the provided type """ - return self.get_typeinfo(cls).serializer + return self.get_type_info(cls).serializer - def get_typeinfo(self, cls, create=True): + def get_type_info(self, cls, create=True): type_info = self._types_info.get(cls) if type_info is not None: if type_info.serializer is None: - self._set_typeinfo(type_info) + self._set_type_info(type_info) return type_info elif not create: return None if cls is NonExistEnum: - return self._get_nonexist_enum_typeinfo() + return self._get_nonexist_enum_type_info() if self.require_registration and not issubclass(cls, Enum): raise TypeUnregisteredError(f"{cls} not registered") logger.info("Type %s not registered", cls) @@ -612,13 +675,13 @@ def get_typeinfo(self, cls, create=True): serializer=serializer, ) - def _set_typeinfo(self, typeinfo): - type_id = typeinfo.type_id & 0xFF + def _set_type_info(self, typeinfo): + type_id = typeinfo.type_id if is_struct_type(type_id): from pyfory.struct import DataClassSerializer, DataClassStubSerializer # Set a stub serializer FIRST to break recursion for self-referencing types. - # get_typeinfo() only calls _set_typeinfo when serializer is None, + # get_type_info() only calls _set_type_info when serializer is None, # so setting stub first prevents re-entry for circular type references. typeinfo.serializer = DataClassStubSerializer(self.fory, typeinfo.cls, xlang=not self.fory.is_py) @@ -703,16 +766,22 @@ def is_registered_by_name(self, cls): typeinfo = self._types_info.get(cls) if typeinfo is None: return False - return TypeId.is_namespaced_type(typeinfo.type_id & 0xFF) + return TypeId.is_namespaced_type(typeinfo.type_id) - def is_registered_by_id(self, cls=None, type_id=None): + def is_registered_by_id(self, cls=None, type_id=None, user_type_id=NO_USER_TYPE_ID): if cls is not None: typeinfo = self._types_info.get(cls) if typeinfo is None: return False - return not TypeId.is_namespaced_type(typeinfo.type_id & 0xFF) + return not TypeId.is_namespaced_type(typeinfo.type_id) else: - return type_id in self._type_id_to_typeinfo + if type_id is None: + return False + if needs_user_type_id(type_id): + if user_type_id in {None, NO_USER_TYPE_ID}: + return False + return user_type_id in self._user_type_id_to_type_info + return type_id in self._type_id_to_type_info def get_registered_name(self, cls): typeinfo = self._types_info.get(cls) @@ -724,108 +793,129 @@ def get_registered_id(self, cls): assert typeinfo is not None, f"{cls} not registered" return typeinfo.type_id - def _load_metabytes_to_typeinfo(self, ns_metabytes, type_metabytes): - typeinfo = self._ns_type_to_typeinfo.get((ns_metabytes, type_metabytes)) + def get_registered_user_type_id(self, cls): + typeinfo = self._types_info.get(cls) + assert typeinfo is not None, f"{cls} not registered" + return typeinfo.user_type_id + + def get_registered_type_ids(self, cls): + typeinfo = self._types_info.get(cls) + assert typeinfo is not None, f"{cls} not registered" + return typeinfo.type_id, typeinfo.user_type_id + + def _load_metabytes_to_type_info(self, ns_metabytes, type_metabytes): + typeinfo = self._ns_type_to_type_info.get((ns_metabytes, type_metabytes)) if typeinfo is not None: return typeinfo ns = ns_metabytes.decode(self.namespace_decoder) typename = type_metabytes.decode(self.typename_decoder) # the hash computed between languages may be different. - typeinfo = self._named_type_to_typeinfo.get((ns, typename)) + typeinfo = self._named_type_to_type_info.get((ns, typename)) if typeinfo is None and typename: alt_typename = typename[0].upper() + typename[1:] - typeinfo = self._named_type_to_typeinfo.get((ns, alt_typename)) + typeinfo = self._named_type_to_type_info.get((ns, alt_typename)) if typeinfo is not None: - self._ns_type_to_typeinfo[(ns_metabytes, type_metabytes)] = typeinfo + self._ns_type_to_type_info[(ns_metabytes, type_metabytes)] = typeinfo return typeinfo cls = load_class(ns + "#" + typename) - typeinfo = self.get_typeinfo(cls) - self._ns_type_to_typeinfo[(ns_metabytes, type_metabytes)] = typeinfo + typeinfo = self.get_type_info(cls) + self._ns_type_to_type_info[(ns_metabytes, type_metabytes)] = typeinfo return typeinfo - def write_typeinfo(self, buffer, typeinfo): + def write_type_info(self, buffer, typeinfo): if typeinfo.dynamic_type: return - # Check if meta share is enabled first - if self.meta_share: + type_id = typeinfo.type_id + buffer.write_uint8(type_id) + if type_id in {TypeId.ENUM, TypeId.STRUCT, TypeId.EXT, TypeId.TYPED_UNION}: + if typeinfo.user_type_id in {None, NO_USER_TYPE_ID}: + raise TypeError(f"user_type_id required for type_id {type_id}") + buffer.write_var_uint32(typeinfo.user_type_id) + return + if type_id in {TypeId.COMPATIBLE_STRUCT, TypeId.NAMED_COMPATIBLE_STRUCT}: self.write_shared_type_meta(buffer, typeinfo) return - type_id = typeinfo.type_id - internal_type_id = type_id & 0xFF - buffer.write_var_uint32(type_id) - if TypeId.is_namespaced_type(internal_type_id): - self.metastring_resolver.write_meta_string_bytes(buffer, typeinfo.namespace_bytes) - self.metastring_resolver.write_meta_string_bytes(buffer, typeinfo.typename_bytes) - - def read_typeinfo(self, buffer): - # Check if meta share is enabled first - if self.meta_share: - return self.read_shared_type_meta(buffer) - - type_id = buffer.read_var_uint32() - internal_type_id = type_id & 0xFF - if TypeId.is_namespaced_type(internal_type_id): + if TypeId.is_namespaced_type(type_id): + if self.meta_share: + self.write_shared_type_meta(buffer, typeinfo) + else: + self.metastring_resolver.write_meta_string_bytes(buffer, typeinfo.namespace_bytes) + self.metastring_resolver.write_meta_string_bytes(buffer, typeinfo.typename_bytes) + + def read_type_info(self, buffer): + type_id = buffer.read_uint8() + if type_id in {TypeId.COMPATIBLE_STRUCT, TypeId.NAMED_COMPATIBLE_STRUCT}: + return self.serialization_context.meta_context.read_shared_type_info_with_type_id(buffer, type_id) + if TypeId.is_namespaced_type(type_id): + if self.meta_share: + return self.serialization_context.meta_context.read_shared_type_info_with_type_id(buffer, type_id) ns_metabytes = self.metastring_resolver.read_meta_string_bytes(buffer) type_metabytes = self.metastring_resolver.read_meta_string_bytes(buffer) - typeinfo = self._ns_type_to_typeinfo.get((ns_metabytes, type_metabytes)) + typeinfo = self._ns_type_to_type_info.get((ns_metabytes, type_metabytes)) if typeinfo is None: ns = ns_metabytes.decode(self.namespace_decoder) typename = type_metabytes.decode(self.typename_decoder) - typeinfo = self._named_type_to_typeinfo.get((ns, typename)) + typeinfo = self._named_type_to_type_info.get((ns, typename)) if typeinfo is None and typename: alt_typename = typename[0].upper() + typename[1:] - typeinfo = self._named_type_to_typeinfo.get((ns, alt_typename)) + typeinfo = self._named_type_to_type_info.get((ns, alt_typename)) if typeinfo is not None: - self._ns_type_to_typeinfo[(ns_metabytes, type_metabytes)] = typeinfo + self._ns_type_to_type_info[(ns_metabytes, type_metabytes)] = typeinfo return typeinfo if not ns and "." in typename: split_ns, split_typename = typename.rsplit(".", 1) - typeinfo = self._named_type_to_typeinfo.get((split_ns, split_typename)) + typeinfo = self._named_type_to_type_info.get((split_ns, split_typename)) if typeinfo is not None: - self._ns_type_to_typeinfo[(ns_metabytes, type_metabytes)] = typeinfo + self._ns_type_to_type_info[(ns_metabytes, type_metabytes)] = typeinfo return typeinfo typename = split_typename ns = split_ns if typename: - matches = [info for (reg_ns, reg_typename), info in self._named_type_to_typeinfo.items() if reg_typename == typename] + matches = [info for (reg_ns, reg_typename), info in self._named_type_to_type_info.items() if reg_typename == typename] if len(matches) == 1: typeinfo = matches[0] - self._ns_type_to_typeinfo[(ns_metabytes, type_metabytes)] = typeinfo + self._ns_type_to_type_info[(ns_metabytes, type_metabytes)] = typeinfo return typeinfo name = ns + "." + typename if ns else typename raise TypeUnregisteredError(f"{name} not registered") return typeinfo - else: - return self._type_id_to_typeinfo.get(type_id) + if type_id in {TypeId.ENUM, TypeId.STRUCT, TypeId.EXT, TypeId.TYPED_UNION}: + user_type_id = buffer.read_var_uint32() + return self.get_type_info_by_id(type_id, user_type_id=user_type_id) + return self.get_type_info_by_id(type_id) - def get_typeinfo_by_id(self, type_id): + def get_type_info_by_id(self, type_id, user_type_id=NO_USER_TYPE_ID): """Get typeinfo by type_id. Never returns None. For unknown ENUM types, returns NonExistEnum typeinfo. For other unknown types, raises TypeUnregisteredError. """ - typeinfo = self._type_id_to_typeinfo.get(type_id) + if needs_user_type_id(type_id): + if user_type_id in {None, NO_USER_TYPE_ID}: + raise TypeUnregisteredError(f"type id {type_id} missing user_type_id") + typeinfo = self._user_type_id_to_type_info.get(user_type_id) + else: + typeinfo = self._type_id_to_type_info.get(type_id) if typeinfo is not None: return typeinfo - base_type_id = type_id & 0xFF - if base_type_id == TypeId.ENUM: - return self._get_nonexist_enum_typeinfo() - raise TypeUnregisteredError(f"type id {type_id} (base {base_type_id}) not registered") + if type_id == TypeId.ENUM: + return self._get_nonexist_enum_type_info() + raise TypeUnregisteredError(f"type id {type_id} (user {user_type_id}) not registered") - def _get_nonexist_enum_typeinfo(self): + def _get_nonexist_enum_type_info(self): """Get or create TypeInfo for NonExistEnum to handle unknown enum types.""" from pyfory.serializer import NonExistEnum, NonExistEnumSerializer typeinfo = self._types_info.get(NonExistEnum) if typeinfo is None: serializer = NonExistEnumSerializer(self.fory) - typeinfo = TypeInfo(NonExistEnum, TypeId.ENUM, serializer, None, None, False) + typeinfo = TypeInfo(NonExistEnum, TypeId.ENUM, NO_USER_TYPE_ID, serializer, None, None, False) self._types_info[NonExistEnum] = typeinfo return typeinfo - def get_typeinfo_by_name(self, namespace, typename): + def get_type_info_by_name(self, namespace, typename): """Get typeinfo by namespace and typename.""" - return self._named_type_to_typeinfo.get((namespace, typename)) + return self._named_type_to_type_info.get((namespace, typename)) def get_meta_compressor(self): return self.meta_compressor @@ -833,13 +923,13 @@ def get_meta_compressor(self): def write_shared_type_meta(self, buffer, typeinfo): """Write shared type meta information.""" meta_context = self.fory.serialization_context.meta_context - meta_context.write_shared_typeinfo(buffer, typeinfo) + meta_context.write_shared_type_info(buffer, typeinfo) def read_shared_type_meta(self, buffer): """Read shared type meta information.""" meta_context = self.serialization_context.meta_context assert meta_context is not None, "Meta context must be set when meta share is enabled" - return meta_context.read_shared_typeinfo(buffer) + return meta_context.read_shared_type_info(buffer) def _build_type_info_from_typedef(self, type_def): """Build TypeInfo from TypeDef using TypeDef's create_serializer method.""" @@ -852,6 +942,7 @@ def _build_type_info_from_typedef(self, type_def): typeinfo = TypeInfo( type_def.cls, type_def.type_id, + type_def.user_type_id, serializer, ns_meta_bytes, type_meta_bytes, @@ -860,7 +951,7 @@ def _build_type_info_from_typedef(self, type_def): ) return typeinfo - def _read_and_build_typeinfo(self, buffer): + def _read_and_build_type_info(self, buffer): """Read TypeDef inline from buffer and build TypeInfo. Used for streaming meta share where TypeDef is written inline. @@ -868,7 +959,7 @@ def _read_and_build_typeinfo(self, buffer): # Read the header (first 8 bytes) to get the type ID header = buffer.read_int64() # Check if we already have this TypeDef cached - type_info = self._meta_shared_typeinfo.get(header) + type_info = self._meta_shared_type_info.get(header) if type_info is not None: # Skip the rest of the TypeDef binary for faster performance skip_typedef(buffer, header) @@ -877,7 +968,7 @@ def _read_and_build_typeinfo(self, buffer): type_def = decode_typedef(buffer, self, header=header) type_info = self._build_type_info_from_typedef(type_def) # Cache the tuple for future use - self._meta_shared_typeinfo[header] = type_info + self._meta_shared_type_info[header] = type_info return type_info def reset(self): diff --git a/python/pyfory/serialization.pyx b/python/pyfory/serialization.pyx index 26c5d54e5f..006f46523b 100644 --- a/python/pyfory/serialization.pyx +++ b/python/pyfory/serialization.pyx @@ -36,7 +36,9 @@ from pyfory.meta.metastring import Encoding from pyfory.types import is_primitive_type from pyfory.policy import DeserializationPolicy, DEFAULT_POLICY from pyfory.includes.libserialization cimport \ - (TypeId, is_namespaced_type, is_type_share_meta, Fory_PyBooleanSequenceWriteToBuffer, Fory_PyFloatSequenceWriteToBuffer) + (TypeId, TypeRegistrationKind, get_type_registration_kind, + is_namespaced_type, is_type_share_meta, + Fory_PyBooleanSequenceWriteToBuffer, Fory_PyFloatSequenceWriteToBuffer) from libc.stdint cimport int8_t, int16_t, int32_t, int64_t, uint64_t from libc.stdint cimport * @@ -269,6 +271,7 @@ cdef int8_t USE_TYPE_NAME = 0 cdef int8_t USE_TYPE_ID = 1 # preserve 0 as flag for type id not set in TypeInfo` cdef int8_t NO_TYPE_ID = 0 +cdef uint32_t NO_USER_TYPE_ID = 0xffffffff cdef int8_t DEFAULT_DYNAMIC_WRITE_META_STR_ID = fmod.DEFAULT_DYNAMIC_WRITE_META_STR_ID cdef int8_t INT64_TYPE_ID = fmod.INT64_TYPE_ID cdef int8_t FLOAT64_TYPE_ID = fmod.FLOAT64_TYPE_ID @@ -282,6 +285,27 @@ cdef int32_t NOT_NULL_STRING_FLAG = fmod.NOT_NULL_STRING_FLAG cdef int32_t SMALL_STRING_THRESHOLD = fmod.SMALL_STRING_THRESHOLD +cdef inline uint64_t _mix64(uint64_t x): + x ^= x >> 33 + x *= 0xff51afd7ed558ccd + x ^= x >> 33 + x *= 0xc4ceb9fe1a85ec53 + x ^= x >> 33 + return x + + +cdef inline int64_t _hash_small_metastring(int64_t v1, + int64_t v2, + int32_t length, + uint8_t encoding): + cdef uint64_t k = 0x9e3779b97f4a7c15 + cdef uint64_t x = ( v1) ^ (( v2) * k) + x ^= ( length) << 56 + cdef uint64_t h = _mix64(x) + h = (h & 0xffffffffffffff00) | encoding + return h + + @cython.final cdef class MetaStringBytes: cdef public bytes data @@ -310,6 +334,9 @@ cdef class MetaStringBytes: return f"MetaStringBytes(data={self.data}, hashcode={self.hashcode})" +EMPTY_META_STRING_BYTES = MetaStringBytes(b"", 0) + + @cython.final cdef class MetaStringResolver: cdef: @@ -337,7 +364,8 @@ cdef class MetaStringResolver: self._c_dynamic_written_enum_string.push_back( metastr_bytes) buffer.write_var_uint32(length << 1) if length <= SMALL_STRING_THRESHOLD: - buffer.write_int8(metastr_bytes.encoding) + if length != 0: + buffer.write_int8(metastr_bytes.encoding) else: buffer.write_int64(metastr_bytes.hashcode) buffer.write_bytes(metastr_bytes.data) @@ -352,15 +380,20 @@ cdef class MetaStringResolver: cdef int64_t v1 = 0, v2 = 0, hashcode cdef PyObject * enum_str_ptr cdef int32_t reader_index - cdef encoding = 0 + cdef int8_t encoding = 0 if length <= SMALL_STRING_THRESHOLD: + if length == 0: + enum_str_ptr = EMPTY_META_STRING_BYTES + self._c_dynamic_id_to_enum_string_vec.push_back(enum_str_ptr) + return enum_str_ptr encoding = buffer.read_int8() if length <= 8: v1 = buffer.read_bytes_as_int64(length) else: v1 = buffer.read_int64() v2 = buffer.read_bytes_as_int64(length - 8) - hashcode = ((v1 * 31 + v2) >> 8 << 8) | encoding + hashcode = _hash_small_metastring( + v1, v2, length, encoding) enum_str_ptr = self._c_hash_to_small_metastring_bytes[hashcode] if enum_str_ptr == NULL: reader_index = buffer.get_reader_index() @@ -390,6 +423,9 @@ cdef class MetaStringResolver: return metastr_bytes cdef int64_t v1 = 0, v2 = 0, hashcode length = len(metastr.encoded_data) + if length == 0: + self._metastr_to_metastr_bytes[metastr] = EMPTY_META_STRING_BYTES + return EMPTY_META_STRING_BYTES if length <= SMALL_STRING_THRESHOLD: data_buf = Buffer(metastr.encoded_data) if length <= 8: @@ -397,12 +433,12 @@ cdef class MetaStringResolver: else: v1 = data_buf.read_int64() v2 = data_buf.read_bytes_as_int64(length - 8) - value_hash = ((v1 * 31 + v2) >> 8 << 8) | metastr.encoding.value + hashcode = _hash_small_metastring( + v1, v2, length, metastr.encoding.value) else: - value_hash = mmh3.hash_buffer(metastr.encoded_data, seed=47)[0] - value_hash = value_hash >> 8 << 8 - value_hash |= metastr.encoding.value & 0xFF - self._metastr_to_metastr_bytes[metastr] = metastr_bytes = MetaStringBytes(metastr.encoded_data, value_hash) + hashcode = mmh3.hash_buffer(metastr.encoded_data, seed=47)[0] + hashcode = (hashcode >> 8 << 8) | (metastr.encoding.value & 0xFF) + self._metastr_to_metastr_bytes[metastr] = metastr_bytes = MetaStringBytes(metastr.encoded_data, hashcode) return metastr_bytes cpdef inline reset_read(self): @@ -422,7 +458,7 @@ cdef class TypeInfo: """ If dynamic_type is true, the serializer will be a dynamic typed serializer and it will write type info when writing the data. - In such cases, the `write_typeinfo` should not write typeinfo. + In such cases, the `write_type_info` should not write typeinfo. In general, if we have 4 type for one class, we will have 5 serializers. For example, we have int8/16/32/64/128 for python `int` type, then we have 6 serializers for python `int`: `Int8/1632/64/128Serializer` for `int8/16/32/64/128` each, and another @@ -434,7 +470,8 @@ cdef class TypeInfo: when serializing the actual data. """ cdef public object cls - cdef public int32_t type_id + cdef public uint8_t type_id + cdef public uint32_t user_type_id cdef public Serializer serializer cdef public MetaStringBytes namespace_bytes cdef public MetaStringBytes typename_bytes @@ -445,6 +482,7 @@ cdef class TypeInfo: self, cls: Union[type, TypeVar] = None, type_id: int = NO_TYPE_ID, + user_type_id: int = NO_USER_TYPE_ID, serializer: Serializer = None, namespace_bytes: MetaStringBytes = None, typename_bytes: MetaStringBytes = None, @@ -452,7 +490,11 @@ cdef class TypeInfo: type_def: object = None ): self.cls = cls - self.type_id = type_id + if type_id is None or type_id < 0: + self.type_id = NO_TYPE_ID + else: + self.type_id = type_id + self.user_type_id = user_type_id self.serializer = serializer self.namespace_bytes = namespace_bytes self.typename_bytes = typename_bytes @@ -460,8 +502,10 @@ cdef class TypeInfo: self.type_def = type_def def __repr__(self): - return f"TypeInfo(cls={self.cls}, type_id={self.type_id}, " \ - f"serializer={self.serializer})" + return ( + f"TypeInfo(cls={self.cls}, type_id={self.type_id}, " + f"user_type_id={self.user_type_id}, serializer={self.serializer})" + ) cpdef str decode_namespace(self): if self.namespace_bytes is None: @@ -500,10 +544,11 @@ cdef class TypeResolver: readonly MetaStringResolver metastring_resolver object _resolver vector[PyObject *] _c_registered_id_to_type_info + flat_hash_map[uint32_t, PyObject *] _c_user_type_id_to_type_info # cls -> TypeInfo flat_hash_map[uint64_t, PyObject *] _c_types_info # hash -> TypeInfo - flat_hash_map[pair[int64_t, int64_t], PyObject *] _c_meta_hash_to_typeinfo + flat_hash_map[pair[int64_t, int64_t], PyObject *] _c_meta_hash_to_type_info MetaStringResolver meta_string_resolver c_bool meta_share readonly SerializationContext serialization_context @@ -518,7 +563,7 @@ cdef class TypeResolver: def initialize(self): self._resolver.initialize() for typeinfo in self._resolver._types_info.values(): - self._populate_typeinfo(typeinfo) + self._populate_type_info(typeinfo) self.serialization_context = self.fory.serialization_context def register( @@ -548,7 +593,7 @@ cdef class TypeResolver: typename=typename, serializer=serializer, ) - self._populate_typeinfo(typeinfo) + self._populate_type_info(typeinfo) def register_union( self, @@ -566,28 +611,48 @@ cdef class TypeResolver: typename=typename, serializer=serializer, ) - self._populate_typeinfo(typeinfo) - - cdef _populate_typeinfo(self, typeinfo): - type_id = typeinfo.type_id - if type_id >= self._c_registered_id_to_type_info.size(): - self._c_registered_id_to_type_info.resize(type_id * 2, NULL) - if type_id > 0 and (self.fory.language == Language.PYTHON or not is_namespaced_type(type_id)): - self._c_registered_id_to_type_info[type_id] = typeinfo + self._populate_type_info(typeinfo) + + cdef _populate_type_info(self, typeinfo): + cdef uint8_t type_id = typeinfo.type_id + if ( + type_id == TypeId.ENUM + or type_id == TypeId.STRUCT + or type_id == TypeId.COMPATIBLE_STRUCT + or type_id == TypeId.EXT + or type_id == TypeId.TYPED_UNION + ): + if typeinfo.user_type_id != NO_USER_TYPE_ID: + self._c_user_type_id_to_type_info[typeinfo.user_type_id] = typeinfo + else: + if type_id >= self._c_registered_id_to_type_info.size(): + self._c_registered_id_to_type_info.resize(type_id * 2, NULL) + if type_id > 0 and (self.fory.language == Language.PYTHON or not is_namespaced_type(type_id)): + self._c_registered_id_to_type_info[type_id] = typeinfo self._c_types_info[ typeinfo.cls] = typeinfo # Resize if load factor >= 0.4 (using integer arithmetic: size/capacity >= 4/10) if self._c_types_info.size() * 10 >= self._c_types_info.bucket_count() * 5: self._c_types_info.rehash(self._c_types_info.size() * 2) if typeinfo.typename_bytes is not None: - self._load_bytes_to_typeinfo(type_id, typeinfo.namespace_bytes, typeinfo.typename_bytes) + self._load_bytes_to_type_info(type_id, typeinfo.namespace_bytes, typeinfo.typename_bytes) def register_serializer(self, cls: Union[type, TypeVar], serializer): - typeinfo1 = self._resolver.get_typeinfo(cls) + typeinfo1 = self._resolver.get_type_info(cls) self._resolver.register_serializer(cls, serializer) - typeinfo2 = self._resolver.get_typeinfo(cls) - if typeinfo1.type_id != typeinfo2.type_id: - self._c_registered_id_to_type_info[typeinfo1.type_id] = NULL - self._populate_typeinfo(typeinfo2) + typeinfo2 = self._resolver.get_type_info(cls) + if typeinfo1.type_id != typeinfo2.type_id or typeinfo1.user_type_id != typeinfo2.user_type_id: + if ( + typeinfo1.type_id == TypeId.ENUM + or typeinfo1.type_id == TypeId.STRUCT + or typeinfo1.type_id == TypeId.COMPATIBLE_STRUCT + or typeinfo1.type_id == TypeId.EXT + or typeinfo1.type_id == TypeId.TYPED_UNION + ): + if typeinfo1.user_type_id != NO_USER_TYPE_ID: + self._c_user_type_id_to_type_info[typeinfo1.user_type_id] = NULL + else: + self._c_registered_id_to_type_info[typeinfo1.type_id] = NULL + self._populate_type_info(typeinfo2) cpdef inline Serializer get_serializer(self, cls): """ @@ -595,9 +660,9 @@ cdef class TypeResolver: ------- Returns or create serializer for the provided type """ - return self.get_typeinfo(cls).serializer + return self.get_type_info(cls).serializer - cpdef inline TypeInfo get_typeinfo(self, cls, create=True): + cpdef inline TypeInfo get_type_info(self, cls, create=True): cdef PyObject * typeinfo_ptr = self._c_types_info[ cls] cdef TypeInfo type_info if typeinfo_ptr != NULL: @@ -605,14 +670,14 @@ cdef class TypeResolver: if type_info.serializer is not None: return type_info else: - type_info.serializer = self._resolver.get_typeinfo(cls).serializer + type_info.serializer = self._resolver.get_type_info(cls).serializer return type_info elif not create: return None else: - type_info = self._resolver.get_typeinfo(cls, create=create) + type_info = self._resolver.get_type_info(cls, create=create) self._c_types_info[ cls] = type_info - self._populate_typeinfo(type_info) + self._populate_type_info(type_info) return type_info cpdef inline is_registered_by_name(self, cls): @@ -627,58 +692,76 @@ cdef class TypeResolver: cpdef inline get_registered_id(self, cls): return self._resolver.get_registered_id(cls) - cdef inline TypeInfo _load_bytes_to_typeinfo( - self, int32_t type_id, MetaStringBytes ns_metabytes, MetaStringBytes type_metabytes): - cdef PyObject * typeinfo_ptr = self._c_meta_hash_to_typeinfo[ + cpdef inline get_registered_user_type_id(self, cls): + return self._resolver.get_registered_user_type_id(cls) + + cpdef inline get_registered_type_ids(self, cls): + return self._resolver.get_registered_type_ids(cls) + + cdef inline TypeInfo _load_bytes_to_type_info( + self, uint8_t type_id, MetaStringBytes ns_metabytes, MetaStringBytes type_metabytes): + cdef PyObject * typeinfo_ptr = self._c_meta_hash_to_type_info[ pair[int64_t, int64_t](ns_metabytes.hashcode, type_metabytes.hashcode)] if typeinfo_ptr != NULL: return typeinfo_ptr - typeinfo = self._resolver._load_metabytes_to_typeinfo(ns_metabytes, type_metabytes) + typeinfo = self._resolver._load_metabytes_to_type_info(ns_metabytes, type_metabytes) typeinfo_ptr = typeinfo - self._c_meta_hash_to_typeinfo[pair[int64_t, int64_t]( + self._c_meta_hash_to_type_info[pair[int64_t, int64_t]( ns_metabytes.hashcode, type_metabytes.hashcode)] = typeinfo_ptr return typeinfo - cpdef inline write_typeinfo(self, Buffer buffer, TypeInfo typeinfo): + cpdef inline write_type_info(self, Buffer buffer, TypeInfo typeinfo): if typeinfo.dynamic_type: return cdef: - int32_t type_id = typeinfo.type_id - int32_t internal_type_id = type_id & 0xFF - - if self.meta_share: + uint8_t type_id = typeinfo.type_id + TypeRegistrationKind reg_kind + buffer.write_uint8(type_id) + if type_id == TypeId.COMPATIBLE_STRUCT or type_id == TypeId.NAMED_COMPATIBLE_STRUCT: self.write_shared_type_meta(buffer, typeinfo) return + reg_kind = get_type_registration_kind(type_id) + if reg_kind == TypeRegistrationKind.BY_ID: + if typeinfo.user_type_id == NO_USER_TYPE_ID: + raise ValueError(f"user_type_id required for type_id {type_id}") + buffer.write_var_uint32(typeinfo.user_type_id) + return + if reg_kind == TypeRegistrationKind.BY_NAME: + if self.meta_share: + self.write_shared_type_meta(buffer, typeinfo) + else: + self.metastring_resolver.write_meta_string_bytes(buffer, typeinfo.namespace_bytes) + self.metastring_resolver.write_meta_string_bytes(buffer, typeinfo.typename_bytes) - buffer.write_var_uint32(type_id) - if is_namespaced_type(internal_type_id): - self.metastring_resolver.write_meta_string_bytes(buffer, typeinfo.namespace_bytes) - self.metastring_resolver.write_meta_string_bytes(buffer, typeinfo.typename_bytes) - - cpdef inline TypeInfo read_typeinfo(self, Buffer buffer): + cpdef inline TypeInfo read_type_info(self, Buffer buffer): cdef: - int32_t type_id = buffer.read_var_uint32() - if type_id < 0: - type_id = -type_id - if type_id >= self._c_registered_id_to_type_info.size(): - raise ValueError(f"Unexpected type_id {type_id}") + uint8_t type_id = buffer.read_uint8() + TypeRegistrationKind reg_kind cdef: - int32_t internal_type_id = type_id & 0xFF + uint32_t user_type_id = NO_USER_TYPE_ID MetaStringBytes namespace_bytes, typename_bytes - if self.meta_share: - return self.serialization_context.meta_context.read_shared_typeinfo_with_type_id(buffer, type_id) - if is_namespaced_type(internal_type_id): + if type_id == TypeId.COMPATIBLE_STRUCT or type_id == TypeId.NAMED_COMPATIBLE_STRUCT: + return self.serialization_context.meta_context.read_shared_type_info_with_type_id(buffer, type_id) + reg_kind = get_type_registration_kind(type_id) + if reg_kind == TypeRegistrationKind.BY_NAME: + if self.meta_share: + return self.serialization_context.meta_context.read_shared_type_info_with_type_id(buffer, type_id) namespace_bytes = self.metastring_resolver.read_meta_string_bytes(buffer) typename_bytes = self.metastring_resolver.read_meta_string_bytes(buffer) - return self._load_bytes_to_typeinfo(type_id, namespace_bytes, typename_bytes) + return self._load_bytes_to_type_info(type_id, namespace_bytes, typename_bytes) + if reg_kind == TypeRegistrationKind.BY_ID: + user_type_id = buffer.read_var_uint32() + return self.get_user_type_info_by_id(user_type_id) + if type_id >= self._c_registered_id_to_type_info.size(): + raise ValueError(f"Unexpected type_id {type_id}") typeinfo_ptr = self._c_registered_id_to_type_info[type_id] if typeinfo_ptr == NULL: raise ValueError(f"Unexpected type_id {type_id}") typeinfo = typeinfo_ptr return typeinfo - cpdef inline TypeInfo get_typeinfo_by_id(self, int32_t type_id): - if type_id >= self._c_registered_id_to_type_info.size() or type_id < 0 or is_namespaced_type(type_id & 0xFF): + cpdef inline TypeInfo get_type_info_by_id(self, uint8_t type_id): + if type_id >= self._c_registered_id_to_type_info.size() or is_namespaced_type(type_id): raise ValueError(f"Unexpected type_id {type_id}") typeinfo_ptr = self._c_registered_id_to_type_info[type_id] if typeinfo_ptr == NULL: @@ -686,11 +769,17 @@ cdef class TypeResolver: typeinfo = typeinfo_ptr return typeinfo - cpdef inline get_typeinfo_by_name(self, namespace, typename): - return self._resolver.get_typeinfo_by_name(namespace=namespace, typename=typename) + cpdef inline TypeInfo get_user_type_info_by_id(self, uint32_t user_type_id): + typeinfo_ptr = self._c_user_type_id_to_type_info[user_type_id] + if typeinfo_ptr == NULL: + raise ValueError(f"Unexpected user_type_id {user_type_id}") + return typeinfo_ptr + + cpdef inline get_type_info_by_name(self, namespace, typename): + return self._resolver.get_type_info_by_name(namespace=namespace, typename=typename) - cpdef inline _set_typeinfo(self, typeinfo): - self._resolver._set_typeinfo(typeinfo) + cpdef inline _set_type_info(self, typeinfo): + self._resolver._set_type_info(typeinfo) cpdef inline get_meta_compressor(self): return self._resolver.get_meta_compressor() @@ -698,17 +787,17 @@ cdef class TypeResolver: cpdef inline write_shared_type_meta(self, Buffer buffer, TypeInfo typeinfo): """write shared type meta information.""" meta_context = self.serialization_context.meta_context - meta_context.write_shared_typeinfo(buffer, typeinfo) + meta_context.write_shared_type_info(buffer, typeinfo) cpdef inline TypeInfo read_shared_type_meta(self, Buffer buffer): """Read shared type meta information.""" meta_context = self.serialization_context.meta_context - typeinfo = meta_context.read_shared_typeinfo(buffer) + typeinfo = meta_context.read_shared_type_info(buffer) return typeinfo - cpdef inline _read_and_build_typeinfo(self, Buffer buffer): + cpdef inline _read_and_build_type_info(self, Buffer buffer): """Read TypeDef inline from buffer and build TypeInfo.""" - return self._resolver._read_and_build_typeinfo(buffer) + return self._resolver._read_and_build_type_info(buffer) cpdef inline reset(self): pass @@ -752,13 +841,11 @@ cdef class MetaContext: self.type_resolver = fory.type_resolver self._read_type_infos = [] - cpdef inline void write_shared_typeinfo(self, Buffer buffer, typeinfo): + cpdef inline void write_shared_type_info(self, Buffer buffer, typeinfo): """write type info with streaming inline TypeDef.""" type_cls = typeinfo.cls - cdef int32_t type_id = typeinfo.type_id - cdef int32_t internal_type_id = type_id & 0xFF - buffer.write_var_uint32(type_id) - if not is_type_share_meta(internal_type_id): + cdef uint8_t type_id = typeinfo.type_id + if not is_type_share_meta(type_id): return cdef uint64_t type_addr = type_cls @@ -774,7 +861,7 @@ cdef class MetaContext: self._c_type_map[type_addr] = index type_def = typeinfo.type_def if type_def is None: - self.type_resolver._set_typeinfo(typeinfo) + self.type_resolver._set_type_info(typeinfo) type_def = typeinfo.type_def # write TypeDef bytes inline instead of deferring to end buffer.write_bytes(type_def.encoded) @@ -783,19 +870,26 @@ cdef class MetaContext: """reset write state.""" self._c_type_map.clear() - cpdef inline add_read_typeinfo(self, type_info): + cpdef inline add_read_type_info(self, type_info): """Add a type info read from peer.""" self._read_type_infos.append(type_info) - cpdef inline read_shared_typeinfo(self, Buffer buffer): + cpdef inline read_shared_type_info(self, Buffer buffer): """Read type info with streaming inline TypeDef.""" - cdef int32_t type_id = buffer.read_var_uint32() - return self.read_shared_typeinfo_with_type_id(buffer, type_id) + cdef uint8_t type_id = buffer.read_uint8() + return self.read_shared_type_info_with_type_id(buffer, type_id) - cpdef inline read_shared_typeinfo_with_type_id(self, Buffer buffer, int32_t type_id): + cpdef inline read_shared_type_info_with_type_id(self, Buffer buffer, uint8_t type_id): """Read shared type info when type_id is already consumed.""" - if not is_type_share_meta(type_id & 0xFF): - return self.type_resolver.get_typeinfo_by_id(type_id) + cdef uint32_t user_type_id = NO_USER_TYPE_ID + cdef TypeRegistrationKind reg_kind = get_type_registration_kind(type_id) + cdef c_bool share_meta = is_type_share_meta(type_id) + if reg_kind == TypeRegistrationKind.BY_ID and not share_meta: + user_type_id = buffer.read_var_uint32() + if not share_meta: + if reg_kind == TypeRegistrationKind.BY_ID: + return self.type_resolver.get_user_type_info_by_id(user_type_id) + return self.type_resolver.get_type_info_by_id(type_id) cdef int32_t index_marker = buffer.read_var_uint32() cdef c_bool is_ref = (index_marker & 1) == 1 @@ -806,7 +900,7 @@ cdef class MetaContext: return self._read_type_infos[index] else: # New type - read TypeDef inline and build TypeInfo - type_info = self.type_resolver._read_and_build_typeinfo(buffer) + type_info = self.type_resolver._read_and_build_type_info(buffer) self._read_type_infos.append(type_info) return type_info @@ -1278,8 +1372,8 @@ cdef class Fory: if self.ref_resolver.write_ref_or_null(buffer, obj): return if typeinfo is None: - typeinfo = self.type_resolver.get_typeinfo(cls) - self.type_resolver.write_typeinfo(buffer, typeinfo) + typeinfo = self.type_resolver.get_type_info(cls) + self.type_resolver.write_type_info(buffer, typeinfo) typeinfo.serializer.write(buffer, obj) cpdef inline write_no_ref(self, Buffer buffer, obj): @@ -1300,8 +1394,8 @@ cdef class Fory: buffer.write_var_uint32(FLOAT64_TYPE_ID) buffer.write_double(obj) return - cdef TypeInfo typeinfo = self.type_resolver.get_typeinfo(cls) - self.type_resolver.write_typeinfo(buffer, typeinfo) + cdef TypeInfo typeinfo = self.type_resolver.get_type_info(cls) + self.type_resolver.write_type_info(buffer, typeinfo) typeinfo.serializer.write(buffer, obj) cpdef inline xwrite_ref( @@ -1323,8 +1417,8 @@ cdef class Fory: cpdef inline xwrite_no_ref( self, Buffer buffer, obj, Serializer serializer=None): if serializer is None: - typeinfo = self.type_resolver.get_typeinfo(type(obj)) - self.type_resolver.write_typeinfo(buffer, typeinfo) + typeinfo = self.type_resolver.get_type_info(type(obj)) + self.type_resolver.write_type_info(buffer, typeinfo) serializer = typeinfo.serializer serializer.xwrite(buffer, obj) @@ -1404,7 +1498,7 @@ cdef class Fory: if ref_id < NOT_NULL_VALUE_FLAG: return ref_resolver.get_read_object() # indicates that the object is first read. - cdef TypeInfo typeinfo = self.type_resolver.read_typeinfo(buffer) + cdef TypeInfo typeinfo = self.type_resolver.read_type_info(buffer) cls = typeinfo.cls if cls is str: return buffer.read_string() @@ -1422,7 +1516,7 @@ cdef class Fory: cpdef inline read_no_ref(self, Buffer buffer): """Deserialize not-null and non-reference object from buffer.""" - cdef TypeInfo typeinfo = self.type_resolver.read_typeinfo(buffer) + cdef TypeInfo typeinfo = self.type_resolver.read_type_info(buffer) cls = typeinfo.cls if cls is str: return buffer.read_string() @@ -1461,7 +1555,7 @@ cdef class Fory: cpdef inline xread_no_ref( self, Buffer buffer, Serializer serializer=None): if serializer is None: - serializer = self.type_resolver.read_typeinfo(buffer).serializer + serializer = self.type_resolver.read_type_info(buffer).serializer # Push -1 to read_ref_ids so reference() can pop it and skip reference tracking # This handles the case where xread_no_ref is called directly without xread_ref if self.ref_resolver.ref_tracking: @@ -1472,7 +1566,7 @@ cdef class Fory: self, Buffer buffer, Serializer serializer): """Internal method to read without pushing to read_ref_ids.""" if serializer is None: - serializer = self.type_resolver.read_typeinfo(buffer).serializer + serializer = self.type_resolver.read_type_info(buffer).serializer self.inc_depth() o = serializer.xread(buffer) self.depth -= 1 @@ -1552,8 +1646,8 @@ cdef class Fory: if self.ref_resolver.write_ref_or_null(buffer, value): return if typeinfo is None: - typeinfo = self.type_resolver.get_typeinfo(type(value)) - self.type_resolver.write_typeinfo(buffer, typeinfo) + typeinfo = self.type_resolver.get_type_info(type(value)) + self.type_resolver.write_type_info(buffer, typeinfo) typeinfo.serializer.write(buffer, value) cpdef inline read_ref_pyobject(self, Buffer buffer): @@ -1562,7 +1656,7 @@ cdef class Fory: if ref_id < NOT_NULL_VALUE_FLAG: return ref_resolver.get_read_object() # indicates that the object is first read. - cdef TypeInfo typeinfo = self.type_resolver.read_typeinfo(buffer) + cdef TypeInfo typeinfo = self.type_resolver.read_type_info(buffer) self.inc_depth() o = typeinfo.serializer.read(buffer) self.depth -= 1 diff --git a/python/pyfory/serializer.py b/python/pyfory/serializer.py index 809352f5ad..fb64d38812 100644 --- a/python/pyfory/serializer.py +++ b/python/pyfory/serializer.py @@ -382,7 +382,7 @@ def xwrite(self, buffer, value): itemsize, ftype, type_id = typecode_dict[value.typecode] view = memoryview(value) nbytes = len(value) * itemsize - buffer.write_var_uint32(type_id) + buffer.write_uint8(type_id) buffer.write_var_uint32(nbytes) if not view.c_contiguous: data = value.tobytes() @@ -401,7 +401,7 @@ def xwrite(self, buffer, value): buffer.write_buffer(swapped) def xread(self, buffer): - type_id = buffer.read_varint32() + type_id = buffer.read_uint8() typecode = typeid_code[type_id] itemsize = typecode_dict[typecode][0] data = buffer.read_bytes_and_size() @@ -507,7 +507,7 @@ def xwrite(self, buffer, value): itemsize, typecode, ftype, type_id = _np_dtypes_dict[value.dtype] view = memoryview(value) nbytes = len(value) * itemsize - buffer.write_var_uint32(type_id) + buffer.write_uint8(type_id) buffer.write_var_uint32(nbytes) if value.dtype == np.dtype("bool") or not view.c_contiguous: buffer.write_bytes(value.tobytes()) diff --git a/python/pyfory/struct.py b/python/pyfory/struct.py index d90faeb984..21a6223189 100644 --- a/python/pyfory/struct.py +++ b/python/pyfory/struct.py @@ -244,11 +244,20 @@ def _extract_field_infos( # Compute runtime ref tracking: field.ref AND global config runtime_ref = meta.ref and global_ref_tracking + # Infer serializer + serializer = infer_field(field_name, unwrapped_type, visitor, types_path=[]) + + # Get type_id from serializer + if serializer is not None: + type_id = fory.type_resolver.get_type_info(serializer.type_).type_id + else: + type_id = TypeId.UNKNOWN + # Compute effective dynamic based on type and mode # - Abstract classes: always True (type info must be written) # - If explicitly set (not None): use that value + # - Xlang mode: write type info for user-defined types # - Native mode: True for object types, False for numeric/str/time types - # - Xlang mode: False for concrete types is_abstract = _is_abstract_type(unwrapped_type) if is_abstract: # Abstract classes always need type info @@ -257,23 +266,15 @@ def _extract_field_infos( # Explicit configuration takes precedence effective_dynamic = meta.dynamic elif xlang: - # Xlang mode: False for concrete types - effective_dynamic = False + # Xlang mode: write type info only when the declared type isn't registered by id. + # Registered-by-id types have stable serializers, so no per-field type info is needed. + effective_dynamic = is_polymorphic_type(type_id) and not fory.type_resolver.is_registered_by_id(unwrapped_type) else: # Native mode: False for numeric/str/time types, True for other object types # Check if the type is a primitive, string, or time type is_non_dynamic_type = is_primitive_type(unwrapped_type) or unwrapped_type in (str, bytes) or unwrapped_type in _time_types effective_dynamic = not is_non_dynamic_type - # Infer serializer - serializer = infer_field(field_name, unwrapped_type, visitor, types_path=[]) - - # Get type_id from serializer - if serializer is not None: - type_id = fory.type_resolver.get_typeinfo(serializer.type_).type_id & 0xFF - else: - type_id = TypeId.UNKNOWN - field_info = FieldInfo( name=field_name, index=index, @@ -314,6 +315,7 @@ def __init__( field_names: List[str] = None, serializers: List[Serializer] = None, nullable_fields: Dict[str, bool] = None, + dynamic_fields: Dict[str, bool] = None, ): super().__init__(fory, clz) self._xlang = xlang @@ -332,7 +334,7 @@ def __init__( self._serializers = serializers self._nullable_fields = nullable_fields or {} self._ref_fields = {} - self._dynamic_fields = {} # Default to empty, will use mode defaults + self._dynamic_fields = dynamic_fields or {} self._field_infos = [] self._field_metas = {} else: @@ -609,7 +611,7 @@ def _gen_write_method(self): # For dynamic=False, get typeinfo for declared type to use its serializer typeinfo_var = f"typeinfo{index}" if not is_dynamic and serializer is not None: - context[typeinfo_var] = self.fory.type_resolver.get_typeinfo(serializer.type_) + context[typeinfo_var] = self.fory.type_resolver.get_type_info(serializer.type_) if is_nullable: # Use gen_write_nullable_basic_stmts for nullable basic types if isinstance(serializer, BooleanSerializer): @@ -988,7 +990,7 @@ def write(self, buffer, value): # For dynamic=False, get typeinfo for declared type typeinfo = None if not is_dynamic and serializer is not None: - typeinfo = self.fory.type_resolver.get_typeinfo(serializer.type_) + typeinfo = self.fory.type_resolver.get_type_info(serializer.type_) if is_nullable: self._write_nullable_field(buffer, field_value, serializer, typeinfo) @@ -1134,7 +1136,7 @@ def xread(self, buffer): return self._replace().xread(buffer) def _replace(self): - typeinfo = self.fory.type_resolver.get_typeinfo(self.type_) + typeinfo = self.fory.type_resolver.get_type_info(self.type_) typeinfo.serializer = DataClassSerializer(self.fory, self.type_, self.xlang) return typeinfo.serializer @@ -1219,7 +1221,7 @@ def visit_customized(self, field_name, type_, types_path=None): return self.fory.type_resolver.get_serializer(type_) # For custom types (dataclasses, etc.), try to get or create serializer # This enables field-level serializer resolution for types like inner structs - typeinfo = self.fory.type_resolver.get_typeinfo(type_, create=False) + typeinfo = self.fory.type_resolver.get_type_info(type_, create=False) if typeinfo is not None: return typeinfo.serializer return None @@ -1269,7 +1271,7 @@ def group_fields(type_resolver, field_names, serializers, nullable_map=None, fie else: type_ids.append( ( - type_resolver.get_typeinfo(serializer.type_).type_id & 0xFF, + type_resolver.get_type_info(serializer.type_).type_id, serializer, field_name, sort_key, @@ -1371,7 +1373,7 @@ def compute_struct_fingerprint(type_resolver, field_names, serializers, nullable # For unknown serializers, use nullable from map (defaults to False for xlang) nullable_flag = "1" if nullable_map.get(field_name, False) else "0" else: - type_id = type_resolver.get_typeinfo(serializer.type_).type_id & 0xFF + type_id = type_resolver.get_type_info(serializer.type_).type_id if is_union_type(type_id): # customized types can't be detected at compile time for some languages type_id = TypeId.UNKNOWN @@ -1477,17 +1479,17 @@ def visit_dict(self, field_name, key_type, value_type, types_path=None): return TypeId.MAP, key_ids, value_ids def visit_customized(self, field_name, type_, types_path=None): - typeinfo = self.fory.type_resolver.get_typeinfo(type_, create=False) + typeinfo = self.fory.type_resolver.get_type_info(type_, create=False) if typeinfo is None: return [TypeId.UNKNOWN] return [typeinfo.type_id] def visit_other(self, field_name, type_, types_path=None): if is_subclass(type_, enum.Enum): - return [self.fory.type_resolver.get_typeinfo(type_).type_id] + return [self.fory.type_resolver.get_type_info(type_).type_id] if type_ not in basic_types and not is_primitive_array_type(type_): return None, None - typeinfo = self.fory.type_resolver.get_typeinfo(type_) + typeinfo = self.fory.type_resolver.get_type_info(type_) return [typeinfo.type_id] diff --git a/python/pyfory/tests/test_field_meta.py b/python/pyfory/tests/test_field_meta.py index d766548d2a..51ba4d2841 100644 --- a/python/pyfory/tests/test_field_meta.py +++ b/python/pyfory/tests/test_field_meta.py @@ -354,8 +354,8 @@ class V2: fory2.serialize(V2(name="b")) # Get the actual serializers after serialization - serializer1 = fory1.type_resolver.get_typeinfo(V1).serializer - serializer2 = fory2.type_resolver.get_typeinfo(V2).serializer + serializer1 = fory1.type_resolver.get_type_info(V1).serializer + serializer2 = fory2.type_resolver.get_type_info(V2).serializer # Fingerprints should be different due to different tag IDs assert serializer1._hash != serializer2._hash @@ -382,8 +382,8 @@ class WithoutRef: fory2.serialize(WithoutRef()) # Get the actual serializers after serialization - serializer1 = fory1.type_resolver.get_typeinfo(WithRef).serializer - serializer2 = fory2.type_resolver.get_typeinfo(WithoutRef).serializer + serializer1 = fory1.type_resolver.get_type_info(WithRef).serializer + serializer2 = fory2.type_resolver.get_type_info(WithoutRef).serializer # Fingerprints should be different due to different ref flags assert serializer1._hash != serializer2._hash diff --git a/python/pyfory/tests/test_metastring_resolver.py b/python/pyfory/tests/test_metastring_resolver.py index 21e3d3b236..adf9b19989 100644 --- a/python/pyfory/tests/test_metastring_resolver.py +++ b/python/pyfory/tests/test_metastring_resolver.py @@ -23,43 +23,46 @@ def test_metastring_resolver(): resolver = MetaStringResolver() encoder = MetaStringEncoder("$", "_") + try: + # Test 1: Regular English string + metastr1 = encoder.encode("hello, world") + metabytes1 = resolver.get_metastr_bytes(metastr1) + buffer = Buffer.allocate(32) + resolver.write_meta_string_bytes(buffer, metabytes1) + assert resolver.read_meta_string_bytes(buffer) == metabytes1 - # Test 1: Regular English string - metastr1 = encoder.encode("hello, world") - metabytes1 = resolver.get_metastr_bytes(metastr1) - buffer = Buffer.allocate(32) - resolver.write_meta_string_bytes(buffer, metabytes1) - assert resolver.read_meta_string_bytes(buffer) == metabytes1 + # Test 2: Manually constructed MetaStringBytes + metabytes2 = MetaStringBytes( + data=b"\xbf\x05\xa4q\xa9\x92S\x96\xa6IOr\x9ch)\x80", + hashcode=-2270219110992250879, + ) + resolver.write_meta_string_bytes(buffer, metabytes2) + assert resolver.read_meta_string_bytes(buffer) == metabytes2 - # Test 2: Manually constructed MetaStringBytes - metabytes2 = MetaStringBytes( - data=b"\xbf\x05\xa4q\xa9\x92S\x96\xa6IOr\x9ch)\x80", - hashcode=-5456063526933366015, - ) - resolver.write_meta_string_bytes(buffer, metabytes2) - assert resolver.read_meta_string_bytes(buffer) == metabytes2 + # Test 3: Empty string + metastr_null = encoder.encode("") + metabytes_null = resolver.get_metastr_bytes(metastr_null) + resolver.write_meta_string_bytes(buffer, metabytes_null) + assert resolver.read_meta_string_bytes(buffer) == metabytes_null - # Test 3: Empty string - metastr_null = encoder.encode("") - metabytes_null = resolver.get_metastr_bytes(metastr_null) - resolver.write_meta_string_bytes(buffer, metabytes_null) - assert resolver.read_meta_string_bytes(buffer) == metabytes_null + # Test 4: Chinese string + metastr_cn = encoder.encode("你好，世界") + metabytes_cn = resolver.get_metastr_bytes(metastr_cn) + resolver.write_meta_string_bytes(buffer, metabytes_cn) + assert resolver.read_meta_string_bytes(buffer) == metabytes_cn - # Test 4: Chinese string - metastr_cn = encoder.encode("你好，世界") - metabytes_cn = resolver.get_metastr_bytes(metastr_cn) - resolver.write_meta_string_bytes(buffer, metabytes_cn) - assert resolver.read_meta_string_bytes(buffer) == metabytes_cn + # Test 5: Japanese string (more than 16 bytes, triggers hash-based encoding) + metastr_jp = encoder.encode("こんにちは世界") + metabytes_jp = resolver.get_metastr_bytes(metastr_jp) + resolver.write_meta_string_bytes(buffer, metabytes_jp) + assert resolver.read_meta_string_bytes(buffer) == metabytes_jp - # Test 5: Japanese string (more than 16 bytes, triggers hash-based encoding) - metastr_jp = encoder.encode("こんにちは世界") - metabytes_jp = resolver.get_metastr_bytes(metastr_jp) - resolver.write_meta_string_bytes(buffer, metabytes_jp) - assert resolver.read_meta_string_bytes(buffer) == metabytes_jp - - # Test 6: Long string (more than 16 bytes, triggers hash-based encoding) - long_str = "hello, world" * 10 - metastr_long = encoder.encode(long_str) - metabytes_long = resolver.get_metastr_bytes(metastr_long) - resolver.write_meta_string_bytes(buffer, metabytes_long) - assert resolver.read_meta_string_bytes(buffer) == metabytes_long + # Test 6: Long string (more than 16 bytes, triggers hash-based encoding) + long_str = "hello, world" * 10 + metastr_long = encoder.encode(long_str) + metabytes_long = resolver.get_metastr_bytes(metastr_long) + resolver.write_meta_string_bytes(buffer, metabytes_long) + assert resolver.read_meta_string_bytes(buffer) == metabytes_long + finally: + resolver.reset_write() + resolver.reset_read() diff --git a/python/pyfory/tests/test_serializer.py b/python/pyfory/tests/test_serializer.py index 119739bc80..e4b4d643ae 100644 --- a/python/pyfory/tests/test_serializer.py +++ b/python/pyfory/tests/test_serializer.py @@ -129,11 +129,11 @@ def test_big_chunk_dict(track_ref): @pytest.mark.parametrize("language", [Language.XLANG, Language.PYTHON]) def test_basic_serializer(language): fory = Fory(language=language, ref=True) - typeinfo = fory.type_resolver.get_typeinfo(datetime.datetime) + typeinfo = fory.type_resolver.get_type_info(datetime.datetime) assert isinstance(typeinfo.serializer, (TimestampSerializer, serialization.TimestampSerializer)) if language == Language.XLANG: assert typeinfo.type_id == TypeId.TIMESTAMP - typeinfo = fory.type_resolver.get_typeinfo(datetime.date) + typeinfo = fory.type_resolver.get_type_info(datetime.date) assert isinstance(typeinfo.serializer, (DateSerializer, serialization.DateSerializer)) if language == Language.XLANG: assert typeinfo.type_id == TypeId.DATE diff --git a/python/pyfory/types.py b/python/pyfory/types.py index 5c12d4af7c..4ca50e2743 100644 --- a/python/pyfory/types.py +++ b/python/pyfory/types.py @@ -420,6 +420,14 @@ def is_map_type(type_): TypeId.NAMED_UNION, } +_user_type_id_required = { + TypeId.ENUM, + TypeId.STRUCT, + TypeId.COMPATIBLE_STRUCT, + TypeId.EXT, + TypeId.TYPED_UNION, +} + def is_polymorphic_type(type_id: int) -> bool: return type_id in _polymorphic_type_ids @@ -438,4 +446,8 @@ def is_union_type(type_or_id) -> bool: type_id = getattr(type_or_id, "type_id", None) if type_id is None or not isinstance(type_id, int): return False - return (type_id & 0xFF) in _union_type_ids + return type_id in _union_type_ids + + +def needs_user_type_id(type_id: int) -> bool: + return type_id in _user_type_id_required diff --git a/python/pyfory/union.py b/python/pyfory/union.py index e36a5e2887..96eb3d1ccf 100644 --- a/python/pyfory/union.py +++ b/python/pyfory/union.py @@ -57,7 +57,7 @@ class UnionSerializer(Serializer): "_alternative_types", "_alternative_serializers", "_case_types", - "_case_typeinfos", + "_case_type_infos", ) def __init__(self, fory, type_, alternative_types): @@ -66,14 +66,14 @@ def __init__(self, fory, type_, alternative_types): if isinstance(alternative_types, dict): self._typing_union = False self._case_types = alternative_types - self._case_typeinfos = {} + self._case_type_infos = {} self._alternative_types = None self._alternative_serializers = None else: self._typing_union = True self._alternative_types = alternative_types self._case_types = None - self._case_typeinfos = None + self._case_type_infos = None self._alternative_serializers = [] for alt_type in alternative_types: serializer = fory.type_resolver.get_serializer(alt_type) @@ -85,7 +85,7 @@ def write(self, buffer, value): return case_id = value.case_id() buffer.write_var_uint32(case_id) - typeinfo = self._get_case_typeinfo(case_id) + typeinfo = self._get_case_type_info(case_id) self.fory.write_ref(buffer, value._value, typeinfo=typeinfo) def read(self, buffer): @@ -101,7 +101,7 @@ def xwrite(self, buffer, value): return case_id = value.case_id() buffer.write_var_uint32(case_id) - typeinfo = self._get_case_typeinfo(case_id) + typeinfo = self._get_case_type_info(case_id) serializer = typeinfo.serializer if serializer.need_to_write_ref: if self.fory.ref_resolver.write_ref_or_null(buffer, value._value): @@ -111,7 +111,7 @@ def xwrite(self, buffer, value): buffer.write_int8(NULL_FLAG) return buffer.write_int8(NOT_NULL_VALUE_FLAG) - self.type_resolver.write_typeinfo(buffer, typeinfo) + self.type_resolver.write_type_info(buffer, typeinfo) serializer.xwrite(buffer, value._value) def xread(self, buffer): @@ -121,14 +121,14 @@ def xread(self, buffer): value = self.fory.xread_ref(buffer) return self._build_union(case_id, value) - def _get_case_typeinfo(self, case_id: int): - typeinfo = self._case_typeinfos.get(case_id) + def _get_case_type_info(self, case_id: int): + typeinfo = self._case_type_infos.get(case_id) if typeinfo is None: case_type = self._case_types.get(case_id) if case_type is None: raise ValueError(f"unknown union case id: {case_id}") - typeinfo = self.type_resolver.get_typeinfo(case_type) - self._case_typeinfos[case_id] = typeinfo + typeinfo = self.type_resolver.get_type_info(case_type) + self._case_type_infos[case_id] = typeinfo return typeinfo def _build_union(self, case_id: int, value: object): @@ -178,13 +178,13 @@ def _xwrite_typing_union(self, buffer, value): raise TypeError(f"Value {value} of type {type(value)} doesn't match any alternative in Union{self._alternative_types}") buffer.write_var_uint32(active_index) - typeinfo = self.type_resolver.get_typeinfo(active_type) - self.type_resolver.write_typeinfo(buffer, typeinfo) + typeinfo = self.type_resolver.get_type_info(active_type) + self.type_resolver.write_type_info(buffer, typeinfo) active_serializer.xwrite(buffer, value) def _xread_typing_union(self, buffer): stored_index = buffer.read_var_uint32() if stored_index >= len(self._alternative_serializers): raise ValueError(f"Union index out of bounds: {stored_index} (max: {len(self._alternative_serializers) - 1})") - typeinfo = self.type_resolver.read_typeinfo(buffer) + typeinfo = self.type_resolver.read_type_info(buffer) return typeinfo.serializer.xread(buffer) diff --git a/rust/fory-core/src/meta/type_meta.rs b/rust/fory-core/src/meta/type_meta.rs index 31e3567787..22d8f1e845 100644 --- a/rust/fory-core/src/meta/type_meta.rs +++ b/rust/fory-core/src/meta/type_meta.rs @@ -34,23 +34,22 @@ use crate::util::to_snake_case; /// UNKNOWN (0) is used for polymorphic types (interfaces) in cross-language serialization. /// Similarly for ENUM and EXT variants, and byte array encodings. fn normalize_type_id_for_eq(type_id: u32) -> u32 { - let low = type_id & 0xff; - match low { + match type_id { // All struct variants and UNKNOWN normalize to STRUCT - _ if low == STRUCT - || low == COMPATIBLE_STRUCT - || low == NAMED_STRUCT - || low == NAMED_COMPATIBLE_STRUCT - || low == UNKNOWN => + _ if type_id == STRUCT + || type_id == COMPATIBLE_STRUCT + || type_id == NAMED_STRUCT + || type_id == NAMED_COMPATIBLE_STRUCT + || type_id == UNKNOWN => { STRUCT } // All enum variants normalize to ENUM - _ if low == ENUM || low == NAMED_ENUM => ENUM, + _ if type_id == ENUM || type_id == NAMED_ENUM => ENUM, // All ext variants normalize to EXT - _ if low == EXT || low == NAMED_EXT => EXT, + _ if type_id == EXT || type_id == NAMED_EXT => EXT, // Byte array encodings normalize to BINARY - _ if low == BINARY || low == INT8_ARRAY || low == UINT8_ARRAY => BINARY, + _ if type_id == BINARY || type_id == INT8_ARRAY || type_id == UINT8_ARRAY => BINARY, // Everything else stays the same _ => type_id, } @@ -74,6 +73,7 @@ const META_SIZE_MASK: i64 = 0xff; const COMPRESS_META_FLAG: i64 = 0b1 << 9; const HAS_FIELDS_META_FLAG: i64 = 0b1 << 8; const NUM_HASH_BITS: i8 = 50; +const NO_USER_TYPE_ID: u32 = u32::MAX; pub static NAMESPACE_ENCODINGS: &[Encoding] = &[ Encoding::Utf8, @@ -97,6 +97,7 @@ static FIELD_NAME_ENCODINGS: &[Encoding] = &[ #[derive(Debug, Eq, Clone)] pub struct FieldType { pub type_id: u32, + pub user_type_id: u32, pub nullable: bool, pub ref_tracking: bool, pub generics: Vec, @@ -106,6 +107,7 @@ impl FieldType { pub fn new(type_id: u32, nullable: bool, generics: Vec) -> Self { FieldType { type_id, + user_type_id: NO_USER_TYPE_ID, nullable, ref_tracking: false, generics, @@ -120,6 +122,7 @@ impl FieldType { ) -> Self { FieldType { type_id, + user_type_id: NO_USER_TYPE_ID, nullable, ref_tracking, generics, @@ -128,6 +131,11 @@ impl FieldType { fn to_bytes(&self, writer: &mut Writer, write_flag: bool, nullable: bool) -> Result<(), Error> { let mut header = self.type_id; + if header == NAMED_ENUM { + header = ENUM; + } else if header == TypeId::NAMED_UNION as u32 || header == TypeId::TYPED_UNION as u32 { + header = TypeId::UNION as u32; + } if write_flag { header <<= 2; if nullable { @@ -136,8 +144,10 @@ impl FieldType { if self.ref_tracking { header |= 1; } + writer.write_var_uint32(header); + } else { + writer.write_u8(header as u8); } - writer.write_var_uint32(header); match self.type_id { x if x == TypeId::LIST as u32 || x == TypeId::SET as u32 => { if let Some(generic) = self.generics.first() { @@ -165,8 +175,12 @@ impl FieldType { read_flag: bool, nullable: Option, ) -> Result { - let header = reader.read_varuint32()?; - let type_id; + let header = if read_flag { + reader.read_varuint32()? + } else { + reader.read_u8()? as u32 + }; + let mut type_id; let _nullable; let _ref_tracking; if read_flag { @@ -178,11 +192,18 @@ impl FieldType { _nullable = nullable.unwrap(); _ref_tracking = false; } + if type_id == NAMED_ENUM { + type_id = ENUM; + } else if type_id == TypeId::NAMED_UNION as u32 || type_id == TypeId::TYPED_UNION as u32 { + type_id = TypeId::UNION as u32; + } + let user_type_id = NO_USER_TYPE_ID; Ok(match type_id { x if x == TypeId::LIST as u32 || x == TypeId::SET as u32 => { let generic = Self::from_bytes(reader, true, None)?; Self { type_id, + user_type_id, nullable: _nullable, ref_tracking: _ref_tracking, generics: vec![generic], @@ -193,6 +214,7 @@ impl FieldType { let val_generic = Self::from_bytes(reader, true, None)?; Self { type_id, + user_type_id, nullable: _nullable, ref_tracking: _ref_tracking, generics: vec![key_generic, val_generic], @@ -200,6 +222,7 @@ impl FieldType { } _ => Self { type_id, + user_type_id, nullable: _nullable, ref_tracking: _ref_tracking, generics: vec![], @@ -399,8 +422,13 @@ impl PartialEq for FieldType { fn eq(&self, other: &Self) -> bool { // Normalize type IDs for comparison to handle cross-language schema evolution. // This allows UNKNOWN (0) polymorphic types to match STRUCT (15) in Rust. - normalize_type_id_for_eq(self.type_id) == normalize_type_id_for_eq(other.type_id) - && self.generics == other.generics + if normalize_type_id_for_eq(self.type_id) != normalize_type_id_for_eq(other.type_id) { + return false; + } + if self.generics != other.generics { + return false; + } + true } } @@ -409,6 +437,7 @@ pub struct TypeMeta { // assigned valid value and used, only during deserializing hash: i64, type_id: u32, + user_type_id: u32, namespace: Rc, type_name: Rc, register_by_name: bool, @@ -419,6 +448,7 @@ pub struct TypeMeta { impl TypeMeta { pub fn new( type_id: u32, + user_type_id: u32, namespace: MetaString, type_name: MetaString, register_by_name: bool, @@ -427,6 +457,7 @@ impl TypeMeta { let mut meta = TypeMeta { hash: 0, type_id, + user_type_id, namespace: Rc::from(namespace), type_name: Rc::from(type_name), register_by_name, @@ -449,6 +480,11 @@ impl TypeMeta { self.type_id } + #[inline(always)] + pub fn get_user_type_id(&self) -> u32 { + self.user_type_id + } + #[inline(always)] pub fn get_hash(&self) -> i64 { self.hash @@ -471,19 +507,16 @@ impl TypeMeta { #[inline(always)] pub fn empty() -> Result { - let mut meta = TypeMeta { + Ok(TypeMeta { hash: 0, type_id: 0, + user_type_id: NO_USER_TYPE_ID, namespace: Rc::from(MetaString::get_empty().clone()), type_name: Rc::from(MetaString::get_empty().clone()), register_by_name: false, field_infos: vec![], bytes: vec![], - }; - let (bytes, meta_hash) = meta.to_bytes()?; - meta.bytes = bytes; - meta.hash = meta_hash; - Ok(meta) + }) } /// Creates a deep clone with new Rc instances. @@ -492,6 +525,7 @@ impl TypeMeta { TypeMeta { hash: self.hash, type_id: self.type_id, + user_type_id: self.user_type_id, namespace: Rc::new((*self.namespace).clone()), type_name: Rc::new((*self.type_name).clone()), register_by_name: self.register_by_name, @@ -502,12 +536,20 @@ impl TypeMeta { pub(crate) fn from_fields( type_id: u32, + user_type_id: u32, namespace: MetaString, type_name: MetaString, register_by_name: bool, field_infos: Vec, ) -> Result { - TypeMeta::new(type_id, namespace, type_name, register_by_name, field_infos) + TypeMeta::new( + type_id, + user_type_id, + namespace, + type_name, + register_by_name, + field_infos, + ) } fn write_name(writer: &mut Writer, name: &MetaString, encodings: &[Encoding]) { @@ -561,7 +603,6 @@ impl TypeMeta { // meta_bytes:| meta_header | fields meta | let mut writer = Writer::from_buffer(&mut buffer); let num_fields = self.field_infos.len(); - let _internal_id = self.type_id & 0xff; // meta_header: | unuse:2 bits | is_register_by_id:1 bit | num_fields:4 bits | let mut meta_header: u8 = min(num_fields, SMALL_NUM_FIELDS_THRESHOLD) as u8; if self.register_by_name { @@ -575,7 +616,13 @@ impl TypeMeta { self.write_namespace(&mut writer); self.write_type_name(&mut writer); } else { - writer.write_var_uint32(self.type_id); + writer.write_u8(self.type_id as u8); + if self.user_type_id == NO_USER_TYPE_ID { + return Err(Error::type_error( + "User type id is required for this type id", + )); + } + writer.write_var_uint32(self.user_type_id); } for field in self.field_infos.iter() { writer.write_bytes(field.to_bytes()?.as_slice()); @@ -618,7 +665,7 @@ impl TypeMeta { } fn get_primitive_type_size(type_id_num: u32) -> i32 { - let type_id = TypeId::try_from(type_id_num as i16).unwrap(); + let type_id = TypeId::try_from(type_id_num as u8).unwrap(); match type_id { TypeId::BOOL => 1, TypeId::INT8 => 1, @@ -712,7 +759,8 @@ impl TypeMeta { if num_fields == SMALL_NUM_FIELDS_THRESHOLD { num_fields += reader.read_varuint32()? as usize; } - let type_id; + let mut type_id; + let mut user_type_id = NO_USER_TYPE_ID; let namespace; let type_name; if register_by_name { @@ -720,7 +768,8 @@ impl TypeMeta { type_name = Self::read_type_name(reader)?; type_id = 0; } else { - type_id = reader.read_varuint32()?; + type_id = reader.read_u8()? as u32; + user_type_id = reader.read_varuint32()?; let empty_name = MetaString::default(); namespace = empty_name.clone(); type_name = empty_name; @@ -736,6 +785,11 @@ impl TypeMeta { if let Some(type_info_current) = type_resolver.get_type_info_by_name(&namespace.original, &type_name.original) { + type_id = type_info_current.get_type_id() as u32; + Self::assign_field_ids(&type_info_current, &mut sorted_field_infos); + } + } else if user_type_id != NO_USER_TYPE_ID { + if let Some(type_info_current) = type_resolver.get_user_type_info_by_id(user_type_id) { Self::assign_field_ids(&type_info_current, &mut sorted_field_infos); } } else if let Some(type_info_current) = type_resolver.get_type_info_by_id(type_id) { @@ -744,6 +798,7 @@ impl TypeMeta { // if no type found, keep all fields id as -1 to be skipped. TypeMeta::new( type_id, + user_type_id, namespace, type_name, register_by_name, diff --git a/rust/fory-core/src/resolver/context.rs b/rust/fory-core/src/resolver/context.rs index 5798038a6e..4f7f08c835 100644 --- a/rust/fory-core/src/resolver/context.rs +++ b/rust/fory-core/src/resolver/context.rs @@ -27,6 +27,7 @@ use crate::resolver::meta_string_resolver::{MetaStringReaderResolver, MetaString use crate::resolver::ref_resolver::{RefReader, RefWriter}; use crate::resolver::type_resolver::{TypeInfo, TypeResolver}; use crate::types; +use crate::TypeId; use std::rc::Rc; /// Thread-local context cache with fast path for single Fory instance. @@ -221,13 +222,13 @@ impl<'a> WriteContext<'a> { .write_type_meta(&mut self.writer, type_id, &self.type_resolver) } - pub fn write_any_typeinfo( + pub fn write_any_type_info( &mut self, fory_type_id: u32, concrete_type_id: std::any::TypeId, ) -> Result, Error> { if types::is_internal_type(fory_type_id) { - self.writer.write_var_uint32(fory_type_id); + self.writer.write_u8(fory_type_id as u8); return self .type_resolver .get_type_info_by_id(fory_type_id) @@ -237,10 +238,14 @@ impl<'a> WriteContext<'a> { let fory_type_id = type_info.get_type_id(); let namespace = type_info.get_namespace(); let type_name = type_info.get_type_name(); - self.writer.write_var_uint32(fory_type_id); + self.writer.write_u8(fory_type_id as u8); // should be compiled to jump table generation - match fory_type_id & 0xff { - types::NAMED_COMPATIBLE_STRUCT | types::COMPATIBLE_STRUCT => { + match fory_type_id { + TypeId::ENUM | TypeId::STRUCT | TypeId::EXT | TypeId::TYPED_UNION => { + let user_type_id = type_info.get_user_type_id(); + self.writer.write_var_uint32(user_type_id); + } + TypeId::COMPATIBLE_STRUCT | TypeId::NAMED_COMPATIBLE_STRUCT => { // Write type meta inline using streaming protocol self.meta_resolver.write_type_meta( &mut self.writer, @@ -248,7 +253,7 @@ impl<'a> WriteContext<'a> { &self.type_resolver, )?; } - types::NAMED_ENUM | types::NAMED_EXT | types::NAMED_STRUCT | types::NAMED_UNION => { + TypeId::NAMED_ENUM | TypeId::NAMED_EXT | TypeId::NAMED_STRUCT | TypeId::NAMED_UNION => { if self.is_share_meta() { // Write type meta inline using streaming protocol self.meta_resolver.write_type_meta( @@ -411,11 +416,17 @@ impl<'a> ReadContext<'a> { .read_type_meta(&mut self.reader, &self.type_resolver) } - pub fn read_any_typeinfo(&mut self) -> Result, Error> { - let fory_type_id = self.reader.read_varuint32()?; + pub fn read_any_type_info(&mut self) -> Result, Error> { + let fory_type_id = self.reader.read_u8()? as u32; // should be compiled to jump table generation - match fory_type_id & 0xff { - types::NAMED_COMPATIBLE_STRUCT | types::COMPATIBLE_STRUCT => { + match fory_type_id { + types::ENUM | types::STRUCT | types::EXT | types::TYPED_UNION => { + let user_type_id = self.reader.read_varuint32()?; + self.type_resolver + .get_user_type_info_by_id(user_type_id) + .ok_or_else(|| Error::type_error("ID harness not found")) + } + types::COMPATIBLE_STRUCT | types::NAMED_COMPATIBLE_STRUCT => { // Read type meta inline using streaming protocol self.read_type_meta() } @@ -426,11 +437,22 @@ impl<'a> ReadContext<'a> { } else { let namespace = self.read_meta_string()?.to_owned(); let type_name = self.read_meta_string()?.to_owned(); - let rc_namespace = Rc::from(namespace); - let rc_type_name = Rc::from(type_name); + let rc_namespace = Rc::from(namespace.clone()); + let rc_type_name = Rc::from(type_name.clone()); self.type_resolver .get_type_info_by_meta_string_name(rc_namespace, rc_type_name) - .ok_or_else(|| Error::type_error("Name harness not found")) + .or_else(|| { + self.type_resolver.get_type_info_by_name( + namespace.original.as_str(), + type_name.original.as_str(), + ) + }) + .ok_or_else(|| { + Error::type_error(format!( + "Name harness not found: namespace='{}', type='{}'", + namespace.original, type_name.original + )) + }) } } _ => self diff --git a/rust/fory-core/src/resolver/meta_resolver.rs b/rust/fory-core/src/resolver/meta_resolver.rs index 598e99084d..2022233b98 100644 --- a/rust/fory-core/src/resolver/meta_resolver.rs +++ b/rust/fory-core/src/resolver/meta_resolver.rs @@ -18,7 +18,7 @@ use crate::buffer::{Reader, Writer}; use crate::error::Error; use crate::meta::TypeMeta; -use crate::resolver::type_resolver::TypeInfo; +use crate::resolver::type_resolver::{TypeInfo, NO_USER_TYPE_ID}; use crate::TypeResolver; use std::collections::HashMap; use std::rc::Rc; @@ -117,34 +117,71 @@ impl MetaReaderResolver { )?); // Try to find local type info - let type_info = if type_meta.get_namespace().original.is_empty() { - // Registered by ID - let type_id = type_meta.get_type_id(); - if let Some(local_type_info) = type_resolver.get_type_info_by_id(type_id) { + let namespace = &type_meta.get_namespace().original; + let type_name = &type_meta.get_type_name().original; + let register_by_name = !namespace.is_empty() || !type_name.is_empty(); + let type_info = if register_by_name { + // Registered by name (namespace can be empty) + if let Some(local_type_info) = + type_resolver.get_type_info_by_name(namespace, type_name) + { // Use local harness with remote metadata Rc::new(TypeInfo::from_remote_meta( type_meta.clone(), Some(local_type_info.get_harness()), + Some(local_type_info.get_type_id() as u32), + Some(local_type_info.get_user_type_id()), )) } else { // No local type found, use stub harness - Rc::new(TypeInfo::from_remote_meta(type_meta.clone(), None)) + Rc::new(TypeInfo::from_remote_meta( + type_meta.clone(), + None, + None, + None, + )) } } else { - // Registered by name - let namespace = &type_meta.get_namespace().original; - let type_name = &type_meta.get_type_name().original; - if let Some(local_type_info) = - type_resolver.get_type_info_by_name(namespace, type_name) + // Registered by ID + let type_id = type_meta.get_type_id(); + let user_type_id = type_meta.get_user_type_id(); + if user_type_id != NO_USER_TYPE_ID { + if let Some(local_type_info) = + type_resolver.get_user_type_info_by_id(user_type_id) + { + // Use local harness with remote metadata + Rc::new(TypeInfo::from_remote_meta( + type_meta.clone(), + Some(local_type_info.get_harness()), + Some(local_type_info.get_type_id() as u32), + Some(local_type_info.get_user_type_id()), + )) + } else { + // No local type found, use stub harness + Rc::new(TypeInfo::from_remote_meta( + type_meta.clone(), + None, + None, + None, + )) + } + } else if let Some(local_type_info) = type_resolver.get_type_info_by_id(type_id) { // Use local harness with remote metadata Rc::new(TypeInfo::from_remote_meta( type_meta.clone(), Some(local_type_info.get_harness()), + Some(local_type_info.get_type_id() as u32), + Some(local_type_info.get_user_type_id()), )) } else { // No local type found, use stub harness - Rc::new(TypeInfo::from_remote_meta(type_meta.clone(), None)) + Rc::new(TypeInfo::from_remote_meta( + type_meta.clone(), + None, + None, + None, + )) } }; diff --git a/rust/fory-core/src/resolver/meta_string_resolver.rs b/rust/fory-core/src/resolver/meta_string_resolver.rs index e557e5e1cf..f174db6344 100644 --- a/rust/fory-core/src/resolver/meta_string_resolver.rs +++ b/rust/fory-core/src/resolver/meta_string_resolver.rs @@ -25,7 +25,7 @@ use crate::buffer::Writer; use crate::error::Error; use crate::meta::{murmurhash3_x64_128, NAMESPACE_DECODER}; use crate::meta::{Encoding, MetaString}; -use crate::{ensure, Reader}; +use crate::Reader; #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct MetaStringBytes { @@ -157,7 +157,7 @@ impl MetaStringWriterResolver { writer.write_var_uint32((len as u32) << 1); if len > Self::SMALL_STRING_THRESHOLD { writer.write_i64(mb_ref.hash_code); - } else { + } else if len != 0 { writer.write_u8(mb_ref.encoding as i16 as u8); } writer.write_bytes(&mb_ref.bytes); @@ -286,16 +286,17 @@ impl MetaStringReaderResolver { reader: &mut Reader, len: usize, ) -> Result<&MetaStringBytes, Error> { - let encoding_val = reader.read_u8()?; if len == 0 { - ensure!( - encoding_val == Encoding::Utf8 as u8, - Error::EncodingError(format!("wrong encoding value: {}", encoding_val).into()) - ); let empty = MetaStringBytes::get_empty(); - // empty must be a static or globally unique instance + let id = self.dynamic_read_id; + self.dynamic_read_id += 1; + if id >= self.dynamic_read.len() { + self.dynamic_read.resize(id * 2 + 1, None); + } + self.dynamic_read[id] = Some(empty as *const MetaStringBytes); return Ok(empty); } + let encoding_val = reader.read_u8()?; let (v1, v2) = if len <= 8 { let v1 = Self::read_bytes_as_u64(reader, len)?; diff --git a/rust/fory-core/src/resolver/type_resolver.rs b/rust/fory-core/src/resolver/type_resolver.rs index b47723c008..df7c8f2fa7 100644 --- a/rust/fory-core/src/resolver/type_resolver.rs +++ b/rust/fory-core/src/resolver/type_resolver.rs @@ -47,6 +47,9 @@ type ReadCompatibleFn = fn(&mut ReadContext, Rc) -> Result) -> Result, Error>; type BuildTypeInfosFn = fn(&TypeResolver) -> Result, Error>; const EMPTY_STRING: String = String::new(); +const INTERNAL_TYPE_ID_LIMIT: usize = 256; +const MAX_USER_TYPE_ID: u32 = 0xfffffffe; +pub(crate) const NO_USER_TYPE_ID: u32 = u32::MAX; #[derive(Clone, Debug)] pub struct Harness { @@ -145,7 +148,8 @@ impl Harness { pub struct TypeInfo { type_def: Rc>, type_meta: Rc, - type_id: u32, + type_id: TypeId, + user_type_id: u32, namespace: Rc, type_name: Rc, register_by_name: bool, @@ -155,11 +159,14 @@ pub struct TypeInfo { impl TypeInfo { fn new( type_id: u32, + user_type_id: u32, namespace: &str, type_name: &str, register_by_name: bool, harness: Harness, ) -> Result { + let type_id = TypeId::try_from(type_id as u8) + .map_err(|_| Error::type_error(format!("Unknown type id {}", type_id)))?; let namespace_meta_string = NAMESPACE_ENCODER.encode_with_encodings(namespace, NAMESPACE_ENCODINGS)?; let type_name_meta_string = @@ -168,6 +175,7 @@ impl TypeInfo { type_def: Rc::from(vec![]), type_meta: Rc::new(TypeMeta::empty()?), type_id, + user_type_id, namespace: Rc::from(namespace_meta_string), type_name: Rc::from(type_name_meta_string), register_by_name, @@ -176,7 +184,10 @@ impl TypeInfo { } fn new_with_type_meta(type_meta: Rc, harness: Harness) -> Result { - let type_id = type_meta.get_type_id(); + let type_id_raw = type_meta.get_type_id(); + let type_id = TypeId::try_from(type_id_raw as u8) + .map_err(|_| Error::type_error(format!("Unknown type id {}", type_id_raw)))?; + let user_type_id = type_meta.get_user_type_id(); let namespace = type_meta.get_namespace(); let type_name = type_meta.get_type_name(); let register_by_name = !namespace.original.is_empty() || !type_name.original.is_empty(); @@ -185,6 +196,7 @@ impl TypeInfo { type_def: Rc::from(type_def_bytes), type_meta, type_id, + user_type_id, namespace, type_name, register_by_name, @@ -193,10 +205,15 @@ impl TypeInfo { } #[inline(always)] - pub fn get_type_id(&self) -> u32 { + pub fn get_type_id(&self) -> TypeId { self.type_id } + #[inline(always)] + pub fn get_user_type_id(&self) -> u32 { + self.user_type_id + } + #[inline(always)] pub fn get_namespace(&self) -> Rc { self.namespace.clone() @@ -234,6 +251,7 @@ impl TypeInfo { type_def: Rc::new((*self.type_def).clone()), type_meta: Rc::new(self.type_meta.deep_clone()), type_id: self.type_id, + user_type_id: self.user_type_id, namespace: Rc::new((*self.namespace).clone()), type_name: Rc::new((*self.type_name).clone()), register_by_name: self.register_by_name, @@ -246,8 +264,12 @@ impl TypeInfo { pub fn from_remote_meta( remote_meta: Rc, local_harness: Option<&Harness>, + type_id_override: Option, + user_type_id_override: Option, ) -> TypeInfo { - let type_id = remote_meta.get_type_id(); + let type_id_raw = type_id_override.unwrap_or_else(|| remote_meta.get_type_id()); + let type_id = TypeId::try_from(type_id_raw as u8).unwrap_or(TypeId::UNKNOWN); + let user_type_id = user_type_id_override.unwrap_or_else(|| remote_meta.get_user_type_id()); let namespace = remote_meta.get_namespace(); let type_name = remote_meta.get_type_name(); let type_def_bytes = remote_meta.get_bytes().to_owned(); @@ -272,6 +294,7 @@ impl TypeInfo { type_def: Rc::from(type_def_bytes), type_meta: remote_meta, type_id, + user_type_id, namespace, type_name, register_by_name, @@ -342,7 +365,8 @@ fn build_struct_type_infos( // Build the main type info let type_meta = TypeMeta::from_fields( - partial_info.type_id, + partial_info.type_id as u32, + partial_info.user_type_id, (*partial_info.namespace).clone(), (*partial_info.type_name).clone(), partial_info.register_by_name, @@ -353,6 +377,7 @@ fn build_struct_type_infos( type_def: Rc::from(type_def_bytes), type_meta: Rc::new(type_meta), type_id: partial_info.type_id, + user_type_id: partial_info.user_type_id, namespace: partial_info.namespace.clone(), type_name: partial_info.type_name.clone(), register_by_name: partial_info.register_by_name, @@ -366,9 +391,7 @@ fn build_struct_type_infos( if type_resolver.compatible && is_enum_type_id(T::fory_static_type_id()) { // Fields are already sorted with IDs assigned by the macro let variants_info = T::fory_variants_fields_info(type_resolver)?; - for (idx, (variant_name, variant_type_id, fields_info)) in - variants_info.into_iter().enumerate() - { + for (variant_name, variant_type_id, fields_info) in variants_info.into_iter() { // Skip empty variant info (unit/unnamed variants) if fields_info.is_empty() { continue; @@ -383,30 +406,41 @@ fn build_struct_type_infos( .encode_with_encodings(&variant_type_name, TYPE_NAME_ENCODINGS)?; TypeMeta::from_fields( TypeId::ENUM as u32, + NO_USER_TYPE_ID, namespace_ms, type_name_ms, true, fields_info.clone(), )? } else { - // add a check to avoid collision with main enum type_id - // since internal id is big alealdy, `74<<8 = 18944` is big enough to avoid collision most of time - let variant_id = (partial_info.type_id << 8) + idx as u32; - - // Check if variant_id conflicts with any already registered type - if let Some(existing_info) = type_resolver.type_info_map_by_id.get(&variant_id) { + if partial_info.user_type_id == NO_USER_TYPE_ID { + return Err(Error::type_error( + "Enum variant metadata requires a user type id", + )); + } + let variant_type_name = format!( + "__fory_enum_variant__{}_{}", + partial_info.user_type_id, variant_name + ); + let namespace_ms = MetaString::get_empty().clone(); + let type_name_ms = TYPE_NAME_ENCODER + .encode_with_encodings(&variant_type_name, TYPE_NAME_ENCODINGS)?; + if type_resolver + .get_type_info_by_name(&namespace_ms.original, &type_name_ms.original) + .is_some() + { return Err(Error::type_error(format!( - "Enum variant type ID {} (calculated from enum type ID {} with variant index {}) conflicts with already registered type ID {}. \ + "Enum variant name {} conflicts with already registered type name. \ Please use a different type ID for the enum to avoid conflicts.", - variant_id, partial_info.type_id, idx, existing_info.type_id + variant_type_name ))); } - TypeMeta::from_fields( - variant_id, - MetaString::get_empty().clone(), - MetaString::get_empty().clone(), - false, + TypeId::ENUM as u32, + NO_USER_TYPE_ID, + namespace_ms, + type_name_ms, + true, fields_info, )? }; @@ -429,7 +463,8 @@ fn build_serializer_type_infos( ) -> Result, Error> { // For ext types, we just build the type info with empty fields let type_meta = TypeMeta::from_fields( - partial_info.type_id, + partial_info.type_id as u32, + partial_info.user_type_id, (*partial_info.namespace).clone(), (*partial_info.type_name).clone(), partial_info.register_by_name, @@ -440,6 +475,7 @@ fn build_serializer_type_infos( type_def: Rc::from(type_def_bytes), type_meta: Rc::new(type_meta), type_id: partial_info.type_id, + user_type_id: partial_info.user_type_id, namespace: partial_info.namespace.clone(), type_name: partial_info.type_name.clone(), register_by_name: partial_info.register_by_name, @@ -451,13 +487,14 @@ fn build_serializer_type_infos( /// TypeResolver is a resolver for fast type/serializer dispatch. pub struct TypeResolver { - type_info_map_by_id: HashMap>, + internal_type_info_by_id: Vec>>, + user_type_info_by_id: HashMap>, type_info_map: HashMap>, type_info_map_by_name: HashMap<(String, String), Rc>, type_info_map_by_meta_string_name: HashMap<(Rc, Rc), Rc>, partial_type_infos: HashMap, // Fast lookup by numeric ID for common types - type_id_index: Vec, + type_id_index: Vec, compatible: bool, xlang: bool, } @@ -468,12 +505,13 @@ pub struct TypeResolver { unsafe impl Send for TypeResolver {} unsafe impl Sync for TypeResolver {} -const NO_TYPE_ID: u32 = 1000000000; +const NO_TYPE_ID: TypeId = TypeId::UNKNOWN; impl Default for TypeResolver { fn default() -> Self { let mut registry = TypeResolver { - type_info_map_by_id: HashMap::new(), + internal_type_info_by_id: vec![None; INTERNAL_TYPE_ID_LIMIT], + user_type_info_by_id: HashMap::new(), type_info_map: HashMap::new(), type_info_map_by_name: HashMap::new(), type_info_map_by_meta_string_name: HashMap::new(), @@ -501,8 +539,19 @@ impl TypeResolver { } #[inline(always)] - pub fn get_type_info_by_id(&self, id: u32) -> Option> { - self.type_info_map_by_id.get(&id).cloned() + pub fn get_type_info_by_id(&self, type_id: u32) -> Option> { + if crate::types::is_internal_type(type_id) { + let index = type_id as usize; + if index < self.internal_type_info_by_id.len() { + return self.internal_type_info_by_id[index].clone(); + } + } + None + } + + #[inline(always)] + pub fn get_user_type_info_by_id(&self, user_type_id: u32) -> Option> { + self.user_type_info_by_id.get(&user_type_id).cloned() } #[inline(always)] @@ -525,12 +574,12 @@ impl TypeResolver { /// Fast path for getting type info by numeric ID (avoids HashMap lookup by TypeId) #[inline(always)] - pub fn get_type_id(&self, type_id: &std::any::TypeId, id: u32) -> Result { + pub fn get_type_id(&self, type_id: &std::any::TypeId, id: u32) -> Result { let id_usize = id as usize; if id_usize < self.type_id_index.len() { - let type_id = self.type_id_index[id_usize]; - if type_id != NO_TYPE_ID { - return Ok(type_id); + let type_id_value = self.type_id_index[id_usize]; + if type_id_value != NO_TYPE_ID { + return Ok(type_id_value); } } Err(Error::type_error(format!( @@ -541,8 +590,7 @@ impl TypeResolver { #[inline(always)] pub fn get_harness(&self, id: u32) -> Option> { - self.type_info_map_by_id - .get(&id) + self.get_type_info_by_id(id) .map(|info| Rc::new(info.get_harness().clone())) } @@ -559,9 +607,8 @@ impl TypeResolver { } #[inline(always)] - pub fn get_ext_harness(&self, id: u32) -> Result, Error> { - self.type_info_map_by_id - .get(&id) + pub fn get_ext_harness(&self, _type_id: u32, user_type_id: u32) -> Result, Error> { + self.get_user_type_info_by_id(user_type_id) .map(|info| Rc::new(info.get_harness().clone())) .ok_or_else(|| Error::type_error("ext type must be registered in both peers")) } @@ -580,7 +627,7 @@ impl TypeResolver { } #[inline(always)] - pub fn get_fory_type_id(&self, rust_type_id: std::any::TypeId) -> Option { + pub fn get_fory_type_id(&self, rust_type_id: std::any::TypeId) -> Option { self.type_info_map .get(&rust_type_id) .map(|info| info.get_type_id()) @@ -682,13 +729,19 @@ impl TypeResolver { _lazy: bool, ) -> Result<(), Error> { let register_by_name = !type_name.is_empty(); - if !register_by_name && id == 0 { - return Err(Error::not_allowed( - "Either id must be non-zero for ID registration, or type_name must be non-empty for name registration", - )); + if !register_by_name && id > MAX_USER_TYPE_ID { + return Err(Error::not_allowed(format!( + "type id must be in range [0, 0xfffffffe], got {}", + id + ))); } let actual_type_id = T::fory_actual_type_id(id, register_by_name, self.compatible, self.xlang); + let user_type_id = if register_by_name || crate::types::is_internal_type(actual_type_id) { + NO_USER_TYPE_ID + } else { + id + }; fn write( this: &dyn Any, @@ -774,6 +827,7 @@ impl TypeResolver { ); let type_info = TypeInfo::new( actual_type_id, + user_type_id, namespace, type_name, register_by_name, @@ -793,8 +847,8 @@ impl TypeResolver { // 1. Internal types (type_id < TypeId::BOUND) as they can be shared // 2. Types registered by name (they use shared type IDs like NAMED_STRUCT) if !register_by_name - && actual_type_id >= TypeId::BOUND as u32 - && self.type_info_map_by_id.contains_key(&actual_type_id) + && !crate::types::is_internal_type(actual_type_id) + && self.user_type_info_by_id.contains_key(&user_type_id) { return Err(Error::type_error(format!( "Type ID {} conflicts with already registered type. Please use a different type ID.", @@ -814,9 +868,20 @@ impl TypeResolver { } self.type_id_index[index] = type_info.type_id; - // Insert partial type info into both maps - self.type_info_map_by_id - .insert(actual_type_id, Rc::new(type_info.clone())); + // Insert partial type info into id maps + if crate::types::is_internal_type(actual_type_id) { + let index = actual_type_id as usize; + if index >= self.internal_type_info_by_id.len() { + return Err(Error::not_allowed(format!( + "Internal type id overflow: {}", + actual_type_id + ))); + } + self.internal_type_info_by_id[index] = Some(Rc::new(type_info.clone())); + } else if user_type_id != NO_USER_TYPE_ID { + self.user_type_info_by_id + .insert(user_type_id, Rc::new(type_info.clone())); + } self.type_info_map .insert(rs_type_id, Rc::new(type_info.clone())); self.partial_type_infos.insert(rs_type_id, type_info); @@ -857,7 +922,14 @@ impl TypeResolver { &mut self, type_id: TypeId, ) -> Result<(), Error> { - self.register_serializer::(type_id as u32, type_id as u32, &EMPTY_STRING, &EMPTY_STRING) + let raw_id = type_id as u32; + if raw_id >= INTERNAL_TYPE_ID_LIMIT as u32 { + return Err(Error::not_allowed(format!( + "Internal type id overflow: {}", + raw_id + ))); + } + self.register_serializer::(raw_id, raw_id, &EMPTY_STRING, &EMPTY_STRING) } fn register_serializer( @@ -868,10 +940,11 @@ impl TypeResolver { type_name: &str, ) -> Result<(), Error> { let register_by_name = !type_name.is_empty(); - if !register_by_name && id == 0 { - return Err(Error::not_allowed( - "Either id must be non-zero for ID registration, or type_name must be non-empty for name registration", - )); + if !register_by_name && id > MAX_USER_TYPE_ID { + return Err(Error::not_allowed(format!( + "type id must be in range [0, 0xfffffffe], got {}", + id + ))); } fn write( @@ -960,8 +1033,14 @@ impl TypeResolver { build_type_infos::, ); + let user_type_id = if register_by_name { + NO_USER_TYPE_ID + } else { + id + }; let type_info = TypeInfo::new( actual_type_id, + user_type_id, namespace, type_name, register_by_name, @@ -977,9 +1056,10 @@ impl TypeResolver { } // Check if type_id conflicts with any already registered type - // Skip check for internal types (type_id < TypeId::BOUND) as they can be shared - if actual_type_id >= TypeId::BOUND as u32 - && self.type_info_map_by_id.contains_key(&actual_type_id) + // Skip check for internal types as they can be shared + if !crate::types::is_internal_type(actual_type_id) + && user_type_id != NO_USER_TYPE_ID + && self.user_type_info_by_id.contains_key(&user_type_id) { return Err(Error::type_error(format!( "Type ID {} conflicts with already registered type. Please use a different type ID.", @@ -987,9 +1067,20 @@ impl TypeResolver { ))); } - // Insert partial type info into both maps - self.type_info_map_by_id - .insert(actual_type_id, Rc::new(type_info.clone())); + // Insert partial type info into id maps + if crate::types::is_internal_type(actual_type_id) { + let index = actual_type_id as usize; + if index >= self.internal_type_info_by_id.len() { + return Err(Error::not_allowed(format!( + "Internal type id overflow: {}", + actual_type_id + ))); + } + self.internal_type_info_by_id[index] = Some(Rc::new(type_info.clone())); + } else if user_type_id != NO_USER_TYPE_ID { + self.user_type_info_by_id + .insert(user_type_id, Rc::new(type_info.clone())); + } self.type_info_map .insert(rs_type_id, Rc::new(type_info.clone())); self.partial_type_infos.insert(rs_type_id, type_info); @@ -1050,7 +1141,8 @@ impl TypeResolver { /// cannot be retrieved or type metadata cannot be serialized. pub(crate) fn build_final_type_resolver(&self) -> Result { // copy all type info from type_resolver to here - let mut type_info_map_by_id = self.type_info_map_by_id.clone(); + let mut internal_type_info_by_id = self.internal_type_info_by_id.clone(); + let mut user_type_info_by_id = self.user_type_info_by_id.clone(); let mut type_info_map = self.type_info_map.clone(); let mut type_info_map_by_name = self.type_info_map_by_name.clone(); let mut type_info_map_by_meta_string_name = self.type_info_map_by_meta_string_name.clone(); @@ -1064,8 +1156,15 @@ impl TypeResolver { // Iterate through all type infos uniformly for (type_rust_id, type_info) in type_infos.iter() { - // Insert into type_info_map_by_id - type_info_map_by_id.insert(type_info.type_id, Rc::new(type_info.clone())); + // Insert into id maps + if crate::types::is_internal_type(type_info.type_id as u32) { + let index = type_info.type_id as usize; + if index < internal_type_info_by_id.len() { + internal_type_info_by_id[index] = Some(Rc::new(type_info.clone())); + } + } else if type_info.user_type_id != NO_USER_TYPE_ID { + user_type_info_by_id.insert(type_info.user_type_id, Rc::new(type_info.clone())); + } // Insert into type_info_map with the TypeId type_info_map.insert(*type_rust_id, Rc::new(type_info.clone())); @@ -1083,7 +1182,8 @@ impl TypeResolver { } Ok(TypeResolver { - type_info_map_by_id, + internal_type_info_by_id, + user_type_info_by_id, type_info_map, type_info_map_by_name, type_info_map_by_meta_string_name, @@ -1133,8 +1233,14 @@ impl TypeResolver { }; // Clone all maps with deep-cloned TypeInfo in new Rc wrappers - let type_info_map_by_id: HashMap> = self - .type_info_map_by_id + let internal_type_info_by_id: Vec>> = self + .internal_type_info_by_id + .iter() + .map(|opt| opt.as_ref().map(&mut get_or_clone_type_info)) + .collect(); + + let user_type_info_by_id: HashMap> = self + .user_type_info_by_id .iter() .map(|(k, v)| (*k, get_or_clone_type_info(v))) .collect(); @@ -1165,7 +1271,8 @@ impl TypeResolver { .collect(); TypeResolver { - type_info_map_by_id, + internal_type_info_by_id, + user_type_info_by_id, type_info_map, type_info_map_by_name, type_info_map_by_meta_string_name, diff --git a/rust/fory-core/src/serializer/any.rs b/rust/fory-core/src/serializer/any.rs index 29828c4d6f..8a7e9f8e44 100644 --- a/rust/fory-core/src/serializer/any.rs +++ b/rust/fory-core/src/serializer/any.rs @@ -21,18 +21,28 @@ use crate::resolver::context::{ReadContext, WriteContext}; use crate::resolver::type_resolver::{TypeInfo, TypeResolver}; use crate::serializer::util::write_dyn_data_generic; use crate::serializer::{ForyDefault, Serializer}; -use crate::types::{RefFlag, RefMode, TypeId, LIST, MAP, SET}; +use crate::types::{RefFlag, RefMode, TypeId}; use std::any::Any; use std::rc::Rc; use std::sync::Arc; +#[inline(always)] +fn resolve_registered_type_id( + type_resolver: &TypeResolver, + concrete_type_id: std::any::TypeId, +) -> Result { + type_resolver + .get_fory_type_id(concrete_type_id) + .ok_or_else(|| Error::type_error("Type not registered")) +} + /// Check if the type info represents a generic container type (LIST, SET, MAP). /// These types cannot be deserialized polymorphically via `Box` because /// different generic instantiations (e.g., `Vec`, `Vec`) share the same type ID. #[inline] fn check_generic_container_type(type_info: &TypeInfo) -> Result<(), Error> { let type_id = type_info.get_type_id(); - if type_id == LIST || type_id == SET || type_id == MAP { + if type_id == TypeId::LIST || type_id == TypeId::SET || type_id == TypeId::MAP { return Err(Error::type_error( "Cannot deserialize generic container types (Vec, HashSet, HashMap) polymorphically \ via Box/Rc/Arc/Weak. The serialization protocol does not preserve the element type \ @@ -51,7 +61,7 @@ pub fn deserialize_any_box(context: &mut ReadContext) -> Result, Er if ref_flag != RefFlag::NotNullValue as i8 { return Err(Error::invalid_ref("Expected NotNullValue for Box")); } - let typeinfo = context.read_any_typeinfo()?; + let typeinfo = context.read_any_type_info()?; // Check for generic container types which cannot be deserialized polymorphically check_generic_container_type(&typeinfo)?; let result = typeinfo @@ -72,14 +82,14 @@ impl Serializer for Box { &self, context: &mut WriteContext, ref_mode: RefMode, - write_typeinfo: bool, + write_type_info: bool, has_generics: bool, ) -> Result<(), Error> { write_box_any( self.as_ref(), context, ref_mode, - write_typeinfo, + write_type_info, has_generics, ) } @@ -124,17 +134,14 @@ impl Serializer for Box { )) } - fn fory_get_type_id(_: &TypeResolver) -> Result { + fn fory_get_type_id(_: &TypeResolver) -> Result { Err(Error::type_error( "Box has no static type ID - use fory_type_id_dyn", )) } - fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { - let concrete_type_id = (**self).type_id(); - type_resolver - .get_fory_type_id(concrete_type_id) - .ok_or_else(|| Error::type_error("Type not registered")) + fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { + resolve_registered_type_id(type_resolver, (**self).type_id()) } fn fory_concrete_type_id(&self) -> std::any::TypeId { @@ -172,15 +179,15 @@ pub fn write_box_any( value: &dyn Any, context: &mut WriteContext, ref_mode: RefMode, - write_typeinfo: bool, + write_type_info: bool, has_generics: bool, ) -> Result<(), Error> { if ref_mode != RefMode::None { context.writer.write_i8(RefFlag::NotNullValue as i8); } let concrete_type_id = value.type_id(); - let typeinfo = if write_typeinfo { - context.write_any_typeinfo(TypeId::UNKNOWN as u32, concrete_type_id)? + let typeinfo = if write_type_info { + context.write_any_type_info(TypeId::UNKNOWN as u32, concrete_type_id)? } else { context.get_type_info(&concrete_type_id)? }; @@ -212,7 +219,7 @@ pub fn read_box_any( read_type_info, Error::invalid_data("Type info must be read for Box") ); - context.read_any_typeinfo()? + context.read_any_type_info()? }; // Check for generic container types which cannot be deserialized polymorphically check_generic_container_type(&typeinfo)?; @@ -245,7 +252,7 @@ impl Serializer for Rc { let concrete_type_id: std::any::TypeId = (**self).type_id(); let write_data_fn = if write_type_info { let typeinfo = - context.write_any_typeinfo(TypeId::UNKNOWN as u32, concrete_type_id)?; + context.write_any_type_info(TypeId::UNKNOWN as u32, concrete_type_id)?; typeinfo.get_harness().get_write_data_fn() } else { context @@ -296,17 +303,14 @@ impl Serializer for Rc { ))) } - fn fory_get_type_id(_: &TypeResolver) -> Result { + fn fory_get_type_id(_: &TypeResolver) -> Result { Err(Error::type_error( "Rc has no static type ID - use fory_type_id_dyn", )) } - fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { - let concrete_type_id = (**self).type_id(); - type_resolver - .get_fory_type_id(concrete_type_id) - .ok_or_else(|| Error::type_error("Type not registered")) + fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { + resolve_registered_type_id(type_resolver, (**self).type_id()) } fn fory_concrete_type_id(&self) -> std::any::TypeId { @@ -365,7 +369,7 @@ pub fn read_rc_any( RefFlag::NotNullValue => { context.inc_depth()?; let typeinfo = if read_type_info { - context.read_any_typeinfo()? + context.read_any_type_info()? } else { type_info.ok_or_else(|| Error::type_error("No type info found for read"))? }; @@ -380,7 +384,7 @@ pub fn read_rc_any( RefFlag::RefValue => { context.inc_depth()?; let typeinfo = if read_type_info { - context.read_any_typeinfo()? + context.read_any_type_info()? } else { type_info.ok_or_else(|| Error::type_error("No type info found for read"))? }; @@ -419,7 +423,7 @@ impl Serializer for Arc { let concrete_type_id: std::any::TypeId = (**self).type_id(); if write_type_info { let typeinfo = - context.write_any_typeinfo(TypeId::UNKNOWN as u32, concrete_type_id)?; + context.write_any_type_info(TypeId::UNKNOWN as u32, concrete_type_id)?; let serializer_fn = typeinfo.get_harness().get_write_data_fn(); serializer_fn(&**self, context, has_generics)?; } else { @@ -471,17 +475,14 @@ impl Serializer for Arc { ))) } - fn fory_get_type_id(_type_resolver: &TypeResolver) -> Result { + fn fory_get_type_id(_type_resolver: &TypeResolver) -> Result { Err(Error::type_error( "Arc has no static type ID - use fory_type_id_dyn", )) } - fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { - let concrete_type_id = (**self).type_id(); - type_resolver - .get_fory_type_id(concrete_type_id) - .ok_or_else(|| Error::type_error("Type not registered")) + fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { + resolve_registered_type_id(type_resolver, (**self).type_id()) } fn fory_concrete_type_id(&self) -> std::any::TypeId { @@ -540,7 +541,7 @@ pub fn read_arc_any( RefFlag::NotNullValue => { context.inc_depth()?; let typeinfo = if read_type_info { - context.read_any_typeinfo()? + context.read_any_type_info()? } else { type_info .ok_or_else(|| Error::type_error("No type info found for read Arc"))? @@ -556,7 +557,7 @@ pub fn read_arc_any( RefFlag::RefValue => { context.inc_depth()?; let typeinfo = if read_type_info { - context.read_any_typeinfo()? + context.read_any_type_info()? } else { type_info .ok_or_else(|| Error::type_error("No type info found for read Arc"))? diff --git a/rust/fory-core/src/serializer/arc.rs b/rust/fory-core/src/serializer/arc.rs index 7aecffa452..4ef4e50d55 100644 --- a/rust/fory-core/src/serializer/arc.rs +++ b/rust/fory-core/src/serializer/arc.rs @@ -129,11 +129,18 @@ impl Serializer for Arc 4 } - fn fory_get_type_id(type_resolver: &TypeResolver) -> Result { + fn fory_get_type_id(type_resolver: &TypeResolver) -> Result { T::fory_get_type_id(type_resolver) } - fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { + fn fory_get_type_info(type_resolver: &TypeResolver) -> Result, Error> { + match type_resolver.get_type_info(&std::any::TypeId::of::()) { + Ok(info) => Ok(info), + Err(e) => Err(Error::enhance_type_error::(e)), + } + } + + fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { (**self).fory_type_id_dyn(type_resolver) } diff --git a/rust/fory-core/src/serializer/array.rs b/rust/fory-core/src/serializer/array.rs index fd07953e19..3e2a6d0666 100644 --- a/rust/fory-core/src/serializer/array.rs +++ b/rust/fory-core/src/serializer/array.rs @@ -159,7 +159,7 @@ where // Read elements if is_same_type { let type_info = if !is_declared { - context.read_any_typeinfo()? + context.read_any_type_info()? } else { let rs_type_id = std::any::TypeId::of::(); context.get_type_resolver().get_type_info(&rs_type_id)? @@ -283,13 +283,13 @@ impl Serializer for [T; N] { } #[inline(always)] - fn fory_get_type_id(_: &TypeResolver) -> Result { - Ok(Self::fory_static_type_id() as u32) + fn fory_get_type_id(_: &TypeResolver) -> Result { + Ok(Self::fory_static_type_id()) } #[inline(always)] - fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { - Ok(Self::fory_static_type_id() as u32) + fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { + Ok(Self::fory_static_type_id()) } #[inline(always)] diff --git a/rust/fory-core/src/serializer/bool.rs b/rust/fory-core/src/serializer/bool.rs index e89d89fcda..edffac5216 100644 --- a/rust/fory-core/src/serializer/bool.rs +++ b/rust/fory-core/src/serializer/bool.rs @@ -42,12 +42,12 @@ impl Serializer for bool { } #[inline(always)] - fn fory_get_type_id(_: &TypeResolver) -> Result { - Ok(TypeId::BOOL as u32) + fn fory_get_type_id(_: &TypeResolver) -> Result { + Ok(TypeId::BOOL) } - fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { - Ok(TypeId::BOOL as u32) + fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { + Ok(TypeId::BOOL) } fn fory_static_type_id() -> TypeId { @@ -61,7 +61,7 @@ impl Serializer for bool { #[inline(always)] fn fory_write_type_info(context: &mut WriteContext) -> Result<(), Error> { - context.writer.write_var_uint32(TypeId::BOOL as u32); + context.writer.write_u8(TypeId::BOOL as u8); Ok(()) } diff --git a/rust/fory-core/src/serializer/box_.rs b/rust/fory-core/src/serializer/box_.rs index 17c2bf0de8..9f992e0ec9 100644 --- a/rust/fory-core/src/serializer/box_.rs +++ b/rust/fory-core/src/serializer/box_.rs @@ -18,9 +18,10 @@ use crate::error::Error; use crate::resolver::context::ReadContext; use crate::resolver::context::WriteContext; -use crate::resolver::type_resolver::TypeResolver; +use crate::resolver::type_resolver::{TypeInfo, TypeResolver}; use crate::serializer::{ForyDefault, Serializer}; use crate::types::TypeId; +use std::rc::Rc; impl Serializer for Box { #[inline(always)] @@ -52,12 +53,20 @@ impl Serializer for Box { } #[inline(always)] - fn fory_get_type_id(type_resolver: &TypeResolver) -> Result { + fn fory_get_type_id(type_resolver: &TypeResolver) -> Result { T::fory_get_type_id(type_resolver) } #[inline(always)] - fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { + fn fory_get_type_info(type_resolver: &TypeResolver) -> Result, Error> { + match type_resolver.get_type_info(&std::any::TypeId::of::()) { + Ok(info) => Ok(info), + Err(e) => Err(Error::enhance_type_error::(e)), + } + } + + #[inline(always)] + fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { (**self).fory_type_id_dyn(type_resolver) } diff --git a/rust/fory-core/src/serializer/collection.rs b/rust/fory-core/src/serializer/collection.rs index b5c16ad3fb..71985a977e 100644 --- a/rust/fory-core/src/serializer/collection.rs +++ b/rust/fory-core/src/serializer/collection.rs @@ -36,7 +36,7 @@ pub fn write_collection_type_info( context: &mut WriteContext, collection_type_id: u32, ) -> Result<(), Error> { - context.writer.write_var_uint32(collection_type_id); + context.writer.write_u8(collection_type_id as u8); Ok(()) } @@ -164,7 +164,7 @@ where "Unable to determine concrete type for polymorphic collection elements", ) })?; - context.write_any_typeinfo(T::fory_static_type_id() as u32, type_id)?; + context.write_any_type_info(T::fory_static_type_id() as u32, type_id)?; } else { T::fory_write_type_info(context)?; } @@ -206,7 +206,7 @@ pub fn read_collection_type_info( context: &mut ReadContext, collection_type_id: u32, ) -> Result<(), Error> { - let remote_collection_type_id = context.reader.read_varuint32()?; + let remote_collection_type_id = context.reader.read_u8()? as u32; if PRIMITIVE_ARRAY_TYPES.contains(&remote_collection_type_id) { return Err(Error::type_error( "Vec belongs to the `number_array` type, \ @@ -236,7 +236,7 @@ where let header = context.reader.read_u8()?; let declared = (header & DECL_ELEMENT_TYPE) != 0; if !declared { - // context.read_any_typeinfo(); + // context.read_any_type_info(); // TODO check whether type info consistent with T T::fory_read_type_info(context)?; } @@ -287,21 +287,9 @@ where // Read elements if is_same_type { let type_info = if !is_declared { - context.read_any_typeinfo()? - } else if T::fory_is_shared_ref() { - let type_id = T::fory_get_type_id(context.get_type_resolver())?; - context - .get_type_resolver() - .get_type_info_by_id(type_id) - .ok_or_else(|| { - Error::type_error(format!( - "Type ID {} not found for shared ref element", - type_id - )) - })? + context.read_any_type_info()? } else { - let rs_type_id = std::any::TypeId::of::(); - context.get_type_resolver().get_type_info(&rs_type_id)? + T::fory_get_type_info(context.get_type_resolver())? }; // All elements are same type if elem_ref_mode == RefMode::None { diff --git a/rust/fory-core/src/serializer/core.rs b/rust/fory-core/src/serializer/core.rs index 98e06b5c12..840807e1ff 100644 --- a/rust/fory-core/src/serializer/core.rs +++ b/rust/fory-core/src/serializer/core.rs @@ -162,7 +162,7 @@ pub trait ForyDefault: Sized { /// // Read your type's fields /// } /// -/// fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { +/// fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { /// Self::fory_get_type_id(type_resolver) /// } /// @@ -378,7 +378,7 @@ pub trait Serializer: 'static { /// Ok(Point { x, y }) /// } /// - /// fn fory_type_id_dyn(&self, type_resolver: &fory_core::TypeResolver) -> Result { + /// fn fory_type_id_dyn(&self, type_resolver: &fory_core::TypeResolver) -> Result { /// Self::fory_get_type_id(type_resolver) /// } /// @@ -431,7 +431,7 @@ pub trait Serializer: 'static { /// Ok(Person { name, age, scores }) /// } /// - /// fn fory_type_id_dyn(&self, type_resolver: &fory_core::TypeResolver) -> Result { + /// fn fory_type_id_dyn(&self, type_resolver: &fory_core::TypeResolver) -> Result { /// Self::fory_get_type_id(type_resolver) /// } /// @@ -490,7 +490,7 @@ pub trait Serializer: 'static { /// ```rust,ignore /// fn fory_write_type_info(context: &mut WriteContext) -> Result<(), Error> { /// let rs_type_id = std::any::TypeId::of::(); - /// context.write_any_typeinfo(Self::fory_static_type_id() as u32, rs_type_id)?; + /// context.write_any_type_info(Self::fory_static_type_id() as u32, rs_type_id)?; /// Ok(()) /// } /// ``` @@ -519,7 +519,7 @@ pub trait Serializer: 'static { { // Serializer for internal types should overwrite this method for faster performance. let rs_type_id = std::any::TypeId::of::(); - context.write_any_typeinfo(Self::fory_static_type_id() as u32, rs_type_id)?; + context.write_any_type_info(Self::fory_static_type_id() as u32, rs_type_id)?; Ok(()) } @@ -725,7 +725,7 @@ pub trait Serializer: 'static { /// Ok(Point { x, y }) /// } /// - /// fn fory_type_id_dyn(&self, type_resolver: &fory_core::TypeResolver) -> Result { + /// fn fory_type_id_dyn(&self, type_resolver: &fory_core::TypeResolver) -> Result { /// Self::fory_get_type_id(type_resolver) /// } /// @@ -779,7 +779,7 @@ pub trait Serializer: 'static { /// Ok(Person { name, age, scores }) /// } /// - /// fn fory_type_id_dyn(&self, type_resolver: &fory_core::TypeResolver) -> Result { + /// fn fory_type_id_dyn(&self, type_resolver: &fory_core::TypeResolver) -> Result { /// Self::fory_get_type_id(type_resolver) /// } /// @@ -830,7 +830,7 @@ pub trait Serializer: 'static { /// Ok(Config { name, timeout }) /// } /// - /// fn fory_type_id_dyn(&self, type_resolver: &fory_core::TypeResolver) -> Result { + /// fn fory_type_id_dyn(&self, type_resolver: &fory_core::TypeResolver) -> Result { /// Self::fory_get_type_id(type_resolver) /// } /// @@ -902,7 +902,7 @@ pub trait Serializer: 'static { /// /// ```rust,ignore /// fn fory_read_type_info(context: &mut ReadContext) -> Result<(), Error> { - /// context.read_any_typeinfo()?; + /// context.read_any_type_info()?; /// Ok(()) /// } /// ``` @@ -930,7 +930,7 @@ pub trait Serializer: 'static { Self: Sized, { // Serializer for internal types should overwrite this method for faster performance. - context.read_any_typeinfo()?; + context.read_any_type_info()?; Ok(()) } @@ -1095,7 +1095,7 @@ pub trait Serializer: 'static { /// /// [`fory_type_id_dyn`]: Serializer::fory_type_id_dyn #[inline(always)] - fn fory_get_type_id(type_resolver: &TypeResolver) -> Result + fn fory_get_type_id(type_resolver: &TypeResolver) -> Result where Self: Sized, { @@ -1105,6 +1105,17 @@ pub trait Serializer: 'static { } } + #[inline(always)] + fn fory_get_type_info(type_resolver: &TypeResolver) -> Result, Error> + where + Self: Sized, + { + match type_resolver.get_type_info(&std::any::TypeId::of::()) { + Ok(info) => Ok(info), + Err(e) => Err(Error::enhance_type_error::(e)), + } + } + /// **[USER IMPLEMENTATION REQUIRED]** Get the runtime type ID for this instance. /// /// This method returns the type ID for the actual runtime type of the instance, @@ -1123,7 +1134,7 @@ pub trait Serializer: 'static { /// For most types, simply delegate to [`fory_get_type_id`]: /// /// ```rust,ignore - /// fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { + /// fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { /// Self::fory_get_type_id(type_resolver) /// } /// ``` @@ -1131,14 +1142,14 @@ pub trait Serializer: 'static { /// For polymorphic types, return the actual runtime type: /// /// ```rust,ignore - /// fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { + /// fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { /// // Get the actual type ID based on runtime type /// self.get_actual_type().fory_get_type_id(type_resolver) /// } /// ``` /// /// [`fory_get_type_id`]: Serializer::fory_get_type_id - fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result; + fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result; /// Get Rust's `std::any::TypeId` for this instance. /// diff --git a/rust/fory-core/src/serializer/datetime.rs b/rust/fory-core/src/serializer/datetime.rs index 3c7c1907db..860a838903 100644 --- a/rust/fory-core/src/serializer/datetime.rs +++ b/rust/fory-core/src/serializer/datetime.rs @@ -54,13 +54,13 @@ impl Serializer for NaiveDateTime { } #[inline(always)] - fn fory_get_type_id(_: &TypeResolver) -> Result { - Ok(TypeId::TIMESTAMP as u32) + fn fory_get_type_id(_: &TypeResolver) -> Result { + Ok(TypeId::TIMESTAMP) } #[inline(always)] - fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { - Ok(TypeId::TIMESTAMP as u32) + fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { + Ok(TypeId::TIMESTAMP) } #[inline(always)] @@ -75,7 +75,7 @@ impl Serializer for NaiveDateTime { #[inline(always)] fn fory_write_type_info(context: &mut WriteContext) -> Result<(), Error> { - context.writer.write_var_uint32(TypeId::TIMESTAMP as u32); + context.writer.write_u8(TypeId::TIMESTAMP as u8); Ok(()) } @@ -108,13 +108,13 @@ impl Serializer for NaiveDate { } #[inline(always)] - fn fory_get_type_id(_: &TypeResolver) -> Result { - Ok(TypeId::DATE as u32) + fn fory_get_type_id(_: &TypeResolver) -> Result { + Ok(TypeId::DATE) } #[inline(always)] - fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { - Ok(TypeId::DATE as u32) + fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { + Ok(TypeId::DATE) } #[inline(always)] @@ -129,7 +129,7 @@ impl Serializer for NaiveDate { #[inline(always)] fn fory_write_type_info(context: &mut WriteContext) -> Result<(), Error> { - context.writer.write_var_uint32(TypeId::DATE as u32); + context.writer.write_u8(TypeId::DATE as u8); Ok(()) } @@ -176,13 +176,13 @@ impl Serializer for Duration { } #[inline(always)] - fn fory_get_type_id(_: &TypeResolver) -> Result { - Ok(TypeId::DURATION as u32) + fn fory_get_type_id(_: &TypeResolver) -> Result { + Ok(TypeId::DURATION) } #[inline(always)] - fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { - Ok(TypeId::DURATION as u32) + fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { + Ok(TypeId::DURATION) } #[inline(always)] @@ -197,7 +197,7 @@ impl Serializer for Duration { #[inline(always)] fn fory_write_type_info(context: &mut WriteContext) -> Result<(), Error> { - context.writer.write_var_uint32(TypeId::DURATION as u32); + context.writer.write_u8(TypeId::DURATION as u8); Ok(()) } diff --git a/rust/fory-core/src/serializer/enum_.rs b/rust/fory-core/src/serializer/enum_.rs index 4496f985b0..2605383efa 100644 --- a/rust/fory-core/src/serializer/enum_.rs +++ b/rust/fory-core/src/serializer/enum_.rs @@ -24,11 +24,11 @@ use crate::types::{RefFlag, RefMode, TypeId}; use crate::TypeResolver; #[inline(always)] -pub fn actual_type_id(type_id: u32, register_by_name: bool, _compatible: bool) -> u32 { +pub fn actual_type_id(_type_id: u32, register_by_name: bool, _compatible: bool) -> u32 { if register_by_name { TypeId::NAMED_ENUM as u32 } else { - (type_id << 8) + TypeId::ENUM as u32 + TypeId::ENUM as u32 } } @@ -51,12 +51,15 @@ pub fn write( #[inline(always)] pub fn write_type_info(context: &mut WriteContext) -> Result<(), Error> { let type_id = T::fory_get_type_id(context.get_type_resolver())?; - context.writer.write_var_uint32(type_id); - let is_named_enum = type_id & 0xff == TypeId::NAMED_ENUM as u32; - if !is_named_enum { + context.writer.write_u8(type_id as u8); + let rs_type_id = std::any::TypeId::of::(); + if type_id == TypeId::ENUM { + let type_info = context.get_type_resolver().get_type_info(&rs_type_id)?; + context + .writer + .write_var_uint32(type_info.get_user_type_id()); return Ok(()); } - let rs_type_id = std::any::TypeId::of::(); if context.is_share_meta() { // Write type meta inline using streaming protocol context.write_type_meta(rs_type_id)?; @@ -101,21 +104,21 @@ pub fn read( #[inline(always)] pub fn read_type_info(context: &mut ReadContext) -> Result<(), Error> { let local_type_id = T::fory_get_type_id(context.get_type_resolver())?; - let remote_type_id = context.reader.read_varuint32()?; + let remote_type_id = context.reader.read_u8()?; ensure!( - local_type_id == remote_type_id, - Error::type_mismatch(local_type_id, remote_type_id) + local_type_id as u8 == remote_type_id, + Error::type_mismatch(local_type_id as u32, remote_type_id as u32) ); - let is_named_enum = local_type_id & 0xff == TypeId::NAMED_ENUM as u32; - if !is_named_enum { - return Ok(()); - } - if context.is_share_meta() { - // Read type meta inline using streaming protocol - let _type_info = context.read_type_meta()?; + if remote_type_id == TypeId::NAMED_ENUM as u8 { + if context.is_share_meta() { + // Read type meta inline using streaming protocol + let _type_info = context.read_type_meta()?; + } else { + let _namespace_msb = context.read_meta_string()?; + let _type_name_msb = context.read_meta_string()?; + } } else { - let _namespace_msb = context.read_meta_string()?; - let _type_name_msb = context.read_meta_string()?; + context.reader.read_varuint32()?; } Ok(()) } diff --git a/rust/fory-core/src/serializer/heap.rs b/rust/fory-core/src/serializer/heap.rs index f74b93997d..623edba302 100644 --- a/rust/fory-core/src/serializer/heap.rs +++ b/rust/fory-core/src/serializer/heap.rs @@ -50,12 +50,12 @@ impl Serializer for BinaryHeap { mem::size_of::() } - fn fory_get_type_id(_: &TypeResolver) -> Result { - Ok(TypeId::SET as u32) + fn fory_get_type_id(_: &TypeResolver) -> Result { + Ok(TypeId::SET) } - fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { - Ok(TypeId::SET as u32) + fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { + Ok(TypeId::SET) } fn fory_static_type_id() -> TypeId { diff --git a/rust/fory-core/src/serializer/list.rs b/rust/fory-core/src/serializer/list.rs index 7bb84cd674..725471157e 100644 --- a/rust/fory-core/src/serializer/list.rs +++ b/rust/fory-core/src/serializer/list.rs @@ -151,22 +151,22 @@ impl Serializer for Vec { } #[inline(always)] - fn fory_get_type_id(_: &TypeResolver) -> Result { + fn fory_get_type_id(_: &TypeResolver) -> Result { let id = get_primitive_type_id::(); if id != TypeId::UNKNOWN { - Ok(id as u32) + Ok(id) } else { - Ok(TypeId::LIST as u32) + Ok(TypeId::LIST) } } #[inline(always)] - fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { + fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { let id = get_primitive_type_id::(); if id != TypeId::UNKNOWN { - Ok(id as u32) + Ok(id) } else { - Ok(TypeId::LIST as u32) + Ok(TypeId::LIST) } } @@ -232,13 +232,13 @@ impl Serializer for VecDeque { } #[inline(always)] - fn fory_get_type_id(_: &TypeResolver) -> Result { - Ok(TypeId::LIST as u32) + fn fory_get_type_id(_: &TypeResolver) -> Result { + Ok(TypeId::LIST) } #[inline(always)] - fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { - Ok(TypeId::LIST as u32) + fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { + Ok(TypeId::LIST) } #[inline(always)] @@ -295,13 +295,13 @@ impl Serializer for LinkedList { } #[inline(always)] - fn fory_get_type_id(_: &TypeResolver) -> Result { - Ok(TypeId::LIST as u32) + fn fory_get_type_id(_: &TypeResolver) -> Result { + Ok(TypeId::LIST) } #[inline(always)] - fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { - Ok(TypeId::LIST as u32) + fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { + Ok(TypeId::LIST) } #[inline(always)] diff --git a/rust/fory-core/src/serializer/map.rs b/rust/fory-core/src/serializer/map.rs index bec1988d98..4e90a1c3d9 100644 --- a/rust/fory-core/src/serializer/map.rs +++ b/rust/fory-core/src/serializer/map.rs @@ -170,8 +170,8 @@ where let val_is_shared_ref = V::fory_is_shared_ref(); // Track the current chunk's key and value types (for polymorphic types) - let mut current_key_type_id: Option = None; - let mut current_val_type_id: Option = None; + let mut current_key_type_id: Option = None; + let mut current_val_type_id: Option = None; for (key, value) in iter { // Handle null key/value entries (write as separate single-entry chunks) @@ -197,7 +197,7 @@ where } else { context.writer.write_u8(chunk_header); if key_is_polymorphic { - context.write_any_typeinfo( + context.write_any_type_info( K::fory_static_type_id() as u32, key.fory_concrete_type_id(), )?; @@ -223,7 +223,7 @@ where } else { context.writer.write_u8(chunk_header); if val_is_polymorphic { - context.write_any_typeinfo( + context.write_any_type_info( V::fory_static_type_id() as u32, value.fory_concrete_type_id(), )?; @@ -242,12 +242,12 @@ where // Get type IDs for polymorphic types let key_type_id = if key_is_polymorphic { - Some(key.fory_type_id_dyn(context.get_type_resolver())?) + Some(key.fory_concrete_type_id()) } else { None }; let val_type_id = if val_is_polymorphic { - Some(value.fory_type_id_dyn(context.get_type_resolver())?) + Some(value.fory_concrete_type_id()) } else { None }; @@ -281,7 +281,7 @@ where } else { // Write type info for key if key_is_polymorphic { - context.write_any_typeinfo( + context.write_any_type_info( K::fory_static_type_id() as u32, key.fory_concrete_type_id(), )?; @@ -299,7 +299,7 @@ where } else { // Write type info for value if val_is_polymorphic { - context.write_any_typeinfo( + context.write_any_type_info( V::fory_static_type_id() as u32, value.fory_concrete_type_id(), )?; @@ -379,7 +379,7 @@ macro_rules! impl_read_map_dyn_ref { // Determine value type info (if any) let value_type_info: Option> = if !value_declared { if val_is_polymorphic { - Some(context.read_any_typeinfo()?) + Some(context.read_any_type_info()?) } else { V::fory_read_type_info(context)?; None @@ -414,7 +414,7 @@ macro_rules! impl_read_map_dyn_ref { let key_type_info: Option> = if !key_declared { if key_is_polymorphic { - Some(context.read_any_typeinfo()?) + Some(context.read_any_type_info()?) } else { K::fory_read_type_info(context)?; None @@ -450,7 +450,7 @@ macro_rules! impl_read_map_dyn_ref { let key_type_info: Option> = if !key_declared { if key_is_polymorphic { - Some(context.read_any_typeinfo()?) + Some(context.read_any_type_info()?) } else { K::fory_read_type_info(context)?; None @@ -460,7 +460,7 @@ macro_rules! impl_read_map_dyn_ref { }; let value_type_info: Option> = if !value_declared { if val_is_polymorphic { - Some(context.read_any_typeinfo()?) + Some(context.read_any_type_info()?) } else { V::fory_read_type_info(context)?; None @@ -646,12 +646,12 @@ impl() } - fn fory_get_type_id(_: &TypeResolver) -> Result { - Ok(TypeId::MAP as u32) + fn fory_get_type_id(_: &TypeResolver) -> Result { + Ok(TypeId::MAP) } - fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { - Ok(TypeId::MAP as u32) + fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { + Ok(TypeId::MAP) } fn fory_static_type_id() -> TypeId @@ -666,7 +666,7 @@ impl Result<(), Error> { - context.writer.write_var_uint32(TypeId::MAP as u32); + context.writer.write_u8(TypeId::MAP as u8); Ok(()) } @@ -796,12 +796,12 @@ impl() } - fn fory_get_type_id(_: &TypeResolver) -> Result { - Ok(TypeId::MAP as u32) + fn fory_get_type_id(_: &TypeResolver) -> Result { + Ok(TypeId::MAP) } - fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { - Ok(TypeId::MAP as u32) + fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { + Ok(TypeId::MAP) } fn fory_static_type_id() -> TypeId @@ -816,7 +816,7 @@ impl Result<(), Error> { - context.writer.write_var_uint32(TypeId::MAP as u32); + context.writer.write_u8(TypeId::MAP as u8); Ok(()) } diff --git a/rust/fory-core/src/serializer/marker.rs b/rust/fory-core/src/serializer/marker.rs index 615107b960..5045080cf7 100644 --- a/rust/fory-core/src/serializer/marker.rs +++ b/rust/fory-core/src/serializer/marker.rs @@ -43,15 +43,15 @@ impl Serializer for PhantomData { } #[inline(always)] - fn fory_get_type_id(_: &TypeResolver) -> Result { + fn fory_get_type_id(_: &TypeResolver) -> Result { // Use NONE - PhantomData has no runtime data, skip can return early - Ok(TypeId::NONE as u32) + Ok(TypeId::NONE) } #[inline(always)] - fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { + fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { // Use NONE - PhantomData has no runtime data, skip can return early - Ok(TypeId::NONE as u32) + Ok(TypeId::NONE) } #[inline(always)] diff --git a/rust/fory-core/src/serializer/mutex.rs b/rust/fory-core/src/serializer/mutex.rs index 8d1530af20..ae8eddf416 100644 --- a/rust/fory-core/src/serializer/mutex.rs +++ b/rust/fory-core/src/serializer/mutex.rs @@ -132,12 +132,20 @@ impl Serializer for Mutex { } #[inline(always)] - fn fory_get_type_id(type_resolver: &TypeResolver) -> Result { + fn fory_get_type_id(type_resolver: &TypeResolver) -> Result { T::fory_get_type_id(type_resolver) } #[inline(always)] - fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { + fn fory_get_type_info(type_resolver: &TypeResolver) -> Result, Error> { + match type_resolver.get_type_info(&std::any::TypeId::of::()) { + Ok(info) => Ok(info), + Err(e) => Err(Error::enhance_type_error::(e)), + } + } + + #[inline(always)] + fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { let guard = self.lock().unwrap(); (*guard).fory_type_id_dyn(type_resolver) } diff --git a/rust/fory-core/src/serializer/number.rs b/rust/fory-core/src/serializer/number.rs index 9a23ad66f3..379d5290c1 100644 --- a/rust/fory-core/src/serializer/number.rs +++ b/rust/fory-core/src/serializer/number.rs @@ -44,13 +44,13 @@ macro_rules! impl_num_serializer { } #[inline(always)] - fn fory_get_type_id(_: &TypeResolver) -> Result { - Ok($field_type as u32) + fn fory_get_type_id(_: &TypeResolver) -> Result { + Ok($field_type) } #[inline(always)] - fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { - Ok($field_type as u32) + fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { + Ok($field_type) } #[inline(always)] diff --git a/rust/fory-core/src/serializer/option.rs b/rust/fory-core/src/serializer/option.rs index 778d42ab9d..6b770f9002 100644 --- a/rust/fory-core/src/serializer/option.rs +++ b/rust/fory-core/src/serializer/option.rs @@ -186,12 +186,12 @@ impl Serializer for Option { } #[inline(always)] - fn fory_get_type_id(type_resolver: &TypeResolver) -> Result { + fn fory_get_type_id(type_resolver: &TypeResolver) -> Result { T::fory_get_type_id(type_resolver) } #[inline(always)] - fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { + fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { match self { Some(val) => val.fory_type_id_dyn(type_resolver), None => T::fory_get_type_id(type_resolver), diff --git a/rust/fory-core/src/serializer/primitive_list.rs b/rust/fory-core/src/serializer/primitive_list.rs index 85805ec200..2fc8029e32 100644 --- a/rust/fory-core/src/serializer/primitive_list.rs +++ b/rust/fory-core/src/serializer/primitive_list.rs @@ -74,7 +74,7 @@ pub fn fory_write_data(this: &[T], context: &mut WriteContext) -> } pub fn fory_write_type_info(context: &mut WriteContext, type_id: TypeId) -> Result<(), Error> { - context.writer.write_var_uint32(type_id as u32); + context.writer.write_u8(type_id as u8); Ok(()) } @@ -107,7 +107,7 @@ pub fn fory_read_data(context: &mut ReadContext) -> Result } pub fn fory_read_type_info(context: &mut ReadContext, type_id: TypeId) -> Result<(), Error> { - let remote_type_id = context.reader.read_varuint32()?; + let remote_type_id = context.reader.read_u8()? as u32; if remote_type_id == TypeId::LIST as u32 { return Err(Error::type_error( "Vec belongs to the `number_array` type, \ diff --git a/rust/fory-core/src/serializer/rc.rs b/rust/fory-core/src/serializer/rc.rs index 4bfd6373f8..991746d558 100644 --- a/rust/fory-core/src/serializer/rc.rs +++ b/rust/fory-core/src/serializer/rc.rs @@ -128,11 +128,18 @@ impl Serializer for Rc { 4 } - fn fory_get_type_id(type_resolver: &TypeResolver) -> Result { + fn fory_get_type_id(type_resolver: &TypeResolver) -> Result { T::fory_get_type_id(type_resolver) } - fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { + fn fory_get_type_info(type_resolver: &TypeResolver) -> Result, Error> { + match type_resolver.get_type_info(&std::any::TypeId::of::()) { + Ok(info) => Ok(info), + Err(e) => Err(Error::enhance_type_error::(e)), + } + } + + fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { (**self).fory_type_id_dyn(type_resolver) } diff --git a/rust/fory-core/src/serializer/refcell.rs b/rust/fory-core/src/serializer/refcell.rs index 64384d9dc4..f8b7013f54 100644 --- a/rust/fory-core/src/serializer/refcell.rs +++ b/rust/fory-core/src/serializer/refcell.rs @@ -130,12 +130,20 @@ impl Serializer for RefCell { } #[inline(always)] - fn fory_get_type_id(type_resolver: &TypeResolver) -> Result { + fn fory_get_type_id(type_resolver: &TypeResolver) -> Result { T::fory_get_type_id(type_resolver) } #[inline(always)] - fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { + fn fory_get_type_info(type_resolver: &TypeResolver) -> Result, Error> { + match type_resolver.get_type_info(&std::any::TypeId::of::()) { + Ok(info) => Ok(info), + Err(e) => Err(Error::enhance_type_error::(e)), + } + } + + #[inline(always)] + fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { (*self.borrow()).fory_type_id_dyn(type_resolver) } diff --git a/rust/fory-core/src/serializer/set.rs b/rust/fory-core/src/serializer/set.rs index 6da010c463..df2ff454bc 100644 --- a/rust/fory-core/src/serializer/set.rs +++ b/rust/fory-core/src/serializer/set.rs @@ -58,12 +58,12 @@ impl Serializer for HashSet< mem::size_of::() } - fn fory_get_type_id(_: &TypeResolver) -> Result { - Ok(TypeId::SET as u32) + fn fory_get_type_id(_: &TypeResolver) -> Result { + Ok(TypeId::SET) } - fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { - Ok(TypeId::SET as u32) + fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { + Ok(TypeId::SET) } fn fory_static_type_id() -> TypeId @@ -113,12 +113,12 @@ impl Serializer for BTreeSet { mem::size_of::() } - fn fory_get_type_id(_: &TypeResolver) -> Result { - Ok(TypeId::SET as u32) + fn fory_get_type_id(_: &TypeResolver) -> Result { + Ok(TypeId::SET) } - fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { - Ok(TypeId::SET as u32) + fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { + Ok(TypeId::SET) } fn fory_static_type_id() -> TypeId diff --git a/rust/fory-core/src/serializer/skip.rs b/rust/fory-core/src/serializer/skip.rs index bb3f902176..f50b7b26ec 100644 --- a/rust/fory-core/src/serializer/skip.rs +++ b/rust/fory-core/src/serializer/skip.rs @@ -40,6 +40,7 @@ pub fn skip_field_value( const UNKNOWN_FIELD_TYPE: FieldType = FieldType { type_id: types::UNKNOWN, + user_type_id: u32::MAX, nullable: true, ref_tracking: false, generics: vec![], @@ -61,8 +62,14 @@ pub fn skip_any_value(context: &mut ReadContext, read_ref_flag: bool) -> Result< } // Read type_id first - let type_id = context.reader.read_varuint32()?; - let internal_id = type_id & 0xff; + let type_id = context.reader.read_u8()? as u32; + let internal_id = type_id; + let _user_type_id = if types::needs_user_type_id(type_id) && type_id != types::COMPATIBLE_STRUCT + { + Some(context.reader.read_varuint32()?) + } else { + None + }; // NONE type has no data - return early if internal_id == types::NONE { @@ -75,6 +82,7 @@ pub fn skip_any_value(context: &mut ReadContext, read_ref_flag: bool) -> Result< types::LIST | types::SET => ( FieldType { type_id, + user_type_id: u32::MAX, nullable: true, ref_tracking: false, generics: vec![UNKNOWN_FIELD_TYPE], @@ -84,6 +92,7 @@ pub fn skip_any_value(context: &mut ReadContext, read_ref_flag: bool) -> Result< types::MAP => ( FieldType { type_id, + user_type_id: u32::MAX, nullable: true, ref_tracking: false, generics: vec![UNKNOWN_FIELD_TYPE, UNKNOWN_FIELD_TYPE], @@ -96,6 +105,7 @@ pub fn skip_any_value(context: &mut ReadContext, read_ref_flag: bool) -> Result< ( FieldType { type_id, + user_type_id: u32::MAX, nullable: true, ref_tracking: false, generics: vec![], @@ -103,13 +113,13 @@ pub fn skip_any_value(context: &mut ReadContext, read_ref_flag: bool) -> Result< Some(type_info), ) } - types::STRUCT | types::NAMED_STRUCT | types::NAMED_UNION => { - // For non-compatible struct types with share_meta enabled, read type meta inline + types::NAMED_ENUM | types::NAMED_EXT | types::NAMED_STRUCT | types::NAMED_UNION => { if context.is_share_meta() { let type_info = context.read_type_meta()?; ( FieldType { type_id, + user_type_id: u32::MAX, nullable: true, ref_tracking: false, generics: vec![], @@ -117,7 +127,6 @@ pub fn skip_any_value(context: &mut ReadContext, read_ref_flag: bool) -> Result< Some(type_info), ) } else { - // Without share_meta, read namespace and type_name let namespace = context.read_meta_string()?.to_owned(); let type_name = context.read_meta_string()?.to_owned(); let rc_namespace = Rc::from(namespace); @@ -129,6 +138,7 @@ pub fn skip_any_value(context: &mut ReadContext, read_ref_flag: bool) -> Result< ( FieldType { type_id, + user_type_id: u32::MAX, nullable: true, ref_tracking: false, generics: vec![], @@ -137,15 +147,25 @@ pub fn skip_any_value(context: &mut ReadContext, read_ref_flag: bool) -> Result< ) } } - _ => ( - FieldType { - type_id, - nullable: true, - ref_tracking: false, - generics: vec![], - }, - None, - ), + _ => { + let type_info = if let Some(user_type_id) = _user_type_id { + context + .get_type_resolver() + .get_user_type_info_by_id(user_type_id) + } else { + None + }; + ( + FieldType { + type_id, + user_type_id: _user_type_id.unwrap_or(u32::MAX), + nullable: true, + ref_tracking: false, + generics: vec![], + }, + type_info, + ) + } }; // Don't read ref flag again in skip_value since we already handled it. // Pass type_info so skip_struct doesn't try to read type_id/meta_index again. @@ -165,9 +185,10 @@ fn skip_collection(context: &mut ReadContext, field_type: &FieldType) -> Result< let default_elem_type = field_type.generics.first().unwrap(); let (type_info, elem_field_type); let elem_type = if is_same_type && !is_declared { - let type_info_rc = context.read_any_typeinfo()?; + let type_info_rc = context.read_any_type_info()?; elem_field_type = FieldType { - type_id: type_info_rc.get_type_id(), + type_id: type_info_rc.get_type_id() as u32, + user_type_id: type_info_rc.get_user_type_id(), nullable: has_null, ref_tracking: false, generics: vec![], @@ -210,9 +231,10 @@ fn skip_map(context: &mut ReadContext, field_type: &FieldType) -> Result<(), Err let value_declared = (header & crate::serializer::map::DECL_VALUE_TYPE) != 0; let (value_type_info, value_field_type); let value_type = if !value_declared { - let type_info = context.read_any_typeinfo()?; + let type_info = context.read_any_type_info()?; value_field_type = FieldType { - type_id: type_info.get_type_id(), + type_id: type_info.get_type_id() as u32, + user_type_id: type_info.get_user_type_id(), nullable: true, ref_tracking: false, generics: vec![], @@ -234,9 +256,10 @@ fn skip_map(context: &mut ReadContext, field_type: &FieldType) -> Result<(), Err let key_declared = (header & crate::serializer::map::DECL_KEY_TYPE) != 0; let (key_type_info, key_field_type); let key_type = if !key_declared { - let type_info = context.read_any_typeinfo()?; + let type_info = context.read_any_type_info()?; key_field_type = FieldType { - type_id: type_info.get_type_id(), + type_id: type_info.get_type_id() as u32, + user_type_id: type_info.get_user_type_id(), nullable: true, ref_tracking: false, generics: vec![], @@ -261,9 +284,10 @@ fn skip_map(context: &mut ReadContext, field_type: &FieldType) -> Result<(), Err // Read key type info if not declared let (key_type_info, key_field_type); let key_type = if !key_declared { - let type_info = context.read_any_typeinfo()?; + let type_info = context.read_any_type_info()?; key_field_type = FieldType { - type_id: type_info.get_type_id(), + type_id: type_info.get_type_id() as u32, + user_type_id: type_info.get_user_type_id(), nullable: true, ref_tracking: false, generics: vec![], @@ -278,9 +302,10 @@ fn skip_map(context: &mut ReadContext, field_type: &FieldType) -> Result<(), Err // Read value type info if not declared let (value_type_info, value_field_type); let value_type = if !value_declared { - let type_info = context.read_any_typeinfo()?; + let type_info = context.read_any_type_info()?; value_field_type = FieldType { - type_id: type_info.get_type_id(), + type_id: type_info.get_type_id() as u32, + user_type_id: type_info.get_user_type_id(), nullable: true, ref_tracking: false, generics: vec![], @@ -310,13 +335,15 @@ fn skip_struct( ) -> Result<(), Error> { let type_info_rc: Option>; let type_info_value = if type_info.is_none() { - let remote_type_id = context.reader.read_varuint32()?; - ensure!( - type_id_num == remote_type_id, - Error::type_mismatch(type_id_num, remote_type_id) - ); - // Read type meta inline using streaming protocol - type_info_rc = Some(context.read_type_meta()?); + let remote_type_info = context.read_any_type_info()?; + let remote_type_id = remote_type_info.get_type_id() as u32; + if type_id_num != types::UNKNOWN && remote_type_id != types::UNKNOWN { + ensure!( + type_id_num == remote_type_id, + Error::type_mismatch(type_id_num, remote_type_id) + ); + } + type_info_rc = Some(remote_type_info); type_info_rc.as_ref().unwrap() } else { type_info.as_ref().unwrap() @@ -335,9 +362,7 @@ fn skip_struct( if ENABLE_FORY_DEBUG_OUTPUT { eprintln!( "[skip_struct] field: {:?}, type_id: {}, internal_id: {}", - field_info.field_name, - field_info.field_type.type_id, - field_info.field_type.type_id & 0xff + field_info.field_name, field_info.field_type.type_id, field_info.field_type.type_id ); } let read_ref_flag = util::field_need_write_ref_into( @@ -357,59 +382,18 @@ fn skip_ext( ) -> Result<(), Error> { let type_info_rc: Option>; let type_info_value = if type_info.is_none() { - let remote_type_id = context.reader.read_varuint32()?; + let remote_type_info = context.read_any_type_info()?; + let remote_type_id = remote_type_info.get_type_id() as u32; ensure!( type_id_num == remote_type_id, Error::type_mismatch(type_id_num, remote_type_id) ); - // Read type meta inline using streaming protocol - type_info_rc = Some(context.read_type_meta()?); + type_info_rc = Some(remote_type_info); type_info_rc.as_ref().unwrap() } else { type_info.as_ref().unwrap() }; - let type_resolver = context.get_type_resolver(); - let type_meta = type_info_value.get_type_meta(); - type_resolver - .get_ext_name_harness(type_meta.get_namespace(), type_meta.get_type_name())? - .get_read_data_fn()(context)?; - Ok(()) -} - -fn skip_user_struct(context: &mut ReadContext, _type_id_num: u32) -> Result<(), Error> { - let remote_type_id = context.reader.read_varuint32()?; - // Read type meta inline using streaming protocol - let type_info = context.read_type_meta()?; - let type_meta = type_info.get_type_meta(); - ensure!( - type_meta.get_type_id() == remote_type_id, - Error::type_mismatch(type_meta.get_type_id(), remote_type_id) - ); - let field_infos = type_meta.get_field_infos().to_vec(); - context.inc_depth()?; - for field_info in field_infos.iter() { - let read_ref_flag = util::field_need_write_ref_into( - field_info.field_type.type_id, - field_info.field_type.nullable, - ); - skip_value(context, &field_info.field_type, read_ref_flag, true, &None)?; - } - context.dec_depth(); - Ok(()) -} - -fn skip_user_ext(context: &mut ReadContext, type_id_num: u32) -> Result<(), Error> { - let remote_type_id = context.reader.read_varuint32()?; - ensure!( - type_id_num == remote_type_id, - Error::type_mismatch(type_id_num, remote_type_id) - ); - context.inc_depth()?; - let type_resolver = context.get_type_resolver(); - type_resolver - .get_ext_harness(type_id_num)? - .get_read_data_fn()(context)?; - context.dec_depth(); + type_info_value.get_harness().get_read_data_fn()(context)?; Ok(()) } @@ -436,32 +420,32 @@ fn skip_value( } let type_id_num = field_type.type_id; - // Check if it's a user-defined type (high bits set, meaning type_id > 255) - if type_id_num > 255 { - let internal_id = type_id_num & 0xff; - // Handle struct-like types including UNKNOWN (0) which is used for polymorphic types - if internal_id == types::COMPATIBLE_STRUCT - || internal_id == types::STRUCT - || internal_id == types::NAMED_STRUCT - || internal_id == types::NAMED_COMPATIBLE_STRUCT - || internal_id == types::UNKNOWN + if type_id_num == types::UNKNOWN { + return skip_any_value(context, false); + } + + // Handle user-defined types (struct/enum/ext/union) + if types::is_user_type(type_id_num) { + if type_id_num == types::COMPATIBLE_STRUCT + || type_id_num == types::STRUCT + || type_id_num == types::NAMED_STRUCT + || type_id_num == types::NAMED_COMPATIBLE_STRUCT + { + return skip_struct(context, type_id_num, type_info); + } else if type_id_num == types::ENUM + || type_id_num == types::NAMED_ENUM + || type_id_num == types::UNION + || type_id_num == types::TYPED_UNION + || type_id_num == types::NAMED_UNION { - // If type_info is provided (from skip_any_value), use skip_struct directly - // which won't try to re-read type_id/meta_index. Otherwise use skip_user_struct. - if type_info.is_some() { - return skip_struct(context, type_id_num, type_info); - } - return skip_user_struct(context, type_id_num); - } else if internal_id == types::ENUM || internal_id == types::NAMED_ENUM { let _ordinal = context.reader.read_varuint32()?; return Ok(()); - } else if internal_id == types::EXT || internal_id == types::NAMED_EXT { - return skip_user_ext(context, type_id_num); + } else if type_id_num == types::EXT || type_id_num == types::NAMED_EXT { + return skip_ext(context, type_id_num, type_info); } else { return Err(Error::type_error(format!( - "Unknown type id: {} (internal_id: {}, type_info provided: {})", + "Unknown type id: {} (type_info provided: {})", type_id_num, - internal_id, type_info.is_some() ))); } @@ -790,6 +774,7 @@ pub fn skip_enum_variant( // Tuple uses collection format but doesn't write type info, so skip directly let field_type = FieldType { type_id: types::LIST, + user_type_id: u32::MAX, nullable: false, ref_tracking: false, generics: vec![UNKNOWN_FIELD_TYPE], @@ -800,12 +785,12 @@ pub fn skip_enum_variant( // Named variant, skip struct-like data using skip_struct // For named variants, we need the type_info which should have been read already if type_info.is_some() { - let type_id = type_info.as_ref().unwrap().get_type_id(); + let type_id = type_info.as_ref().unwrap().get_type_id() as u32; skip_struct(context, type_id, type_info) } else { // If no type_info provided, read it inline using streaming protocol let type_info_rc = context.read_type_meta()?; - let type_id = type_info_rc.get_type_id(); + let type_id = type_info_rc.get_type_id() as u32; let type_info_opt = Some(type_info_rc); skip_struct(context, type_id, &type_info_opt) } diff --git a/rust/fory-core/src/serializer/string.rs b/rust/fory-core/src/serializer/string.rs index ac2c56d98a..1893cba9a8 100644 --- a/rust/fory-core/src/serializer/string.rs +++ b/rust/fory-core/src/serializer/string.rs @@ -66,13 +66,13 @@ impl Serializer for String { } #[inline(always)] - fn fory_get_type_id(_: &TypeResolver) -> Result { - Ok(TypeId::STRING as u32) + fn fory_get_type_id(_: &TypeResolver) -> Result { + Ok(TypeId::STRING) } #[inline(always)] - fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { - Ok(TypeId::STRING as u32) + fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { + Ok(TypeId::STRING) } #[inline(always)] @@ -90,7 +90,7 @@ impl Serializer for String { #[inline(always)] fn fory_write_type_info(context: &mut WriteContext) -> Result<(), Error> { - context.writer.write_var_uint32(TypeId::STRING as u32); + context.writer.write_u8(TypeId::STRING as u8); Ok(()) } diff --git a/rust/fory-core/src/serializer/struct_.rs b/rust/fory-core/src/serializer/struct_.rs index 89216d547b..51dd6a0f09 100644 --- a/rust/fory-core/src/serializer/struct_.rs +++ b/rust/fory-core/src/serializer/struct_.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use crate::ensure; use crate::error::Error; use crate::resolver::context::{ReadContext, WriteContext}; use crate::serializer::Serializer; @@ -24,69 +23,31 @@ use crate::util::ENABLE_FORY_DEBUG_OUTPUT; use std::any::Any; #[inline(always)] -pub fn actual_type_id(type_id: u32, register_by_name: bool, compatible: bool) -> u32 { +pub fn actual_type_id(_type_id: u32, register_by_name: bool, compatible: bool) -> u32 { if compatible { if register_by_name { TypeId::NAMED_COMPATIBLE_STRUCT as u32 } else { - (type_id << 8) + TypeId::COMPATIBLE_STRUCT as u32 + TypeId::COMPATIBLE_STRUCT as u32 } } else if register_by_name { TypeId::NAMED_STRUCT as u32 } else { - (type_id << 8) + TypeId::STRUCT as u32 + TypeId::STRUCT as u32 } } #[inline(always)] pub fn write_type_info(context: &mut WriteContext) -> Result<(), Error> { - let type_id = T::fory_get_type_id(context.get_type_resolver())?; - context.writer.write_var_uint32(type_id); let rs_type_id = std::any::TypeId::of::(); - - if type_id & 0xff == TypeId::NAMED_STRUCT as u32 { - if context.is_share_meta() { - // Write type meta inline using streaming protocol - context.write_type_meta(rs_type_id)?; - } else { - let type_info = context.get_type_resolver().get_type_info(&rs_type_id)?; - let namespace = type_info.get_namespace(); - let type_name = type_info.get_type_name(); - context.write_meta_string_bytes(namespace)?; - context.write_meta_string_bytes(type_name)?; - } - } else if type_id & 0xff == TypeId::NAMED_COMPATIBLE_STRUCT as u32 - || type_id & 0xff == TypeId::COMPATIBLE_STRUCT as u32 - { - // Write type meta inline using streaming protocol - context.write_type_meta(rs_type_id)?; - } + let type_id = T::fory_get_type_id(context.get_type_resolver())?; + context.write_any_type_info(type_id as u32, rs_type_id)?; Ok(()) } #[inline(always)] pub fn read_type_info(context: &mut ReadContext) -> Result<(), Error> { - let remote_type_id = context.reader.read_varuint32()?; - let local_type_id = T::fory_get_type_id(context.get_type_resolver())?; - ensure!( - local_type_id == remote_type_id, - Error::type_mismatch(local_type_id, remote_type_id) - ); - - if local_type_id & 0xff == TypeId::NAMED_STRUCT as u32 { - if context.is_share_meta() { - // Read type meta inline using streaming protocol - let _type_info = context.read_type_meta()?; - } else { - let _namespace_msb = context.read_meta_string()?; - let _type_name_msb = context.read_meta_string()?; - } - } else if local_type_id & 0xff == TypeId::NAMED_COMPATIBLE_STRUCT as u32 - || local_type_id & 0xff == TypeId::COMPATIBLE_STRUCT as u32 - { - // Read type meta inline using streaming protocol - let _type_info = context.read_type_meta()?; - } + context.read_any_type_info()?; Ok(()) } diff --git a/rust/fory-core/src/serializer/trait_object.rs b/rust/fory-core/src/serializer/trait_object.rs index 65d9059938..f5b9ddf5b4 100644 --- a/rust/fory-core/src/serializer/trait_object.rs +++ b/rust/fory-core/src/serializer/trait_object.rs @@ -25,6 +25,7 @@ use crate::resolver::type_resolver::{TypeInfo, TypeResolver}; use crate::serializer::{ForyDefault, Serializer}; use crate::types::RefMode; use crate::RefFlag; +use crate::TypeId; use std::rc::Rc; /// Helper macro for common type resolution and downcasting pattern @@ -43,24 +44,6 @@ macro_rules! downcast_and_serialize { }}; } -/// Helper macro for common type resolution and deserialization pattern -#[macro_export] -macro_rules! resolve_and_deserialize { - ($fory_type_id:expr, $context:expr, $constructor:expr, $trait_name:ident, $($impl_type:ty),+) => {{ - $( - if let Some(registered_type_id) = $context.get_type_resolver().get_fory_type_id(std::any::TypeId::of::<$impl_type>()) { - if $fory_type_id == registered_type_id { - let concrete_obj = <$impl_type as fory_core::Serializer>::fory_read_data($context)?; - return Ok($constructor(concrete_obj)); - } - } - )* - Err(fory_core::Error::type_error( - format!("Type ID {} not registered for trait {}", $fory_type_id, stringify!($trait_name)) - )) - }}; -} - /// Macro to register trait object conversions for custom traits. /// /// This macro automatically generates serializers for `Box` trait objects. @@ -173,7 +156,7 @@ macro_rules! register_trait_type { } #[inline(always)] - fn fory_type_id_dyn(&self, type_resolver: &fory_core::TypeResolver) -> Result { + fn fory_type_id_dyn(&self, type_resolver: &fory_core::TypeResolver) -> Result { let any_ref = ::as_any(&**self); let concrete_type_id = any_ref.type_id(); type_resolver @@ -237,7 +220,7 @@ macro_rules! register_trait_type { $crate::not_allowed!("fory_read_data should not be called directly on polymorphic Box trait object", stringify!($trait_name)) } - fn fory_get_type_id(_type_resolver: &fory_core::TypeResolver) -> Result { + fn fory_get_type_id(_type_resolver: &fory_core::TypeResolver) -> Result { $crate::not_allowed!("fory_get_type_id should not be called directly on polymorphic Box trait object", stringify!($trait_name)) } @@ -376,14 +359,29 @@ macro_rules! read_ptr_trait_object { fory_core::RefFlag::NotNullValue => { $context.inc_depth()?; let typeinfo = if $read_type_info { - $context.read_any_typeinfo()? + $context.read_any_type_info()? } else { $type_info.ok_or_else(|| fory_core::Error::type_error("No type info found for read"))? }; let fory_type_id = typeinfo.get_type_id(); + let user_type_id = typeinfo.get_user_type_id(); + let registered_by_name = typeinfo.is_registered_by_name(); + let namespace = typeinfo.get_namespace(); + let type_name = typeinfo.get_type_name(); + let matches_type = |local_info: &fory_core::TypeInfo| -> bool { + if registered_by_name { + local_info.is_registered_by_name() + && local_info.get_namespace().original == namespace.original + && local_info.get_type_name().original == type_name.original + } else if user_type_id != u32::MAX { + local_info.get_user_type_id() == user_type_id + } else { + local_info.get_type_id() == fory_type_id + } + }; $( - if let Some(registered_type_id) = $context.get_type_resolver().get_fory_type_id(std::any::TypeId::of::<$impl_type>()) { - if fory_type_id == registered_type_id { + if let Ok(local_info) = $context.get_type_resolver().get_type_info(&std::any::TypeId::of::<$impl_type>()) { + if matches_type(&local_info) { let concrete_obj = <$impl_type as fory_core::Serializer>::fory_read_data($context)?; $context.dec_depth(); let ptr = $constructor_expr(concrete_obj) as $pointer_type; @@ -392,21 +390,38 @@ macro_rules! read_ptr_trait_object { } )* $context.dec_depth(); - Err(fory_core::Error::type_error( - format!("Type ID {} not registered for trait {}", fory_type_id, stringify!($trait_name)) - )) + Err(fory_core::Error::type_error(format!( + "Type ID {} not registered for trait {}", + fory_type_id as u32, + stringify!($trait_name) + ))) } fory_core::RefFlag::RefValue => { $context.inc_depth()?; let typeinfo = if $read_type_info { - $context.read_any_typeinfo()? + $context.read_any_type_info()? } else { $type_info.ok_or_else(|| fory_core::Error::type_error("No type info found for read"))? }; let fory_type_id = typeinfo.get_type_id(); + let user_type_id = typeinfo.get_user_type_id(); + let registered_by_name = typeinfo.is_registered_by_name(); + let namespace = typeinfo.get_namespace(); + let type_name = typeinfo.get_type_name(); + let matches_type = |local_info: &fory_core::TypeInfo| -> bool { + if registered_by_name { + local_info.is_registered_by_name() + && local_info.get_namespace().original == namespace.original + && local_info.get_type_name().original == type_name.original + } else if user_type_id != u32::MAX { + local_info.get_user_type_id() == user_type_id + } else { + local_info.get_type_id() == fory_type_id + } + }; $( - if let Some(registered_type_id) = $context.get_type_resolver().get_fory_type_id(std::any::TypeId::of::<$impl_type>()) { - if fory_type_id == registered_type_id { + if let Ok(local_info) = $context.get_type_resolver().get_type_info(&std::any::TypeId::of::<$impl_type>()) { + if matches_type(&local_info) { let concrete_obj = <$impl_type as fory_core::Serializer>::fory_read_data($context)?; $context.dec_depth(); let ptr = $constructor_expr(concrete_obj) as $pointer_type; @@ -417,9 +432,11 @@ macro_rules! read_ptr_trait_object { } )* $context.dec_depth(); - Err(fory_core::Error::type_error( - format!("Type ID {} not registered for trait {}", fory_type_id, stringify!($trait_name)) - )) + Err(fory_core::Error::type_error(format!( + "Type ID {} not registered for trait {}", + fory_type_id as u32, + stringify!($trait_name) + ))) } } }}; @@ -435,7 +452,7 @@ macro_rules! impl_smart_pointer_serializer { let any_obj = ::as_any(&*self.0); let concrete_type_id = any_obj.type_id(); let typeinfo = if write_type_info { - context.write_any_typeinfo(fory_core::TypeId::UNKNOWN as u32, concrete_type_id)? + context.write_any_type_info(fory_core::TypeId::UNKNOWN as u32, concrete_type_id)? } else { context.get_type_info(&concrete_type_id)? }; @@ -485,8 +502,8 @@ macro_rules! impl_smart_pointer_serializer { } #[inline(always)] - fn fory_get_type_id(_type_resolver: &fory_core::TypeResolver) -> Result { - Ok(fory_core::TypeId::STRUCT as u32) + fn fory_get_type_id(_type_resolver: &fory_core::TypeResolver) -> Result { + Ok(fory_core::TypeId::STRUCT) } #[inline(always)] @@ -515,7 +532,7 @@ macro_rules! impl_smart_pointer_serializer { } #[inline(always)] - fn fory_type_id_dyn(&self, type_resolver: &fory_core::TypeResolver) -> Result { + fn fory_type_id_dyn(&self, type_resolver: &fory_core::TypeResolver) -> Result { let any_obj = ::as_any(&*self.0); let concrete_type_id = any_obj.type_id(); type_resolver @@ -608,7 +625,7 @@ impl Serializer for Box { let fory_type_id_dyn = self.fory_type_id_dyn(context.get_type_resolver())?; let concrete_type_id = (**self).fory_concrete_type_id(); if write_type_info { - context.write_any_typeinfo(fory_type_id_dyn, concrete_type_id)?; + context.write_any_type_info(fory_type_id_dyn as u32, concrete_type_id)?; }; self.fory_write_data_generic(context, has_generics) } @@ -628,7 +645,7 @@ impl Serializer for Box { } #[inline(always)] - fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { + fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { (**self).fory_type_id_dyn(type_resolver) } @@ -650,7 +667,7 @@ impl Serializer for Box { #[inline(always)] fn fory_read_type_info(context: &mut ReadContext) -> Result<(), Error> { - context.read_any_typeinfo()?; + context.read_any_type_info()?; Ok(()) } @@ -706,7 +723,7 @@ fn read_box_seralizer( read_type_info, Error::invalid_data("Type info must be read for Box") ); - context.read_any_typeinfo()? + context.read_any_type_info()? }; let harness = typeinfo.get_harness(); let boxed_any = harness.get_read_data_fn()(context)?; diff --git a/rust/fory-core/src/serializer/tuple.rs b/rust/fory-core/src/serializer/tuple.rs index 4ab5a1b240..97cf78b20e 100644 --- a/rust/fory-core/src/serializer/tuple.rs +++ b/rust/fory-core/src/serializer/tuple.rs @@ -44,15 +44,15 @@ impl Serializer for () { } #[inline(always)] - fn fory_get_type_id(_: &TypeResolver) -> Result { + fn fory_get_type_id(_: &TypeResolver) -> Result { // Use NONE - unit type has no runtime data, skip can return early - Ok(TypeId::NONE as u32) + Ok(TypeId::NONE) } #[inline(always)] - fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { + fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { // Use NONE - unit type has no runtime data, skip can return early - Ok(TypeId::NONE as u32) + Ok(TypeId::NONE) } #[inline(always)] @@ -170,13 +170,13 @@ impl Serializer for (T0,) { } #[inline(always)] - fn fory_get_type_id(_: &TypeResolver) -> Result { - Ok(TypeId::LIST as u32) + fn fory_get_type_id(_: &TypeResolver) -> Result { + Ok(TypeId::LIST) } #[inline(always)] - fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { - Ok(TypeId::LIST as u32) + fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { + Ok(TypeId::LIST) } #[inline(always)] @@ -438,13 +438,13 @@ macro_rules! impl_tuple_serializer { } #[inline(always)] - fn fory_get_type_id(_: &TypeResolver) -> Result { - Ok(TypeId::LIST as u32) + fn fory_get_type_id(_: &TypeResolver) -> Result { + Ok(TypeId::LIST) } #[inline(always)] - fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { - Ok(TypeId::LIST as u32) + fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { + Ok(TypeId::LIST) } #[inline(always)] diff --git a/rust/fory-core/src/serializer/unsigned_number.rs b/rust/fory-core/src/serializer/unsigned_number.rs index d3a3c02a84..597e31434b 100644 --- a/rust/fory-core/src/serializer/unsigned_number.rs +++ b/rust/fory-core/src/serializer/unsigned_number.rs @@ -45,13 +45,13 @@ macro_rules! impl_xlang_unsigned_num_serializer { } #[inline(always)] - fn fory_get_type_id(_: &TypeResolver) -> Result { - Ok($field_type as u32) + fn fory_get_type_id(_: &TypeResolver) -> Result { + Ok($field_type) } #[inline(always)] - fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { - Ok($field_type as u32) + fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { + Ok($field_type) } #[inline(always)] @@ -111,13 +111,13 @@ macro_rules! impl_rust_unsigned_num_serializer { } #[inline(always)] - fn fory_get_type_id(_: &TypeResolver) -> Result { - Ok($field_type as u32) + fn fory_get_type_id(_: &TypeResolver) -> Result { + Ok($field_type) } #[inline(always)] - fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { - Ok($field_type as u32) + fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { + Ok($field_type) } #[inline(always)] diff --git a/rust/fory-core/src/serializer/util.rs b/rust/fory-core/src/serializer/util.rs index c6aab15178..a712af5e9c 100644 --- a/rust/fory-core/src/serializer/util.rs +++ b/rust/fory-core/src/serializer/util.rs @@ -25,10 +25,11 @@ use crate::types::{is_user_type, ENUM, NAMED_ENUM, UNION}; #[inline(always)] pub(crate) fn read_basic_type_info(context: &mut ReadContext) -> Result<(), Error> { let local_type_id = T::fory_get_type_id(context.get_type_resolver())?; - let remote_type_id = context.reader.read_varuint32()?; + let local_type_id_u32 = local_type_id as u32; + let remote_type_id = context.reader.read_u8()? as u32; ensure!( - local_type_id == remote_type_id, - Error::type_mismatch(local_type_id, remote_type_id) + local_type_id_u32 == remote_type_id, + Error::type_mismatch(local_type_id_u32, remote_type_id) ); Ok(()) } @@ -42,11 +43,10 @@ pub(crate) fn read_basic_type_info(context: &mut ReadContext) -> /// Keep as const fn for compile time evaluation or constant folding #[inline] pub const fn field_need_read_type_info(type_id: u32) -> bool { - let internal_type_id = type_id & 0xff; - if internal_type_id == ENUM || internal_type_id == NAMED_ENUM || internal_type_id == UNION { + if type_id == ENUM || type_id == NAMED_ENUM || type_id == UNION { return false; } - is_user_type(internal_type_id) + is_user_type(type_id) } /// Keep as const fn for compile time evaluation or constant folding @@ -83,7 +83,7 @@ pub fn write_dyn_data_generic( let any_value = value.as_any(); let concrete_type_id = any_value.type_id(); let serializer_fn = context - .write_any_typeinfo(T::fory_static_type_id() as u32, concrete_type_id)? + .write_any_type_info(T::fory_static_type_id() as u32, concrete_type_id)? .get_harness() .get_write_data_fn(); serializer_fn(any_value, context, has_generics) diff --git a/rust/fory-core/src/serializer/weak.rs b/rust/fory-core/src/serializer/weak.rs index 14ef35d521..0bdd96aca0 100644 --- a/rust/fory-core/src/serializer/weak.rs +++ b/rust/fory-core/src/serializer/weak.rs @@ -417,11 +417,18 @@ impl Serializer for RcWeak { 4 } - fn fory_get_type_id(type_resolver: &TypeResolver) -> Result { + fn fory_get_type_id(type_resolver: &TypeResolver) -> Result { T::fory_get_type_id(type_resolver) } - fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { + fn fory_get_type_info(type_resolver: &TypeResolver) -> Result, Error> { + match type_resolver.get_type_info(&std::any::TypeId::of::()) { + Ok(info) => Ok(info), + Err(e) => Err(Error::enhance_type_error::(e)), + } + } + + fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { if let Some(rc) = self.upgrade() { (*rc).fory_type_id_dyn(type_resolver) } else { @@ -453,7 +460,7 @@ fn read_rc_weak( T::fory_read_with_type_info(context, RefMode::None, type_info)? } else { if read_type_info { - context.read_any_typeinfo()?; + context.read_any_type_info()?; } T::fory_read_data(context)? }; @@ -583,11 +590,18 @@ impl Serializer for ArcWeak 4 } - fn fory_get_type_id(type_resolver: &TypeResolver) -> Result { + fn fory_get_type_id(type_resolver: &TypeResolver) -> Result { T::fory_get_type_id(type_resolver) } - fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { + fn fory_get_type_info(type_resolver: &TypeResolver) -> Result, Error> { + match type_resolver.get_type_info(&std::any::TypeId::of::()) { + Ok(info) => Ok(info), + Err(e) => Err(Error::enhance_type_error::(e)), + } + } + + fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { if let Some(arc) = self.upgrade() { (*arc).fory_type_id_dyn(type_resolver) } else { @@ -619,7 +633,7 @@ fn read_arc_weak( T::fory_read_with_type_info(context, RefMode::None, type_info)? } else { if read_type_info { - context.read_any_typeinfo()?; + context.read_any_type_info()?; } T::fory_read_data(context)? }; diff --git a/rust/fory-core/src/types.rs b/rust/fory-core/src/types.rs index 391eef992f..825fb0664a 100644 --- a/rust/fory-core/src/types.rs +++ b/rust/fory-core/src/types.rs @@ -96,7 +96,7 @@ impl RefMode { #[derive(Clone, Copy, Debug, PartialEq, TryFromPrimitive, IntoPrimitive)] #[allow(non_camel_case_types)] -#[repr(i16)] +#[repr(u8)] pub enum TypeId { // Unknown/polymorphic type marker. UNKNOWN = 0, @@ -474,6 +474,14 @@ pub const fn is_user_type(type_id: u32) -> bool { ) } +#[inline(always)] +pub const fn needs_user_type_id(type_id: u32) -> bool { + matches!( + type_id, + ENUM | STRUCT | COMPATIBLE_STRUCT | EXT | TYPED_UNION + ) +} + pub fn compute_field_hash(hash: u32, id: i16) -> u32 { let mut new_hash: u64 = (hash as u64) * 31 + (id as u64); while new_hash >= MAX_UNT32 { @@ -531,29 +539,11 @@ impl TryFrom for Language { // every object start with i8 i16 reference flag and type flag pub const SIZE_OF_REF_AND_TYPE: usize = mem::size_of::() + mem::size_of::(); -/// Formats a combined type ID into a human-readable string. -/// -/// Combined type IDs have the format: `(registered_id << 8) + internal_type_id`. -/// This function extracts both parts and formats them for debugging. -/// -/// For internal types (type_id < BOUND), returns just the type name. -/// For user-registered types, returns format like "registered_id=3(STRUCT)". +/// Formats a type ID into a human-readable string. /// -/// # Examples -/// ``` -/// use fory_core::types::format_type_id; -/// -/// // Internal type (e.g., BOOL = 1) -/// assert_eq!(format_type_id(1), "BOOL"); -/// -/// // User registered struct with id=3: (3 << 8) + 25 = 793 -/// assert_eq!(format_type_id(793), "registered_id=3(STRUCT)"); -/// ``` +/// Type IDs are internal Fory type IDs in the range 0..=255. pub fn format_type_id(type_id: u32) -> String { - let internal_type_id = type_id & 0xff; - let registered_id = type_id >> 8; - - let type_name = match internal_type_id { + let type_name = match type_id { 0 => "UNKNOWN", 1 => "BOOL", 2 => "INT8", @@ -619,24 +609,14 @@ pub fn format_type_id(type_id: u32) -> String { _ => "UNKNOWN_TYPE", }; - // If it's a pure internal type (no registered_id), just return the type name - if registered_id == 0 { - type_name.to_string() - } else { - // For user-registered types, show both the registered ID and internal type - format!("registered_id={}({})", registered_id, type_name) - } + type_name.to_string() } -/// Computes the actual type ID for extension types. -/// -/// Extension types combine a user-registered type ID with an internal EXT or NAMED_EXT marker. -/// The format is: `(type_id << 8) + internal_type_id`. -pub fn get_ext_actual_type_id(type_id: u32, register_by_name: bool) -> u32 { - (type_id << 8) - + if register_by_name { - TypeId::NAMED_EXT as u32 - } else { - TypeId::EXT as u32 - } +/// Returns the internal type ID for extension types. +pub fn get_ext_actual_type_id(_type_id: u32, register_by_name: bool) -> u32 { + if register_by_name { + TypeId::NAMED_EXT as u32 + } else { + TypeId::EXT as u32 + } } diff --git a/rust/fory-derive/src/object/derive_enum.rs b/rust/fory-derive/src/object/derive_enum.rs index 7872a2f9e6..67a296f611 100644 --- a/rust/fory-derive/src/object/derive_enum.rs +++ b/rust/fory-derive/src/object/derive_enum.rs @@ -43,7 +43,7 @@ pub fn gen_actual_type_id(data_enum: &DataEnum) -> TokenStream { if register_by_name { fory_core::types::TypeId::NAMED_UNION as u32 } else { - (type_id << 8) | (fory_core::types::TypeId::TYPED_UNION as u32) + fory_core::types::TypeId::TYPED_UNION as u32 } } else { fory_core::serializer::enum_::actual_type_id(type_id, register_by_name, compatible) @@ -462,7 +462,7 @@ pub fn gen_write_type_info(data_enum: &DataEnum) -> TokenStream { quote! { if context.is_xlang() { let rs_type_id = std::any::TypeId::of::(); - context.write_any_typeinfo(fory_core::types::UNKNOWN, rs_type_id)?; + context.write_any_type_info(fory_core::types::UNKNOWN, rs_type_id)?; Ok(()) } else { fory_core::serializer::enum_::write_type_info::(context) @@ -964,10 +964,13 @@ pub fn gen_read_type_info(data_enum: &DataEnum) -> TokenStream { quote! { if context.is_xlang() { let expected_type_id = Self::fory_get_type_id(context.get_type_resolver())?; - let type_info = context.read_any_typeinfo()?; + let type_info = context.read_any_type_info()?; let remote_type_id = type_info.get_type_id(); if remote_type_id != expected_type_id { - return Err(fory_core::error::Error::type_mismatch(expected_type_id, remote_type_id)); + return Err(fory_core::error::Error::type_mismatch( + expected_type_id as u32, + remote_type_id as u32, + )); } Ok(()) } else { diff --git a/rust/fory-derive/src/object/misc.rs b/rust/fory-derive/src/object/misc.rs index 1371e02cb8..0aae55a096 100644 --- a/rust/fory-derive/src/object/misc.rs +++ b/rust/fory-derive/src/object/misc.rs @@ -158,6 +158,7 @@ pub fn gen_field_fields_info(source_fields: &[SourceField<'_>]) -> TokenStream { #name, fory_core::meta::FieldType { type_id: #type_id_ts, + user_type_id: u32::MAX, nullable: #nullable, ref_tracking: #ref_tracking, generics: Vec::new() @@ -180,6 +181,7 @@ pub fn gen_field_fields_info(source_fields: &[SourceField<'_>]) -> TokenStream { #name, fory_core::meta::FieldType { type_id: #type_id_ts, + user_type_id: u32::MAX, nullable: #nullable, ref_tracking: #ref_tracking, generics: Vec::new() @@ -207,10 +209,12 @@ pub fn gen_field_fields_info(source_fields: &[SourceField<'_>]) -> TokenStream { quote! { fory_core::meta::FieldInfo::new_with_id(#field_id, #name, fory_core::meta::FieldType { type_id: fory_core::types::TypeId::LIST as u32, + user_type_id: u32::MAX, nullable: #nullable, ref_tracking: #ref_tracking, generics: vec![fory_core::meta::FieldType { type_id: fory_core::types::TypeId::UNKNOWN as u32, + user_type_id: u32::MAX, nullable: false, ref_tracking: false, generics: Vec::new() @@ -226,12 +230,14 @@ pub fn gen_field_fields_info(source_fields: &[SourceField<'_>]) -> TokenStream { quote! { fory_core::meta::FieldInfo::new_with_id(#field_id, #name, fory_core::meta::FieldType { type_id: fory_core::types::TypeId::MAP as u32, + user_type_id: u32::MAX, nullable: #nullable, ref_tracking: #ref_tracking, generics: vec![ #key_generic_token, fory_core::meta::FieldType { type_id: fory_core::types::TypeId::UNKNOWN as u32, + user_type_id: u32::MAX, nullable: false, ref_tracking: false, generics: Vec::new() @@ -244,6 +250,7 @@ pub fn gen_field_fields_info(source_fields: &[SourceField<'_>]) -> TokenStream { quote! { fory_core::meta::FieldInfo::new_with_id(#field_id, #name, fory_core::meta::FieldType { type_id: fory_core::types::TypeId::UNKNOWN as u32, + user_type_id: u32::MAX, nullable: #nullable, ref_tracking: #ref_tracking, generics: Vec::new() diff --git a/rust/fory-derive/src/object/read.rs b/rust/fory-derive/src/object/read.rs index a76156cd7b..2635f3b718 100644 --- a/rust/fory-derive/src/object/read.rs +++ b/rust/fory-derive/src/object/read.rs @@ -879,7 +879,7 @@ pub fn gen_read(_struct_ident: &Ident) -> TokenStream { } if context.is_compatible() { let type_info = if read_type_info { - context.read_any_typeinfo()? + context.read_any_type_info()? } else { let rs_type_id = std::any::TypeId::of::(); context.get_type_info(&rs_type_id)? @@ -1011,8 +1011,66 @@ pub(crate) fn gen_read_compatible_with_construction( crate::util::ok_self_construction(is_tuple, &assign_ts) }; + let variant_field_remap = if let Some(variant) = variant_ident { + let variant_name = variant.to_string(); + quote! { + if let Ok(variants_info) = + ::fory_variants_fields_info( + context.get_type_resolver(), + ) + { + if let Some((_, _, local_fields)) = + variants_info.iter().find(|(name, _, _)| name == #variant_name) + { + let field_index_by_name: std::collections::HashMap<_, _> = local_fields + .iter() + .enumerate() + .filter(|(_, f)| !f.field_name.is_empty()) + .map(|(i, f)| (f.field_name.clone(), (i, f))) + .collect(); + + let field_index_by_id: std::collections::HashMap<_, _> = local_fields + .iter() + .enumerate() + .filter(|(_, f)| f.field_id >= 0) + .map(|(i, f)| (f.field_id, (i, f))) + .collect(); + + for field in fields.iter_mut() { + let local_match = if field.field_id >= 0 && field.field_name.is_empty() { + field_index_by_id.get(&field.field_id).copied() + } else { + let snake_case_name = + fory_core::util::to_snake_case(&field.field_name); + field_index_by_name.get(&snake_case_name).copied() + }; + + match local_match { + Some((sorted_index, local_info)) => { + if field.field_name.is_empty() { + field.field_name = local_info.field_name.clone(); + } + if field.field_type != local_info.field_type { + field.field_id = -1; + } else { + field.field_id = sorted_index as i16; + } + } + None => { + field.field_id = -1; + } + } + } + } + } + } + } else { + quote! {} + }; + quote! { - let fields = type_info.get_type_meta().get_field_infos().clone(); + let mut fields = type_info.get_type_meta().get_field_infos().clone(); + #variant_field_remap #(#declare_ts)* let meta = context.get_type_info(&std::any::TypeId::of::())?.get_type_meta(); let local_type_hash = meta.get_hash(); diff --git a/rust/fory-derive/src/object/serializer.rs b/rust/fory-derive/src/object/serializer.rs index d8cb5ec180..5cdea36e39 100644 --- a/rust/fory-derive/src/object/serializer.rs +++ b/rust/fory-derive/src/object/serializer.rs @@ -189,13 +189,15 @@ pub fn derive_serializer(ast: &syn::DeriveInput, attrs: ForyAttrs) -> TokenStrea impl #impl_generics fory_core::Serializer for #name #ty_generics #where_clause { #[inline(always)] - fn fory_get_type_id(type_resolver: &fory_core::resolver::type_resolver::TypeResolver) -> Result { - type_resolver.get_type_id(&std::any::TypeId::of::(), #type_idx) - .map_err(fory_core::error::Error::enhance_type_error::) + fn fory_get_type_id(type_resolver: &fory_core::resolver::type_resolver::TypeResolver) -> Result { + let type_id = type_resolver + .get_type_id(&std::any::TypeId::of::(), #type_idx) + .map_err(fory_core::error::Error::enhance_type_error::)?; + Ok(type_id) } #[inline(always)] - fn fory_type_id_dyn(&self, type_resolver: &fory_core::resolver::type_resolver::TypeResolver) -> Result { + fn fory_type_id_dyn(&self, type_resolver: &fory_core::resolver::type_resolver::TypeResolver) -> Result { Self::fory_get_type_id(type_resolver) } diff --git a/rust/fory-derive/src/object/util.rs b/rust/fory-derive/src/object/util.rs index 291fdd99e0..5080a4bc9b 100644 --- a/rust/fory-derive/src/object/util.rs +++ b/rust/fory-derive/src/object/util.rs @@ -507,12 +507,11 @@ pub(super) fn generic_tree_to_tokens(node: &TypeNode) -> TokenStream { fory_core::meta::FieldType::new( fory_core::types::TypeId::LIST as u32, true, - vec![fory_core::meta::FieldType { - type_id: fory_core::types::TypeId::UNKNOWN as u32, - nullable: true, - ref_tracking: false, - generics: vec![], - }] + vec![fory_core::meta::FieldType::new( + fory_core::types::TypeId::UNKNOWN as u32, + true, + vec![] + )] ) }; } @@ -576,12 +575,11 @@ pub(super) fn generic_tree_to_tokens(node: &TypeNode) -> TokenStream { fory_core::meta::FieldType::new( fory_core::types::TypeId::LIST as u32, true, - vec![fory_core::meta::FieldType { - type_id: fory_core::types::TypeId::UNKNOWN as u32, - nullable: true, - ref_tracking: false, - generics: vec![], - }] + vec![fory_core::meta::FieldType::new( + fory_core::types::TypeId::UNKNOWN as u32, + true, + vec![] + )] ) }; } @@ -633,18 +631,14 @@ pub(super) fn generic_tree_to_tokens(node: &TypeNode) -> TokenStream { ts } else { quote! { - <#ty as fory_core::serializer::Serializer>::fory_get_type_id(type_resolver)? + <#ty as fory_core::serializer::Serializer>::fory_get_type_id(type_resolver)? as u32 } }; quote! { { let mut type_id = #get_type_id; - let internal_type_id = type_id & 0xff; - if internal_type_id == fory_core::types::TypeId::TYPED_UNION as u32 - || internal_type_id == fory_core::types::TypeId::NAMED_UNION as u32 { - type_id = fory_core::types::TypeId::UNION as u32; - } + let mut user_type_id = u32::MAX; let mut generics = vec![#(#children_tokens),*] as Vec; // For tuples and sets, if no generic info is available, add UNKNOWN element // This handles type aliases to tuples where we can't detect the tuple at macro time @@ -657,15 +651,32 @@ pub(super) fn generic_tree_to_tokens(node: &TypeNode) -> TokenStream { vec![] )); } - let is_custom = !fory_core::types::is_internal_type(type_id & 0xff); + let is_custom = !fory_core::types::is_internal_type(type_id); if is_custom { + let type_info = <#ty as fory_core::serializer::Serializer>::fory_get_type_info(type_resolver)?; + type_id = type_info.get_type_id() as u32; + user_type_id = type_info.get_user_type_id(); + if type_id == fory_core::types::TypeId::TYPED_UNION as u32 + || type_id == fory_core::types::TypeId::NAMED_UNION as u32 { + type_id = fory_core::types::TypeId::UNION as u32; + user_type_id = u32::MAX; + } if type_resolver.is_xlang() && generics.len() > 0 { return Err(fory_core::error::Error::unsupported("serialization of generic structs and enums is not supported in xlang mode")); } else { generics = vec![]; } + } else if type_id == fory_core::types::TypeId::TYPED_UNION as u32 + || type_id == fory_core::types::TypeId::NAMED_UNION as u32 { + type_id = fory_core::types::TypeId::UNION as u32; + } + fory_core::meta::FieldType { + type_id, + user_type_id, + nullable: #nullable, + ref_tracking: false, + generics, } - fory_core::meta::FieldType::new(type_id, #nullable, generics) } } } @@ -1036,7 +1047,10 @@ pub(crate) fn get_type_id_by_name(ty: &str) -> u32 { } fn get_primitive_type_size(type_id_num: u32) -> i32 { - let type_id = TypeId::try_from(type_id_num as i16).unwrap(); + if type_id_num > u8::MAX as u32 { + return 0; + } + let type_id = TypeId::try_from(type_id_num as u8).unwrap(); match type_id { TypeId::BOOL => 1, TypeId::INT8 => 1, diff --git a/rust/fory/src/lib.rs b/rust/fory/src/lib.rs index d34ad701be..6c88b6c45a 100644 --- a/rust/fory/src/lib.rs +++ b/rust/fory/src/lib.rs @@ -839,7 +839,7 @@ //! Ok(Self { value, name }) //! } //! -//! fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { +//! fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { //! Self::fory_get_type_id(type_resolver) //! } //! diff --git a/rust/tests/tests/compatible/test_struct.rs b/rust/tests/tests/compatible/test_struct.rs index a7c16e2a9c..8e9c9461c0 100644 --- a/rust/tests/tests/compatible/test_struct.rs +++ b/rust/tests/tests/compatible/test_struct.rs @@ -645,7 +645,10 @@ fn test_struct_with_generic() { }) } - fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { + fn fory_type_id_dyn( + &self, + type_resolver: &TypeResolver, + ) -> Result { Self::fory_get_type_id(type_resolver) } diff --git a/rust/tests/tests/test_cross_language.rs b/rust/tests/tests/test_cross_language.rs index 487c57ad99..9b1c06b388 100644 --- a/rust/tests/tests/test_cross_language.rs +++ b/rust/tests/tests/test_cross_language.rs @@ -608,7 +608,7 @@ impl Serializer for MyExt { fn fory_type_id_dyn( &self, type_resolver: &TypeResolver, - ) -> Result { + ) -> Result { Self::fory_get_type_id(type_resolver) } diff --git a/rust/tests/tests/test_ext.rs b/rust/tests/tests/test_ext.rs index f93f80a605..2cf0cc7b4a 100644 --- a/rust/tests/tests/test_ext.rs +++ b/rust/tests/tests/test_ext.rs @@ -74,7 +74,7 @@ fn test_use() { fn fory_type_id_dyn( &self, type_resolver: &TypeResolver, - ) -> Result { + ) -> Result { Self::fory_get_type_id(type_resolver) } diff --git a/rust/tests/tests/test_fory.rs b/rust/tests/tests/test_fory.rs index 1b22b9b8fa..1c8be4cb46 100644 --- a/rust/tests/tests/test_fory.rs +++ b/rust/tests/tests/test_fory.rs @@ -288,7 +288,7 @@ fn test_unregistered_type_error_message() { } #[test] -fn test_type_mismatch_error_shows_registered_id() { +fn test_type_mismatch_error_shows_type_name() { use fory_core::error::Error; use fory_core::types::{format_type_id, TypeId}; @@ -296,32 +296,32 @@ fn test_type_mismatch_error_shows_registered_id() { let formatted = format_type_id(TypeId::BOOL as u32); assert_eq!(formatted, "BOOL"); - // Test user registered struct with id=3: (3 << 8) + STRUCT - let struct_type_id = (3 << 8) + TypeId::STRUCT as u32; + // Test struct type + let struct_type_id = TypeId::STRUCT as u32; let formatted = format_type_id(struct_type_id); - assert_eq!(formatted, "registered_id=3(STRUCT)"); + assert_eq!(formatted, "STRUCT"); - // Test user registered enum with id=1: (1 << 8) + ENUM - let enum_type_id = (1 << 8) + TypeId::ENUM as u32; + // Test enum type + let enum_type_id = TypeId::ENUM as u32; let formatted = format_type_id(enum_type_id); - assert_eq!(formatted, "registered_id=1(ENUM)"); + assert_eq!(formatted, "ENUM"); - // Test user registered EXT with id=3: (3 << 8) + EXT - let ext_type_id = (3 << 8) + TypeId::EXT as u32; + // Test EXT type + let ext_type_id = TypeId::EXT as u32; let formatted = format_type_id(ext_type_id); - assert_eq!(formatted, "registered_id=3(EXT)"); + assert_eq!(formatted, "EXT"); // Test error message format let err = Error::type_mismatch(struct_type_id, enum_type_id); let err_str = format!("{}", err); assert!( - err_str.contains("registered_id=3(STRUCT)"), - "error should contain registered_id=3(STRUCT), got: {}", + err_str.contains("STRUCT"), + "error should contain STRUCT, got: {}", err_str ); assert!( - err_str.contains("registered_id=1(ENUM)"), - "error should contain registered_id=1(ENUM), got: {}", + err_str.contains("ENUM"), + "error should contain ENUM, got: {}", err_str ); // Check the message contains "local" and "remote" for clarity diff --git a/rust/tests/tests/test_meta.rs b/rust/tests/tests/test_meta.rs index ec8f586da4..4d32296b8c 100644 --- a/rust/tests/tests/test_meta.rs +++ b/rust/tests/tests/test_meta.rs @@ -16,11 +16,13 @@ // under the License. use fory_core::meta::{FieldInfo, FieldType, MetaString, TypeMeta}; +use fory_core::types::TypeId; #[test] fn test_meta_hash() { let meta = TypeMeta::new( 42, + 1, MetaString::get_empty().clone(), MetaString::get_empty().clone(), false, @@ -28,7 +30,8 @@ fn test_meta_hash() { field_id: 43, field_name: "f1".to_string(), field_type: FieldType { - type_id: 44, + type_id: TypeId::BOOL as u32, + user_type_id: u32::MAX, nullable: true, ref_tracking: false, generics: vec![], diff --git a/scala/src/main/scala/org/apache/fory/serializer/scala/RangeSerializer.scala b/scala/src/main/scala/org/apache/fory/serializer/scala/RangeSerializer.scala index 32b6722777..fb0754b015 100644 --- a/scala/src/main/scala/org/apache/fory/serializer/scala/RangeSerializer.scala +++ b/scala/src/main/scala/org/apache/fory/serializer/scala/RangeSerializer.scala @@ -73,8 +73,8 @@ class NumericRangeSerializer[A, T <: NumericRange[A]](fory: Fory, cls: Class[T]) override def write(buffer: MemoryBuffer, value: T): Unit = { val cls = value.start.getClass val resolver = fory.getClassResolver - val classInfo = resolver.getClassInfo(cls) - resolver.writeClassInfo(buffer, classInfo) + val classInfo = resolver.getTypeInfo(cls) + resolver.writeTypeInfo(buffer, classInfo) val serializer = classInfo.getSerializer.asInstanceOf[Serializer[A]] serializer.write(buffer, value.start) serializer.write(buffer, value.end) @@ -84,7 +84,7 @@ class NumericRangeSerializer[A, T <: NumericRange[A]](fory: Fory, cls: Class[T]) override def read(buffer: MemoryBuffer) = { val resolver = fory.getClassResolver - val classInfo = resolver.readClassInfo(buffer) + val classInfo = resolver.readTypeInfo(buffer) val serializer = classInfo.getSerializer.asInstanceOf[Serializer[A]] val start = serializer.read(buffer) val end = serializer.read(buffer)