Enable packing and unpacking structures with specified gaps between fields

Author: kagalenko-m-b

src/StructIO.jl

@@ -70,7 +70,7 @@ end
 
 @pure function packed_sizeof(T::DataType, ::Type{Packed})
     @assert fieldcount(T) != 0 && isbitstype(T)
-    return sum(packed_sizeof, T.types)
+    return sum(packed_sizeof, T.types) + sum(field_gaps(T))
 end
 
 @pure function packed_sizeof(T::DataType)
@@ -103,7 +103,6 @@ Return the size (in bytes) of a field within `T` in memory
     end
 end
 
-
 """
     @io <type definition>
         ...
@@ -136,11 +135,20 @@ macro io(typ, annotations...)
         T = T.args[1]
     end
 
+    if alignment == :align_packed
+        gap_values =  strip_gap_nodes!(typ)
+    end
+    
     ret = Expr(:toplevel, :(Base.@__doc__ $(typ)))
     strat = (alignment == :align_default ? StructIO.Default : StructIO.Packed)
     push!(ret.args, :($StructIO.packing_strategy(::Type{T}) where {T <: $T} = $strat))
+    if strat == Packed
+         push!(ret.args, :($StructIO.field_gaps(::Type{T})  where {T <: $T} = $gap_values))
+    end
+    
     return esc(ret)
 end
+field_gaps(::Type{T}) where T = UInt[]
 
 """
     unsafe_unpack(io, T, target, endianness, ::Type{Default})
@@ -242,7 +250,7 @@ function unsafe_pack(io, source::Ref{T}, endianness, ::Type{Default}) where {T}
 end
 
 # `Packed` packing strategy override for `unsafe_unpack`
-function unsafe_unpack(io, T, target, endianness, ::Type{Packed})
+function unsafe_unpack(io, T, target, endianness, ::Type{Packed}; gaps=field_gaps(T))
     # If this type cannot be subdivided, packing strategy means nothing, so
     # hand it off to the `Default` packing strategy method
     if fieldcount(T) == 0
@@ -255,12 +263,15 @@ function unsafe_unpack(io, T, target, endianness, ::Type{Packed})
         # Unpack this field into `target` at the appropriate offset
         fT = fieldtype(T, i)
         target_i = target_ptr + fieldoffset(T, i)
+        isempty(gaps) || skip(io, gaps[i])
         unsafe_unpack(io, fT, target_i, endianness, Packed)
     end
 end
 
 # `Packed` packing strategy override for `unsafe_pack`
-function unsafe_pack(io, source::Ref{T}, endianness, ::Type{Packed}) where {T}
+function unsafe_pack(
+    io, source::Ref{T}, endianness, ::Type{Packed}; gaps=field_gaps(T)
+    ) where {T}
     # If this type cannot be subdivided, packing strategy means nothing, so
     # hand it off to the `Default` packing strategy method
     if fieldcount(T) == 0
@@ -272,6 +283,7 @@ function unsafe_pack(io, source::Ref{T}, endianness, ::Type{Packed}) where {T}
         # Unpack this field into `target` at the appropriate offset
         fT = fieldtype(T, i)
         f = Ref{fT}(getfield(source[], fieldname(T, i)))
+        isempty(gaps) || skip(io, gaps[i])
         unsafe_pack(io, f, endianness, Packed)
     end
 end
@@ -311,4 +323,48 @@ function pack(io::IO, source::T, endianness::Symbol = :NativeEndian) where {T}
     return nothing
 end
 
+function strip_gap_nodes!(strct_expr::Expr)
+    @assert strct_expr.head == :struct
+    strct_nodes = strct_expr.args[3].args
+    K = length(strct_nodes)
+    gap_locations = UInt[]
+    # accumulator, in case several gap specifications occur in unbroken succession
+    gap_value = zero(UInt)
+    # gaps before structure fields
+    field_gap_values = UInt[]
+    for k in 1:K
+        f = strct_nodes[k]
+        !isa(f, LineNumberNode) || continue
+        if f.head === :(::)
+            # collect positions of gap nodes, when encountering a structure
+            # field, 
+            if f.args[1] === :_ 
+                if isa(f.args[2], Integer) && f.args[2] >= 0
+                    push!(gap_locations, k)
+                    gap_value += UInt(f.args[2])
+                else
+                    error("incorrect gap specification: $f")
+                end
+            elseif isa(f.args[1], Symbol)
+                
+                push!(field_gap_values, gap_value)
+                gap_value = zero(UInt)
+            else
+                error("unsupported structure field specificaion: $f")
+            end
+        end
+    end
+    if all(iszero, field_gap_values)
+        field_gap_values = UInt[]
+    end
+    # remove gap specifications from the expression tree
+    for k in length(gap_locations):-1:1
+        deleteat!(strct_nodes, gap_locations[k])
+    end
+    strct_expr.args[3].args = strct_nodes
+
+    return field_gap_values
+end
+
+
 end # module

test/runtests.jl

@@ -32,6 +32,32 @@ end
     y::T
 end align_packed
 
+# Structs with gaps
+@io struct RawData
+    A::UInt32
+    dummy_1::UInt8
+    dummy_2::UInt16
+    B::UInt16
+    dummy_3::UInt32
+    C::UInt128
+    dummy_4::UInt16
+    dummy_5::UInt64
+    D::UInt8
+end align_packed
+
+@io struct GappedStruct
+    A::UInt32
+    _::1
+    _::2
+    B::UInt16
+    _::4
+    C::UInt128
+    _::2
+    _::8
+    _D::UInt8
+end align_packed
+
+
 # Also test documenting a type
 """
 This is a docstring
@@ -167,3 +193,27 @@ end
 @testset "Documentation" begin
     @test string(@doc ParametricType) == "This is a docstring\n"
 end
+
+@testset "Struct with gaps" begin
+    A,B,C,D = 1,2,3,4
+    gapped_struct =  GappedStruct(A, B, C, D)
+    raw_data = RawData(UInt32(A),0xBE, 0xBEEF,UInt16(B),0xDEADBEEF,UInt128(C),
+                       0xBEEF, 0xDEADBEEFDEADBEEF, UInt8(D));
+    unpacked_raw_data = RawData(UInt32(A),0x00, 0x0000,UInt16(B),0x00000000,UInt128(C),
+                       0x0000, 0x0000000000000000, UInt8(D));
+    #
+    @test packed_sizeof(RawData) == packed_sizeof(GappedStruct)
+	for endian in [:LittleEndian, :BigEndian]
+        buf = IOBuffer()
+        pack(buf, raw_data, endian)
+        seekstart(buf)
+        @test unpack(buf, GappedStruct, endian) == gapped_struct
+        #
+        buf = IOBuffer(zeros(UInt8, packed_sizeof(GappedStruct)), read=true, write=true)
+        pack(buf, gapped_struct, endian)
+        seekstart(buf)
+        @test unpack(buf, GappedStruct, endian) == gapped_struct
+        seekstart(buf)
+        @test unpack(buf, RawData, endian) == unpacked_raw_data
+    end
+end