Offset pack

src/StructIO.jl

@@ -50,6 +50,7 @@
 
 # Alignment traits
 abstract type PackingStrategy end
+struct Offset <: PackingStrategy; end
 struct Packed <: PackingStrategy; end
 struct Default <: PackingStrategy; end
 
@@ -73,6 +74,14 @@
     return sum(packed_sizeof, T.types)
 end
 
+@pure function packed_sizeof(T::DataType, ::Type{Offset})
+    @assert fieldcount(T) != 0 && isbitstype(T)
+    f_offsets = StructIO.stream_offset.(T, 1:fieldcount(T))
+    max_offset,idx_max = findmax(f_offsets)
+
+    return max_offset + packed_sizeof(T.types[idx_max])
+end
+# Offset struct may be packed with gaps
 @pure function packed_sizeof(T::DataType)
     if fieldcount(T) == 0
         return packed_sizeof(T, Default)
@@ -81,6 +90,8 @@
     end
 end
 
+stream_offset(::Type{T}, idx::Integer) where T = fieldoffset(T, idx)
+
 """
     fieldsize(T::DataType, field_idx)
 
@@ -135,10 +146,22 @@
     if isexpr(T, :curly)
         T = T.args[1]
     end
-
+    if alignment == :align_default
+        strat = StructIO.Default
+    elseif alignment == :align_packed
+        strat = StructIO.Packed
+    elseif alignment == :align_offset
+        strat = StructIO.Offset
+        field_offsets =  parse_annotations!(typ)
+    else
+         throw(ArgumentError("unknown alignment specification"))
+    end
     ret = Expr(:toplevel, :(Base.@__doc__ $(typ)))
-    strat = (alignment == :align_default ? StructIO.Default : StructIO.Packed)
     push!(ret.args, :($StructIO.packing_strategy(::Type{T}) where {T <: $T} = $strat))
+    if strat == Offset
+        push!(ret.args, Expr(:(=), :($StructIO.stream_offset(::Type{T},idx::Integer)
+                                     where {T <: $T}), :($field_offsets[idx])))
+    end
     return esc(ret)
 end
 
@@ -259,6 +282,23 @@
     end
 end
 
+# `Offset` packing strategy override for `unsafe_unpack`
+function unsafe_unpack(io, T, target, endianness, ::Type{Offset})
+    if fieldcount(T) == 0
+        return unsafe_unpack(io, T, target, endianness, Default)
+    end
+    f_gaps,idx = field_gaps(T)
+    @assert all(x->x>=0, f_gaps) "packed fields of structure $T overlap"
+    target_ptr = Base.unsafe_convert(Ptr{Cvoid}, target)
+    for i = 1:fieldcount(T)
+        k = idx[i]
+        fT = fieldtype(T, k)
+        target_i = target_ptr + fieldoffset(T, k)
+        skip(io, f_gaps[i])
+        unsafe_unpack(io, fT, target_i, endianness, Packed)
+    end
+end
+
 # `Packed` packing strategy override for `unsafe_pack`
 function unsafe_pack(io, source::Ref{T}, endianness, ::Type{Packed}) where {T}
     # If this type cannot be subdivided, packing strategy means nothing, so
@@ -276,6 +316,13 @@
     end
 end
 
+# `Offset` packing strategy override for `unsafe_pack`
+function unsafe_pack(io, source::Ref{T}, endianness, ::Type{Offset}) where {T}
+    @assert all(x->x>=0, field_gaps(T)[1]) "packed fields of structure $T overlap"
+    @warn "not yet implemented"
+    # unsafe_pack(io, source, endianness, Packed)
+end
+
 """
     unpack(io::IO, T::Type, endianness::Symbol = :NativeEndian)
 
@@ -311,4 +358,43 @@
     return nothing
 end
 
+is_declaration(f::Expr) = (f.head === :(::)) && isa.(f.args[1],Symbol)
+function is_annotation(f::Expr)
+    c = (f.head === :call) && (f.args[1] === :~)
+    c = c && is_declaration(f.args[2]) && isa(f.args[3], Integer)
+end
+
+function parse_annotations!(strct_expr)
+    @assert strct_expr.head == :struct
+    K = length(strct_expr.args[3].args)
+    field_offsets = UInt[]
+    for k in 1:length(strct_expr.args[3].args)
+        f = strct_expr.args[3].args[k]
+        !isa(f, LineNumberNode) || continue
+        is_annotation(f) || throw(ArgumentError("all fields must have offset annotations"))
+        push!(field_offsets, f.args[3])
+        strct_expr.args[3].args[k] = f.args[2]
+    end
+
+    return field_offsets
+end
+
+"""
+Compute the gaps between the fields of a structure when packed.
+
+"""
+function field_gaps(::Type{T}) where {T}
+    f_offsets = Int.(StructIO.stream_offset.(T, 1:fieldcount(T)))
+    # Do not assume that the offsets are in ascending order
+    idx = sortperm(f_offsets)
+    gaps = zeros(Int, fieldcount(T))
+    gaps[1] = f_offsets[idx[1]]
+    for k in 2:fieldcount(T)
+        gaps[k] = (f_offsets[idx[k]] - f_offsets[idx[k - 1]]
+                   - sizeof(fieldtype(T, idx[k - 1])))
+    end
+
+    return gaps,idx
+end
+
 end # module

test/runtests.jl

@@ -42,6 +42,31 @@ This is a docstring
     C::T
 end
 
+@io struct RawData
+    A::UInt32       # offset 0
+    dummy_1::UInt16 # offset 4
+    B::UInt16       # offset 6
+    dummy_2::UInt32 # offset 8
+    C::UInt128      # offset 12
+    dummy_3::UInt16 # offset 28
+    dummy_4::UInt32 # offset 30
+    D::UInt8        # offset 34
+end align_packed
+
+@io struct OffsetStruct
+	A::UInt32  ~ 0
+    C::UInt128 ~ 12
+    B::UInt16  ~ 6
+    D::UInt8   ~ 34
+end align_offset
+
+@io struct OverlappingFields
+	A::UInt32  ~ 0
+    B::UInt16  ~ 6
+    D::UInt8   ~ 27
+    C::UInt128 ~ 12
+end align_offset
+
 @testset "unpack()" begin
     # Test native unpacking
     buf = IOBuffer()
@@ -167,3 +192,24 @@ end
 @testset "Documentation" begin
     @test string(@doc ParametricType) == "This is a docstring\n"
 end
+
+@testset "Offset alignment" begin
+    A,B,C,D = 1,2,3,4
+    raw_data = RawData(UInt32(A),0xBEEF,UInt16(B),0xDEADBEEF,UInt128(C),
+                       0xBEEF, 0xDEADBEEF, UInt8(D));
+    missing_annotation = :(@io struct MissingAnnotation
+	                           x::Int ~ 0
+                               y::Int
+	                           z::Int ~ 6
+	                       end align_offset)
+    #
+	@test_throws Exception macroexpand(@__MODULE__, missing_annotation)
+    @test packed_sizeof(RawData) == packed_sizeof(OffsetStruct)
+	for endian in [:LittleEndian, :BigEndian]
+        buf = IOBuffer()
+        pack(buf, raw_data, endian)
+        seekstart(buf)
+        @test unpack(buf, OffsetStruct, endian) == OffsetStruct(A, C, B, D)
+    end
+    @test_throws AssertionError unpack(IOBuffer(), OverlappingFields)
+end