diff --git a/certstore/cbor_gen.go b/certstore/cbor_gen.go new file mode 100644 index 00000000..daef095f --- /dev/null +++ b/certstore/cbor_gen.go @@ -0,0 +1,175 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + +package certstore + +import ( + "fmt" + "io" + "math" + "sort" + + gpbft "github.com/filecoin-project/go-f3/gpbft" + cid "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +var _ = xerrors.Errorf +var _ = cid.Undef +var _ = math.E +var _ = sort.Sort + +var lengthBufSnapshotHeader = []byte{132} + +func (t *SnapshotHeader) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write(lengthBufSnapshotHeader); err != nil { + return err + } + + // t.Version (uint64) (uint64) + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Version)); err != nil { + return err + } + + // t.FirstInstance (uint64) (uint64) + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.FirstInstance)); err != nil { + return err + } + + // t.LatestInstance (uint64) (uint64) + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.LatestInstance)); err != nil { + return err + } + + // t.InitialPowerTable (gpbft.PowerEntries) (slice) + if len(t.InitialPowerTable) > 8192 { + return xerrors.Errorf("Slice value in field t.InitialPowerTable was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.InitialPowerTable))); err != nil { + return err + } + for _, v := range t.InitialPowerTable { + if err := v.MarshalCBOR(cw); err != nil { + return err + } + + } + return nil +} + +func (t *SnapshotHeader) UnmarshalCBOR(r io.Reader) (err error) { + *t = SnapshotHeader{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 4 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.Version (uint64) (uint64) + + { + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Version = uint64(extra) + + } + // t.FirstInstance (uint64) (uint64) + + { + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.FirstInstance = uint64(extra) + + } + // t.LatestInstance (uint64) (uint64) + + { + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.LatestInstance = uint64(extra) + + } + // t.InitialPowerTable (gpbft.PowerEntries) (slice) + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 8192 { + return fmt.Errorf("t.InitialPowerTable: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.InitialPowerTable = make([]gpbft.PowerEntry, extra) + } + + for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + + if err := t.InitialPowerTable[i].UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.InitialPowerTable[i]: %w", err) + } + + } + + } + } + return nil +} diff --git a/certstore/snapshot.go b/certstore/snapshot.go new file mode 100644 index 00000000..05127380 --- /dev/null +++ b/certstore/snapshot.go @@ -0,0 +1,184 @@ +package certstore + +import ( + "bytes" + "context" + "encoding/binary" + "errors" + "fmt" + "hash" + "io" + + "github.com/filecoin-project/go-f3/certs" + "github.com/filecoin-project/go-f3/gpbft" + "github.com/filecoin-project/go-state-types/cbor" + cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + "github.com/multiformats/go-multihash" + "golang.org/x/crypto/blake2b" +) + +var ErrUnknownLatestCertificate = errors.New("latest certificate is not known") + +// ExportLatestSnapshot exports an F3 snapshot that includes the finality certificate chain until the current `latestCertificate`. +// +// Checkout the snapshot format specification at +func (cs *Store) ExportLatestSnapshot(ctx context.Context, writer io.Writer) (cid.Cid, *SnapshotHeader, error) { + if cs.latestCertificate == nil { + return cid.Undef, nil, ErrUnknownLatestCertificate + } + return cs.ExportSnapshot(ctx, cs.latestCertificate.GPBFTInstance, writer) +} + +// ExportSnapshot exports an F3 snapshot that includes the finality certificate chain from the `Store.firstInstance` to the specified `lastInstance`. +// +// Checkout the snapshot format specification at +func (cs *Store) ExportSnapshot(ctx context.Context, latestInstance uint64, writer io.Writer) (cid.Cid, *SnapshotHeader, error) { + hasher, err := blake2b.New256(nil) + if err != nil { + return cid.Undef, nil, err + } + hashWriter := hashWriter{hasher, writer} + initialPowerTable, err := cs.GetPowerTable(ctx, cs.firstInstance) + if err != nil { + return cid.Undef, nil, fmt.Errorf("failed to get initial power table at instance %d: %w", cs.firstInstance, err) + } + header := SnapshotHeader{1, cs.firstInstance, latestInstance, initialPowerTable} + if _, err := header.WriteTo(hashWriter); err != nil { + return cid.Undef, nil, fmt.Errorf("failed to write snapshot header: %w", err) + } + for i := cs.firstInstance; i <= latestInstance; i++ { + cert, err := cs.ds.Get(ctx, cs.keyForCert(i)) + if err != nil { + return cid.Undef, nil, fmt.Errorf("failed to get certificate at instance %d:: %w", i, err) + } + buffer := bytes.NewBuffer(cert) + if _, err := writeSnapshotBlockBytes(hashWriter, buffer); err != nil { + return cid.Undef, nil, err + } + } + hash := hashWriter.hasher.Sum(nil) + mh, err := multihash.Encode(hash, multihash.BLAKE2B_MIN+31) + if err != nil { + return cid.Undef, nil, err + } + + return cid.NewCidV1(cid.Raw, mh), &header, nil +} + +type hashWriter struct { + hasher hash.Hash + writer io.Writer +} + +func (w hashWriter) Write(p []byte) (n int, err error) { + if _, err := w.hasher.Write(p); err != nil { + return 0, err + } + return w.writer.Write(p) +} + +type SnapshotReader interface { + io.Reader + io.ByteReader +} + +// ImportSnapshotToDatastore imports an F3 snapshot into the specified Datastore +// +// Checkout the snapshot format specification at +func ImportSnapshotToDatastore(ctx context.Context, snapshot SnapshotReader, ds datastore.Datastore) error { + return importSnapshotToDatastoreWithTestingPowerTableFrequency(ctx, snapshot, ds, 0) +} + +func importSnapshotToDatastoreWithTestingPowerTableFrequency(ctx context.Context, snapshot SnapshotReader, ds datastore.Datastore, testingPowerTableFrequency uint64) error { + headerBytes, err := readSnapshotBlockBytes(snapshot) + if err != nil { + return err + } + var header SnapshotHeader + err = header.UnmarshalCBOR(bytes.NewReader(headerBytes)) + if err != nil { + return fmt.Errorf("failed to decode snapshot header: %w", err) + } + cs, err := OpenOrCreateStore(ctx, ds, header.FirstInstance, header.InitialPowerTable) + if testingPowerTableFrequency > 0 { + cs.powerTableFrequency = testingPowerTableFrequency + } + if err != nil { + return err + } + pt := header.InitialPowerTable + for { + certBytes, err := readSnapshotBlockBytes(snapshot) + if err == io.EOF { + break + } else if err != nil { + return fmt.Errorf("failed to decode finality certificate: %w", err) + } + var cert certs.FinalityCertificate + cert.UnmarshalCBOR(bytes.NewReader(certBytes)) + if err = cs.Put(ctx, &cert); err != nil { + return err + } + if pt, err = certs.ApplyPowerTableDiffs(pt, cert.PowerTableDelta); err != nil { + return err + } + if (cert.GPBFTInstance+1)%cs.powerTableFrequency == 0 { + if err := cs.putPowerTable(ctx, cert.GPBFTInstance+1, pt); err != nil { + return err + } + } + } + return nil +} + +type SnapshotHeader struct { + Version uint64 + FirstInstance uint64 + LatestInstance uint64 + InitialPowerTable gpbft.PowerEntries +} + +func (h *SnapshotHeader) WriteTo(w io.Writer) (int64, error) { + return writeSnapshotCborEncodedBlock(w, h) +} + +// writeSnapshotCborEncodedBlock writes CBOR-encoded header or data block with a varint-encoded length prefix +func writeSnapshotCborEncodedBlock(writer io.Writer, block cbor.Marshaler) (int64, error) { + var buffer bytes.Buffer + if err := block.MarshalCBOR(&buffer); err != nil { + return 0, err + } + return writeSnapshotBlockBytes(writer, &buffer) +} + +// writeSnapshotBlockBytes writes header or data block with a varint-encoded length prefix +func writeSnapshotBlockBytes(writer io.Writer, buffer *bytes.Buffer) (int64, error) { + buf := make([]byte, 8) + n := binary.PutUvarint(buf, uint64(buffer.Len())) + len1, err := bytes.NewBuffer(buf[:n]).WriteTo(writer) + if err != nil { + return 0, err + } + len2, err := buffer.WriteTo(writer) + if err != nil { + return 0, err + } + return len1 + len2, nil +} + +func readSnapshotBlockBytes(reader SnapshotReader) ([]byte, error) { + n1, err := binary.ReadUvarint(reader) + if err != nil { + return nil, err + } + buf := make([]byte, n1) + n2, err := reader.Read(buf) + if err != nil { + return nil, err + } + if n2 != int(n1) { + return nil, fmt.Errorf("incomplete block, %d bytes expected, %d bytes got", n1, n2) + } + return buf, nil +} diff --git a/certstore/snapshot_test.go b/certstore/snapshot_test.go new file mode 100644 index 00000000..83c1be93 --- /dev/null +++ b/certstore/snapshot_test.go @@ -0,0 +1,150 @@ +package certstore + +import ( + "bytes" + "context" + "math/rand" + "testing" + "time" + + "github.com/filecoin-project/go-f3/certchain" + "github.com/filecoin-project/go-f3/gpbft" + "github.com/filecoin-project/go-f3/internal/clock" + "github.com/filecoin-project/go-f3/internal/consensus" + "github.com/filecoin-project/go-f3/manifest" + "github.com/filecoin-project/go-f3/sim/signing" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + "github.com/multiformats/go-multihash" + "github.com/stretchr/testify/require" + "go.uber.org/zap/buffer" + "golang.org/x/crypto/blake2b" +) + +func Test_SnapshotExportImportRoundTrip(t *testing.T) { + const ( + seed = 1427 + certChainLength = 150 + testingPowerTableFreqency = uint64(23) + ) + + ctx, clk := clock.WithMockClock(context.Background()) + m := manifest.LocalDevnetManifest() + m.InitialInstance = 100 + signVerifier := signing.NewFakeBackend() + rng := rand.New(rand.NewSource(seed * 23)) + generatePublicKey := func(id gpbft.ActorID) gpbft.PubKey { + //TODO: add the ability to evolve public key across instances. Fake signing + // backed does not support this. + + // Use allow instead of GenerateKey for a reproducible key generation. + return signVerifier.Allow(int(id)) + } + initialPowerTable := generatePowerTable(t, rng, generatePublicKey, nil) + + ec := consensus.NewFakeEC( + consensus.WithClock(clk), + consensus.WithSeed(seed*13), + consensus.WithBootstrapEpoch(m.BootstrapEpoch), + consensus.WithECPeriod(m.EC.Period), + consensus.WithInitialPowerTable(initialPowerTable), + consensus.WithEvolvingPowerTable( + func(epoch int64, entries gpbft.PowerEntries) gpbft.PowerEntries { + if epoch == m.BootstrapEpoch-m.EC.Finality { + return initialPowerTable + } + rng := rand.New(rand.NewSource(epoch * seed)) + next := generatePowerTable(t, rng, generatePublicKey, entries) + return next + }, + ), + ) + + subject, err := certchain.New( + certchain.WithSeed(seed), + certchain.WithSignVerifier(signVerifier), + certchain.WithManifest(m), + certchain.WithEC(ec), + ) + require.NoError(t, err) + + // The mock clock is buried into context passed to fake EC. The face EC will + // refuse to generate a chain if the clock is not advanced. Advance it + // sufficiently to never be bothered by it again. + // + // The fake EC and its relationship with clock needs to be reworked: Clock should + // ideally be passed as an option, and its absence should mean "advance the clock + // as needed". Because, we do not always care about controlling the progress of + // chain generated by fake EC. + clk.Add(200 * time.Hour) + + generatedChain, err := subject.Generate(ctx, certChainLength) + require.NoError(t, err) + + ds1 := datastore.NewMapDatastore() + cs, err := OpenOrCreateStore(ctx, ds1, generatedChain[0].GPBFTInstance, initialPowerTable) + cs.powerTableFrequency = testingPowerTableFreqency + require.NoError(t, err) + + for _, cert := range generatedChain { + cs.Put(ctx, cert) + } + + snapshot := buffer.Buffer{} + c, _, err := cs.ExportLatestSnapshot(ctx, &snapshot) + require.NoError(t, err) + require.NotEqual(t, c, cid.Undef) + require.Equal(t, int(c.Prefix().Version), 1) + require.Equal(t, int(c.Prefix().Codec), cid.Raw) + require.Equal(t, int(c.Prefix().MhType), 0xb220) + hash := blake2b.Sum256(snapshot.Bytes()) + mh, err := multihash.Encode(hash[:], multihash.BLAKE2B_MIN+31) + require.NoError(t, err) + require.Equal(t, c.Hash(), multihash.Multihash(mh)) + + ds2 := datastore.NewMapDatastore() + err = importSnapshotToDatastoreWithTestingPowerTableFrequency(ctx, bytes.NewReader(snapshot.Bytes()), ds2, testingPowerTableFreqency) + require.NoError(t, err) + + require.Equal(t, ds1, ds2) + + ds3 := datastore.NewMapDatastore() + err = ImportSnapshotToDatastore(ctx, bytes.NewReader(snapshot.Bytes()), ds3) + require.NoError(t, err) + + require.NotEqual(t, ds1, ds3) +} + +func generatePowerTable(t *testing.T, rng *rand.Rand, generatePublicKey func(id gpbft.ActorID) gpbft.PubKey, previousEntries gpbft.PowerEntries) gpbft.PowerEntries { + const ( + maxEntries = 100 + maxPower = 1 << 20 + minPower = 0 // Pick a sufficiently low power to facilitate entries with zero scaled power. + actorIDOffset = 1413 + powerChangeProbability = 0.2 + ) + + size := rng.Intn(maxEntries) + entries := make(gpbft.PowerEntries, 0, size) + for i := range size { + var entry gpbft.PowerEntry + if i < previousEntries.Len() { + entry = previousEntries[i] + changedPower := rng.Float64() > powerChangeProbability + if changedPower { + entry.Power = gpbft.NewStoragePower(int64(rng.Intn(maxPower) + minPower)) + } + } else { + id := gpbft.ActorID(uint64(actorIDOffset + i)) + entry = gpbft.PowerEntry{ + ID: id, + Power: gpbft.NewStoragePower(int64(rng.Intn(maxPower) + minPower)), + PubKey: generatePublicKey(id), + } + } + entries = append(entries, entry) + } + next := gpbft.NewPowerTable() + require.NoError(t, next.Add(entries...)) + return next.Entries +} diff --git a/f3.go b/f3.go index 892433c2..268390a7 100644 --- a/f3.go +++ b/f3.go @@ -113,25 +113,35 @@ func (m *F3) Broadcast(ctx context.Context, signatureBuilder *gpbft.SignatureBui } func (m *F3) GetLatestCert(context.Context) (*certs.FinalityCertificate, error) { - if state := m.state.Load(); state != nil { - return state.cs.Latest(), nil + cs, err := m.GetCertStore() + if err != nil { + return nil, err } - return nil, ErrF3NotRunning + return cs.Latest(), nil } func (m *F3) GetCert(ctx context.Context, instance uint64) (*certs.FinalityCertificate, error) { - if state := m.state.Load(); state != nil { - return state.cs.Get(ctx, instance) + cs, err := m.GetCertStore() + if err != nil { + return nil, err + } + return cs.Get(ctx, instance) +} + +func (m *F3) GetCertStore() (*certstore.Store, error) { + if state := m.state.Load(); state != nil && state.cs != nil { + return state.cs, nil } return nil, ErrF3NotRunning } // GetPowerTableByInstance returns the power table (committee) used to validate the specified instance. func (m *F3) GetPowerTableByInstance(ctx context.Context, instance uint64) (gpbft.PowerEntries, error) { - if state := m.state.Load(); state != nil { - return state.cs.GetPowerTable(ctx, instance) + cs, err := m.GetCertStore() + if err != nil { + return nil, err } - return nil, ErrF3NotRunning + return cs.GetPowerTable(ctx, instance) } // computeBootstrapDelay returns the time at which the F3 instance specified by diff --git a/gen/main.go b/gen/main.go index 47318200..f2e14f04 100644 --- a/gen/main.go +++ b/gen/main.go @@ -6,6 +6,7 @@ import ( "github.com/filecoin-project/go-f3/certexchange" "github.com/filecoin-project/go-f3/certs" + "github.com/filecoin-project/go-f3/certstore" "github.com/filecoin-project/go-f3/chainexchange" "github.com/filecoin-project/go-f3/gpbft" gen "github.com/whyrusleeping/cbor-gen" @@ -47,6 +48,11 @@ func main() { chainexchange.Message{}, ) }) + eg.Go(func() error { + return gen.WriteTupleEncodersToFile("../certstore/cbor_gen.go", "certstore", + certstore.SnapshotHeader{}, + ) + }) if err := eg.Wait(); err != nil { fmt.Printf("Failed to complete cborg_gen: %v\n", err) os.Exit(1)