From f5b7866287cbe5675db0363472ada7a461a675f8 Mon Sep 17 00:00:00 2001 From: Keagan McClelland Date: Fri, 8 Dec 2023 12:50:43 -0800 Subject: [PATCH 01/20] htlcswitch: define state machine for quiescence htlcswitch: add sendOwedStfu method to quiescer --- go.mod | 2 +- go.sum | 4 +- htlcswitch/quiescer.go | 339 ++++++++++++++++++++++++++++++++++++ htlcswitch/quiescer_test.go | 247 ++++++++++++++++++++++++++ 4 files changed, 589 insertions(+), 3 deletions(-) create mode 100644 htlcswitch/quiescer.go create mode 100644 htlcswitch/quiescer_test.go diff --git a/go.mod b/go.mod index 6e2bd9f77d1..d0ff97ac7cc 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/lightningnetwork/lightning-onion v1.2.1-0.20240712235311-98bd56499dfb github.com/lightningnetwork/lnd/cert v1.2.2 github.com/lightningnetwork/lnd/clock v1.1.1 - github.com/lightningnetwork/lnd/fn v1.2.3 + github.com/lightningnetwork/lnd/fn v1.2.5 github.com/lightningnetwork/lnd/healthcheck v1.2.6 github.com/lightningnetwork/lnd/kvdb v1.4.11 github.com/lightningnetwork/lnd/queue v1.1.1 diff --git a/go.sum b/go.sum index 86c1c8a21ae..2ea42fd8ce0 100644 --- a/go.sum +++ b/go.sum @@ -456,8 +456,8 @@ github.com/lightningnetwork/lnd/cert v1.2.2 h1:71YK6hogeJtxSxw2teq3eGeuy4rHGKcFf github.com/lightningnetwork/lnd/cert v1.2.2/go.mod h1:jQmFn/Ez4zhDgq2hnYSw8r35bqGVxViXhX6Cd7HXM6U= github.com/lightningnetwork/lnd/clock v1.1.1 h1:OfR3/zcJd2RhH0RU+zX/77c0ZiOnIMsDIBjgjWdZgA0= github.com/lightningnetwork/lnd/clock v1.1.1/go.mod h1:mGnAhPyjYZQJmebS7aevElXKTFDuO+uNFFfMXK1W8xQ= -github.com/lightningnetwork/lnd/fn v1.2.3 h1:Q1OrgNSgQynVheBNa16CsKVov1JI5N2AR6G07x9Mles= -github.com/lightningnetwork/lnd/fn v1.2.3/go.mod h1:SyFohpVrARPKH3XVAJZlXdVe+IwMYc4OMAvrDY32kw0= +github.com/lightningnetwork/lnd/fn v1.2.5 h1:pGMz0BDUxrhvOtShD4FIysdVy+ulfFAnFvTKjZO5Pp8= +github.com/lightningnetwork/lnd/fn v1.2.5/go.mod h1:SyFohpVrARPKH3XVAJZlXdVe+IwMYc4OMAvrDY32kw0= github.com/lightningnetwork/lnd/healthcheck v1.2.6 h1:1sWhqr93GdkWy4+6U7JxBfcyZIE78MhIHTJZfPx7qqI= github.com/lightningnetwork/lnd/healthcheck v1.2.6/go.mod h1:Mu02um4CWY/zdTOvFje7WJgJcHyX2zq/FG3MhOAiGaQ= github.com/lightningnetwork/lnd/kvdb v1.4.11 h1:fk1HMVFrsVK3xqU7q+JWHRgBltw/a2qIg1E3zazMb/8= diff --git a/htlcswitch/quiescer.go b/htlcswitch/quiescer.go new file mode 100644 index 00000000000..9bde04c322b --- /dev/null +++ b/htlcswitch/quiescer.go @@ -0,0 +1,339 @@ +package htlcswitch + +import ( + "fmt" + "sync" + + "github.com/lightningnetwork/lnd/fn" + "github.com/lightningnetwork/lnd/lntypes" + "github.com/lightningnetwork/lnd/lnwire" +) + +var ( + // ErrInvalidStfu indicates that the Stfu we have received is invalid. + // This can happen in instances where we have not sent Stfu but we have + // received one with the initiator field set to false. + ErrInvalidStfu = fmt.Errorf("stfu received is invalid") + + // ErrStfuAlreadySent indicates that this channel has already sent an + // Stfu message for this negotiation. + ErrStfuAlreadySent = fmt.Errorf("stfu already sent") + + // ErrStfuAlreadyRcvd indicates that this channel has already received + // an Stfu message for this negotiation. + ErrStfuAlreadyRcvd = fmt.Errorf("stfu already received") + + // ErrNoQuiescenceInitiator indicates that the caller has requested the + // quiescence initiator for a channel that is not yet quiescent. + ErrNoQuiescenceInitiator = fmt.Errorf( + "indeterminate quiescence initiator: channel is not quiescent", + ) + + // ErrPendingRemoteUpdates indicates that we have received an Stfu while + // the remote party has issued updates that are not yet bilaterally + // committed. + ErrPendingRemoteUpdates = fmt.Errorf( + "stfu received with pending remote updates", + ) + + // ErrPendingLocalUpdates indicates that we are attempting to send an + // Stfu while we have issued updates that are not yet bilaterally + // committed. + ErrPendingLocalUpdates = fmt.Errorf( + "stfu send attempted with pending local updates", + ) +) + +// QuiescerCfg is a config structure used to initialize a quiescer giving it the +// appropriate functionality to interact with the channel state that the +// quiescer must syncrhonize with. +type QuiescerCfg struct { + // chanID marks what channel we are managing the state machine for. This + // is important because the quiescer needs to know the ChannelID to + // construct the Stfu message. + chanID lnwire.ChannelID + + // channelInitiator indicates which ChannelParty originally opened the + // channel. This is used to break ties when both sides of the channel + // send Stfu claiming to be the initiator. + channelInitiator lntypes.ChannelParty + + // sendMsg is a function that can be used to send an Stfu message over + // the wire. + sendMsg func(lnwire.Stfu) error +} + +// Quiescer is a state machine that tracks progression through the quiescence +// protocol. +type Quiescer struct { + cfg QuiescerCfg + + // localInit indicates whether our path through this state machine was + // initiated by our node. This can be true or false independently of + // remoteInit. + localInit bool + + // remoteInit indicates whether we received Stfu from our peer where the + // message indicated that the remote node believes it was the initiator. + // This can be true or false independently of localInit. + remoteInit bool + + // sent tracks whether or not we have emitted Stfu for sending. + sent bool + + // received tracks whether or not we have received Stfu from our peer. + received bool + + sync.RWMutex +} + +// NewQuiescer creates a new quiescer for the given channel. +func NewQuiescer(cfg QuiescerCfg) Quiescer { + return Quiescer{ + cfg: cfg, + } +} + +// RecvStfu is called when we receive an Stfu message from the remote. +func (q *Quiescer) RecvStfu(msg lnwire.Stfu, + numPendingRemoteUpdates uint64) error { + + q.Lock() + defer q.Unlock() + + return q.recvStfu(msg, numPendingRemoteUpdates) +} + +// recvStfu is called when we receive an Stfu message from the remote. +func (q *Quiescer) recvStfu(msg lnwire.Stfu, + numPendingRemoteUpdates uint64) error { + + // At the time of this writing, this check that we have already received + // an Stfu is not strictly necessary, according to the specification. + // However, it is fishy if we do and it is unclear how we should handle + // such a case so we will err on the side of caution. + if q.received { + return fmt.Errorf("%w for channel %v", ErrStfuAlreadyRcvd, + q.cfg.chanID) + } + + // We need to check that the Stfu we are receiving is valid. + if !q.sent && !msg.Initiator { + return fmt.Errorf("%w for channel %v", ErrInvalidStfu, + q.cfg.chanID) + } + + if !q.canRecvStfu(numPendingRemoteUpdates) { + return fmt.Errorf("%w for channel %v", ErrPendingRemoteUpdates, + q.cfg.chanID) + } + + q.received = true + + // If the remote party sets the initiator bit to true then we will + // remember that they are making a claim to the initiator role. This + // does not necessarily mean they will get it, though. + q.remoteInit = msg.Initiator + + return nil +} + +// MakeStfu is called when we are ready to send an Stfu message. It returns the +// Stfu message to be sent. +func (q *Quiescer) MakeStfu( + numPendingLocalUpdates uint64) fn.Result[lnwire.Stfu] { + + q.RLock() + defer q.RUnlock() + + return q.makeStfu(numPendingLocalUpdates) +} + +// makeStfu is called when we are ready to send an Stfu message. It returns the +// Stfu message to be sent. +func (q *Quiescer) makeStfu( + numPendingLocalUpdates uint64) fn.Result[lnwire.Stfu] { + + if q.sent { + return fn.Errf[lnwire.Stfu]("%w for channel %v", + ErrStfuAlreadySent, q.cfg.chanID) + } + + if !q.canSendStfu(numPendingLocalUpdates) { + return fn.Errf[lnwire.Stfu]("%w for channel %v", + ErrPendingLocalUpdates, q.cfg.chanID) + } + + stfu := lnwire.Stfu{ + ChanID: q.cfg.chanID, + Initiator: q.localInit, + } + + return fn.Ok(stfu) +} + +// OweStfu returns true if we owe the other party an Stfu. We owe the remote an +// Stfu when we have received but not yet sent an Stfu, or we are the initiator +// but have not yet sent an Stfu. +func (q *Quiescer) OweStfu() bool { + q.RLock() + defer q.RUnlock() + + return q.oweStfu() +} + +// oweStfu returns true if we owe the other party an Stfu. We owe the remote an +// Stfu when we have received but not yet sent an Stfu, or we are the initiator +// but have not yet sent an Stfu. +func (q *Quiescer) oweStfu() bool { + return q.received && !q.sent +} + +// NeedStfu returns true if the remote owes us an Stfu. They owe us an Stfu when +// we have sent but not yet received an Stfu. +func (q *Quiescer) NeedStfu() bool { + q.RLock() + defer q.RUnlock() + + return q.needStfu() +} + +// needStfu returns true if the remote owes us an Stfu. They owe us an Stfu when +// we have sent but not yet received an Stfu. +func (q *Quiescer) needStfu() bool { + q.RLock() + defer q.RUnlock() + + return q.sent && !q.received +} + +// IsQuiescent returns true if the state machine has been driven all the way to +// completion. If this returns true, processes that depend on channel quiescence +// may proceed. +func (q *Quiescer) IsQuiescent() bool { + q.RLock() + defer q.RUnlock() + + return q.isQuiescent() +} + +// isQuiescent returns true if the state machine has been driven all the way to +// completion. If this returns true, processes that depend on channel quiescence +// may proceed. +func (q *Quiescer) isQuiescent() bool { + return q.sent && q.received +} + +// QuiescenceInitiator determines which ChannelParty is the initiator of +// quiescence for the purposes of downstream protocols. If the channel is not +// currently quiescent, this method will return ErrNoQuiescenceInitiator. +func (q *Quiescer) QuiescenceInitiator() fn.Result[lntypes.ChannelParty] { + q.RLock() + defer q.RUnlock() + + return q.quiescenceInitiator() +} + +// quiescenceInitiator determines which ChannelParty is the initiator of +// quiescence for the purposes of downstream protocols. If the channel is not +// currently quiescent, this method will return ErrNoQuiescenceInitiator. +func (q *Quiescer) quiescenceInitiator() fn.Result[lntypes.ChannelParty] { + switch { + case !q.isQuiescent(): + return fn.Err[lntypes.ChannelParty](ErrNoQuiescenceInitiator) + + case q.localInit && q.remoteInit: + // In the case of a tie, the channel initiator wins. + return fn.Ok(q.cfg.channelInitiator) + + case q.localInit: + return fn.Ok(lntypes.Local) + + case q.remoteInit: + return fn.Ok(lntypes.Remote) + } + + // unreachable + return fn.Err[lntypes.ChannelParty](ErrNoQuiescenceInitiator) +} + +// CanSendUpdates returns true if we haven't yet sent an Stfu which would mark +// the end of our ability to send updates. +func (q *Quiescer) CanSendUpdates() bool { + q.RLock() + defer q.RUnlock() + + return q.canSendUpdates() +} + +// canSendUpdates returns true if we haven't yet sent an Stfu which would mark +// the end of our ability to send updates. +func (q *Quiescer) canSendUpdates() bool { + return !q.sent && !q.localInit +} + +// CanRecvUpdates returns true if we haven't yet received an Stfu which would +// mark the end of the remote's ability to send updates. +func (q *Quiescer) CanRecvUpdates() bool { + q.RLock() + defer q.RUnlock() + + return q.canRecvUpdates() +} + +// canRecvUpdates returns true if we haven't yet received an Stfu which would +// mark the end of the remote's ability to send updates. +func (q *Quiescer) canRecvUpdates() bool { + return !q.received +} + +// CanSendStfu returns true if we can send an Stfu. +func (q *Quiescer) CanSendStfu(numPendingLocalUpdates uint64) bool { + q.RLock() + defer q.RUnlock() + + return q.canSendStfu(numPendingLocalUpdates) +} + +// canSendStfu returns true if we can send an Stfu. +func (q *Quiescer) canSendStfu(numPendingLocalUpdates uint64) bool { + return numPendingLocalUpdates == 0 && !q.sent +} + +// CanRecvStfu returns true if we can receive an Stfu. +func (q *Quiescer) CanRecvStfu(numPendingRemoteUpdates uint64) bool { + q.RLock() + defer q.RUnlock() + + return q.canRecvStfu(numPendingRemoteUpdates) +} + +// canRecvStfu returns true if we can receive an Stfu. +func (q *Quiescer) canRecvStfu(numPendingRemoteUpdates uint64) bool { + return numPendingRemoteUpdates == 0 && !q.received +} + +// SendOwedStfu sends Stfu if it owes one. It returns an error if the state +// machine is in an invalid state. +func (q *Quiescer) SendOwedStfu(numPendingLocalUpdates uint64) error { + q.Lock() + defer q.Unlock() + + return q.sendOwedStfu(numPendingLocalUpdates) +} + +// sendOwedStfu sends Stfu if it owes one. It returns an error if the state +// machine is in an invalid state. +func (q *Quiescer) sendOwedStfu(numPendingLocalUpdates uint64) error { + if !q.oweStfu() || !q.canSendStfu(numPendingLocalUpdates) { + return nil + } + + err := q.makeStfu(numPendingLocalUpdates).Sink(q.cfg.sendMsg) + + if err == nil { + q.sent = true + } + + return err +} diff --git a/htlcswitch/quiescer_test.go b/htlcswitch/quiescer_test.go new file mode 100644 index 00000000000..914f64b3c57 --- /dev/null +++ b/htlcswitch/quiescer_test.go @@ -0,0 +1,247 @@ +package htlcswitch + +import ( + "bytes" + "testing" + + "github.com/lightningnetwork/lnd/fn" + "github.com/lightningnetwork/lnd/lntypes" + "github.com/lightningnetwork/lnd/lnwire" + "github.com/stretchr/testify/require" +) + +var cid = lnwire.ChannelID(bytes.Repeat([]byte{0x00}, 32)) + +type quiescerTestHarness struct { + pendingUpdates lntypes.Dual[uint64] + quiescer Quiescer + conn <-chan lnwire.Stfu +} + +func initQuiescerTestHarness() *quiescerTestHarness { + conn := make(chan lnwire.Stfu, 1) + harness := &quiescerTestHarness{ + pendingUpdates: lntypes.Dual[uint64]{}, + conn: conn, + } + + harness.quiescer = NewQuiescer(QuiescerCfg{ + chanID: cid, + sendMsg: func(msg lnwire.Stfu) error { + conn <- msg + return nil + }, + }) + + return harness +} + +// TestQuiescerDoubleRecvInvalid ensures that we get an error response when we +// receive the Stfu message twice during the lifecycle of the quiescer. +func TestQuiescerDoubleRecvInvalid(t *testing.T) { + t.Parallel() + + harness := initQuiescerTestHarness() + + msg := lnwire.Stfu{ + ChanID: cid, + Initiator: true, + } + + err := harness.quiescer.RecvStfu(msg, harness.pendingUpdates.Remote) + require.NoError(t, err) + err = harness.quiescer.RecvStfu(msg, harness.pendingUpdates.Remote) + require.Error(t, err, ErrStfuAlreadyRcvd) +} + +// TestQuiescerPendingUpdatesRecvInvalid ensures that we get an error if we +// receive the Stfu message while the Remote party has panding updates on the +// channel. +func TestQuiescerPendingUpdatesRecvInvalid(t *testing.T) { + t.Parallel() + + harness := initQuiescerTestHarness() + + msg := lnwire.Stfu{ + ChanID: cid, + Initiator: true, + } + + harness.pendingUpdates.SetForParty(lntypes.Remote, 1) + err := harness.quiescer.RecvStfu(msg, harness.pendingUpdates.Remote) + require.ErrorIs(t, err, ErrPendingRemoteUpdates) +} + +// TestQuiescenceRemoteInit ensures that we can successfully traverse the state +// graph of quiescence beginning with the Remote party initiating quiescence. +func TestQuiescenceRemoteInit(t *testing.T) { + t.Parallel() + + harness := initQuiescerTestHarness() + + msg := lnwire.Stfu{ + ChanID: cid, + Initiator: true, + } + + harness.pendingUpdates.SetForParty(lntypes.Local, 1) + + err := harness.quiescer.RecvStfu(msg, harness.pendingUpdates.Remote) + require.NoError(t, err) + + err = harness.quiescer.SendOwedStfu(harness.pendingUpdates.Local) + require.NoError(t, err) + + select { + case <-harness.conn: + t.Fatalf("stfu sent when not expected") + default: + } + + harness.pendingUpdates.SetForParty(lntypes.Local, 0) + err = harness.quiescer.SendOwedStfu(harness.pendingUpdates.Local) + require.NoError(t, err) + + select { + case msg := <-harness.conn: + require.False(t, msg.Initiator) + default: + t.Fatalf("stfu not sent when expected") + } +} + +// TestQuiescenceInitiator ensures that the quiescenceInitiator is the Remote +// party when we have a receive first traversal of the quiescer's state graph. +func TestQuiescenceInitiator(t *testing.T) { + t.Parallel() + + harness := initQuiescerTestHarness() + require.True(t, harness.quiescer.QuiescenceInitiator().IsErr()) + + // Receive + msg := lnwire.Stfu{ + ChanID: cid, + Initiator: true, + } + require.NoError( + t, harness.quiescer.RecvStfu( + msg, harness.pendingUpdates.Remote, + ), + ) + require.True(t, harness.quiescer.QuiescenceInitiator().IsErr()) + + // Send + require.NoError( + t, harness.quiescer.SendOwedStfu(harness.pendingUpdates.Local), + ) + require.Equal( + t, harness.quiescer.QuiescenceInitiator(), + fn.Ok(lntypes.Remote), + ) +} + +// TestQuiescenceCantReceiveUpdatesAfterStfu tests that we can receive channel +// updates prior to but not after we receive Stfu. +func TestQuiescenceCantReceiveUpdatesAfterStfu(t *testing.T) { + t.Parallel() + + harness := initQuiescerTestHarness() + require.True(t, harness.quiescer.CanRecvUpdates()) + + msg := lnwire.Stfu{ + ChanID: cid, + Initiator: true, + } + require.NoError( + t, harness.quiescer.RecvStfu( + msg, harness.pendingUpdates.Remote, + ), + ) + require.False(t, harness.quiescer.CanRecvUpdates()) +} + +// TestQuiescenceCantSendUpdatesAfterStfu tests that we can send channel updates +// prior to but not after we send Stfu. +func TestQuiescenceCantSendUpdatesAfterStfu(t *testing.T) { + t.Parallel() + + harness := initQuiescerTestHarness() + require.True(t, harness.quiescer.CanSendUpdates()) + + msg := lnwire.Stfu{ + ChanID: cid, + Initiator: true, + } + + err := harness.quiescer.RecvStfu(msg, harness.pendingUpdates.Remote) + require.NoError(t, err) + + err = harness.quiescer.SendOwedStfu(harness.pendingUpdates.Local) + require.NoError(t, err) + + require.False(t, harness.quiescer.CanSendUpdates()) +} + +// TestQuiescenceStfuNotNeededAfterRecv tests that after we receive an Stfu we +// do not needStfu either before or after receiving it if we do not initiate +// quiescence. +func TestQuiescenceStfuNotNeededAfterRecv(t *testing.T) { + t.Parallel() + + harness := initQuiescerTestHarness() + + msg := lnwire.Stfu{ + ChanID: cid, + Initiator: true, + } + require.False(t, harness.quiescer.NeedStfu()) + + require.NoError( + t, harness.quiescer.RecvStfu( + msg, harness.pendingUpdates.Remote, + ), + ) + + require.False(t, harness.quiescer.NeedStfu()) +} + +// TestQuiescenceInappropriateMakeStfuReturnsErr ensures that we cannot call +// makeStfu at times when it would be a protocol violation to send it. +func TestQuiescenceInappropriateMakeStfuReturnsErr(t *testing.T) { + t.Parallel() + + harness := initQuiescerTestHarness() + + harness.pendingUpdates.SetForParty(lntypes.Local, 1) + + require.True( + t, harness.quiescer.MakeStfu( + harness.pendingUpdates.Local, + ).IsErr(), + ) + + harness.pendingUpdates.SetForParty(lntypes.Local, 0) + msg := lnwire.Stfu{ + ChanID: cid, + Initiator: true, + } + require.NoError( + t, harness.quiescer.RecvStfu( + msg, harness.pendingUpdates.Remote, + ), + ) + require.True( + t, harness.quiescer.MakeStfu( + harness.pendingUpdates.Local, + ).IsOk(), + ) + + require.NoError( + t, harness.quiescer.SendOwedStfu(harness.pendingUpdates.Local), + ) + require.True( + t, harness.quiescer.MakeStfu( + harness.pendingUpdates.Local, + ).IsErr(), + ) +} From 2ece1fdc5469c689e12ca5b45f642ed607eb4561 Mon Sep 17 00:00:00 2001 From: Keagan McClelland Date: Mon, 11 Dec 2023 21:41:31 -0800 Subject: [PATCH 02/20] htlcswitch: implement stfu response htlcswitch: use quiescer SendOwedStfu method in link stfu implementation --- htlcswitch/link.go | 71 +++++++++++++++++++++++++++++++++++++++ htlcswitch/linkfailure.go | 8 +++++ 2 files changed, 79 insertions(+) diff --git a/htlcswitch/link.go b/htlcswitch/link.go index 344bf77a4a9..46f06c9abe1 100644 --- a/htlcswitch/link.go +++ b/htlcswitch/link.go @@ -392,6 +392,10 @@ type channelLink struct { // our next CommitSig. incomingCommitHooks hookMap + // quiescer is the state machine that tracks where this channel is with + // respect to the quiescence protocol. + quiescer Quiescer + // ContextGuard is a helper that encapsulates a wait group and quit // channel and allows contexts that either block or cancel on those // depending on the use case. @@ -467,6 +471,16 @@ func NewChannelLink(cfg ChannelLinkConfig, cfg.MaxFeeExposure = DefaultMaxFeeExposure } + quiescerCfg := QuiescerCfg{ + chanID: lnwire.NewChanIDFromOutPoint( + channel.ChannelPoint(), + ), + channelInitiator: channel.Initiator(), + sendMsg: func(s lnwire.Stfu) error { + return cfg.Peer.SendMessage(false, &s) + }, + } + return &channelLink{ cfg: cfg, channel: channel, @@ -476,6 +490,7 @@ func NewChannelLink(cfg ChannelLinkConfig, flushHooks: newHookMap(), outgoingCommitHooks: newHookMap(), incomingCommitHooks: newHookMap(), + quiescer: NewQuiescer(quiescerCfg), ContextGuard: fn.NewContextGuard(), } } @@ -2325,6 +2340,19 @@ func (l *channelLink) handleUpstreamMsg(msg lnwire.Message) { } } + // If we need to send out an Stfu, this would be the time to do + // so. + pendingOnLocal := l.channel.NumPendingUpdates( + lntypes.Local, lntypes.Local, + ) + pendingOnRemote := l.channel.NumPendingUpdates( + lntypes.Local, lntypes.Remote, + ) + err = l.quiescer.SendOwedStfu(pendingOnLocal + pendingOnRemote) + if err != nil { + l.stfuFailf("sendOwedStfu: %v", err.Error()) + } + // Now that we have finished processing the incoming CommitSig // and sent out our RevokeAndAck, we invoke the flushHooks if // the channel state is clean. @@ -2458,6 +2486,12 @@ func (l *channelLink) handleUpstreamMsg(msg lnwire.Message) { // Update the mailbox's feerate as well. l.mailBox.SetFeeRate(fee) + case *lnwire.Stfu: + err := l.handleStfu(msg) + if err != nil { + l.stfuFailf("handleStfu: %v", err.Error()) + } + // In the case where we receive a warning message from our peer, just // log it and move on. We choose not to disconnect from our peer, // although we "MAY" do so according to the specification. @@ -2490,6 +2524,43 @@ func (l *channelLink) handleUpstreamMsg(msg lnwire.Message) { } +// handleStfu implements the top-level logic for handling the Stfu message from +// our peer. +func (l *channelLink) handleStfu(stfu *lnwire.Stfu) error { + pendingOnLocal := l.channel.NumPendingUpdates( + lntypes.Remote, lntypes.Local, + ) + pendingOnRemote := l.channel.NumPendingUpdates( + lntypes.Remote, lntypes.Remote, + ) + err := l.quiescer.RecvStfu(*stfu, pendingOnLocal+pendingOnRemote) + if err != nil { + return err + } + + // If we can immediately send an Stfu response back, we will. + pendingOnLocal = l.channel.NumPendingUpdates( + lntypes.Local, lntypes.Local, + ) + pendingOnRemote = l.channel.NumPendingUpdates( + lntypes.Local, lntypes.Remote, + ) + + return l.quiescer.SendOwedStfu(pendingOnLocal + pendingOnRemote) +} + +// stfuFailf fails the link in the case where the requirements of the quiescence +// protocol are violated. In all cases we opt to drop the connection as only +// link state (as opposed to channel state) is affected. +func (l *channelLink) stfuFailf(format string, args ...interface{}) { + l.failf(LinkFailureError{ + code: ErrStfuViolation, + FailureAction: LinkFailureDisconnect, + PermanentFailure: false, + Warning: true, + }, format, args...) +} + // ackDownStreamPackets is responsible for removing htlcs from a link's mailbox // for packets delivered from server, and cleaning up any circuits closed by // signing a previous commitment txn. This method ensures that the circuits are diff --git a/htlcswitch/linkfailure.go b/htlcswitch/linkfailure.go index 47f8065f766..495bd46fc26 100644 --- a/htlcswitch/linkfailure.go +++ b/htlcswitch/linkfailure.go @@ -51,6 +51,12 @@ const ( // circuit map. This is non-fatal and will resolve itself (usually // within several minutes). ErrCircuitError + + // ErrStfuViolation indicates that the quiescence protocol has been + // violated, either because Stfu has been sent/received at an invalid + // time, or that an update has been sent/received while the channel is + // quiesced. + ErrStfuViolation ) // LinkFailureAction is an enum-like type that describes the action that should @@ -122,6 +128,8 @@ func (e LinkFailureError) Error() string { return "unable to resume channel, recovery required" case ErrCircuitError: return "non-fatal circuit map error" + case ErrStfuViolation: + return "quiescence protocol executed improperly" default: return "unknown error" } From c9debea4082604083e781eb5465e0c6e14cc3e90 Mon Sep 17 00:00:00 2001 From: Keagan McClelland Date: Fri, 9 Aug 2024 15:57:06 -0700 Subject: [PATCH 03/20] lnwire: add IsChannelUpdate function to distinguish channel updates --- lnwire/message.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/lnwire/message.go b/lnwire/message.go index a758db000db..68b09692e55 100644 --- a/lnwire/message.go +++ b/lnwire/message.go @@ -63,6 +63,25 @@ const ( MsgKickoffSig = 777 ) +// IsChannelUpdate is a filter function that discerns channel update messages +// from the other messages in the Lightning Network Protocol. +func (t MessageType) IsChannelUpdate() bool { + switch t { + case MsgUpdateAddHTLC: + return true + case MsgUpdateFulfillHTLC: + return true + case MsgUpdateFailHTLC: + return true + case MsgUpdateFailMalformedHTLC: + return true + case MsgUpdateFee: + return true + default: + return false + } +} + // ErrorEncodeMessage is used when failed to encode the message payload. func ErrorEncodeMessage(err error) error { return fmt.Errorf("failed to encode message to buffer, got %w", err) From 44c87ef1d74870323570517fd71193da0956f928 Mon Sep 17 00:00:00 2001 From: Keagan McClelland Date: Tue, 12 Dec 2023 13:11:05 -0800 Subject: [PATCH 04/20] htlcswitch: bounce packets when quiescent --- htlcswitch/link.go | 49 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/htlcswitch/link.go b/htlcswitch/link.go index 46f06c9abe1..b56b0897f9e 100644 --- a/htlcswitch/link.go +++ b/htlcswitch/link.go @@ -643,20 +643,37 @@ func (l *channelLink) WaitForShutdown() { // actively accept requests to forward HTLC's. We're able to forward HTLC's if // we are eligible to update AND the channel isn't currently flushing the // outgoing half of the channel. +// +// NOTE: MUST NOT be called from the main event loop. func (l *channelLink) EligibleToForward() bool { - return l.EligibleToUpdate() && - !l.IsFlushing(Outgoing) + l.RLock() + defer l.RUnlock() + + return l.eligibleToForward() +} + +// eligibleToForward returns a bool indicating if the channel is able to +// actively accept requests to forward HTLC's. We're able to forward HTLC's if +// we are eligible to update AND the channel isn't currently flushing the +// outgoing half of the channel. +// +// NOTE: MUST be called from the main event loop. +func (l *channelLink) eligibleToForward() bool { + return l.eligibleToUpdate() && !l.IsFlushing(Outgoing) } -// EligibleToUpdate returns a bool indicating if the channel is able to update +// eligibleToUpdate returns a bool indicating if the channel is able to update // channel state. We're able to update channel state if we know the remote // party's next revocation point. Otherwise, we can't initiate new channel // state. We also require that the short channel ID not be the all-zero source // ID, meaning that the channel has had its ID finalized. -func (l *channelLink) EligibleToUpdate() bool { +// +// NOTE: MUST be called from the main event loop. +func (l *channelLink) eligibleToUpdate() bool { return l.channel.RemoteNextRevocation() != nil && - l.ShortChanID() != hop.Source && - l.isReestablished() + l.channel.ShortChanID() != hop.Source && + l.isReestablished() && + l.quiescer.CanSendUpdates() } // EnableAdds sets the ChannelUpdateHandler state to allow UpdateAddHtlc's in @@ -1599,13 +1616,14 @@ func (l *channelLink) handleDownstreamUpdateAdd(pkt *htlcPacket) error { return errors.New("not an UpdateAddHTLC packet") } - // If we are flushing the link in the outgoing direction we can't add - // new htlcs to the link and we need to bounce it - if l.IsFlushing(Outgoing) { + // If we are flushing the link in the outgoing direction or we have + // already sent Stfu, then we can't add new htlcs to the link and we + // need to bounce it. + if l.IsFlushing(Outgoing) || !l.quiescer.CanSendUpdates() { l.mailBox.FailAdd(pkt) return NewDetailedLinkError( - &lnwire.FailPermanentChannelFailure{}, + &lnwire.FailTemporaryChannelFailure{}, OutgoingFailureLinkNotEligible, ) } @@ -1708,6 +1726,15 @@ func (l *channelLink) handleDownstreamUpdateAdd(pkt *htlcPacket) error { // // TODO(roasbeef): add sync ntfn to ensure switch always has consistent view? func (l *channelLink) handleDownstreamPkt(pkt *htlcPacket) { + if pkt.htlc.MsgType().IsChannelUpdate() && + !l.quiescer.CanSendUpdates() { + + l.log.Warnf("unable to process channel update. "+ + "ChannelID=%v is quiescent.", l.ChanID) + + return + } + switch htlc := pkt.htlc.(type) { case *lnwire.UpdateAddHTLC: // Handle add message. The returned error can be ignored, @@ -3364,7 +3391,7 @@ func (l *channelLink) updateChannelFee(feePerKw chainfee.SatPerKWeight) error { // We skip sending the UpdateFee message if the channel is not // currently eligible to forward messages. - if !l.EligibleToUpdate() { + if !l.eligibleToUpdate() { l.log.Debugf("skipping fee update for inactive channel") return nil } From 6d30ab6c4fc4f921db62f69df37d810498a47f67 Mon Sep 17 00:00:00 2001 From: Keagan McClelland Date: Tue, 12 Dec 2023 13:16:58 -0800 Subject: [PATCH 05/20] htlcswitch: drop connection if link updates after stfu --- htlcswitch/link.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/htlcswitch/link.go b/htlcswitch/link.go index b56b0897f9e..cf9ba8ba97c 100644 --- a/htlcswitch/link.go +++ b/htlcswitch/link.go @@ -1979,6 +1979,13 @@ func (l *channelLink) cleanupSpuriousResponse(pkt *htlcPacket) { // updates from the upstream peer. The upstream peer is the peer whom we have a // direct channel with, updating our respective commitment chains. func (l *channelLink) handleUpstreamMsg(msg lnwire.Message) { + // First check if the message is an update and we are capable of + // receiving updates right now. + if msg.MsgType().IsChannelUpdate() && !l.quiescer.CanRecvUpdates() { + l.stfuFailf("update received after stfu: %T", msg) + return + } + switch msg := msg.(type) { case *lnwire.UpdateAddHTLC: if l.IsFlushing(Incoming) { From 7a5b55a4730c757c2f45549582ba9be00f36c2b2 Mon Sep 17 00:00:00 2001 From: Keagan McClelland Date: Tue, 12 Dec 2023 14:15:59 -0800 Subject: [PATCH 06/20] lnwire: signal that we support quiescence --- lnwire/features.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lnwire/features.go b/lnwire/features.go index bc6204f4244..eb00d869067 100644 --- a/lnwire/features.go +++ b/lnwire/features.go @@ -171,6 +171,16 @@ const ( // sender-generated preimages according to BOLT XX. AMPOptional FeatureBit = 31 + // QuiescenceRequired is a required feature bit that denotes that a + // connection established with this node must support the quiescence + // protocol if it wants to have a channel relationship. + QuiescenceRequired FeatureBit = 34 + + // QuiescenceOptional is an optional feature bit that denotes that a + // connection established with this node is permitted to use the + // quiescence protocol. + QuiescenceOptional FeatureBit = 35 + // ExplicitChannelTypeRequired is a required bit that denotes that a // connection established with this node is to use explicit channel // commitment types for negotiation instead of the existing implicit @@ -335,6 +345,8 @@ var Features = map[FeatureBit]string{ WumboChannelsOptional: "wumbo-channels", AMPRequired: "amp", AMPOptional: "amp", + QuiescenceRequired: "quiescence", + QuiescenceOptional: "quiescence", PaymentMetadataOptional: "payment-metadata", PaymentMetadataRequired: "payment-metadata", ExplicitChannelTypeOptional: "explicit-commitment-type", From 99f5ca40185792df6e78ada7bd31f4133ee0fbda Mon Sep 17 00:00:00 2001 From: Keagan McClelland Date: Tue, 12 Mar 2024 12:05:58 -0700 Subject: [PATCH 07/20] lnrpc add new RPC 'Quiesce' to protobuf definitions --- lnrpc/devrpc/dev.pb.go | 176 ++++++++++++++++++++++++++++++---- lnrpc/devrpc/dev.pb.gw.go | 81 ++++++++++++++++ lnrpc/devrpc/dev.pb.json.go | 25 +++++ lnrpc/devrpc/dev.proto | 18 ++++ lnrpc/devrpc/dev.swagger.json | 70 ++++++++++++++ lnrpc/devrpc/dev.yaml | 3 + lnrpc/devrpc/dev_grpc.pb.go | 42 ++++++++ lntest/rpc/harness_rpc.go | 4 +- 8 files changed, 398 insertions(+), 21 deletions(-) diff --git a/lnrpc/devrpc/dev.pb.go b/lnrpc/devrpc/dev.pb.go index 890f4e31714..d8de47fc8a3 100644 --- a/lnrpc/devrpc/dev.pb.go +++ b/lnrpc/devrpc/dev.pb.go @@ -59,6 +59,103 @@ func (*ImportGraphResponse) Descriptor() ([]byte, []int) { return file_devrpc_dev_proto_rawDescGZIP(), []int{0} } +type QuiescenceRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The channel point of the channel we wish to quiesce + ChanId *lnrpc.ChannelPoint `protobuf:"bytes,1,opt,name=chan_id,json=chanId,proto3" json:"chan_id,omitempty"` +} + +func (x *QuiescenceRequest) Reset() { + *x = QuiescenceRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_devrpc_dev_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *QuiescenceRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*QuiescenceRequest) ProtoMessage() {} + +func (x *QuiescenceRequest) ProtoReflect() protoreflect.Message { + mi := &file_devrpc_dev_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use QuiescenceRequest.ProtoReflect.Descriptor instead. +func (*QuiescenceRequest) Descriptor() ([]byte, []int) { + return file_devrpc_dev_proto_rawDescGZIP(), []int{1} +} + +func (x *QuiescenceRequest) GetChanId() *lnrpc.ChannelPoint { + if x != nil { + return x.ChanId + } + return nil +} + +type QuiescenceResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Indicates whether or not we hold the initiator role or not once the + // negotiation completes + Initiator bool `protobuf:"varint,1,opt,name=initiator,proto3" json:"initiator,omitempty"` +} + +func (x *QuiescenceResponse) Reset() { + *x = QuiescenceResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_devrpc_dev_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *QuiescenceResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*QuiescenceResponse) ProtoMessage() {} + +func (x *QuiescenceResponse) ProtoReflect() protoreflect.Message { + mi := &file_devrpc_dev_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use QuiescenceResponse.ProtoReflect.Descriptor instead. +func (*QuiescenceResponse) Descriptor() ([]byte, []int) { + return file_devrpc_dev_proto_rawDescGZIP(), []int{2} +} + +func (x *QuiescenceResponse) GetInitiator() bool { + if x != nil { + return x.Initiator + } + return false +} + var File_devrpc_dev_proto protoreflect.FileDescriptor var file_devrpc_dev_proto_rawDesc = []byte{ @@ -66,15 +163,26 @@ var file_devrpc_dev_proto_rawDesc = []byte{ 0x74, 0x6f, 0x12, 0x06, 0x64, 0x65, 0x76, 0x72, 0x70, 0x63, 0x1a, 0x0f, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x6e, 0x69, 0x6e, 0x67, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x15, 0x0a, 0x13, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x32, 0x46, 0x0a, 0x03, 0x44, 0x65, 0x76, 0x12, 0x3f, 0x0a, 0x0b, 0x49, 0x6d, 0x70, - 0x6f, 0x72, 0x74, 0x47, 0x72, 0x61, 0x70, 0x68, 0x12, 0x13, 0x2e, 0x6c, 0x6e, 0x72, 0x70, 0x63, - 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x47, 0x72, 0x61, 0x70, 0x68, 0x1a, 0x1b, 0x2e, - 0x64, 0x65, 0x76, 0x72, 0x70, 0x63, 0x2e, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x47, 0x72, 0x61, - 0x70, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x2e, 0x5a, 0x2c, 0x67, 0x69, - 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x6e, 0x69, - 0x6e, 0x67, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x2f, 0x6c, 0x6e, 0x64, 0x2f, 0x6c, 0x6e, - 0x72, 0x70, 0x63, 0x2f, 0x64, 0x65, 0x76, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, + 0x73, 0x65, 0x22, 0x41, 0x0a, 0x11, 0x51, 0x75, 0x69, 0x65, 0x73, 0x63, 0x65, 0x6e, 0x63, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2c, 0x0a, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x5f, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x6c, 0x6e, 0x72, 0x70, 0x63, + 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x06, 0x63, + 0x68, 0x61, 0x6e, 0x49, 0x64, 0x22, 0x32, 0x0a, 0x12, 0x51, 0x75, 0x69, 0x65, 0x73, 0x63, 0x65, + 0x6e, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x69, + 0x6e, 0x69, 0x74, 0x69, 0x61, 0x74, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, + 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x74, 0x6f, 0x72, 0x32, 0x88, 0x01, 0x0a, 0x03, 0x44, 0x65, + 0x76, 0x12, 0x3f, 0x0a, 0x0b, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x47, 0x72, 0x61, 0x70, 0x68, + 0x12, 0x13, 0x2e, 0x6c, 0x6e, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, + 0x47, 0x72, 0x61, 0x70, 0x68, 0x1a, 0x1b, 0x2e, 0x64, 0x65, 0x76, 0x72, 0x70, 0x63, 0x2e, 0x49, + 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x40, 0x0a, 0x07, 0x51, 0x75, 0x69, 0x65, 0x73, 0x63, 0x65, 0x12, 0x19, 0x2e, + 0x64, 0x65, 0x76, 0x72, 0x70, 0x63, 0x2e, 0x51, 0x75, 0x69, 0x65, 0x73, 0x63, 0x65, 0x6e, 0x63, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x64, 0x65, 0x76, 0x72, 0x70, + 0x63, 0x2e, 0x51, 0x75, 0x69, 0x65, 0x73, 0x63, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x2e, 0x5a, 0x2c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, + 0x6f, 0x6d, 0x2f, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x6e, 0x69, 0x6e, 0x67, 0x6e, 0x65, 0x74, 0x77, + 0x6f, 0x72, 0x6b, 0x2f, 0x6c, 0x6e, 0x64, 0x2f, 0x6c, 0x6e, 0x72, 0x70, 0x63, 0x2f, 0x64, 0x65, + 0x76, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -89,19 +197,25 @@ func file_devrpc_dev_proto_rawDescGZIP() []byte { return file_devrpc_dev_proto_rawDescData } -var file_devrpc_dev_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_devrpc_dev_proto_msgTypes = make([]protoimpl.MessageInfo, 3) var file_devrpc_dev_proto_goTypes = []interface{}{ (*ImportGraphResponse)(nil), // 0: devrpc.ImportGraphResponse - (*lnrpc.ChannelGraph)(nil), // 1: lnrpc.ChannelGraph + (*QuiescenceRequest)(nil), // 1: devrpc.QuiescenceRequest + (*QuiescenceResponse)(nil), // 2: devrpc.QuiescenceResponse + (*lnrpc.ChannelPoint)(nil), // 3: lnrpc.ChannelPoint + (*lnrpc.ChannelGraph)(nil), // 4: lnrpc.ChannelGraph } var file_devrpc_dev_proto_depIdxs = []int32{ - 1, // 0: devrpc.Dev.ImportGraph:input_type -> lnrpc.ChannelGraph - 0, // 1: devrpc.Dev.ImportGraph:output_type -> devrpc.ImportGraphResponse - 1, // [1:2] is the sub-list for method output_type - 0, // [0:1] is the sub-list for method input_type - 0, // [0:0] is the sub-list for extension type_name - 0, // [0:0] is the sub-list for extension extendee - 0, // [0:0] is the sub-list for field type_name + 3, // 0: devrpc.QuiescenceRequest.chan_id:type_name -> lnrpc.ChannelPoint + 4, // 1: devrpc.Dev.ImportGraph:input_type -> lnrpc.ChannelGraph + 1, // 2: devrpc.Dev.Quiesce:input_type -> devrpc.QuiescenceRequest + 0, // 3: devrpc.Dev.ImportGraph:output_type -> devrpc.ImportGraphResponse + 2, // 4: devrpc.Dev.Quiesce:output_type -> devrpc.QuiescenceResponse + 3, // [3:5] is the sub-list for method output_type + 1, // [1:3] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name } func init() { file_devrpc_dev_proto_init() } @@ -122,6 +236,30 @@ func file_devrpc_dev_proto_init() { return nil } } + file_devrpc_dev_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*QuiescenceRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_devrpc_dev_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*QuiescenceResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -129,7 +267,7 @@ func file_devrpc_dev_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_devrpc_dev_proto_rawDesc, NumEnums: 0, - NumMessages: 1, + NumMessages: 3, NumExtensions: 0, NumServices: 1, }, diff --git a/lnrpc/devrpc/dev.pb.gw.go b/lnrpc/devrpc/dev.pb.gw.go index d702804cd6a..10b0a60c0d5 100644 --- a/lnrpc/devrpc/dev.pb.gw.go +++ b/lnrpc/devrpc/dev.pb.gw.go @@ -66,6 +66,40 @@ func local_request_Dev_ImportGraph_0(ctx context.Context, marshaler runtime.Mars } +func request_Dev_Quiesce_0(ctx context.Context, marshaler runtime.Marshaler, client DevClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { + var protoReq QuiescenceRequest + var metadata runtime.ServerMetadata + + newReader, berr := utilities.IOReaderFactory(req.Body) + if berr != nil { + return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", berr) + } + if err := marshaler.NewDecoder(newReader()).Decode(&protoReq); err != nil && err != io.EOF { + return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) + } + + msg, err := client.Quiesce(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) + return msg, metadata, err + +} + +func local_request_Dev_Quiesce_0(ctx context.Context, marshaler runtime.Marshaler, server DevServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { + var protoReq QuiescenceRequest + var metadata runtime.ServerMetadata + + newReader, berr := utilities.IOReaderFactory(req.Body) + if berr != nil { + return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", berr) + } + if err := marshaler.NewDecoder(newReader()).Decode(&protoReq); err != nil && err != io.EOF { + return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) + } + + msg, err := server.Quiesce(ctx, &protoReq) + return msg, metadata, err + +} + // RegisterDevHandlerServer registers the http handlers for service Dev to "mux". // UnaryRPC :call DevServer directly. // StreamingRPC :currently unsupported pending https://github.com/grpc/grpc-go/issues/906. @@ -95,6 +129,29 @@ func RegisterDevHandlerServer(ctx context.Context, mux *runtime.ServeMux, server }) + mux.Handle("POST", pattern_Dev_Quiesce_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { + ctx, cancel := context.WithCancel(req.Context()) + defer cancel() + var stream runtime.ServerTransportStream + ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) + inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) + rctx, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/devrpc.Dev/Quiesce", runtime.WithHTTPPathPattern("/v2/dev/quiesce")) + if err != nil { + runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) + return + } + resp, md, err := local_request_Dev_Quiesce_0(rctx, inboundMarshaler, server, req, pathParams) + md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) + ctx = runtime.NewServerMetadataContext(ctx, md) + if err != nil { + runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) + return + } + + forward_Dev_Quiesce_0(ctx, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) + + }) + return nil } @@ -156,13 +213,37 @@ func RegisterDevHandlerClient(ctx context.Context, mux *runtime.ServeMux, client }) + mux.Handle("POST", pattern_Dev_Quiesce_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { + ctx, cancel := context.WithCancel(req.Context()) + defer cancel() + inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) + rctx, err := runtime.AnnotateContext(ctx, mux, req, "/devrpc.Dev/Quiesce", runtime.WithHTTPPathPattern("/v2/dev/quiesce")) + if err != nil { + runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) + return + } + resp, md, err := request_Dev_Quiesce_0(rctx, inboundMarshaler, client, req, pathParams) + ctx = runtime.NewServerMetadataContext(ctx, md) + if err != nil { + runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) + return + } + + forward_Dev_Quiesce_0(ctx, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) + + }) + return nil } var ( pattern_Dev_ImportGraph_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v2", "dev", "importgraph"}, "")) + + pattern_Dev_Quiesce_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v2", "dev", "quiesce"}, "")) ) var ( forward_Dev_ImportGraph_0 = runtime.ForwardResponseMessage + + forward_Dev_Quiesce_0 = runtime.ForwardResponseMessage ) diff --git a/lnrpc/devrpc/dev.pb.json.go b/lnrpc/devrpc/dev.pb.json.go index 954917a1a3f..2163a13de65 100644 --- a/lnrpc/devrpc/dev.pb.json.go +++ b/lnrpc/devrpc/dev.pb.json.go @@ -46,4 +46,29 @@ func RegisterDevJSONCallbacks(registry map[string]func(ctx context.Context, } callback(string(respBytes), nil) } + + registry["devrpc.Dev.Quiesce"] = func(ctx context.Context, + conn *grpc.ClientConn, reqJSON string, callback func(string, error)) { + + req := &QuiescenceRequest{} + err := marshaler.Unmarshal([]byte(reqJSON), req) + if err != nil { + callback("", err) + return + } + + client := NewDevClient(conn) + resp, err := client.Quiesce(ctx, req) + if err != nil { + callback("", err) + return + } + + respBytes, err := marshaler.Marshal(resp) + if err != nil { + callback("", err) + return + } + callback(string(respBytes), nil) + } } diff --git a/lnrpc/devrpc/dev.proto b/lnrpc/devrpc/dev.proto index 502fbadc8b9..4b4fe778fde 100644 --- a/lnrpc/devrpc/dev.proto +++ b/lnrpc/devrpc/dev.proto @@ -30,7 +30,25 @@ service Dev { used for development. */ rpc ImportGraph (lnrpc.ChannelGraph) returns (ImportGraphResponse); + + /* + Quiesce instructs a channel to initiate the quiescence (stfu) protocol. This + RPC is for testing purposes only. The commit that adds it will be removed + once interop is confirmed. + */ + rpc Quiesce (QuiescenceRequest) returns (QuiescenceResponse); } message ImportGraphResponse { } + +message QuiescenceRequest { + // The channel point of the channel we wish to quiesce + lnrpc.ChannelPoint chan_id = 1; +} + +message QuiescenceResponse { + // Indicates whether or not we hold the initiator role or not once the + // negotiation completes + bool initiator = 1; +} diff --git a/lnrpc/devrpc/dev.swagger.json b/lnrpc/devrpc/dev.swagger.json index 16e16d7be8e..b540a8e4ce8 100644 --- a/lnrpc/devrpc/dev.swagger.json +++ b/lnrpc/devrpc/dev.swagger.json @@ -48,12 +48,63 @@ "Dev" ] } + }, + "/v2/dev/quiesce": { + "post": { + "summary": "Quiesce instructs a channel to initiate the quiescence (stfu) protocol. This\nRPC is for testing purposes only. The commit that adds it will be removed\nonce interop is confirmed.", + "operationId": "Dev_Quiesce", + "responses": { + "200": { + "description": "A successful response.", + "schema": { + "$ref": "#/definitions/devrpcQuiescenceResponse" + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "$ref": "#/definitions/rpcStatus" + } + } + }, + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/devrpcQuiescenceRequest" + } + } + ], + "tags": [ + "Dev" + ] + } } }, "definitions": { "devrpcImportGraphResponse": { "type": "object" }, + "devrpcQuiescenceRequest": { + "type": "object", + "properties": { + "chan_id": { + "$ref": "#/definitions/lnrpcChannelPoint", + "title": "The channel point of the channel we wish to quiesce" + } + } + }, + "devrpcQuiescenceResponse": { + "type": "object", + "properties": { + "initiator": { + "type": "boolean", + "title": "Indicates whether or not we hold the initiator role or not once the\nnegotiation completes" + } + } + }, "lnrpcChannelEdge": { "type": "object", "properties": { @@ -116,6 +167,25 @@ }, "description": "Returns a new instance of the directed channel graph." }, + "lnrpcChannelPoint": { + "type": "object", + "properties": { + "funding_txid_bytes": { + "type": "string", + "format": "byte", + "description": "Txid of the funding transaction. When using REST, this field must be\nencoded as base64." + }, + "funding_txid_str": { + "type": "string", + "description": "Hex-encoded string representing the byte-reversed hash of the funding\ntransaction." + }, + "output_index": { + "type": "integer", + "format": "int64", + "title": "The index of the output of the funding transaction" + } + } + }, "lnrpcFeature": { "type": "object", "properties": { diff --git a/lnrpc/devrpc/dev.yaml b/lnrpc/devrpc/dev.yaml index 18c4e26b405..cb849dc6384 100644 --- a/lnrpc/devrpc/dev.yaml +++ b/lnrpc/devrpc/dev.yaml @@ -6,3 +6,6 @@ http: - selector: devrpc.Dev.ImportGraph post: "/v2/dev/importgraph" body: "*" + - selector: devrpc.Dev.Quiesce + post: "/v2/dev/quiesce" + body: "*" diff --git a/lnrpc/devrpc/dev_grpc.pb.go b/lnrpc/devrpc/dev_grpc.pb.go index 1744c12a3ba..1eb6266fbe4 100644 --- a/lnrpc/devrpc/dev_grpc.pb.go +++ b/lnrpc/devrpc/dev_grpc.pb.go @@ -23,6 +23,10 @@ type DevClient interface { // ImportGraph imports a ChannelGraph into the graph database. Should only be // used for development. ImportGraph(ctx context.Context, in *lnrpc.ChannelGraph, opts ...grpc.CallOption) (*ImportGraphResponse, error) + // Quiesce instructs a channel to initiate the quiescence (stfu) protocol. This + // RPC is for testing purposes only. The commit that adds it will be removed + // once interop is confirmed. + Quiesce(ctx context.Context, in *QuiescenceRequest, opts ...grpc.CallOption) (*QuiescenceResponse, error) } type devClient struct { @@ -42,6 +46,15 @@ func (c *devClient) ImportGraph(ctx context.Context, in *lnrpc.ChannelGraph, opt return out, nil } +func (c *devClient) Quiesce(ctx context.Context, in *QuiescenceRequest, opts ...grpc.CallOption) (*QuiescenceResponse, error) { + out := new(QuiescenceResponse) + err := c.cc.Invoke(ctx, "/devrpc.Dev/Quiesce", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // DevServer is the server API for Dev service. // All implementations must embed UnimplementedDevServer // for forward compatibility @@ -50,6 +63,10 @@ type DevServer interface { // ImportGraph imports a ChannelGraph into the graph database. Should only be // used for development. ImportGraph(context.Context, *lnrpc.ChannelGraph) (*ImportGraphResponse, error) + // Quiesce instructs a channel to initiate the quiescence (stfu) protocol. This + // RPC is for testing purposes only. The commit that adds it will be removed + // once interop is confirmed. + Quiesce(context.Context, *QuiescenceRequest) (*QuiescenceResponse, error) mustEmbedUnimplementedDevServer() } @@ -60,6 +77,9 @@ type UnimplementedDevServer struct { func (UnimplementedDevServer) ImportGraph(context.Context, *lnrpc.ChannelGraph) (*ImportGraphResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method ImportGraph not implemented") } +func (UnimplementedDevServer) Quiesce(context.Context, *QuiescenceRequest) (*QuiescenceResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Quiesce not implemented") +} func (UnimplementedDevServer) mustEmbedUnimplementedDevServer() {} // UnsafeDevServer may be embedded to opt out of forward compatibility for this service. @@ -91,6 +111,24 @@ func _Dev_ImportGraph_Handler(srv interface{}, ctx context.Context, dec func(int return interceptor(ctx, in, info, handler) } +func _Dev_Quiesce_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(QuiescenceRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DevServer).Quiesce(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/devrpc.Dev/Quiesce", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DevServer).Quiesce(ctx, req.(*QuiescenceRequest)) + } + return interceptor(ctx, in, info, handler) +} + // Dev_ServiceDesc is the grpc.ServiceDesc for Dev service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -102,6 +140,10 @@ var Dev_ServiceDesc = grpc.ServiceDesc{ MethodName: "ImportGraph", Handler: _Dev_ImportGraph_Handler, }, + { + MethodName: "Quiesce", + Handler: _Dev_Quiesce_Handler, + }, }, Streams: []grpc.StreamDesc{}, Metadata: "devrpc/dev.proto", diff --git a/lntest/rpc/harness_rpc.go b/lntest/rpc/harness_rpc.go index 0640581dcd8..2e08a84947c 100644 --- a/lntest/rpc/harness_rpc.go +++ b/lntest/rpc/harness_rpc.go @@ -43,7 +43,7 @@ type HarnessRPC struct { ChainKit chainrpc.ChainKitClient NeutrinoKit neutrinorpc.NeutrinoKitClient Peer peersrpc.PeersClient - DevRPC devrpc.DevClient + Dev devrpc.DevClient // Name is the HarnessNode's name. Name string @@ -75,7 +75,7 @@ func NewHarnessRPC(ctxt context.Context, t *testing.T, c *grpc.ClientConn, ChainKit: chainrpc.NewChainKitClient(c), NeutrinoKit: neutrinorpc.NewNeutrinoKitClient(c), Peer: peersrpc.NewPeersClient(c), - DevRPC: devrpc.NewDevClient(c), + Dev: devrpc.NewDevClient(c), Name: name, } From a085b59814a797bc9aa93d511d351cd4fe7ccf76 Mon Sep 17 00:00:00 2001 From: Keagan McClelland Date: Tue, 12 Mar 2024 12:16:01 -0700 Subject: [PATCH 08/20] lnd: implement new Quiesce RPC with link operation stub --- lnrpc/devrpc/config_active.go | 2 ++ lnrpc/devrpc/dev_server.go | 26 ++++++++++++++++++++++++++ subrpcserver_config.go | 4 ++++ 3 files changed, 32 insertions(+) diff --git a/lnrpc/devrpc/config_active.go b/lnrpc/devrpc/config_active.go index 6fc274f2e90..da5cd5be97b 100644 --- a/lnrpc/devrpc/config_active.go +++ b/lnrpc/devrpc/config_active.go @@ -6,6 +6,7 @@ package devrpc import ( "github.com/btcsuite/btcd/chaincfg" "github.com/lightningnetwork/lnd/channeldb" + "github.com/lightningnetwork/lnd/htlcswitch" ) // Config is the primary configuration struct for the DEV RPC server. It @@ -16,4 +17,5 @@ import ( type Config struct { ActiveNetParams *chaincfg.Params GraphDB *channeldb.ChannelGraph + Switch *htlcswitch.Switch } diff --git a/lnrpc/devrpc/dev_server.go b/lnrpc/devrpc/dev_server.go index 662c0d08d9f..4b7ebcd1e6e 100644 --- a/lnrpc/devrpc/dev_server.go +++ b/lnrpc/devrpc/dev_server.go @@ -40,6 +40,10 @@ var ( Entity: "offchain", Action: "write", }}, + "/devrpc.Dev/Quiesce": {{ + Entity: "offchain", + Action: "write", + }}, } ) @@ -342,3 +346,25 @@ func (s *Server) ImportGraph(ctx context.Context, return &ImportGraphResponse{}, nil } + +// Quiesce initiates the quiescence process for the channel with the given +// channel ID. This method will block until the channel is fully quiesced. +func (s *Server) Quiesce(_ context.Context, in *QuiescenceRequest) ( + *QuiescenceResponse, error) { + + txid, err := lnrpc.GetChanPointFundingTxid(in.ChanId) + if err != nil { + return nil, err + } + + op := wire.NewOutPoint(txid, in.ChanId.OutputIndex) + cid := lnwire.NewChanIDFromOutPoint(*op) + _, err = s.cfg.Switch.GetLink(cid) + if err != nil { + return nil, err + } + + // TODO(proofofkeags): Add Link operation for initiating quiescence and + // implement the rest of this in those terms + return nil, fmt.Errorf("TODO(proofofkeags): Implement") +} diff --git a/subrpcserver_config.go b/subrpcserver_config.go index a4ee6d1a16b..9e9295931b8 100644 --- a/subrpcserver_config.go +++ b/subrpcserver_config.go @@ -346,6 +346,10 @@ func (s *subRPCServerConfigs) PopulateDependencies(cfg *Config, reflect.ValueOf(graphDB), ) + subCfgValue.FieldByName("Switch").Set( + reflect.ValueOf(htlcSwitch), + ) + case *peersrpc.Config: subCfgValue := extractReflectValue(subCfg) From 70e3804121dcdc5e9594e6ed72d31891113e3c38 Mon Sep 17 00:00:00 2001 From: Keagan McClelland Date: Tue, 12 Mar 2024 11:43:48 -0700 Subject: [PATCH 09/20] htlcswitch: add link operation for initiating quiescence --- htlcswitch/interfaces.go | 12 ++++++++++++ htlcswitch/link.go | 16 ++++++++++++++++ htlcswitch/mock.go | 8 ++++++++ peer/test_utils.go | 9 +++++++++ 4 files changed, 45 insertions(+) diff --git a/htlcswitch/interfaces.go b/htlcswitch/interfaces.go index 3dd70247d29..a3f4ff9a9a3 100644 --- a/htlcswitch/interfaces.go +++ b/htlcswitch/interfaces.go @@ -170,6 +170,18 @@ type ChannelUpdateHandler interface { // will only ever be called once. If no CommitSig is owed in the // argument's LinkDirection, then we will call this hook immediately. OnCommitOnce(LinkDirection, func()) + + // InitStfu allows us to initiate quiescence on this link. It returns + // a receive only channel that will block until quiescence has been + // achieved, or definitively fails. The return value is the + // ChannelParty who holds the role of initiator or Err if the operation + // fails. + // + // This operation has been added to allow channels to be quiesced via + // RPC. It may be removed or reworked in the future as RPC initiated + // quiescence is a holdover until we have downstream protocols that use + // it. + InitStfu() <-chan fn.Result[lntypes.ChannelParty] } // CommitHookID is a value that is used to uniquely identify hooks in the diff --git a/htlcswitch/link.go b/htlcswitch/link.go index cf9ba8ba97c..442522de149 100644 --- a/htlcswitch/link.go +++ b/htlcswitch/link.go @@ -737,6 +737,22 @@ func (l *channelLink) OnCommitOnce(direction LinkDirection, hook func()) { } } +// InitStfu allows us to initiate quiescence on this link. It returns a receive +// only channel that will block until quiescence has been achieved, or +// definitively fails. +// +// This operation has been added to allow channels to be quiesced via RPC. It +// may be removed or reworked in the future as RPC initiated quiescence is a +// holdover until we have downstream protocols that use it. +func (l *channelLink) InitStfu() <-chan fn.Result[lntypes.ChannelParty] { + // TODO(proofofkeags): Implement + c := make(chan fn.Result[lntypes.ChannelParty], 1) + + c <- fn.Errf[lntypes.ChannelParty]("InitStfu not yet implemented") + + return c +} + // isReestablished returns true if the link has successfully completed the // channel reestablishment dance. func (l *channelLink) isReestablished() bool { diff --git a/htlcswitch/mock.go b/htlcswitch/mock.go index 6de60b38a16..37bf4c6ef2d 100644 --- a/htlcswitch/mock.go +++ b/htlcswitch/mock.go @@ -950,6 +950,14 @@ func (f *mockChannelLink) OnFlushedOnce(func()) { func (f *mockChannelLink) OnCommitOnce(LinkDirection, func()) { // TODO(proofofkeags): Implement } +func (f *mockChannelLink) InitStfu() <-chan fn.Result[lntypes.ChannelParty] { + // TODO(proofofkeags): Implement + c := make(chan fn.Result[lntypes.ChannelParty], 1) + + c <- fn.Errf[lntypes.ChannelParty]("InitStfu not implemented") + + return c +} func (f *mockChannelLink) FundingCustomBlob() fn.Option[tlv.Blob] { return fn.None[tlv.Blob]() diff --git a/peer/test_utils.go b/peer/test_utils.go index 1a623559088..9034bb5a962 100644 --- a/peer/test_utils.go +++ b/peer/test_utils.go @@ -24,6 +24,7 @@ import ( "github.com/lightningnetwork/lnd/keychain" "github.com/lightningnetwork/lnd/lntest/channels" "github.com/lightningnetwork/lnd/lntest/mock" + "github.com/lightningnetwork/lnd/lntypes" "github.com/lightningnetwork/lnd/lnwallet" "github.com/lightningnetwork/lnd/lnwallet/chainfee" "github.com/lightningnetwork/lnd/lnwire" @@ -477,6 +478,14 @@ func (m *mockUpdateHandler) OnCommitOnce( hook() } +func (m *mockUpdateHandler) InitStfu() <-chan fn.Result[lntypes.ChannelParty] { + // TODO(proofofkeags): Implement + c := make(chan fn.Result[lntypes.ChannelParty], 1) + + c <- fn.Errf[lntypes.ChannelParty]("InitStfu not yet implemented") + + return c +} func newMockConn(t *testing.T, expectedMessages int) *mockMessageConn { return &mockMessageConn{ From bca15164290bb679412a059c309a7b4e6f941b60 Mon Sep 17 00:00:00 2001 From: Keagan McClelland Date: Tue, 12 Mar 2024 12:30:33 -0700 Subject: [PATCH 10/20] lnd: finish Quiesce implementation using new link op --- lnrpc/devrpc/dev_server.go | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/lnrpc/devrpc/dev_server.go b/lnrpc/devrpc/dev_server.go index 4b7ebcd1e6e..ad135e8dfb3 100644 --- a/lnrpc/devrpc/dev_server.go +++ b/lnrpc/devrpc/dev_server.go @@ -18,8 +18,10 @@ import ( "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" "github.com/lightningnetwork/lnd/channeldb" "github.com/lightningnetwork/lnd/channeldb/models" + "github.com/lightningnetwork/lnd/fn" "github.com/lightningnetwork/lnd/lncfg" "github.com/lightningnetwork/lnd/lnrpc" + "github.com/lightningnetwork/lnd/lntypes" "github.com/lightningnetwork/lnd/lnwire" "google.golang.org/grpc" "gopkg.in/macaroon-bakery.v2/bakery" @@ -60,6 +62,7 @@ type ServerShell struct { type Server struct { started int32 // To be used atomically. shutdown int32 // To be used atomically. + quit chan struct{} // Required by the grpc-gateway/v2 library for forward compatibility. // Must be after the atomically used variables to not break struct @@ -82,7 +85,8 @@ func New(cfg *Config) (*Server, lnrpc.MacaroonPerms, error) { // We don't create any new macaroons for this subserver, instead reuse // existing onchain/offchain permissions. server := &Server{ - cfg: cfg, + quit: make(chan struct{}), + cfg: cfg, } return server, macPermissions, nil @@ -107,6 +111,8 @@ func (s *Server) Stop() error { return nil } + close(s.quit) + return nil } @@ -359,12 +365,20 @@ func (s *Server) Quiesce(_ context.Context, in *QuiescenceRequest) ( op := wire.NewOutPoint(txid, in.ChanId.OutputIndex) cid := lnwire.NewChanIDFromOutPoint(*op) - _, err = s.cfg.Switch.GetLink(cid) + ln, err := s.cfg.Switch.GetLink(cid) if err != nil { return nil, err } - // TODO(proofofkeags): Add Link operation for initiating quiescence and - // implement the rest of this in those terms - return nil, fmt.Errorf("TODO(proofofkeags): Implement") + select { + case result := <-ln.InitStfu(): + mkResp := func(b lntypes.ChannelParty) *QuiescenceResponse { + return &QuiescenceResponse{Initiator: b.IsLocal()} + } + + return fn.MapOk(mkResp)(result).Unpack() + + case <-s.quit: + return nil, fmt.Errorf("server shutting down") + } } From 7255b7357cd66e77324049cf5063e49d0d4a92d8 Mon Sep 17 00:00:00 2001 From: Keagan McClelland Date: Tue, 12 Mar 2024 12:32:43 -0700 Subject: [PATCH 11/20] htlcswitch: implement InitStfu link operation --- htlcswitch/link.go | 39 ++++++++- htlcswitch/quiescer.go | 65 ++++++++++++++- htlcswitch/quiescer_test.go | 155 +++++++++++++++++++++++++++++++++--- 3 files changed, 244 insertions(+), 15 deletions(-) diff --git a/htlcswitch/link.go b/htlcswitch/link.go index 442522de149..449c811790f 100644 --- a/htlcswitch/link.go +++ b/htlcswitch/link.go @@ -396,6 +396,11 @@ type channelLink struct { // respect to the quiescence protocol. quiescer Quiescer + // quiescenceReqs is a queue of requests to quiesce this link. The + // members of the queue are send-only channels we should call back with + // the result. + quiescenceReqs chan StfuReq + // ContextGuard is a helper that encapsulates a wait group and quit // channel and allows contexts that either block or cancel on those // depending on the use case. @@ -481,6 +486,10 @@ func NewChannelLink(cfg ChannelLinkConfig, }, } + quiescenceReqs := make( + chan fn.Req[fn.Unit, fn.Result[lntypes.ChannelParty]], 1, + ) + return &channelLink{ cfg: cfg, channel: channel, @@ -491,6 +500,7 @@ func NewChannelLink(cfg ChannelLinkConfig, outgoingCommitHooks: newHookMap(), incomingCommitHooks: newHookMap(), quiescer: NewQuiescer(quiescerCfg), + quiescenceReqs: quiescenceReqs, ContextGuard: fn.NewContextGuard(), } } @@ -745,12 +755,17 @@ func (l *channelLink) OnCommitOnce(direction LinkDirection, hook func()) { // may be removed or reworked in the future as RPC initiated quiescence is a // holdover until we have downstream protocols that use it. func (l *channelLink) InitStfu() <-chan fn.Result[lntypes.ChannelParty] { - // TODO(proofofkeags): Implement - c := make(chan fn.Result[lntypes.ChannelParty], 1) + req, out := fn.NewReq[fn.Unit, fn.Result[lntypes.ChannelParty]]( + fn.Unit{}, + ) - c <- fn.Errf[lntypes.ChannelParty]("InitStfu not yet implemented") + select { + case l.quiescenceReqs <- req: + case <-l.Quit: + req.Resolve(fn.Err[lntypes.ChannelParty](ErrLinkShuttingDown)) + } - return c + return out } // isReestablished returns true if the link has successfully completed the @@ -1498,6 +1513,22 @@ func (l *channelLink) htlcManager() { ) } + case qReq := <-l.quiescenceReqs: + l.quiescer.InitStfu(qReq) + + pendingOnLocal := l.channel.NumPendingUpdates( + lntypes.Local, lntypes.Local, + ) + pendingOnRemote := l.channel.NumPendingUpdates( + lntypes.Local, lntypes.Remote, + ) + if err := l.quiescer.SendOwedStfu( + pendingOnLocal + pendingOnRemote, + ); err != nil { + l.stfuFailf("%s", err.Error()) + qReq.Resolve(fn.Err[lntypes.ChannelParty](err)) + } + case <-l.Quit: return } diff --git a/htlcswitch/quiescer.go b/htlcswitch/quiescer.go index 9bde04c322b..300988c7a63 100644 --- a/htlcswitch/quiescer.go +++ b/htlcswitch/quiescer.go @@ -44,6 +44,8 @@ var ( ) ) +type StfuReq = fn.Req[fn.Unit, fn.Result[lntypes.ChannelParty]] + // QuiescerCfg is a config structure used to initialize a quiescer giving it the // appropriate functionality to interact with the channel state that the // quiescer must syncrhonize with. @@ -84,6 +86,10 @@ type Quiescer struct { // received tracks whether or not we have received Stfu from our peer. received bool + // activeQuiescenceRequest is a possibly None Request that we should + // resolve when we complete quiescence. + activeQuiescenceReq fn.Option[StfuReq] + sync.RWMutex } @@ -135,6 +141,10 @@ func (q *Quiescer) recvStfu(msg lnwire.Stfu, // does not necessarily mean they will get it, though. q.remoteInit = msg.Initiator + // Since we just received an Stfu, we may have a newly quiesced state. + // If so, we will try to resolve any outstanding StfuReqs. + q.tryResolveStfuReq() + return nil } @@ -186,7 +196,7 @@ func (q *Quiescer) OweStfu() bool { // Stfu when we have received but not yet sent an Stfu, or we are the initiator // but have not yet sent an Stfu. func (q *Quiescer) oweStfu() bool { - return q.received && !q.sent + return (q.received || q.localInit) && !q.sent } // NeedStfu returns true if the remote owes us an Stfu. They owe us an Stfu when @@ -333,7 +343,60 @@ func (q *Quiescer) sendOwedStfu(numPendingLocalUpdates uint64) error { if err == nil { q.sent = true + + // Since we just sent an Stfu, we may have a newly quiesced + // state. If so, we will try to resolve any outstanding + // StfuReqs. + q.tryResolveStfuReq() } return err } + +// TryResolveStfuReq attempts to resolve the active quiescence request if the +// state machine has reached a quiescent state. +func (q *Quiescer) TryResolveStfuReq() { + q.Lock() + defer q.Unlock() + + q.tryResolveStfuReq() +} + +// tryResolveStfuReq attempts to resolve the active quiescence request if the +// state machine has reached a quiescent state. +func (q *Quiescer) tryResolveStfuReq() { + q.activeQuiescenceReq.WhenSome( + func(req StfuReq) { + if q.isQuiescent() { + req.Resolve(q.quiescenceInitiator()) + q.activeQuiescenceReq = fn.None[StfuReq]() + } + }, + ) +} + +// InitStfu instructs the quiescer that we intend to begin a quiescence +// negotiation where we are the initiator. We don't yet send stfu yet because +// we need to wait for the link to give us a valid opportunity to do so. +func (q *Quiescer) InitStfu(req StfuReq) { + q.Lock() + defer q.Unlock() + + q.initStfu(req) +} + +// initStfu instructs the quiescer that we intend to begin a quiescence +// negotiation where we are the initiator. We don't yet send stfu yet because +// we need to wait for the link to give us a valid opportunity to do so. +func (q *Quiescer) initStfu(req StfuReq) { + if q.localInit { + req.Resolve(fn.Errf[lntypes.ChannelParty]( + "quiescence already requested", + )) + + return + } + + q.localInit = true + q.activeQuiescenceReq = fn.Some(req) +} diff --git a/htlcswitch/quiescer_test.go b/htlcswitch/quiescer_test.go index 914f64b3c57..5c6e2fa4c82 100644 --- a/htlcswitch/quiescer_test.go +++ b/htlcswitch/quiescer_test.go @@ -18,7 +18,9 @@ type quiescerTestHarness struct { conn <-chan lnwire.Stfu } -func initQuiescerTestHarness() *quiescerTestHarness { +func initQuiescerTestHarness( + channelInitiator lntypes.ChannelParty) *quiescerTestHarness { + conn := make(chan lnwire.Stfu, 1) harness := &quiescerTestHarness{ pendingUpdates: lntypes.Dual[uint64]{}, @@ -26,7 +28,8 @@ func initQuiescerTestHarness() *quiescerTestHarness { } harness.quiescer = NewQuiescer(QuiescerCfg{ - chanID: cid, + chanID: cid, + channelInitiator: channelInitiator, sendMsg: func(msg lnwire.Stfu) error { conn <- msg return nil @@ -41,7 +44,7 @@ func initQuiescerTestHarness() *quiescerTestHarness { func TestQuiescerDoubleRecvInvalid(t *testing.T) { t.Parallel() - harness := initQuiescerTestHarness() + harness := initQuiescerTestHarness(lntypes.Local) msg := lnwire.Stfu{ ChanID: cid, @@ -60,7 +63,7 @@ func TestQuiescerDoubleRecvInvalid(t *testing.T) { func TestQuiescerPendingUpdatesRecvInvalid(t *testing.T) { t.Parallel() - harness := initQuiescerTestHarness() + harness := initQuiescerTestHarness(lntypes.Local) msg := lnwire.Stfu{ ChanID: cid, @@ -77,7 +80,7 @@ func TestQuiescerPendingUpdatesRecvInvalid(t *testing.T) { func TestQuiescenceRemoteInit(t *testing.T) { t.Parallel() - harness := initQuiescerTestHarness() + harness := initQuiescerTestHarness(lntypes.Local) msg := lnwire.Stfu{ ChanID: cid, @@ -110,12 +113,61 @@ func TestQuiescenceRemoteInit(t *testing.T) { } } +func TestQuiescenceLocalInit(t *testing.T) { + t.Parallel() + + harness := initQuiescerTestHarness(lntypes.Local) + + msg := lnwire.Stfu{ + ChanID: cid, + Initiator: true, + } + harness.pendingUpdates.SetForParty(lntypes.Local, 1) + + stfuReq, stfuRes := fn.NewReq[fn.Unit, fn.Result[lntypes.ChannelParty]]( + fn.Unit{}, + ) + harness.quiescer.InitStfu(stfuReq) + + harness.pendingUpdates.SetForParty(lntypes.Local, 1) + err := harness.quiescer.SendOwedStfu(harness.pendingUpdates.Local) + require.NoError(t, err) + + select { + case <-harness.conn: + t.Fatalf("stfu sent when not expected") + default: + } + + harness.pendingUpdates.SetForParty(lntypes.Local, 0) + err = harness.quiescer.SendOwedStfu(harness.pendingUpdates.Local) + require.NoError(t, err) + + select { + case msg := <-harness.conn: + require.True(t, msg.Initiator) + default: + t.Fatalf("stfu not sent when expected") + } + + err = harness.quiescer.RecvStfu(msg, harness.pendingUpdates.Remote) + require.NoError(t, err) + + select { + case party := <-stfuRes: + require.Equal(t, fn.Ok(lntypes.Local), party) + default: + t.Fatalf("quiescence request not resolved") + } +} + // TestQuiescenceInitiator ensures that the quiescenceInitiator is the Remote // party when we have a receive first traversal of the quiescer's state graph. func TestQuiescenceInitiator(t *testing.T) { t.Parallel() - harness := initQuiescerTestHarness() + // Remote Initiated + harness := initQuiescerTestHarness(lntypes.Local) require.True(t, harness.quiescer.QuiescenceInitiator().IsErr()) // Receive @@ -138,6 +190,48 @@ func TestQuiescenceInitiator(t *testing.T) { t, harness.quiescer.QuiescenceInitiator(), fn.Ok(lntypes.Remote), ) + + // Local Initiated + harness = initQuiescerTestHarness(lntypes.Local) + require.True(t, harness.quiescer.quiescenceInitiator().IsErr()) + + req, res := fn.NewReq[fn.Unit, fn.Result[lntypes.ChannelParty]]( + fn.Unit{}, + ) + harness.quiescer.initStfu(req) + req2, res2 := fn.NewReq[fn.Unit, fn.Result[lntypes.ChannelParty]]( + fn.Unit{}, + ) + harness.quiescer.initStfu(req2) + select { + case initiator := <-res2: + require.True(t, initiator.IsErr()) + default: + t.Fatal("quiescence request not resolved") + } + + require.NoError( + t, harness.quiescer.sendOwedStfu(harness.pendingUpdates.Local), + ) + require.True(t, harness.quiescer.quiescenceInitiator().IsErr()) + + msg = lnwire.Stfu{ + ChanID: cid, + Initiator: false, + } + require.NoError( + t, harness.quiescer.recvStfu( + msg, harness.pendingUpdates.Remote, + ), + ) + require.True(t, harness.quiescer.quiescenceInitiator().IsOk()) + + select { + case initiator := <-res: + require.Equal(t, fn.Ok(lntypes.Local), initiator) + default: + t.Fatal("quiescence request not resolved") + } } // TestQuiescenceCantReceiveUpdatesAfterStfu tests that we can receive channel @@ -145,7 +239,7 @@ func TestQuiescenceInitiator(t *testing.T) { func TestQuiescenceCantReceiveUpdatesAfterStfu(t *testing.T) { t.Parallel() - harness := initQuiescerTestHarness() + harness := initQuiescerTestHarness(lntypes.Local) require.True(t, harness.quiescer.CanRecvUpdates()) msg := lnwire.Stfu{ @@ -165,7 +259,7 @@ func TestQuiescenceCantReceiveUpdatesAfterStfu(t *testing.T) { func TestQuiescenceCantSendUpdatesAfterStfu(t *testing.T) { t.Parallel() - harness := initQuiescerTestHarness() + harness := initQuiescerTestHarness(lntypes.Local) require.True(t, harness.quiescer.CanSendUpdates()) msg := lnwire.Stfu{ @@ -188,7 +282,7 @@ func TestQuiescenceCantSendUpdatesAfterStfu(t *testing.T) { func TestQuiescenceStfuNotNeededAfterRecv(t *testing.T) { t.Parallel() - harness := initQuiescerTestHarness() + harness := initQuiescerTestHarness(lntypes.Local) msg := lnwire.Stfu{ ChanID: cid, @@ -210,7 +304,7 @@ func TestQuiescenceStfuNotNeededAfterRecv(t *testing.T) { func TestQuiescenceInappropriateMakeStfuReturnsErr(t *testing.T) { t.Parallel() - harness := initQuiescerTestHarness() + harness := initQuiescerTestHarness(lntypes.Local) harness.pendingUpdates.SetForParty(lntypes.Local, 1) @@ -245,3 +339,44 @@ func TestQuiescenceInappropriateMakeStfuReturnsErr(t *testing.T) { ).IsErr(), ) } + +// TestQuiescerTieBreaker ensures that if both parties attempt to claim the +// initiator role that the result of the negotiation breaks the tie using the +// channel initiator. +func TestQuiescerTieBreaker(t *testing.T) { + t.Parallel() + + for _, initiator := range []lntypes.ChannelParty{ + lntypes.Local, lntypes.Remote, + } { + harness := initQuiescerTestHarness(initiator) + + msg := lnwire.Stfu{ + ChanID: cid, + Initiator: true, + } + + req, res := fn.NewReq[fn.Unit, fn.Result[lntypes.ChannelParty]]( + fn.Unit{}, + ) + + harness.quiescer.InitStfu(req) + require.NoError( + t, harness.quiescer.RecvStfu( + msg, harness.pendingUpdates.Remote, + ), + ) + require.NoError( + t, harness.quiescer.SendOwedStfu( + harness.pendingUpdates.Local, + ), + ) + + select { + case party := <-res: + require.Equal(t, fn.Ok(initiator), party) + default: + t.Fatal("quiescence party unavailable") + } + } +} From 77fd8c6a213fa1d4d7d2309d1377e8b27d433062 Mon Sep 17 00:00:00 2001 From: Keagan McClelland Date: Tue, 12 Mar 2024 12:26:01 -0700 Subject: [PATCH 12/20] itest+lntest: add itest for Quiesce RPC method --- itest/list_on_test.go | 4 +++ itest/lnd_quiescence_test.go | 48 ++++++++++++++++++++++++++++++++++++ lntest/rpc/lnd.go | 15 +++++++++++ 3 files changed, 67 insertions(+) create mode 100644 itest/lnd_quiescence_test.go diff --git a/itest/list_on_test.go b/itest/list_on_test.go index 7560fb16be1..fe6c373855b 100644 --- a/itest/list_on_test.go +++ b/itest/list_on_test.go @@ -710,4 +710,8 @@ var allTestCases = []*lntest.TestCase{ Name: "experimental endorsement", TestFunc: testExperimentalEndorsement, }, + { + Name: "quiescence", + TestFunc: testQuiescence, + }, } diff --git a/itest/lnd_quiescence_test.go b/itest/lnd_quiescence_test.go new file mode 100644 index 00000000000..7c2c274a212 --- /dev/null +++ b/itest/lnd_quiescence_test.go @@ -0,0 +1,48 @@ +package itest + +import ( + "github.com/btcsuite/btcd/btcutil" + "github.com/lightningnetwork/lnd/lnrpc" + "github.com/lightningnetwork/lnd/lnrpc/devrpc" + "github.com/lightningnetwork/lnd/lnrpc/routerrpc" + "github.com/lightningnetwork/lnd/lntest" + "github.com/stretchr/testify/require" +) + +// testQuiescence tests whether we can come to agreement on quiescence of a +// channel. We initiate quiescence via RPC and if it succeeds we verify that +// the expected initiator is the resulting initiator. +// +// NOTE FOR REVIEW: this could be improved by blasting the channel with HTLC +// traffic on both sides to increase the surface area of the change under test. +func testQuiescence(ht *lntest.HarnessTest) { + alice, bob := ht.Alice, ht.Bob + + chanPoint := ht.OpenChannel(bob, alice, lntest.OpenChannelParams{ + Amt: btcutil.Amount(1000000), + }) + defer ht.CloseChannel(bob, chanPoint) + + res := alice.RPC.Quiesce(&devrpc.QuiescenceRequest{ + ChanId: chanPoint, + }) + + require.True(ht, res.Initiator) + + req := &routerrpc.SendPaymentRequest{ + Dest: ht.Alice.PubKey[:], + Amt: 100, + PaymentHash: ht.Random32Bytes(), + FinalCltvDelta: finalCltvDelta, + TimeoutSeconds: 60, + FeeLimitMsat: noFeeLimitMsat, + } + + ht.SendPaymentAssertFail( + ht.Bob, req, + // This fails with insufficient balance because the bandwidth + // manager reports 0 bandwidth if a link is not eligible for + // forwarding, which is the case during quiescence. + lnrpc.PaymentFailureReason_FAILURE_REASON_INSUFFICIENT_BALANCE, + ) +} diff --git a/lntest/rpc/lnd.go b/lntest/rpc/lnd.go index f0ed52fd88f..1657caac8dd 100644 --- a/lntest/rpc/lnd.go +++ b/lntest/rpc/lnd.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/lightningnetwork/lnd/lnrpc" + "github.com/lightningnetwork/lnd/lnrpc/devrpc" "github.com/stretchr/testify/require" ) @@ -730,3 +731,17 @@ func (h *HarnessRPC) LookupHtlcResolutionAssertErr( return err } + +// Quiesce makes an RPC call to the node's Quiesce method and returns the +// response. +func (h *HarnessRPC) Quiesce( + req *devrpc.QuiescenceRequest) *devrpc.QuiescenceResponse { + + ctx, cancel := context.WithTimeout(h.runCtx, DefaultTimeout) + defer cancel() + + res, err := h.Dev.Quiesce(ctx, req) + h.NoError(err, "Quiesce returned an error") + + return res +} From 4fbab45a5fcd3a85672e15ef68898ff2ab511276 Mon Sep 17 00:00:00 2001 From: Keagan McClelland Date: Mon, 8 Apr 2024 18:17:00 -0700 Subject: [PATCH 13/20] htlcswitch: defer processRemoteAdds when quiescent In this commit we defer processRemoteAdds using a new mechanism on the quiescer where we capture a closure that needs to be run. We do this because we need to avoid the scenario where we send back immediate resolutions to the newly added HTLCs when quiescent as it is a protocol violation. It is not enough for us to simply defer sending the messages since the purpose of quiescence itself is to have well-defined and agreed upon channel state. If, for whatever reason, the node (or connection) is restarted between when these hooks are captured and when they are ultimately run, they will be resolved by the resolveFwdPkgs logic when the link comes back up. In a future commit we will explicitly call the quiescer's resume method when it is OK for htlc traffic to commence. --- htlcswitch/link.go | 16 +++++++++++++- htlcswitch/quiescer.go | 44 +++++++++++++++++++++++++++++++++++++ htlcswitch/quiescer_test.go | 35 +++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 1 deletion(-) diff --git a/htlcswitch/link.go b/htlcswitch/link.go index 449c811790f..980f4dcb6d2 100644 --- a/htlcswitch/link.go +++ b/htlcswitch/link.go @@ -2500,8 +2500,22 @@ func (l *channelLink) handleUpstreamMsg(msg lnwire.Message) { } } + // If we can send updates then we can process adds in case we + // are the exit hop and need to send back resolutions, or in + // case there are validity issues with the packets. Otherwise + // we defer the action until resume. + // + // We are free to process the settles and fails without this + // check since processing those can't result in further updates + // to this channel link. + if l.quiescer.CanSendUpdates() { + l.processRemoteAdds(fwdPkg) + } else { + l.quiescer.OnResume(func() { + l.processRemoteAdds(fwdPkg) + }) + } l.processRemoteSettleFails(fwdPkg) - l.processRemoteAdds(fwdPkg) // If the link failed during processing the adds, we must // return to ensure we won't attempted to update the state diff --git a/htlcswitch/quiescer.go b/htlcswitch/quiescer.go index 300988c7a63..3fa30597ba8 100644 --- a/htlcswitch/quiescer.go +++ b/htlcswitch/quiescer.go @@ -90,6 +90,11 @@ type Quiescer struct { // resolve when we complete quiescence. activeQuiescenceReq fn.Option[StfuReq] + // resumeQueue is a slice of hooks that will be called when the quiescer + // is resumed. These are actions that needed to be deferred while the + // channel was quiescent. + resumeQueue []func() + sync.RWMutex } @@ -400,3 +405,42 @@ func (q *Quiescer) initStfu(req StfuReq) { q.localInit = true q.activeQuiescenceReq = fn.Some(req) } + +// OnResume accepts a no return closure that will run when the quiescer is +// resumed. +func (q *Quiescer) OnResume(hook func()) { + q.Lock() + defer q.Unlock() + + q.onResume(hook) +} + +// onResume accepts a no return closure that will run when the quiescer is +// resumed. +func (q *Quiescer) onResume(hook func()) { + q.resumeQueue = append(q.resumeQueue, hook) +} + +// Resume runs all of the deferred actions that have accumulated while the +// channel has been quiescent and then resets the quiescer state to its initial +// state. +func (q *Quiescer) Resume() { + q.Lock() + defer q.Unlock() + + q.resume() +} + +// resume runs all of the deferred actions that have accumulated while the +// channel has been quiescent and then resets the quiescer state to its initial +// state. +func (q *Quiescer) resume() { + for _, hook := range q.resumeQueue { + hook() + } + q.localInit = false + q.remoteInit = false + q.sent = false + q.received = false + q.resumeQueue = nil +} diff --git a/htlcswitch/quiescer_test.go b/htlcswitch/quiescer_test.go index 5c6e2fa4c82..77cf9e45be1 100644 --- a/htlcswitch/quiescer_test.go +++ b/htlcswitch/quiescer_test.go @@ -380,3 +380,38 @@ func TestQuiescerTieBreaker(t *testing.T) { } } } + +// TestQuiescerResume ensures that the hooks that are attached to the quiescer +// are called when we call the resume method and no earlier. +func TestQuiescerResume(t *testing.T) { + t.Parallel() + + harness := initQuiescerTestHarness(lntypes.Local) + + msg := lnwire.Stfu{ + ChanID: cid, + Initiator: true, + } + + require.NoError( + t, harness.quiescer.RecvStfu( + msg, harness.pendingUpdates.Remote, + ), + ) + require.NoError( + t, harness.quiescer.SendOwedStfu( + harness.pendingUpdates.Local, + ), + ) + + require.True(t, harness.quiescer.IsQuiescent()) + var resumeHooksCalled = false + harness.quiescer.OnResume(func() { + resumeHooksCalled = true + }) + require.False(t, resumeHooksCalled) + + harness.quiescer.Resume() + require.True(t, resumeHooksCalled) + require.False(t, harness.quiescer.IsQuiescent()) +} From 5906ca2537ecf4640f9d3a02bf0492c76afaf8c8 Mon Sep 17 00:00:00 2001 From: Keagan McClelland Date: Tue, 9 Apr 2024 17:48:56 -0700 Subject: [PATCH 14/20] htlcswitch: add test for deferred processing remote adds when quiescent --- htlcswitch/link_test.go | 89 +++++++++++++++++++++++++++++++++++++++++ htlcswitch/mock.go | 12 +++++- 2 files changed, 99 insertions(+), 2 deletions(-) diff --git a/htlcswitch/link_test.go b/htlcswitch/link_test.go index c72a255384d..72597234034 100644 --- a/htlcswitch/link_test.go +++ b/htlcswitch/link_test.go @@ -7540,3 +7540,92 @@ func TestLinkFlushHooksCalled(t *testing.T) { ctx.receiveRevAndAckAliceToBob() assertHookCalled(true) } + +// TestLinkQuiescenceExitHopProcessingDeferred ensures that we do not send back +// htlc resolution messages in the case where the link is quiescent AND we are +// the exit hop. This is needed because we handle exit hop processing in the +// link instead of the switch and we process htlc resolutions when we receive +// a RevokeAndAck. Because of this we need to ensure that we hold off on +// processing the remote adds when we are quiescent. Later, when the channel +// update traffic is allowed to resume, we will need to verify that the actions +// we didn't run during the initial RevokeAndAck are run. +func TestLinkQuiescenceExitHopProcessingDeferred(t *testing.T) { + t.Parallel() + + // Initialize two channel state machines for testing. + alice, bob, err := createMirroredChannel( + t, btcutil.SatoshiPerBitcoin, btcutil.SatoshiPerBitcoin, + ) + require.NoError(t, err) + + // Build a single edge network to test channel quiescence. + network := newTwoHopNetwork( + t, alice.channel, bob.channel, testStartingHeight, + ) + aliceLink := network.aliceChannelLink + bobLink := network.bobChannelLink + + // Generate an invoice for Bob so that Alice can pay him. + htlcID := uint64(0) + htlc, invoice := generateHtlcAndInvoice(t, htlcID) + err = network.bobServer.registry.AddInvoice( + nil, *invoice, htlc.PaymentHash, + ) + require.NoError(t, err) + + // Establish a payment circuit for Alice + circuit := &PaymentCircuit{ + Incoming: CircuitKey{ + HtlcID: htlcID, + }, + PaymentHash: htlc.PaymentHash, + } + circuitMap := network.aliceServer.htlcSwitch.circuits + _, err = circuitMap.CommitCircuits(circuit) + require.NoError(t, err) + + // Add a switch packet to Alice's switch so that she can initialize the + // payment attempt. + err = aliceLink.handleSwitchPacket(&htlcPacket{ + incomingHTLCID: htlcID, + htlc: htlc, + circuit: circuit, + }) + require.NoError(t, err) + + // give alice enough time to fire the update_add + // TODO(proofofkeags): make this not depend on a flakey sleep. + <-time.After(time.Millisecond) + + // bob initiates stfu which he can do immediately since he doesn't have + // local updates + <-bobLink.InitStfu() + + // wait for other possible messages to play out + <-time.After(1 * time.Second) + + ensureNoUpdateAfterStfu := func(t *testing.T, trace []lnwire.Message) { + stfuReceived := false + for _, msg := range trace { + if msg.MsgType() == lnwire.MsgStfu { + stfuReceived = true + continue + } + + if stfuReceived && msg.MsgType().IsChannelUpdate() { + t.Fatalf("channel update after stfu: %v", + msg.MsgType()) + } + } + } + + network.aliceServer.protocolTraceMtx.Lock() + ensureNoUpdateAfterStfu(t, network.aliceServer.protocolTrace) + network.aliceServer.protocolTraceMtx.Unlock() + + network.bobServer.protocolTraceMtx.Lock() + ensureNoUpdateAfterStfu(t, network.bobServer.protocolTrace) + network.bobServer.protocolTraceMtx.Unlock() + + // TODO(proofofkeags): make sure these actions are run on resume. +} diff --git a/htlcswitch/mock.go b/htlcswitch/mock.go index 37bf4c6ef2d..0a3364ae271 100644 --- a/htlcswitch/mock.go +++ b/htlcswitch/mock.go @@ -153,8 +153,10 @@ type mockServer struct { t testing.TB - name string - messages chan lnwire.Message + name string + messages chan lnwire.Message + protocolTraceMtx sync.Mutex + protocolTrace []lnwire.Message id [33]byte htlcSwitch *Switch @@ -289,6 +291,10 @@ func (s *mockServer) Start() error { for { select { case msg := <-s.messages: + s.protocolTraceMtx.Lock() + s.protocolTrace = append(s.protocolTrace, msg) + s.protocolTraceMtx.Unlock() + var shouldSkip bool for _, interceptor := range s.interceptorFuncs { @@ -627,6 +633,8 @@ func (s *mockServer) readHandler(message lnwire.Message) error { targetChan = msg.ChanID case *lnwire.UpdateFee: targetChan = msg.ChanID + case *lnwire.Stfu: + targetChan = msg.ChanID default: return fmt.Errorf("unknown message type: %T", msg) } From 48ee643c0dd365ec0f6528056491ed41555647fa Mon Sep 17 00:00:00 2001 From: Keagan McClelland Date: Fri, 9 Aug 2024 20:25:44 -0700 Subject: [PATCH 15/20] htlcswitch: implement noop quiescer In this commit we implement a noop quiescer that we will use when the feature hasn't been negotiated. This will make it far easier to manage quiescence operations without having a number of if statements in the link logic. --- htlcswitch/quiescer.go | 131 ++++++++++++++++++++++++++---------- htlcswitch/quiescer_test.go | 8 ++- 2 files changed, 102 insertions(+), 37 deletions(-) diff --git a/htlcswitch/quiescer.go b/htlcswitch/quiescer.go index 3fa30597ba8..560448bfd6b 100644 --- a/htlcswitch/quiescer.go +++ b/htlcswitch/quiescer.go @@ -46,6 +46,51 @@ var ( type StfuReq = fn.Req[fn.Unit, fn.Result[lntypes.ChannelParty]] +// Quiescer is the public interface of the quiescence mechanism. Callers of the +// quiescence API should not need any methods besides the ones detailed here. +type Quiescer interface { + // IsQuiescent returns true if the state machine has been driven all the + // way to completion. If this returns true, processes that depend on + // channel quiescence may proceed. + IsQuiescent() bool + + // QuiescenceInitiator determines which ChannelParty is the initiator of + // quiescence for the purposes of downstream protocols. If the channel + // is not currently quiescent, this method will return + // ErrNoDownstreamLeader. + QuiescenceInitiator() fn.Result[lntypes.ChannelParty] + + // InitStfu instructs the quiescer that we intend to begin a quiescence + // negotiation where we are the initiator. We don't yet send stfu yet + // because we need to wait for the link to give us a valid opportunity + // to do so. + InitStfu(req StfuReq) + + // RecvStfu is called when we receive an Stfu message from the remote. + RecvStfu(stfu lnwire.Stfu, numRemotePendingUpdates uint64) error + + // CanRecvUpdates returns true if we haven't yet received an Stfu which + // would mark the end of the remote's ability to send updates. + CanRecvUpdates() bool + + // CanSendUpdates returns true if we haven't yet sent an Stfu which + // would mark the end of our ability to send updates. + CanSendUpdates() bool + + // SendOwedStfu sends Stfu if it owes one. It returns an error if the + // state machine is in an invalid state. + SendOwedStfu(numPendingLocalUpdates uint64) error + + // OnResume accepts a no return closure that will run when the quiescer + // is resumed. + OnResume(hook func()) + + // Resume runs all of the deferred actions that have accumulated while + // the channel has been quiescent and then resets the quiescer state to + // its initial state. + Resume() +} + // QuiescerCfg is a config structure used to initialize a quiescer giving it the // appropriate functionality to interact with the channel state that the // quiescer must syncrhonize with. @@ -65,9 +110,9 @@ type QuiescerCfg struct { sendMsg func(lnwire.Stfu) error } -// Quiescer is a state machine that tracks progression through the quiescence -// protocol. -type Quiescer struct { +// QuiescerLive is a state machine that tracks progression through the +// quiescence protocol. +type QuiescerLive struct { cfg QuiescerCfg // localInit indicates whether our path through this state machine was @@ -100,13 +145,13 @@ type Quiescer struct { // NewQuiescer creates a new quiescer for the given channel. func NewQuiescer(cfg QuiescerCfg) Quiescer { - return Quiescer{ + return &QuiescerLive{ cfg: cfg, } } // RecvStfu is called when we receive an Stfu message from the remote. -func (q *Quiescer) RecvStfu(msg lnwire.Stfu, +func (q *QuiescerLive) RecvStfu(msg lnwire.Stfu, numPendingRemoteUpdates uint64) error { q.Lock() @@ -116,7 +161,7 @@ func (q *Quiescer) RecvStfu(msg lnwire.Stfu, } // recvStfu is called when we receive an Stfu message from the remote. -func (q *Quiescer) recvStfu(msg lnwire.Stfu, +func (q *QuiescerLive) recvStfu(msg lnwire.Stfu, numPendingRemoteUpdates uint64) error { // At the time of this writing, this check that we have already received @@ -155,7 +200,7 @@ func (q *Quiescer) recvStfu(msg lnwire.Stfu, // MakeStfu is called when we are ready to send an Stfu message. It returns the // Stfu message to be sent. -func (q *Quiescer) MakeStfu( +func (q *QuiescerLive) MakeStfu( numPendingLocalUpdates uint64) fn.Result[lnwire.Stfu] { q.RLock() @@ -166,7 +211,7 @@ func (q *Quiescer) MakeStfu( // makeStfu is called when we are ready to send an Stfu message. It returns the // Stfu message to be sent. -func (q *Quiescer) makeStfu( +func (q *QuiescerLive) makeStfu( numPendingLocalUpdates uint64) fn.Result[lnwire.Stfu] { if q.sent { @@ -190,7 +235,7 @@ func (q *Quiescer) makeStfu( // OweStfu returns true if we owe the other party an Stfu. We owe the remote an // Stfu when we have received but not yet sent an Stfu, or we are the initiator // but have not yet sent an Stfu. -func (q *Quiescer) OweStfu() bool { +func (q *QuiescerLive) OweStfu() bool { q.RLock() defer q.RUnlock() @@ -200,13 +245,13 @@ func (q *Quiescer) OweStfu() bool { // oweStfu returns true if we owe the other party an Stfu. We owe the remote an // Stfu when we have received but not yet sent an Stfu, or we are the initiator // but have not yet sent an Stfu. -func (q *Quiescer) oweStfu() bool { +func (q *QuiescerLive) oweStfu() bool { return (q.received || q.localInit) && !q.sent } // NeedStfu returns true if the remote owes us an Stfu. They owe us an Stfu when // we have sent but not yet received an Stfu. -func (q *Quiescer) NeedStfu() bool { +func (q *QuiescerLive) NeedStfu() bool { q.RLock() defer q.RUnlock() @@ -215,7 +260,7 @@ func (q *Quiescer) NeedStfu() bool { // needStfu returns true if the remote owes us an Stfu. They owe us an Stfu when // we have sent but not yet received an Stfu. -func (q *Quiescer) needStfu() bool { +func (q *QuiescerLive) needStfu() bool { q.RLock() defer q.RUnlock() @@ -225,7 +270,7 @@ func (q *Quiescer) needStfu() bool { // IsQuiescent returns true if the state machine has been driven all the way to // completion. If this returns true, processes that depend on channel quiescence // may proceed. -func (q *Quiescer) IsQuiescent() bool { +func (q *QuiescerLive) IsQuiescent() bool { q.RLock() defer q.RUnlock() @@ -235,14 +280,14 @@ func (q *Quiescer) IsQuiescent() bool { // isQuiescent returns true if the state machine has been driven all the way to // completion. If this returns true, processes that depend on channel quiescence // may proceed. -func (q *Quiescer) isQuiescent() bool { +func (q *QuiescerLive) isQuiescent() bool { return q.sent && q.received } // QuiescenceInitiator determines which ChannelParty is the initiator of // quiescence for the purposes of downstream protocols. If the channel is not // currently quiescent, this method will return ErrNoQuiescenceInitiator. -func (q *Quiescer) QuiescenceInitiator() fn.Result[lntypes.ChannelParty] { +func (q *QuiescerLive) QuiescenceInitiator() fn.Result[lntypes.ChannelParty] { q.RLock() defer q.RUnlock() @@ -252,7 +297,7 @@ func (q *Quiescer) QuiescenceInitiator() fn.Result[lntypes.ChannelParty] { // quiescenceInitiator determines which ChannelParty is the initiator of // quiescence for the purposes of downstream protocols. If the channel is not // currently quiescent, this method will return ErrNoQuiescenceInitiator. -func (q *Quiescer) quiescenceInitiator() fn.Result[lntypes.ChannelParty] { +func (q *QuiescerLive) quiescenceInitiator() fn.Result[lntypes.ChannelParty] { switch { case !q.isQuiescent(): return fn.Err[lntypes.ChannelParty](ErrNoQuiescenceInitiator) @@ -274,7 +319,7 @@ func (q *Quiescer) quiescenceInitiator() fn.Result[lntypes.ChannelParty] { // CanSendUpdates returns true if we haven't yet sent an Stfu which would mark // the end of our ability to send updates. -func (q *Quiescer) CanSendUpdates() bool { +func (q *QuiescerLive) CanSendUpdates() bool { q.RLock() defer q.RUnlock() @@ -283,13 +328,13 @@ func (q *Quiescer) CanSendUpdates() bool { // canSendUpdates returns true if we haven't yet sent an Stfu which would mark // the end of our ability to send updates. -func (q *Quiescer) canSendUpdates() bool { +func (q *QuiescerLive) canSendUpdates() bool { return !q.sent && !q.localInit } // CanRecvUpdates returns true if we haven't yet received an Stfu which would // mark the end of the remote's ability to send updates. -func (q *Quiescer) CanRecvUpdates() bool { +func (q *QuiescerLive) CanRecvUpdates() bool { q.RLock() defer q.RUnlock() @@ -298,12 +343,12 @@ func (q *Quiescer) CanRecvUpdates() bool { // canRecvUpdates returns true if we haven't yet received an Stfu which would // mark the end of the remote's ability to send updates. -func (q *Quiescer) canRecvUpdates() bool { +func (q *QuiescerLive) canRecvUpdates() bool { return !q.received } // CanSendStfu returns true if we can send an Stfu. -func (q *Quiescer) CanSendStfu(numPendingLocalUpdates uint64) bool { +func (q *QuiescerLive) CanSendStfu(numPendingLocalUpdates uint64) bool { q.RLock() defer q.RUnlock() @@ -311,12 +356,12 @@ func (q *Quiescer) CanSendStfu(numPendingLocalUpdates uint64) bool { } // canSendStfu returns true if we can send an Stfu. -func (q *Quiescer) canSendStfu(numPendingLocalUpdates uint64) bool { +func (q *QuiescerLive) canSendStfu(numPendingLocalUpdates uint64) bool { return numPendingLocalUpdates == 0 && !q.sent } // CanRecvStfu returns true if we can receive an Stfu. -func (q *Quiescer) CanRecvStfu(numPendingRemoteUpdates uint64) bool { +func (q *QuiescerLive) CanRecvStfu(numPendingRemoteUpdates uint64) bool { q.RLock() defer q.RUnlock() @@ -324,13 +369,13 @@ func (q *Quiescer) CanRecvStfu(numPendingRemoteUpdates uint64) bool { } // canRecvStfu returns true if we can receive an Stfu. -func (q *Quiescer) canRecvStfu(numPendingRemoteUpdates uint64) bool { +func (q *QuiescerLive) canRecvStfu(numPendingRemoteUpdates uint64) bool { return numPendingRemoteUpdates == 0 && !q.received } // SendOwedStfu sends Stfu if it owes one. It returns an error if the state // machine is in an invalid state. -func (q *Quiescer) SendOwedStfu(numPendingLocalUpdates uint64) error { +func (q *QuiescerLive) SendOwedStfu(numPendingLocalUpdates uint64) error { q.Lock() defer q.Unlock() @@ -339,7 +384,7 @@ func (q *Quiescer) SendOwedStfu(numPendingLocalUpdates uint64) error { // sendOwedStfu sends Stfu if it owes one. It returns an error if the state // machine is in an invalid state. -func (q *Quiescer) sendOwedStfu(numPendingLocalUpdates uint64) error { +func (q *QuiescerLive) sendOwedStfu(numPendingLocalUpdates uint64) error { if !q.oweStfu() || !q.canSendStfu(numPendingLocalUpdates) { return nil } @@ -360,7 +405,7 @@ func (q *Quiescer) sendOwedStfu(numPendingLocalUpdates uint64) error { // TryResolveStfuReq attempts to resolve the active quiescence request if the // state machine has reached a quiescent state. -func (q *Quiescer) TryResolveStfuReq() { +func (q *QuiescerLive) TryResolveStfuReq() { q.Lock() defer q.Unlock() @@ -369,7 +414,7 @@ func (q *Quiescer) TryResolveStfuReq() { // tryResolveStfuReq attempts to resolve the active quiescence request if the // state machine has reached a quiescent state. -func (q *Quiescer) tryResolveStfuReq() { +func (q *QuiescerLive) tryResolveStfuReq() { q.activeQuiescenceReq.WhenSome( func(req StfuReq) { if q.isQuiescent() { @@ -383,7 +428,7 @@ func (q *Quiescer) tryResolveStfuReq() { // InitStfu instructs the quiescer that we intend to begin a quiescence // negotiation where we are the initiator. We don't yet send stfu yet because // we need to wait for the link to give us a valid opportunity to do so. -func (q *Quiescer) InitStfu(req StfuReq) { +func (q *QuiescerLive) InitStfu(req StfuReq) { q.Lock() defer q.Unlock() @@ -393,7 +438,7 @@ func (q *Quiescer) InitStfu(req StfuReq) { // initStfu instructs the quiescer that we intend to begin a quiescence // negotiation where we are the initiator. We don't yet send stfu yet because // we need to wait for the link to give us a valid opportunity to do so. -func (q *Quiescer) initStfu(req StfuReq) { +func (q *QuiescerLive) initStfu(req StfuReq) { if q.localInit { req.Resolve(fn.Errf[lntypes.ChannelParty]( "quiescence already requested", @@ -408,7 +453,7 @@ func (q *Quiescer) initStfu(req StfuReq) { // OnResume accepts a no return closure that will run when the quiescer is // resumed. -func (q *Quiescer) OnResume(hook func()) { +func (q *QuiescerLive) OnResume(hook func()) { q.Lock() defer q.Unlock() @@ -417,14 +462,14 @@ func (q *Quiescer) OnResume(hook func()) { // onResume accepts a no return closure that will run when the quiescer is // resumed. -func (q *Quiescer) onResume(hook func()) { +func (q *QuiescerLive) onResume(hook func()) { q.resumeQueue = append(q.resumeQueue, hook) } // Resume runs all of the deferred actions that have accumulated while the // channel has been quiescent and then resets the quiescer state to its initial // state. -func (q *Quiescer) Resume() { +func (q *QuiescerLive) Resume() { q.Lock() defer q.Unlock() @@ -434,7 +479,7 @@ func (q *Quiescer) Resume() { // resume runs all of the deferred actions that have accumulated while the // channel has been quiescent and then resets the quiescer state to its initial // state. -func (q *Quiescer) resume() { +func (q *QuiescerLive) resume() { for _, hook := range q.resumeQueue { hook() } @@ -444,3 +489,21 @@ func (q *Quiescer) resume() { q.received = false q.resumeQueue = nil } + +type quiescerNoop struct{} + +var _ Quiescer = (*quiescerNoop)(nil) + +func (q *quiescerNoop) InitStfu(req StfuReq) { + req.Resolve(fn.Errf[lntypes.ChannelParty]("quiescence not supported")) +} +func (q *quiescerNoop) RecvStfu(_ lnwire.Stfu, _ uint64) error { return nil } +func (q *quiescerNoop) CanRecvUpdates() bool { return true } +func (q *quiescerNoop) CanSendUpdates() bool { return true } +func (q *quiescerNoop) SendOwedStfu(_ uint64) error { return nil } +func (q *quiescerNoop) IsQuiescent() bool { return false } +func (q *quiescerNoop) OnResume(hook func()) { hook() } +func (q *quiescerNoop) Resume() {} +func (q *quiescerNoop) QuiescenceInitiator() fn.Result[lntypes.ChannelParty] { + return fn.Err[lntypes.ChannelParty](ErrNoQuiescenceInitiator) +} diff --git a/htlcswitch/quiescer_test.go b/htlcswitch/quiescer_test.go index 77cf9e45be1..3a5438792de 100644 --- a/htlcswitch/quiescer_test.go +++ b/htlcswitch/quiescer_test.go @@ -14,7 +14,7 @@ var cid = lnwire.ChannelID(bytes.Repeat([]byte{0x00}, 32)) type quiescerTestHarness struct { pendingUpdates lntypes.Dual[uint64] - quiescer Quiescer + quiescer *QuiescerLive conn <-chan lnwire.Stfu } @@ -27,14 +27,16 @@ func initQuiescerTestHarness( conn: conn, } - harness.quiescer = NewQuiescer(QuiescerCfg{ + quiescer, _ := NewQuiescer(QuiescerCfg{ chanID: cid, channelInitiator: channelInitiator, sendMsg: func(msg lnwire.Stfu) error { conn <- msg return nil }, - }) + }).(*QuiescerLive) + + harness.quiescer = quiescer return harness } From 111c9b05f338abbcc029ee478fe298611597118d Mon Sep 17 00:00:00 2001 From: Keagan McClelland Date: Fri, 9 Aug 2024 20:33:34 -0700 Subject: [PATCH 16/20] htlcswitch+peer: allow the disabling of quiescence Here we add a flag where we can disable quiescence. This will be used in the case where the feature is not negotiated with our peer. --- feature/default_sets.go | 4 ++++ feature/manager.go | 6 ++++++ htlcswitch/link.go | 27 ++++++++++++++++++--------- lncfg/protocol.go | 5 +++++ lncfg/protocol_integration.go | 8 ++++++++ peer/brontide.go | 6 ++++++ server.go | 2 ++ 7 files changed, 49 insertions(+), 9 deletions(-) diff --git a/feature/default_sets.go b/feature/default_sets.go index 616abc8ba38..9aee982ca7b 100644 --- a/feature/default_sets.go +++ b/feature/default_sets.go @@ -84,6 +84,10 @@ var defaultSetDesc = setDesc{ SetNodeAnn: {}, // N SetInvoice: {}, // 9 }, + lnwire.QuiescenceOptional: { + SetInit: {}, // I + SetNodeAnn: {}, // N + }, lnwire.ShutdownAnySegwitOptional: { SetInit: {}, // I SetNodeAnn: {}, // N diff --git a/feature/manager.go b/feature/manager.go index e0bcfc96bb0..95388321633 100644 --- a/feature/manager.go +++ b/feature/manager.go @@ -63,6 +63,9 @@ type Config struct { // NoRouteBlinding unsets route blinding feature bits. NoRouteBlinding bool + // NoQuiescence unsets quiescence feature bits. + NoQuiescence bool + // NoTaprootOverlay unsets the taproot overlay channel feature bits. NoTaprootOverlay bool @@ -199,6 +202,9 @@ func newManager(cfg Config, desc setDesc) (*Manager, error) { raw.Unset(lnwire.Bolt11BlindedPathsOptional) raw.Unset(lnwire.Bolt11BlindedPathsRequired) } + if cfg.NoQuiescence { + raw.Unset(lnwire.QuiescenceOptional) + } if cfg.NoTaprootOverlay { raw.Unset(lnwire.SimpleTaprootOverlayChansOptional) raw.Unset(lnwire.SimpleTaprootOverlayChansRequired) diff --git a/htlcswitch/link.go b/htlcswitch/link.go index 980f4dcb6d2..d2d8802a3d3 100644 --- a/htlcswitch/link.go +++ b/htlcswitch/link.go @@ -283,6 +283,10 @@ type ChannelLinkConfig struct { // invalid. DisallowRouteBlinding bool + // DisallowQuiescence is a flag that can be used to disable the + // quiescence protocol. + DisallowQuiescence bool + // MaxFeeExposure is the threshold in milli-satoshis after which we'll // restrict the flow of HTLCs and fee updates. MaxFeeExposure lnwire.MilliSatoshi @@ -476,14 +480,19 @@ func NewChannelLink(cfg ChannelLinkConfig, cfg.MaxFeeExposure = DefaultMaxFeeExposure } - quiescerCfg := QuiescerCfg{ - chanID: lnwire.NewChanIDFromOutPoint( - channel.ChannelPoint(), - ), - channelInitiator: channel.Initiator(), - sendMsg: func(s lnwire.Stfu) error { - return cfg.Peer.SendMessage(false, &s) - }, + var qsm Quiescer + if !cfg.DisallowQuiescence { + qsm = NewQuiescer(QuiescerCfg{ + chanID: lnwire.NewChanIDFromOutPoint( + channel.ChannelPoint(), + ), + channelInitiator: channel.Initiator(), + sendMsg: func(s lnwire.Stfu) error { + return cfg.Peer.SendMessage(false, &s) + }, + }) + } else { + qsm = &quiescerNoop{} } quiescenceReqs := make( @@ -499,7 +508,7 @@ func NewChannelLink(cfg ChannelLinkConfig, flushHooks: newHookMap(), outgoingCommitHooks: newHookMap(), incomingCommitHooks: newHookMap(), - quiescer: NewQuiescer(quiescerCfg), + quiescer: qsm, quiescenceReqs: quiescenceReqs, ContextGuard: fn.NewContextGuard(), } diff --git a/lncfg/protocol.go b/lncfg/protocol.go index 80809f49d66..5852d032e08 100644 --- a/lncfg/protocol.go +++ b/lncfg/protocol.go @@ -141,6 +141,11 @@ func (l *ProtocolOptions) NoExperimentalEndorsement() bool { return l.NoExperimentalEndorsementOption } +// NoQuiescence returns true if quiescence is disabled. +func (l *ProtocolOptions) NoQuiescence() bool { + return true +} + // CustomMessageOverrides returns the set of protocol messages that we override // to allow custom handling. func (p ProtocolOptions) CustomMessageOverrides() []uint16 { diff --git a/lncfg/protocol_integration.go b/lncfg/protocol_integration.go index 52cc658c3ba..e9f32d9dfb9 100644 --- a/lncfg/protocol_integration.go +++ b/lncfg/protocol_integration.go @@ -73,6 +73,9 @@ type ProtocolOptions struct { // NoExperimentalEndorsementOption disables experimental endorsement. NoExperimentalEndorsementOption bool `long:"no-experimental-endorsement" description:"do not forward experimental endorsement signals"` + // NoQuiescenceOption disables quiescence for all channels. + NoQuiescenceOption bool `long:"no-quiescence" description:"do not allow or advertise quiescence for any channel"` + // CustomMessage allows the custom message APIs to handle messages with // the provided protocol numbers, which fall outside the custom message // number range. @@ -136,6 +139,11 @@ func (l *ProtocolOptions) NoExperimentalEndorsement() bool { return l.NoExperimentalEndorsementOption } +// NoQuiescence returns true if quiescence is disabled. +func (l *ProtocolOptions) NoQuiescence() bool { + return l.NoQuiescenceOption +} + // CustomMessageOverrides returns the set of protocol messages that we override // to allow custom handling. func (l ProtocolOptions) CustomMessageOverrides() []uint16 { diff --git a/peer/brontide.go b/peer/brontide.go index 17bef3234e2..3bae1be1bc3 100644 --- a/peer/brontide.go +++ b/peer/brontide.go @@ -411,6 +411,10 @@ type Config struct { // invalid. DisallowRouteBlinding bool + // DisallowQuiescence is a flag that indicates whether the Brontide + // should have the quiescence feature disabled. + DisallowQuiescence bool + // MaxFeeExposure limits the number of outstanding fees in a channel. // This value will be passed to created links. MaxFeeExposure lnwire.MilliSatoshi @@ -1324,6 +1328,8 @@ func (p *Brontide) addLink(chanPoint *wire.OutPoint, DisallowRouteBlinding: p.cfg.DisallowRouteBlinding, MaxFeeExposure: p.cfg.MaxFeeExposure, ShouldFwdExpEndorsement: p.cfg.ShouldFwdExpEndorsement, + DisallowQuiescence: p.cfg.DisallowQuiescence || + !p.remoteFeatures.HasFeature(lnwire.QuiescenceOptional), } // Before adding our new link, purge the switch of any pending or live diff --git a/server.go b/server.go index 3b8224f5b04..c186d365607 100644 --- a/server.go +++ b/server.go @@ -587,6 +587,7 @@ func newServer(cfg *Config, listenAddrs []net.Addr, NoTaprootOverlay: !cfg.ProtocolOptions.TaprootOverlayChans, NoRouteBlinding: cfg.ProtocolOptions.NoRouteBlinding(), NoExperimentalEndorsement: cfg.ProtocolOptions.NoExperimentalEndorsement(), + NoQuiescence: cfg.ProtocolOptions.NoQuiescence(), }) if err != nil { return nil, err @@ -4214,6 +4215,7 @@ func (s *server) peerConnected(conn net.Conn, connReq *connmgr.ConnReq, RequestAlias: s.aliasMgr.RequestAlias, AddLocalAlias: s.aliasMgr.AddLocalAlias, DisallowRouteBlinding: s.cfg.ProtocolOptions.NoRouteBlinding(), + DisallowQuiescence: s.cfg.ProtocolOptions.NoQuiescence(), MaxFeeExposure: thresholdMSats, Quit: s.quit, AuxLeafStore: s.implCfg.AuxLeafStore, From a4c49a88f1ce340c31ec8bb7b1b7ca9eb0d2ca08 Mon Sep 17 00:00:00 2001 From: Keagan McClelland Date: Mon, 11 Nov 2024 16:33:50 -0700 Subject: [PATCH 17/20] htlcswitch: add quiescence timeout that is aborted by Resume --- htlcswitch/link.go | 4 +++ htlcswitch/quiescer.go | 64 +++++++++++++++++++++++++++++++++++++ htlcswitch/quiescer_test.go | 56 ++++++++++++++++++++++++++++++++ 3 files changed, 124 insertions(+) diff --git a/htlcswitch/link.go b/htlcswitch/link.go index d2d8802a3d3..c62b6f16392 100644 --- a/htlcswitch/link.go +++ b/htlcswitch/link.go @@ -490,6 +490,10 @@ func NewChannelLink(cfg ChannelLinkConfig, sendMsg: func(s lnwire.Stfu) error { return cfg.Peer.SendMessage(false, &s) }, + timeoutDuration: defaultQuiescenceTimeout, + onTimeout: func() { + cfg.Peer.Disconnect(ErrQuiescenceTimeout) + }, }) } else { qsm = &quiescerNoop{} diff --git a/htlcswitch/quiescer.go b/htlcswitch/quiescer.go index 560448bfd6b..4b3518ee2f5 100644 --- a/htlcswitch/quiescer.go +++ b/htlcswitch/quiescer.go @@ -3,6 +3,7 @@ package htlcswitch import ( "fmt" "sync" + "time" "github.com/lightningnetwork/lnd/fn" "github.com/lightningnetwork/lnd/lntypes" @@ -42,8 +43,16 @@ var ( ErrPendingLocalUpdates = fmt.Errorf( "stfu send attempted with pending local updates", ) + + // ErrQuiescenceTimeout indicates that the quiescer has been quiesced + // beyond the allotted time. + ErrQuiescenceTimeout = fmt.Errorf( + "quiescence timeout", + ) ) +const defaultQuiescenceTimeout = 30 * time.Second + type StfuReq = fn.Req[fn.Unit, fn.Result[lntypes.ChannelParty]] // Quiescer is the public interface of the quiescence mechanism. Callers of the @@ -108,6 +117,17 @@ type QuiescerCfg struct { // sendMsg is a function that can be used to send an Stfu message over // the wire. sendMsg func(lnwire.Stfu) error + + // timeoutDuration is the Duration that we will wait from the moment the + // channel is considered quiescent before we call the onTimeout function + timeoutDuration time.Duration + + // onTimeout is a function that will be called in the event that the + // Quiescer has not been resumed before the timeout is reached. If + // Quiescer.Resume is called before the timeout has been raeached, then + // onTimeout will not be called until the quiescer reaches a quiescent + // state again. + onTimeout func() } // QuiescerLive is a state machine that tracks progression through the @@ -140,6 +160,10 @@ type QuiescerLive struct { // channel was quiescent. resumeQueue []func() + // timeoutTimer is a field that is used to hold onto the timeout job + // when we reach quiescence. + timeoutTimer *time.Timer + sync.RWMutex } @@ -195,6 +219,10 @@ func (q *QuiescerLive) recvStfu(msg lnwire.Stfu, // If so, we will try to resolve any outstanding StfuReqs. q.tryResolveStfuReq() + if q.isQuiescent() { + q.startTimeout() + } + return nil } @@ -398,6 +426,10 @@ func (q *QuiescerLive) sendOwedStfu(numPendingLocalUpdates uint64) error { // state. If so, we will try to resolve any outstanding // StfuReqs. q.tryResolveStfuReq() + + if q.isQuiescent() { + q.startTimeout() + } } return err @@ -480,6 +512,10 @@ func (q *QuiescerLive) Resume() { // channel has been quiescent and then resets the quiescer state to its initial // state. func (q *QuiescerLive) resume() { + // since we are resuming we want to cancel the quiescence timeout + // action. + q.cancelTimeout() + for _, hook := range q.resumeQueue { hook() } @@ -490,6 +526,34 @@ func (q *QuiescerLive) resume() { q.resumeQueue = nil } +// startTimeout starts the timeout function that fires if the quiescer remains +// in a quiesced state for too long. If this function is called multiple times +// only the last one will have an effect. +func (q *QuiescerLive) startTimeout() { + if q.cfg.onTimeout == nil { + return + } + + old := q.timeoutTimer + + q.timeoutTimer = time.AfterFunc(q.cfg.timeoutDuration, q.cfg.onTimeout) + + if old != nil { + old.Stop() + } +} + +// cancelTimeout cancels the timeout function that would otherwise fire if the +// quiescer remains in a quiesced state too long. If this function is called +// before startTimeout or after another call to cancelTimeout, the effect will +// be a noop. +func (q *QuiescerLive) cancelTimeout() { + if q.timeoutTimer != nil { + q.timeoutTimer.Stop() + q.timeoutTimer = nil + } +} + type quiescerNoop struct{} var _ Quiescer = (*quiescerNoop)(nil) diff --git a/htlcswitch/quiescer_test.go b/htlcswitch/quiescer_test.go index 3a5438792de..08e201ddd05 100644 --- a/htlcswitch/quiescer_test.go +++ b/htlcswitch/quiescer_test.go @@ -3,6 +3,7 @@ package htlcswitch import ( "bytes" "testing" + "time" "github.com/lightningnetwork/lnd/fn" "github.com/lightningnetwork/lnd/lntypes" @@ -417,3 +418,58 @@ func TestQuiescerResume(t *testing.T) { require.True(t, resumeHooksCalled) require.False(t, harness.quiescer.IsQuiescent()) } + +func TestQuiescerTimeoutTriggers(t *testing.T) { + t.Parallel() + + harness := initQuiescerTestHarness(lntypes.Local) + + msg := lnwire.Stfu{ + ChanID: cid, + Initiator: true, + } + + timeoutGate := make(chan struct{}) + + harness.quiescer.cfg.timeoutDuration = time.Second + harness.quiescer.cfg.onTimeout = func() { close(timeoutGate) } + + err := harness.quiescer.RecvStfu(msg, harness.pendingUpdates.Remote) + require.NoError(t, err) + err = harness.quiescer.SendOwedStfu(harness.pendingUpdates.Local) + require.NoError(t, err) + + select { + case <-timeoutGate: + case <-time.After(2 * harness.quiescer.cfg.timeoutDuration): + t.Fatal("quiescence timeout did not trigger") + } +} + +func TestQuiescerTimeoutAborts(t *testing.T) { + t.Parallel() + + harness := initQuiescerTestHarness(lntypes.Local) + + msg := lnwire.Stfu{ + ChanID: cid, + Initiator: true, + } + + timeoutGate := make(chan struct{}) + + harness.quiescer.cfg.timeoutDuration = time.Second + harness.quiescer.cfg.onTimeout = func() { close(timeoutGate) } + + err := harness.quiescer.RecvStfu(msg, harness.pendingUpdates.Remote) + require.NoError(t, err) + err = harness.quiescer.SendOwedStfu(harness.pendingUpdates.Local) + require.NoError(t, err) + harness.quiescer.Resume() + + select { + case <-timeoutGate: + t.Fatal("quiescence timeout triggered despite being resumed") + case <-time.After(2 * harness.quiescer.cfg.timeoutDuration): + } +} From ac0c24aa7bd60c644739182f1feabc3885c70da7 Mon Sep 17 00:00:00 2001 From: Keagan McClelland Date: Tue, 12 Nov 2024 15:06:02 -0700 Subject: [PATCH 18/20] htlcswitch: don't pass pending update counts into quiescer This change simplifies some of the quiescer responsibilities in favor of making the link check whether or not it has a clean state to be able to send or receive an stfu. This change was made on the basis that the only use the quiescer makes of this information is to assess that it is or is not zero. Further the difficulty of checking this condition in the link is barely more burdensome than selecting the proper information to pass to the quiescer anyway. --- htlcswitch/link.go | 69 ++++++++-------- htlcswitch/quiescer.go | 66 +++++++-------- htlcswitch/quiescer_test.go | 159 +++++++----------------------------- 3 files changed, 93 insertions(+), 201 deletions(-) diff --git a/htlcswitch/link.go b/htlcswitch/link.go index c62b6f16392..3eb398c1af1 100644 --- a/htlcswitch/link.go +++ b/htlcswitch/link.go @@ -1529,17 +1529,15 @@ func (l *channelLink) htlcManager() { case qReq := <-l.quiescenceReqs: l.quiescer.InitStfu(qReq) - pendingOnLocal := l.channel.NumPendingUpdates( - lntypes.Local, lntypes.Local, - ) - pendingOnRemote := l.channel.NumPendingUpdates( - lntypes.Local, lntypes.Remote, - ) - if err := l.quiescer.SendOwedStfu( - pendingOnLocal + pendingOnRemote, - ); err != nil { - l.stfuFailf("%s", err.Error()) - qReq.Resolve(fn.Err[lntypes.ChannelParty](err)) + if l.noDanglingUpdates(lntypes.Local) { + err := l.quiescer.SendOwedStfu() + if err != nil { + l.stfuFailf( + "SendOwedStfu: %s", err.Error(), + ) + res := fn.Err[lntypes.ChannelParty](err) + qReq.Resolve(res) + } } case <-l.Quit: @@ -2436,15 +2434,11 @@ func (l *channelLink) handleUpstreamMsg(msg lnwire.Message) { // If we need to send out an Stfu, this would be the time to do // so. - pendingOnLocal := l.channel.NumPendingUpdates( - lntypes.Local, lntypes.Local, - ) - pendingOnRemote := l.channel.NumPendingUpdates( - lntypes.Local, lntypes.Remote, - ) - err = l.quiescer.SendOwedStfu(pendingOnLocal + pendingOnRemote) - if err != nil { - l.stfuFailf("sendOwedStfu: %v", err.Error()) + if l.noDanglingUpdates(lntypes.Local) { + err = l.quiescer.SendOwedStfu() + if err != nil { + l.stfuFailf("sendOwedStfu: %v", err.Error()) + } } // Now that we have finished processing the incoming CommitSig @@ -2635,26 +2629,20 @@ func (l *channelLink) handleUpstreamMsg(msg lnwire.Message) { // handleStfu implements the top-level logic for handling the Stfu message from // our peer. func (l *channelLink) handleStfu(stfu *lnwire.Stfu) error { - pendingOnLocal := l.channel.NumPendingUpdates( - lntypes.Remote, lntypes.Local, - ) - pendingOnRemote := l.channel.NumPendingUpdates( - lntypes.Remote, lntypes.Remote, - ) - err := l.quiescer.RecvStfu(*stfu, pendingOnLocal+pendingOnRemote) + if !l.noDanglingUpdates(lntypes.Remote) { + return ErrPendingRemoteUpdates + } + err := l.quiescer.RecvStfu(*stfu) if err != nil { return err } // If we can immediately send an Stfu response back, we will. - pendingOnLocal = l.channel.NumPendingUpdates( - lntypes.Local, lntypes.Local, - ) - pendingOnRemote = l.channel.NumPendingUpdates( - lntypes.Local, lntypes.Remote, - ) + if l.noDanglingUpdates(lntypes.Local) { + return l.quiescer.SendOwedStfu() + } - return l.quiescer.SendOwedStfu(pendingOnLocal + pendingOnRemote) + return nil } // stfuFailf fails the link in the case where the requirements of the quiescence @@ -2669,6 +2657,19 @@ func (l *channelLink) stfuFailf(format string, args ...interface{}) { }, format, args...) } +// noDanglingUpdates returns true when there are 0 updates that were originally +// issued by whose on either the Local or Remote commitment transaction. +func (l *channelLink) noDanglingUpdates(whose lntypes.ChannelParty) bool { + pendingOnLocal := l.channel.NumPendingUpdates( + whose, lntypes.Local, + ) + pendingOnRemote := l.channel.NumPendingUpdates( + whose, lntypes.Remote, + ) + + return pendingOnLocal == 0 && pendingOnRemote == 0 +} + // ackDownStreamPackets is responsible for removing htlcs from a link's mailbox // for packets delivered from server, and cleaning up any circuits closed by // signing a previous commitment txn. This method ensures that the circuits are diff --git a/htlcswitch/quiescer.go b/htlcswitch/quiescer.go index 4b3518ee2f5..7e6269b1202 100644 --- a/htlcswitch/quiescer.go +++ b/htlcswitch/quiescer.go @@ -76,7 +76,7 @@ type Quiescer interface { InitStfu(req StfuReq) // RecvStfu is called when we receive an Stfu message from the remote. - RecvStfu(stfu lnwire.Stfu, numRemotePendingUpdates uint64) error + RecvStfu(stfu lnwire.Stfu) error // CanRecvUpdates returns true if we haven't yet received an Stfu which // would mark the end of the remote's ability to send updates. @@ -88,7 +88,7 @@ type Quiescer interface { // SendOwedStfu sends Stfu if it owes one. It returns an error if the // state machine is in an invalid state. - SendOwedStfu(numPendingLocalUpdates uint64) error + SendOwedStfu() error // OnResume accepts a no return closure that will run when the quiescer // is resumed. @@ -175,19 +175,15 @@ func NewQuiescer(cfg QuiescerCfg) Quiescer { } // RecvStfu is called when we receive an Stfu message from the remote. -func (q *QuiescerLive) RecvStfu(msg lnwire.Stfu, - numPendingRemoteUpdates uint64) error { - +func (q *QuiescerLive) RecvStfu(msg lnwire.Stfu) error { q.Lock() defer q.Unlock() - return q.recvStfu(msg, numPendingRemoteUpdates) + return q.recvStfu(msg) } // recvStfu is called when we receive an Stfu message from the remote. -func (q *QuiescerLive) recvStfu(msg lnwire.Stfu, - numPendingRemoteUpdates uint64) error { - +func (q *QuiescerLive) recvStfu(msg lnwire.Stfu) error { // At the time of this writing, this check that we have already received // an Stfu is not strictly necessary, according to the specification. // However, it is fishy if we do and it is unclear how we should handle @@ -203,7 +199,7 @@ func (q *QuiescerLive) recvStfu(msg lnwire.Stfu, q.cfg.chanID) } - if !q.canRecvStfu(numPendingRemoteUpdates) { + if !q.canRecvStfu() { return fmt.Errorf("%w for channel %v", ErrPendingRemoteUpdates, q.cfg.chanID) } @@ -228,26 +224,22 @@ func (q *QuiescerLive) recvStfu(msg lnwire.Stfu, // MakeStfu is called when we are ready to send an Stfu message. It returns the // Stfu message to be sent. -func (q *QuiescerLive) MakeStfu( - numPendingLocalUpdates uint64) fn.Result[lnwire.Stfu] { - +func (q *QuiescerLive) MakeStfu() fn.Result[lnwire.Stfu] { q.RLock() defer q.RUnlock() - return q.makeStfu(numPendingLocalUpdates) + return q.makeStfu() } // makeStfu is called when we are ready to send an Stfu message. It returns the // Stfu message to be sent. -func (q *QuiescerLive) makeStfu( - numPendingLocalUpdates uint64) fn.Result[lnwire.Stfu] { - +func (q *QuiescerLive) makeStfu() fn.Result[lnwire.Stfu] { if q.sent { return fn.Errf[lnwire.Stfu]("%w for channel %v", ErrStfuAlreadySent, q.cfg.chanID) } - if !q.canSendStfu(numPendingLocalUpdates) { + if !q.canSendStfu() { return fn.Errf[lnwire.Stfu]("%w for channel %v", ErrPendingLocalUpdates, q.cfg.chanID) } @@ -380,44 +372,44 @@ func (q *QuiescerLive) CanSendStfu(numPendingLocalUpdates uint64) bool { q.RLock() defer q.RUnlock() - return q.canSendStfu(numPendingLocalUpdates) + return q.canSendStfu() } // canSendStfu returns true if we can send an Stfu. -func (q *QuiescerLive) canSendStfu(numPendingLocalUpdates uint64) bool { - return numPendingLocalUpdates == 0 && !q.sent +func (q *QuiescerLive) canSendStfu() bool { + return !q.sent } // CanRecvStfu returns true if we can receive an Stfu. -func (q *QuiescerLive) CanRecvStfu(numPendingRemoteUpdates uint64) bool { +func (q *QuiescerLive) CanRecvStfu() bool { q.RLock() defer q.RUnlock() - return q.canRecvStfu(numPendingRemoteUpdates) + return q.canRecvStfu() } // canRecvStfu returns true if we can receive an Stfu. -func (q *QuiescerLive) canRecvStfu(numPendingRemoteUpdates uint64) bool { - return numPendingRemoteUpdates == 0 && !q.received +func (q *QuiescerLive) canRecvStfu() bool { + return !q.received } // SendOwedStfu sends Stfu if it owes one. It returns an error if the state // machine is in an invalid state. -func (q *QuiescerLive) SendOwedStfu(numPendingLocalUpdates uint64) error { +func (q *QuiescerLive) SendOwedStfu() error { q.Lock() defer q.Unlock() - return q.sendOwedStfu(numPendingLocalUpdates) + return q.sendOwedStfu() } // sendOwedStfu sends Stfu if it owes one. It returns an error if the state // machine is in an invalid state. -func (q *QuiescerLive) sendOwedStfu(numPendingLocalUpdates uint64) error { - if !q.oweStfu() || !q.canSendStfu(numPendingLocalUpdates) { +func (q *QuiescerLive) sendOwedStfu() error { + if !q.oweStfu() || !q.canSendStfu() { return nil } - err := q.makeStfu(numPendingLocalUpdates).Sink(q.cfg.sendMsg) + err := q.makeStfu().Sink(q.cfg.sendMsg) if err == nil { q.sent = true @@ -561,13 +553,13 @@ var _ Quiescer = (*quiescerNoop)(nil) func (q *quiescerNoop) InitStfu(req StfuReq) { req.Resolve(fn.Errf[lntypes.ChannelParty]("quiescence not supported")) } -func (q *quiescerNoop) RecvStfu(_ lnwire.Stfu, _ uint64) error { return nil } -func (q *quiescerNoop) CanRecvUpdates() bool { return true } -func (q *quiescerNoop) CanSendUpdates() bool { return true } -func (q *quiescerNoop) SendOwedStfu(_ uint64) error { return nil } -func (q *quiescerNoop) IsQuiescent() bool { return false } -func (q *quiescerNoop) OnResume(hook func()) { hook() } -func (q *quiescerNoop) Resume() {} +func (q *quiescerNoop) RecvStfu(_ lnwire.Stfu) error { return nil } +func (q *quiescerNoop) CanRecvUpdates() bool { return true } +func (q *quiescerNoop) CanSendUpdates() bool { return true } +func (q *quiescerNoop) SendOwedStfu() error { return nil } +func (q *quiescerNoop) IsQuiescent() bool { return false } +func (q *quiescerNoop) OnResume(hook func()) { hook() } +func (q *quiescerNoop) Resume() {} func (q *quiescerNoop) QuiescenceInitiator() fn.Result[lntypes.ChannelParty] { return fn.Err[lntypes.ChannelParty](ErrNoQuiescenceInitiator) } diff --git a/htlcswitch/quiescer_test.go b/htlcswitch/quiescer_test.go index 08e201ddd05..da08909d57c 100644 --- a/htlcswitch/quiescer_test.go +++ b/htlcswitch/quiescer_test.go @@ -14,9 +14,8 @@ import ( var cid = lnwire.ChannelID(bytes.Repeat([]byte{0x00}, 32)) type quiescerTestHarness struct { - pendingUpdates lntypes.Dual[uint64] - quiescer *QuiescerLive - conn <-chan lnwire.Stfu + quiescer *QuiescerLive + conn <-chan lnwire.Stfu } func initQuiescerTestHarness( @@ -24,8 +23,7 @@ func initQuiescerTestHarness( conn := make(chan lnwire.Stfu, 1) harness := &quiescerTestHarness{ - pendingUpdates: lntypes.Dual[uint64]{}, - conn: conn, + conn: conn, } quiescer, _ := NewQuiescer(QuiescerCfg{ @@ -54,30 +52,12 @@ func TestQuiescerDoubleRecvInvalid(t *testing.T) { Initiator: true, } - err := harness.quiescer.RecvStfu(msg, harness.pendingUpdates.Remote) + err := harness.quiescer.RecvStfu(msg) require.NoError(t, err) - err = harness.quiescer.RecvStfu(msg, harness.pendingUpdates.Remote) + err = harness.quiescer.RecvStfu(msg) require.Error(t, err, ErrStfuAlreadyRcvd) } -// TestQuiescerPendingUpdatesRecvInvalid ensures that we get an error if we -// receive the Stfu message while the Remote party has panding updates on the -// channel. -func TestQuiescerPendingUpdatesRecvInvalid(t *testing.T) { - t.Parallel() - - harness := initQuiescerTestHarness(lntypes.Local) - - msg := lnwire.Stfu{ - ChanID: cid, - Initiator: true, - } - - harness.pendingUpdates.SetForParty(lntypes.Remote, 1) - err := harness.quiescer.RecvStfu(msg, harness.pendingUpdates.Remote) - require.ErrorIs(t, err, ErrPendingRemoteUpdates) -} - // TestQuiescenceRemoteInit ensures that we can successfully traverse the state // graph of quiescence beginning with the Remote party initiating quiescence. func TestQuiescenceRemoteInit(t *testing.T) { @@ -90,22 +70,10 @@ func TestQuiescenceRemoteInit(t *testing.T) { Initiator: true, } - harness.pendingUpdates.SetForParty(lntypes.Local, 1) - - err := harness.quiescer.RecvStfu(msg, harness.pendingUpdates.Remote) + err := harness.quiescer.RecvStfu(msg) require.NoError(t, err) - err = harness.quiescer.SendOwedStfu(harness.pendingUpdates.Local) - require.NoError(t, err) - - select { - case <-harness.conn: - t.Fatalf("stfu sent when not expected") - default: - } - - harness.pendingUpdates.SetForParty(lntypes.Local, 0) - err = harness.quiescer.SendOwedStfu(harness.pendingUpdates.Local) + err = harness.quiescer.SendOwedStfu() require.NoError(t, err) select { @@ -125,25 +93,13 @@ func TestQuiescenceLocalInit(t *testing.T) { ChanID: cid, Initiator: true, } - harness.pendingUpdates.SetForParty(lntypes.Local, 1) stfuReq, stfuRes := fn.NewReq[fn.Unit, fn.Result[lntypes.ChannelParty]]( fn.Unit{}, ) harness.quiescer.InitStfu(stfuReq) - harness.pendingUpdates.SetForParty(lntypes.Local, 1) - err := harness.quiescer.SendOwedStfu(harness.pendingUpdates.Local) - require.NoError(t, err) - - select { - case <-harness.conn: - t.Fatalf("stfu sent when not expected") - default: - } - - harness.pendingUpdates.SetForParty(lntypes.Local, 0) - err = harness.quiescer.SendOwedStfu(harness.pendingUpdates.Local) + err := harness.quiescer.SendOwedStfu() require.NoError(t, err) select { @@ -153,7 +109,7 @@ func TestQuiescenceLocalInit(t *testing.T) { t.Fatalf("stfu not sent when expected") } - err = harness.quiescer.RecvStfu(msg, harness.pendingUpdates.Remote) + err = harness.quiescer.RecvStfu(msg) require.NoError(t, err) select { @@ -178,17 +134,11 @@ func TestQuiescenceInitiator(t *testing.T) { ChanID: cid, Initiator: true, } - require.NoError( - t, harness.quiescer.RecvStfu( - msg, harness.pendingUpdates.Remote, - ), - ) + require.NoError(t, harness.quiescer.RecvStfu(msg)) require.True(t, harness.quiescer.QuiescenceInitiator().IsErr()) // Send - require.NoError( - t, harness.quiescer.SendOwedStfu(harness.pendingUpdates.Local), - ) + require.NoError(t, harness.quiescer.SendOwedStfu()) require.Equal( t, harness.quiescer.QuiescenceInitiator(), fn.Ok(lntypes.Remote), @@ -214,7 +164,7 @@ func TestQuiescenceInitiator(t *testing.T) { } require.NoError( - t, harness.quiescer.sendOwedStfu(harness.pendingUpdates.Local), + t, harness.quiescer.sendOwedStfu(), ) require.True(t, harness.quiescer.quiescenceInitiator().IsErr()) @@ -222,11 +172,7 @@ func TestQuiescenceInitiator(t *testing.T) { ChanID: cid, Initiator: false, } - require.NoError( - t, harness.quiescer.recvStfu( - msg, harness.pendingUpdates.Remote, - ), - ) + require.NoError(t, harness.quiescer.recvStfu(msg)) require.True(t, harness.quiescer.quiescenceInitiator().IsOk()) select { @@ -249,11 +195,7 @@ func TestQuiescenceCantReceiveUpdatesAfterStfu(t *testing.T) { ChanID: cid, Initiator: true, } - require.NoError( - t, harness.quiescer.RecvStfu( - msg, harness.pendingUpdates.Remote, - ), - ) + require.NoError(t, harness.quiescer.RecvStfu(msg)) require.False(t, harness.quiescer.CanRecvUpdates()) } @@ -270,10 +212,10 @@ func TestQuiescenceCantSendUpdatesAfterStfu(t *testing.T) { Initiator: true, } - err := harness.quiescer.RecvStfu(msg, harness.pendingUpdates.Remote) + err := harness.quiescer.RecvStfu(msg) require.NoError(t, err) - err = harness.quiescer.SendOwedStfu(harness.pendingUpdates.Local) + err = harness.quiescer.SendOwedStfu() require.NoError(t, err) require.False(t, harness.quiescer.CanSendUpdates()) @@ -293,11 +235,7 @@ func TestQuiescenceStfuNotNeededAfterRecv(t *testing.T) { } require.False(t, harness.quiescer.NeedStfu()) - require.NoError( - t, harness.quiescer.RecvStfu( - msg, harness.pendingUpdates.Remote, - ), - ) + require.NoError(t, harness.quiescer.RecvStfu(msg)) require.False(t, harness.quiescer.NeedStfu()) } @@ -309,38 +247,15 @@ func TestQuiescenceInappropriateMakeStfuReturnsErr(t *testing.T) { harness := initQuiescerTestHarness(lntypes.Local) - harness.pendingUpdates.SetForParty(lntypes.Local, 1) - - require.True( - t, harness.quiescer.MakeStfu( - harness.pendingUpdates.Local, - ).IsErr(), - ) - - harness.pendingUpdates.SetForParty(lntypes.Local, 0) msg := lnwire.Stfu{ ChanID: cid, Initiator: true, } - require.NoError( - t, harness.quiescer.RecvStfu( - msg, harness.pendingUpdates.Remote, - ), - ) - require.True( - t, harness.quiescer.MakeStfu( - harness.pendingUpdates.Local, - ).IsOk(), - ) + require.NoError(t, harness.quiescer.RecvStfu(msg)) + require.True(t, harness.quiescer.MakeStfu().IsOk()) - require.NoError( - t, harness.quiescer.SendOwedStfu(harness.pendingUpdates.Local), - ) - require.True( - t, harness.quiescer.MakeStfu( - harness.pendingUpdates.Local, - ).IsErr(), - ) + require.NoError(t, harness.quiescer.SendOwedStfu()) + require.True(t, harness.quiescer.MakeStfu().IsErr()) } // TestQuiescerTieBreaker ensures that if both parties attempt to claim the @@ -364,16 +279,8 @@ func TestQuiescerTieBreaker(t *testing.T) { ) harness.quiescer.InitStfu(req) - require.NoError( - t, harness.quiescer.RecvStfu( - msg, harness.pendingUpdates.Remote, - ), - ) - require.NoError( - t, harness.quiescer.SendOwedStfu( - harness.pendingUpdates.Local, - ), - ) + require.NoError(t, harness.quiescer.RecvStfu(msg)) + require.NoError(t, harness.quiescer.SendOwedStfu()) select { case party := <-res: @@ -396,16 +303,8 @@ func TestQuiescerResume(t *testing.T) { Initiator: true, } - require.NoError( - t, harness.quiescer.RecvStfu( - msg, harness.pendingUpdates.Remote, - ), - ) - require.NoError( - t, harness.quiescer.SendOwedStfu( - harness.pendingUpdates.Local, - ), - ) + require.NoError(t, harness.quiescer.RecvStfu(msg)) + require.NoError(t, harness.quiescer.SendOwedStfu()) require.True(t, harness.quiescer.IsQuiescent()) var resumeHooksCalled = false @@ -434,9 +333,9 @@ func TestQuiescerTimeoutTriggers(t *testing.T) { harness.quiescer.cfg.timeoutDuration = time.Second harness.quiescer.cfg.onTimeout = func() { close(timeoutGate) } - err := harness.quiescer.RecvStfu(msg, harness.pendingUpdates.Remote) + err := harness.quiescer.RecvStfu(msg) require.NoError(t, err) - err = harness.quiescer.SendOwedStfu(harness.pendingUpdates.Local) + err = harness.quiescer.SendOwedStfu() require.NoError(t, err) select { @@ -461,9 +360,9 @@ func TestQuiescerTimeoutAborts(t *testing.T) { harness.quiescer.cfg.timeoutDuration = time.Second harness.quiescer.cfg.onTimeout = func() { close(timeoutGate) } - err := harness.quiescer.RecvStfu(msg, harness.pendingUpdates.Remote) + err := harness.quiescer.RecvStfu(msg) require.NoError(t, err) - err = harness.quiescer.SendOwedStfu(harness.pendingUpdates.Local) + err = harness.quiescer.SendOwedStfu() require.NoError(t, err) harness.quiescer.Resume() From 127e4fff28cef52f870a82836473a63eb86bf1de Mon Sep 17 00:00:00 2001 From: Keagan McClelland Date: Tue, 12 Nov 2024 15:49:24 -0700 Subject: [PATCH 19/20] htlcswitch: add logging to quiescer --- htlcswitch/quiescer.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/htlcswitch/quiescer.go b/htlcswitch/quiescer.go index 7e6269b1202..5a762215765 100644 --- a/htlcswitch/quiescer.go +++ b/htlcswitch/quiescer.go @@ -5,6 +5,8 @@ import ( "sync" "time" + "github.com/btcsuite/btclog/v2" + "github.com/lightningnetwork/lnd/build" "github.com/lightningnetwork/lnd/fn" "github.com/lightningnetwork/lnd/lntypes" "github.com/lightningnetwork/lnd/lnwire" @@ -135,6 +137,9 @@ type QuiescerCfg struct { type QuiescerLive struct { cfg QuiescerCfg + // log is a quiescer-scoped logging instance. + log btclog.Logger + // localInit indicates whether our path through this state machine was // initiated by our node. This can be true or false independently of // remoteInit. @@ -169,8 +174,11 @@ type QuiescerLive struct { // NewQuiescer creates a new quiescer for the given channel. func NewQuiescer(cfg QuiescerCfg) Quiescer { + logPrefix := fmt.Sprintf("Quiescer(%v):", cfg.chanID) + return &QuiescerLive{ cfg: cfg, + log: build.NewPrefixLog(logPrefix, log), } } @@ -504,6 +512,8 @@ func (q *QuiescerLive) Resume() { // channel has been quiescent and then resets the quiescer state to its initial // state. func (q *QuiescerLive) resume() { + q.log.Debug("quiescence terminated, resuming htlc traffic") + // since we are resuming we want to cancel the quiescence timeout // action. q.cancelTimeout() From debc43daf255a8dd21e984c1cbeff320e3c37128 Mon Sep 17 00:00:00 2001 From: Keagan McClelland Date: Mon, 18 Nov 2024 11:55:06 -0700 Subject: [PATCH 20/20] docs: add quiescence to release notes --- docs/release-notes/release-notes-0.19.0.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/release-notes/release-notes-0.19.0.md b/docs/release-notes/release-notes-0.19.0.md index f7d2fe594e0..4429f5cf5c9 100644 --- a/docs/release-notes/release-notes-0.19.0.md +++ b/docs/release-notes/release-notes-0.19.0.md @@ -67,6 +67,10 @@ signal relay was added. This signal has *no impact* on routing, and is deployed experimentally to assist ongoing channel jamming research. +* Add initial support for [quiescence](https://github.com/lightningnetwork/lnd/pull/8270). + This is a protocol gadget required for Dynamic Commitments and Splicing that + will be added later. + ## Functional Enhancements * [Add ability](https://github.com/lightningnetwork/lnd/pull/8998) to paginate wallet transactions. @@ -216,6 +220,7 @@ The underlying functionality between those two options remain the same. * Elle Mouton * George Tsagkarelis * hieblmi +* Keagan McClelland * Oliver Gugger * Pins * Viktor Tigerström