Track output confirmations, spends and maintain an internal UTXO set

common/utxo.h

@@ -26,6 +26,12 @@ struct utxo {
 	/* Optional unilateral close information, NULL if this is just
 	 * a HD key */
 	struct unilateral_close_info *close_info;
+
+	/* NULL if we haven't seen it in a block, otherwise the block it's in */
+	const int *blockheight;
+
+	/* NULL if not spent yet, otherwise, the block the spending transaction is in */
+	const int *spendheight;
 };
 
 void towire_utxo(u8 **pptr, const struct utxo *utxo);

lightningd/Makefile

@@ -73,7 +73,6 @@ LIGHTNINGD_SRC :=				\
 	lightningd/peer_control.c		\
 	lightningd/peer_htlcs.c			\
 	lightningd/subd.c			\
-	lightningd/txfilter.c			\
 	lightningd/watch.c
 
 # Source files without corresponding headers

lightningd/chaintopology.c

@@ -1,4 +1,5 @@
 #include "bitcoin/block.h"
+#include "bitcoin/script.h"
 #include "bitcoin/tx.h"
 #include "bitcoind.h"
 #include "chaintopology.h"
@@ -77,7 +78,7 @@ static void filter_block_txs(struct chain_topology *topo, struct block *b)
 		satoshi_owned = 0;
 		if (txfilter_match(topo->bitcoind->ld->owned_txfilter, tx)) {
 			wallet_extract_owned_outputs(topo->bitcoind->ld->wallet,
-						     tx, &satoshi_owned);
+						     tx, b, &satoshi_owned);
 		}
 
 		/* We did spends first, in case that tells us to watch tx. */
@@ -348,18 +349,53 @@ static void updates_complete(struct chain_topology *topo)
 	next_topology_timer(topo);
 }
 
-static void add_tip(struct chain_topology *topo, struct block *b)
+/**
+ * topo_update_spends -- Tell the wallet about all spent outpoints
+ */
+static void topo_update_spends(struct chain_topology *topo, struct block *b)
 {
-	/* Only keep the transactions we care about. */
-	filter_block_txs(topo, b);
+	for (size_t i = 0; i < tal_count(b->full_txs); i++) {
+		const struct bitcoin_tx *tx = b->full_txs[i];
+		for (size_t j = 0; j < tal_count(tx->input); j++) {
+			const struct bitcoin_tx_input *input = &tx->input[j];
+			wallet_outpoint_spend(topo->wallet, b->height,
+					      &input->txid,
+					      input->index);
+		}
+	}
+}
 
-	block_map_add(&topo->block_map, b);
+static void topo_add_utxos(struct chain_topology *topo, struct block *b)
+{
+	for (size_t i = 0; i < tal_count(b->full_txs); i++) {
+		const struct bitcoin_tx *tx = b->full_txs[i];
+		for (size_t j = 0; j < tal_count(tx->output); j++) {
+			const struct bitcoin_tx_output *output = &tx->output[j];
+			if (is_p2wsh(output->script, NULL)) {
+				wallet_utxoset_add(topo->wallet, tx, j,
+						   b->height, i, output->script,
+						   output->amount);
+			}
+		}
+	}
+}
 
+static void add_tip(struct chain_topology *topo, struct block *b)
+{
 	/* Attach to tip; b is now the tip. */
 	assert(b->height == topo->tip->height + 1);
 	b->prev = topo->tip;
 	topo->tip->next = b;
 	topo->tip = b;
+	wallet_block_add(topo->wallet, b);
+
+	topo_add_utxos(topo, b);
+	topo_update_spends(topo, b);
+
+	/* Only keep the transactions we care about. */
+	filter_block_txs(topo, b);
+
+	block_map_add(&topo->block_map, b);
 }
 
 static struct block *new_block(struct chain_topology *topo,
@@ -403,6 +439,7 @@ static void remove_tip(struct chain_topology *topo)
 	for (i = 0; i < n; i++)
 		txwatch_fire(topo, b->txs[i], 0);
 
+	wallet_block_remove(topo->wallet, b);
 	tal_free(b);
 }
 
@@ -471,6 +508,10 @@ static void get_init_blockhash(struct bitcoind *bitcoind, u32 blockcount,
 	else
 		topo->first_blocknum -= 100;
 
+	/* Rollback to the given blockheight, so we start track
+	 * correctly again */
+	wallet_blocks_rollback(topo->wallet, topo->first_blocknum - 1);
+
 	/* Get up to speed with topology. */
 	bitcoind_getblockhash(bitcoind, topo->first_blocknum,
 			      get_init_block, topo);
@@ -666,7 +707,7 @@ struct chain_topology *new_topology(struct lightningd *ld, struct log *log)
 	topo->default_fee_rate = 40000;
 	topo->override_fee_rate = NULL;
 	topo->bitcoind = new_bitcoind(topo, ld, log);
-
+	topo->wallet = ld->wallet;
 	return topo;
 }
 

lightningd/chaintopology.h

@@ -86,6 +86,9 @@ struct chain_topology {
 	u32 feerate[NUM_FEERATES];
 	bool startup;
 
+	/* Where to store blockchain info. */
+	struct wallet *wallet;
+
 	/* Where to log things. */
 	struct log *log;
 

lightningd/lightningd.c

@@ -321,6 +321,7 @@ int main(int argc, char *argv[])
 	/* Initialize wallet, now that we are in the correct directory */
 	ld->wallet = wallet_new(ld, ld->log, &ld->timers);
 	ld->owned_txfilter = txfilter_new(ld);
+	ld->topology->wallet = ld->wallet;
 
 	/* Set up HSM. */
 	hsm_init(ld, newdir);

lightningd/lightningd.h

@@ -7,8 +7,8 @@
 #include <ccan/time/time.h>
 #include <ccan/timer/timer.h>
 #include <lightningd/htlc_end.h>
-#include <lightningd/txfilter.h>
 #include <stdio.h>
+#include <wallet/txfilter.h>
 #include <wallet/wallet.h>
 
 /* BOLT #1:

lightningd/onchain_control.c

@@ -226,6 +226,8 @@ static void onchain_add_utxo(struct channel *channel, const u8 *msg)
 	u->status = output_state_available;
 	u->close_info->channel_id = channel->dbid;
 	u->close_info->peer_id = channel->peer->id;
+	u->blockheight = NULL;
+	u->spendheight = NULL;
 
 	if (!fromwire_onchain_add_utxo(msg, &u->txid, &u->outnum,
 				       &u->close_info->commitment_point,
@@ -457,4 +459,3 @@ enum watch_result funding_spent(struct channel *channel,
 	/* We keep watching until peer finally deleted, for reorgs. */
 	return KEEP_WATCHING;
 }
-

lightningd/opening_control.c

@@ -354,7 +354,7 @@ static void opening_funder_finished(struct subd *openingd, const u8 *resp,
 		      tal_hex(msg, resp));
 
 	/* Extract the change output and add it to the DB */
-	wallet_extract_owned_outputs(ld->wallet, fundingtx, &change_satoshi);
+	wallet_extract_owned_outputs(ld->wallet, fundingtx, NULL, &change_satoshi);
 
 	/* Send it out and watch for confirms. */
 	broadcast_tx(ld->topology, channel, fundingtx, funding_broadcast_failed);

tests/test_lightningd.py

@@ -3763,6 +3763,45 @@ def test_peerinfo(self):
         assert l1.rpc.getpeer(l2.info['id'])['alias'] == l1.rpc.listnodes(l2.info['id'])['nodes'][0]['alias']
         assert l1.rpc.getpeer(l2.info['id'])['color'] == l1.rpc.listnodes(l2.info['id'])['nodes'][0]['color']
 
+    @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1")
+    def test_blockchaintrack(self):
+        """Check that we track the blockchain correctly across reorgs
+        """
+        l1 = self.node_factory.get_node()
+        btc = l1.bitcoin
+        addr = l1.rpc.newaddr()['address']
+
+        ######################################################################
+        # First failure scenario: rollback on startup doesn't work,
+        # and we try to add a block twice when rescanning:
+        l1.restart()
+
+        # At height 442 we receive an incoming payment
+        hashes = btc.rpc.generate(9)
+        btc.rpc.sendtoaddress(addr, 1)
+        time.sleep(1)  # mempool is still unpredictable
+        btc.rpc.generate(1)
+
+        l1.daemon.wait_for_log(r'Owning')
+        outputs = l1.rpc.listfunds()['outputs']
+        assert len(outputs) == 1
+
+        ######################################################################
+        # Second failure scenario: perform a 20 block reorg
+        btc.rpc.generate(10)
+        blockheight = btc.rpc.getblockcount()
+        wait_for(lambda: l1.rpc.dev_blockheight()['blockheight'] == blockheight)
+
+        # Now reorg out with a longer fork of 21 blocks
+        btc.rpc.invalidateblock(hashes[0])
+        hashes = btc.rpc.generate(21)
+
+        blockheight = btc.rpc.getblockcount()
+        wait_for(lambda: l1.rpc.dev_blockheight()['blockheight'] == blockheight)
+
+        # Our funds got reorged out, we should not have any funds that are confirmed
+        assert [o for o in l1.rpc.listfunds()['outputs'] if o['status'] != "unconfirmed"] == []
+
 
 if __name__ == '__main__':
     unittest.main(verbosity=2)

wallet/Makefile

@@ -7,6 +7,7 @@ wallet-wrongdir:
 WALLET_LIB_SRC :=		\
 	wallet/db.c		\
 	wallet/invoices.c	\
+	wallet/txfilter.c	\
 	wallet/wallet.c		\
 	wallet/walletrpc.c
 

wallet/db.c

@@ -202,6 +202,34 @@ char *dbmigrations[] = {
     "UPDATE channels SET STATE = 8 WHERE state > 8;",
     /* Add bolt11 to invoices table*/
     "ALTER TABLE invoices ADD bolt11 TEXT;",
+    /* What do we think the head of the blockchain looks like? Used
+     * primarily to track confirmations across restarts and making
+     * sure we handle reorgs correctly. */
+    "CREATE TABLE blocks (height INT, hash BLOB, prev_hash BLOB, UNIQUE(height));",
+    /* ON DELETE CASCADE would have been nice for confirmation_height,
+     * so that we automatically delete outputs that fall off the
+     * blockchain and then we rediscover them if they are included
+     * again. However, we have the their_unilateral/to_us which we
+     * can't simply recognize from the chain without additional
+     * hints. So we just mark them as unconfirmed should the block
+     * die. */
+    "ALTER TABLE outputs ADD COLUMN confirmation_height INTEGER REFERENCES blocks(height) ON DELETE SET NULL;",
+    "ALTER TABLE outputs ADD COLUMN spend_height INTEGER REFERENCES blocks(height) ON DELETE SET NULL;",
+    /* Create a covering index that covers both fields */
+    "CREATE INDEX output_height_idx ON outputs (confirmation_height, spend_height);",
+    "CREATE TABLE utxoset ("
+    " txid BLOB,"
+    " outnum INT,"
+    " blockheight INT REFERENCES blocks(height) ON DELETE CASCADE,"
+    " spendheight INT REFERENCES blocks(height) ON DELETE SET NULL,"
+    " txindex INT,"
+    " scriptpubkey BLOB,"
+    " satoshis BIGINT,"
+    " PRIMARY KEY(txid, outnum));",
+    "CREATE INDEX short_channel_id ON utxoset (blockheight, txindex, outnum)",
+    /* Necessary index for long rollbacks of the blockchain, otherwise we're
+     * doing table scans for every block removed. */
+    "CREATE INDEX utxoset_spend ON utxoset (spendheight)",
     NULL,
 };
 

wallet/test/run-wallet.c

@@ -254,6 +254,17 @@ struct json_result *new_json_result(const tal_t *ctx UNNEEDED)
 /* Generated stub for null_response */
 struct json_result *null_response(const tal_t *ctx UNNEEDED)
 { fprintf(stderr, "null_response called!\n"); abort(); }
+/* Generated stub for outpointfilter_add */
+void outpointfilter_add(struct outpointfilter *of UNNEEDED,
+			const struct bitcoin_txid *txid UNNEEDED, const u32 outnum UNNEEDED)
+{ fprintf(stderr, "outpointfilter_add called!\n"); abort(); }
+/* Generated stub for outpointfilter_matches */
+bool outpointfilter_matches(struct outpointfilter *of UNNEEDED,
+			    const struct bitcoin_txid *txid UNNEEDED, const u32 outnum UNNEEDED)
+{ fprintf(stderr, "outpointfilter_matches called!\n"); abort(); }
+/* Generated stub for outpointfilter_new */
+struct outpointfilter *outpointfilter_new(tal_t *ctx UNNEEDED)
+{ fprintf(stderr, "outpointfilter_new called!\n"); abort(); }
 /* Generated stub for peer_accept_channel */
 u8 *peer_accept_channel(struct lightningd *ld UNNEEDED,
 			const struct pubkey *peer_id UNNEEDED,