diff --git a/common/utxo.h b/common/utxo.h index a1ba5e4ee021..c31d40641edd 100644 --- a/common/utxo.h +++ b/common/utxo.h @@ -26,6 +26,12 @@ struct utxo { /* Optional unilateral close information, NULL if this is just * a HD key */ struct unilateral_close_info *close_info; + + /* NULL if we haven't seen it in a block, otherwise the block it's in */ + const int *blockheight; + + /* NULL if not spent yet, otherwise, the block the spending transaction is in */ + const int *spendheight; }; void towire_utxo(u8 **pptr, const struct utxo *utxo); diff --git a/lightningd/Makefile b/lightningd/Makefile index 3ebb582d9750..4780730c6ee0 100644 --- a/lightningd/Makefile +++ b/lightningd/Makefile @@ -73,7 +73,6 @@ LIGHTNINGD_SRC := \ lightningd/peer_control.c \ lightningd/peer_htlcs.c \ lightningd/subd.c \ - lightningd/txfilter.c \ lightningd/watch.c # Source files without corresponding headers diff --git a/lightningd/chaintopology.c b/lightningd/chaintopology.c index ebf633da0a88..11683d870ed7 100644 --- a/lightningd/chaintopology.c +++ b/lightningd/chaintopology.c @@ -1,4 +1,5 @@ #include "bitcoin/block.h" +#include "bitcoin/script.h" #include "bitcoin/tx.h" #include "bitcoind.h" #include "chaintopology.h" @@ -77,7 +78,7 @@ static void filter_block_txs(struct chain_topology *topo, struct block *b) satoshi_owned = 0; if (txfilter_match(topo->bitcoind->ld->owned_txfilter, tx)) { wallet_extract_owned_outputs(topo->bitcoind->ld->wallet, - tx, &satoshi_owned); + tx, b, &satoshi_owned); } /* We did spends first, in case that tells us to watch tx. */ @@ -348,18 +349,53 @@ static void updates_complete(struct chain_topology *topo) next_topology_timer(topo); } -static void add_tip(struct chain_topology *topo, struct block *b) +/** + * topo_update_spends -- Tell the wallet about all spent outpoints + */ +static void topo_update_spends(struct chain_topology *topo, struct block *b) { - /* Only keep the transactions we care about. */ - filter_block_txs(topo, b); + for (size_t i = 0; i < tal_count(b->full_txs); i++) { + const struct bitcoin_tx *tx = b->full_txs[i]; + for (size_t j = 0; j < tal_count(tx->input); j++) { + const struct bitcoin_tx_input *input = &tx->input[j]; + wallet_outpoint_spend(topo->wallet, b->height, + &input->txid, + input->index); + } + } +} - block_map_add(&topo->block_map, b); +static void topo_add_utxos(struct chain_topology *topo, struct block *b) +{ + for (size_t i = 0; i < tal_count(b->full_txs); i++) { + const struct bitcoin_tx *tx = b->full_txs[i]; + for (size_t j = 0; j < tal_count(tx->output); j++) { + const struct bitcoin_tx_output *output = &tx->output[j]; + if (is_p2wsh(output->script, NULL)) { + wallet_utxoset_add(topo->wallet, tx, j, + b->height, i, output->script, + output->amount); + } + } + } +} +static void add_tip(struct chain_topology *topo, struct block *b) +{ /* Attach to tip; b is now the tip. */ assert(b->height == topo->tip->height + 1); b->prev = topo->tip; topo->tip->next = b; topo->tip = b; + wallet_block_add(topo->wallet, b); + + topo_add_utxos(topo, b); + topo_update_spends(topo, b); + + /* Only keep the transactions we care about. */ + filter_block_txs(topo, b); + + block_map_add(&topo->block_map, b); } static struct block *new_block(struct chain_topology *topo, @@ -403,6 +439,7 @@ static void remove_tip(struct chain_topology *topo) for (i = 0; i < n; i++) txwatch_fire(topo, b->txs[i], 0); + wallet_block_remove(topo->wallet, b); tal_free(b); } @@ -471,6 +508,10 @@ static void get_init_blockhash(struct bitcoind *bitcoind, u32 blockcount, else topo->first_blocknum -= 100; + /* Rollback to the given blockheight, so we start track + * correctly again */ + wallet_blocks_rollback(topo->wallet, topo->first_blocknum - 1); + /* Get up to speed with topology. */ bitcoind_getblockhash(bitcoind, topo->first_blocknum, get_init_block, topo); @@ -666,7 +707,7 @@ struct chain_topology *new_topology(struct lightningd *ld, struct log *log) topo->default_fee_rate = 40000; topo->override_fee_rate = NULL; topo->bitcoind = new_bitcoind(topo, ld, log); - + topo->wallet = ld->wallet; return topo; } diff --git a/lightningd/chaintopology.h b/lightningd/chaintopology.h index 512ed50ba0fd..a6fc8419612b 100644 --- a/lightningd/chaintopology.h +++ b/lightningd/chaintopology.h @@ -86,6 +86,9 @@ struct chain_topology { u32 feerate[NUM_FEERATES]; bool startup; + /* Where to store blockchain info. */ + struct wallet *wallet; + /* Where to log things. */ struct log *log; diff --git a/lightningd/lightningd.c b/lightningd/lightningd.c index 31db45c43258..fd8cf3ec833d 100644 --- a/lightningd/lightningd.c +++ b/lightningd/lightningd.c @@ -321,6 +321,7 @@ int main(int argc, char *argv[]) /* Initialize wallet, now that we are in the correct directory */ ld->wallet = wallet_new(ld, ld->log, &ld->timers); ld->owned_txfilter = txfilter_new(ld); + ld->topology->wallet = ld->wallet; /* Set up HSM. */ hsm_init(ld, newdir); diff --git a/lightningd/lightningd.h b/lightningd/lightningd.h index 9b6b6fa52619..f738920b0e8a 100644 --- a/lightningd/lightningd.h +++ b/lightningd/lightningd.h @@ -7,8 +7,8 @@ #include #include #include -#include #include +#include #include /* BOLT #1: diff --git a/lightningd/onchain_control.c b/lightningd/onchain_control.c index e4d27180d7da..c2cbad27061e 100644 --- a/lightningd/onchain_control.c +++ b/lightningd/onchain_control.c @@ -226,6 +226,8 @@ static void onchain_add_utxo(struct channel *channel, const u8 *msg) u->status = output_state_available; u->close_info->channel_id = channel->dbid; u->close_info->peer_id = channel->peer->id; + u->blockheight = NULL; + u->spendheight = NULL; if (!fromwire_onchain_add_utxo(msg, &u->txid, &u->outnum, &u->close_info->commitment_point, @@ -457,4 +459,3 @@ enum watch_result funding_spent(struct channel *channel, /* We keep watching until peer finally deleted, for reorgs. */ return KEEP_WATCHING; } - diff --git a/lightningd/opening_control.c b/lightningd/opening_control.c index a750d901bedd..6b9f81d1296e 100644 --- a/lightningd/opening_control.c +++ b/lightningd/opening_control.c @@ -354,7 +354,7 @@ static void opening_funder_finished(struct subd *openingd, const u8 *resp, tal_hex(msg, resp)); /* Extract the change output and add it to the DB */ - wallet_extract_owned_outputs(ld->wallet, fundingtx, &change_satoshi); + wallet_extract_owned_outputs(ld->wallet, fundingtx, NULL, &change_satoshi); /* Send it out and watch for confirms. */ broadcast_tx(ld->topology, channel, fundingtx, funding_broadcast_failed); diff --git a/lightningd/txfilter.c b/lightningd/txfilter.c deleted file mode 100644 index 77997023b398..000000000000 --- a/lightningd/txfilter.c +++ /dev/null @@ -1,54 +0,0 @@ -#include "txfilter.h" - -#include -#include -#include - -struct txfilter { - u8 **scriptpubkeys; -}; - - - -struct txfilter *txfilter_new(const tal_t *ctx) -{ - struct txfilter *filter = tal(ctx, struct txfilter); - filter->scriptpubkeys = tal_arr(filter, u8*, 0); - return filter; -} - -void txfilter_add_scriptpubkey(struct txfilter *filter, u8 *script) -{ - size_t count = tal_count(filter->scriptpubkeys); - tal_resize(&filter->scriptpubkeys, count + 1); - filter->scriptpubkeys[count] = tal_dup_arr(filter, u8, script, tal_len(script), 0); -} - -void txfilter_add_derkey(struct txfilter *filter, u8 derkey[PUBKEY_DER_LEN]) -{ - tal_t *tmpctx = tal_tmpctx(filter); - u8 *skp, *p2sh; - - skp = scriptpubkey_p2wpkh_derkey(tmpctx, derkey); - p2sh = scriptpubkey_p2sh(tmpctx, skp); - - txfilter_add_scriptpubkey(filter, take(skp)); - txfilter_add_scriptpubkey(filter, take(p2sh)); - - tal_free(tmpctx); -} - - -bool txfilter_match(const struct txfilter *filter, const struct bitcoin_tx *tx) -{ - u8 *oscript; - for (size_t i = 0; i < tal_count(tx->output); i++) { - oscript = tx->output[i].script; - - for (size_t j = 0; j < tal_count(filter->scriptpubkeys); j++) { - if (scripteq(oscript, filter->scriptpubkeys[j])) - return true; - } - } - return false; -} diff --git a/tests/test_lightningd.py b/tests/test_lightningd.py index be544f9695de..6e486bb1b27e 100644 --- a/tests/test_lightningd.py +++ b/tests/test_lightningd.py @@ -3763,6 +3763,45 @@ def test_peerinfo(self): assert l1.rpc.getpeer(l2.info['id'])['alias'] == l1.rpc.listnodes(l2.info['id'])['nodes'][0]['alias'] assert l1.rpc.getpeer(l2.info['id'])['color'] == l1.rpc.listnodes(l2.info['id'])['nodes'][0]['color'] + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") + def test_blockchaintrack(self): + """Check that we track the blockchain correctly across reorgs + """ + l1 = self.node_factory.get_node() + btc = l1.bitcoin + addr = l1.rpc.newaddr()['address'] + + ###################################################################### + # First failure scenario: rollback on startup doesn't work, + # and we try to add a block twice when rescanning: + l1.restart() + + # At height 442 we receive an incoming payment + hashes = btc.rpc.generate(9) + btc.rpc.sendtoaddress(addr, 1) + time.sleep(1) # mempool is still unpredictable + btc.rpc.generate(1) + + l1.daemon.wait_for_log(r'Owning') + outputs = l1.rpc.listfunds()['outputs'] + assert len(outputs) == 1 + + ###################################################################### + # Second failure scenario: perform a 20 block reorg + btc.rpc.generate(10) + blockheight = btc.rpc.getblockcount() + wait_for(lambda: l1.rpc.dev_blockheight()['blockheight'] == blockheight) + + # Now reorg out with a longer fork of 21 blocks + btc.rpc.invalidateblock(hashes[0]) + hashes = btc.rpc.generate(21) + + blockheight = btc.rpc.getblockcount() + wait_for(lambda: l1.rpc.dev_blockheight()['blockheight'] == blockheight) + + # Our funds got reorged out, we should not have any funds that are confirmed + assert [o for o in l1.rpc.listfunds()['outputs'] if o['status'] != "unconfirmed"] == [] + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/wallet/Makefile b/wallet/Makefile index 66468e7e1339..a12988e31e3e 100644 --- a/wallet/Makefile +++ b/wallet/Makefile @@ -7,6 +7,7 @@ wallet-wrongdir: WALLET_LIB_SRC := \ wallet/db.c \ wallet/invoices.c \ + wallet/txfilter.c \ wallet/wallet.c \ wallet/walletrpc.c diff --git a/wallet/db.c b/wallet/db.c index d3e815447357..9ccc2d029c61 100644 --- a/wallet/db.c +++ b/wallet/db.c @@ -202,6 +202,34 @@ char *dbmigrations[] = { "UPDATE channels SET STATE = 8 WHERE state > 8;", /* Add bolt11 to invoices table*/ "ALTER TABLE invoices ADD bolt11 TEXT;", + /* What do we think the head of the blockchain looks like? Used + * primarily to track confirmations across restarts and making + * sure we handle reorgs correctly. */ + "CREATE TABLE blocks (height INT, hash BLOB, prev_hash BLOB, UNIQUE(height));", + /* ON DELETE CASCADE would have been nice for confirmation_height, + * so that we automatically delete outputs that fall off the + * blockchain and then we rediscover them if they are included + * again. However, we have the their_unilateral/to_us which we + * can't simply recognize from the chain without additional + * hints. So we just mark them as unconfirmed should the block + * die. */ + "ALTER TABLE outputs ADD COLUMN confirmation_height INTEGER REFERENCES blocks(height) ON DELETE SET NULL;", + "ALTER TABLE outputs ADD COLUMN spend_height INTEGER REFERENCES blocks(height) ON DELETE SET NULL;", + /* Create a covering index that covers both fields */ + "CREATE INDEX output_height_idx ON outputs (confirmation_height, spend_height);", + "CREATE TABLE utxoset (" + " txid BLOB," + " outnum INT," + " blockheight INT REFERENCES blocks(height) ON DELETE CASCADE," + " spendheight INT REFERENCES blocks(height) ON DELETE SET NULL," + " txindex INT," + " scriptpubkey BLOB," + " satoshis BIGINT," + " PRIMARY KEY(txid, outnum));", + "CREATE INDEX short_channel_id ON utxoset (blockheight, txindex, outnum)", + /* Necessary index for long rollbacks of the blockchain, otherwise we're + * doing table scans for every block removed. */ + "CREATE INDEX utxoset_spend ON utxoset (spendheight)", NULL, }; diff --git a/wallet/test/run-wallet.c b/wallet/test/run-wallet.c index d28360c8f24a..55d8d2bae8e0 100644 --- a/wallet/test/run-wallet.c +++ b/wallet/test/run-wallet.c @@ -254,6 +254,17 @@ struct json_result *new_json_result(const tal_t *ctx UNNEEDED) /* Generated stub for null_response */ struct json_result *null_response(const tal_t *ctx UNNEEDED) { fprintf(stderr, "null_response called!\n"); abort(); } +/* Generated stub for outpointfilter_add */ +void outpointfilter_add(struct outpointfilter *of UNNEEDED, + const struct bitcoin_txid *txid UNNEEDED, const u32 outnum UNNEEDED) +{ fprintf(stderr, "outpointfilter_add called!\n"); abort(); } +/* Generated stub for outpointfilter_matches */ +bool outpointfilter_matches(struct outpointfilter *of UNNEEDED, + const struct bitcoin_txid *txid UNNEEDED, const u32 outnum UNNEEDED) +{ fprintf(stderr, "outpointfilter_matches called!\n"); abort(); } +/* Generated stub for outpointfilter_new */ +struct outpointfilter *outpointfilter_new(tal_t *ctx UNNEEDED) +{ fprintf(stderr, "outpointfilter_new called!\n"); abort(); } /* Generated stub for peer_accept_channel */ u8 *peer_accept_channel(struct lightningd *ld UNNEEDED, const struct pubkey *peer_id UNNEEDED, diff --git a/wallet/txfilter.c b/wallet/txfilter.c new file mode 100644 index 000000000000..d74bef0bdd7a --- /dev/null +++ b/wallet/txfilter.c @@ -0,0 +1,127 @@ +#include "txfilter.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct txfilter { + u8 **scriptpubkeys; +}; + +struct outpointfilter_entry { + struct bitcoin_txid txid; + u32 outnum; +}; + +static size_t outpoint_hash(const struct outpointfilter_entry *out) +{ + struct siphash24_ctx ctx; + siphash24_init(&ctx, siphash_seed()); + siphash24_update(&ctx, &out->txid, sizeof(out->txid)); + siphash24_u32(&ctx, out->outnum); + return siphash24_done(&ctx); +} + +static bool outpoint_eq(const struct outpointfilter_entry *o1, + const struct outpointfilter_entry *o2) +{ + return structeq(&o1->txid, &o2->txid) && o1->outnum == o2->outnum; +} + +static const struct outpointfilter_entry *outpoint_keyof(const struct outpointfilter_entry *out) +{ + return out; +} + +HTABLE_DEFINE_TYPE(struct outpointfilter_entry, outpoint_keyof, outpoint_hash, outpoint_eq, + outpointset); + +struct outpointfilter { + struct outpointset *set; +}; + +struct txfilter *txfilter_new(const tal_t *ctx) +{ + struct txfilter *filter = tal(ctx, struct txfilter); + filter->scriptpubkeys = tal_arr(filter, u8*, 0); + return filter; +} + +void txfilter_add_scriptpubkey(struct txfilter *filter, u8 *script) +{ + size_t count = tal_count(filter->scriptpubkeys); + tal_resize(&filter->scriptpubkeys, count + 1); + filter->scriptpubkeys[count] = tal_dup_arr(filter, u8, script, tal_len(script), 0); +} + +void txfilter_add_derkey(struct txfilter *filter, u8 derkey[PUBKEY_DER_LEN]) +{ + tal_t *tmpctx = tal_tmpctx(filter); + u8 *skp, *p2sh; + + skp = scriptpubkey_p2wpkh_derkey(tmpctx, derkey); + p2sh = scriptpubkey_p2sh(tmpctx, skp); + + txfilter_add_scriptpubkey(filter, take(skp)); + txfilter_add_scriptpubkey(filter, take(p2sh)); + + tal_free(tmpctx); +} + + +bool txfilter_match(const struct txfilter *filter, const struct bitcoin_tx *tx) +{ + u8 *oscript; + for (size_t i = 0; i < tal_count(tx->output); i++) { + oscript = tx->output[i].script; + + for (size_t j = 0; j < tal_count(filter->scriptpubkeys); j++) { + if (scripteq(oscript, filter->scriptpubkeys[j])) + return true; + } + } + return false; +} + +void outpointfilter_add(struct outpointfilter *of, const struct bitcoin_txid *txid, const u32 outnum) +{ + struct outpointfilter_entry *op; + if (outpointfilter_matches(of, txid, outnum)) + return; + /* Have to mark the entries as notleak since they'll not be + * pointed to by anything other than the htable */ + op = notleak(tal(of->set, struct outpointfilter_entry)); + op->txid = *txid; + op->outnum = outnum; + outpointset_add(of->set, op); +} + +bool outpointfilter_matches(struct outpointfilter *of, const struct bitcoin_txid *txid, const u32 outnum) +{ + struct outpointfilter_entry op; + op.txid = *txid; + op.outnum = outnum; + return outpointset_get(of->set, &op) != NULL; +} + +void outpointfilter_remove(struct outpointfilter *of, const struct bitcoin_txid *txid, const u32 outnum) +{ + struct outpointfilter_entry op; + op.txid = *txid; + op.outnum = outnum; + outpointset_del(of->set, &op); +} + +struct outpointfilter *outpointfilter_new(tal_t *ctx) +{ + struct outpointfilter *opf = tal(ctx, struct outpointfilter); + opf->set = tal(opf, struct outpointset); + outpointset_init(opf->set); + return opf; +} diff --git a/lightningd/txfilter.h b/wallet/txfilter.h similarity index 60% rename from lightningd/txfilter.h rename to wallet/txfilter.h index a30f0f76a68c..fb8db02f811f 100644 --- a/lightningd/txfilter.h +++ b/wallet/txfilter.h @@ -8,6 +8,11 @@ struct txfilter; +/** + * outpointfilter -- Simple filter that keeps track of outpoints + */ +struct outpointfilter; + /** * txfilter_new -- Construct and initialize a new txfilter */ @@ -33,4 +38,26 @@ bool txfilter_match(const struct txfilter *filter, const struct bitcoin_tx *tx); */ void txfilter_add_scriptpubkey(struct txfilter *filter, u8 *script); +/** + * outpointfilter_new -- Create a new outpointfilter + */ +struct outpointfilter *outpointfilter_new(tal_t *ctx); + +/** + * outpointfilter_add -- Add an outpoint to the filter + */ +void outpointfilter_add(struct outpointfilter *of, + const struct bitcoin_txid *txid, const u32 outnum); + +/** + * outpointfilter_matches -- Are we tracking this outpoint? + */ +bool outpointfilter_matches(struct outpointfilter *of, + const struct bitcoin_txid *txid, const u32 outnum); +/** + * outpointfilter_remove -- Do not match this outpoint in the future + */ +void outpointfilter_remove(struct outpointfilter *of, + const struct bitcoin_txid *txid, const u32 outnum); + #endif /* LIGHTNING_LIGHTNINGD_TXFILTER_H */ diff --git a/wallet/wallet.c b/wallet/wallet.c index bc562e22453d..8a0378e973ac 100644 --- a/wallet/wallet.c +++ b/wallet/wallet.c @@ -15,6 +15,34 @@ #define SQLITE_MAX_UINT 0x7FFFFFFFFFFFFFFF #define DIRECTION_INCOMING 0 #define DIRECTION_OUTGOING 1 +/* How many blocks must a UTXO entry be buried under to be considered old enough + * to prune? */ +#define UTXO_PRUNE_DEPTH 144 + +static void outpointfilters_init(struct wallet *w) +{ + sqlite3_stmt *stmt; + struct utxo **utxos = wallet_get_utxos(NULL, w, output_state_any); + struct bitcoin_txid txid; + u32 outnum; + + w->owned_outpoints = outpointfilter_new(w); + for (size_t i = 0; i < tal_count(utxos); i++) + outpointfilter_add(w->owned_outpoints, &utxos[i]->txid, utxos[i]->outnum); + + tal_free(utxos); + + w->utxoset_outpoints = outpointfilter_new(w); + stmt = db_prepare(w->db, "SELECT txid, outnum FROM utxoset WHERE spendheight is NULL"); + + while (sqlite3_step(stmt) == SQLITE_ROW) { + sqlite3_column_sha256_double(stmt, 0, &txid.shad); + outnum = sqlite3_column_int(stmt, 1); + outpointfilter_add(w->utxoset_outpoints, &txid, outnum); + } + + sqlite3_finalize(stmt); +} struct wallet *wallet_new(struct lightningd *ld, struct log *log, struct timers *timers) @@ -26,6 +54,10 @@ struct wallet *wallet_new(struct lightningd *ld, wallet->bip32_base = NULL; wallet->invoices = invoices_new(wallet, wallet->db, log, timers); list_head_init(&wallet->unstored_payments); + + db_begin_transaction(wallet->db); + outpointfilters_init(wallet); + db_commit_transaction(wallet->db); return wallet; } @@ -35,7 +67,18 @@ bool wallet_add_utxo(struct wallet *w, struct utxo *utxo, { sqlite3_stmt *stmt; - stmt = db_prepare(w->db, "INSERT INTO outputs (prev_out_tx, prev_out_index, value, type, status, keyindex, channel_id, peer_id, commitment_point) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?);"); + stmt = db_prepare(w->db, "INSERT INTO outputs (" + "prev_out_tx, " + "prev_out_index, " + "value, " + "type, " + "status, " + "keyindex, " + "channel_id, " + "peer_id, " + "commitment_point, " + "confirmation_height, " + "spend_height) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"); sqlite3_bind_blob(stmt, 1, &utxo->txid, sizeof(utxo->txid), SQLITE_TRANSIENT); sqlite3_bind_int(stmt, 2, utxo->outnum); sqlite3_bind_int64(stmt, 3, utxo->amount); @@ -51,6 +94,19 @@ bool wallet_add_utxo(struct wallet *w, struct utxo *utxo, sqlite3_bind_null(stmt, 8); sqlite3_bind_null(stmt, 9); } + + if (utxo->blockheight) { + sqlite3_bind_int(stmt, 10, *utxo->blockheight); + } else + sqlite3_bind_null(stmt, 10); + + if (utxo->spendheight) + sqlite3_bind_int(stmt, 11, *utxo->spendheight); + else + sqlite3_bind_null(stmt, 11); + + /* May fail if we already know about the tx, e.g., because + * it's change or some internal tx. */ return db_exec_prepared_mayfail(w->db, stmt); } @@ -61,6 +117,7 @@ bool wallet_add_utxo(struct wallet *w, struct utxo *utxo, */ static bool wallet_stmt2output(sqlite3_stmt *stmt, struct utxo *utxo) { + int *blockheight, *spendheight; sqlite3_column_sha256_double(stmt, 0, &utxo->txid.shad); utxo->outnum = sqlite3_column_int(stmt, 1); utxo->amount = sqlite3_column_int64(stmt, 2); @@ -76,6 +133,21 @@ static bool wallet_stmt2output(sqlite3_stmt *stmt, struct utxo *utxo) utxo->close_info = NULL; } + utxo->blockheight = NULL; + utxo->spendheight = NULL; + + if (sqlite3_column_type(stmt, 9) != SQLITE_NULL) { + blockheight = tal(utxo, int); + *blockheight = sqlite3_column_int(stmt, 9); + utxo->blockheight = blockheight; + } + + if (sqlite3_column_type(stmt, 10) != SQLITE_NULL) { + spendheight = tal(utxo, int); + *spendheight = sqlite3_column_int(stmt, 10); + utxo->spendheight = spendheight; + } + return true; } @@ -110,7 +182,7 @@ struct utxo **wallet_get_utxos(const tal_t *ctx, struct wallet *w, const enum ou sqlite3_stmt *stmt = db_prepare( w->db, "SELECT prev_out_tx, prev_out_index, value, type, status, keyindex, " - "channel_id, peer_id, commitment_point " + "channel_id, peer_id, commitment_point, confirmation_height, spend_height " "FROM outputs WHERE status=?1 OR ?1=255"); sqlite3_bind_int(stmt, 1, state); @@ -941,8 +1013,26 @@ void wallet_peer_delete(struct wallet *w, u64 peer_dbid) db_exec_prepared(w->db, stmt); } +static void wallet_output_confirm(struct wallet *w, + const struct bitcoin_txid *txid, + const u32 outnum, + const u32 confirmation_height) +{ + sqlite3_stmt *stmt; + assert(confirmation_height > 0); + stmt = db_prepare(w->db, + "UPDATE outputs " + "SET confirmation_height = ? " + "WHERE prev_out_tx = ? AND prev_out_index = ?"); + sqlite3_bind_int(stmt, 1, confirmation_height); + sqlite3_bind_sha256_double(stmt, 2, &txid->shad); + sqlite3_bind_int(stmt, 3, outnum); + + db_exec_prepared(w->db, stmt); +} + int wallet_extract_owned_outputs(struct wallet *w, const struct bitcoin_tx *tx, - u64 *total_satoshi) + const struct block *block, u64 *total_satoshi) { int num_utxos = 0; for (size_t output = 0; output < tal_count(tx->output); output++) { @@ -962,6 +1052,10 @@ int wallet_extract_owned_outputs(struct wallet *w, const struct bitcoin_tx *tx, bitcoin_txid(tx, &utxo->txid); utxo->outnum = output; utxo->close_info = NULL; + + utxo->blockheight = block?&block->height:NULL; + utxo->spendheight = NULL; + log_debug(w->log, "Owning output %zu %"PRIu64" (%s) txid %s", output, tx->output[output].amount, is_p2sh ? "P2SH" : "SEGWIT", @@ -969,9 +1063,18 @@ int wallet_extract_owned_outputs(struct wallet *w, const struct bitcoin_tx *tx, &utxo->txid)); if (!wallet_add_utxo(w, utxo, is_p2sh ? p2sh_wpkh : our_change)) { + /* In case we already know the output, make + * sure we actually track its + * blockheight. This can happen when we grab + * the output from a transaction we created + * outselves. */ + if (block) + wallet_output_confirm(w, &utxo->txid, utxo->outnum, block->height); tal_free(utxo); - return -1; + continue; } + outpointfilter_add(w->owned_outpoints, &utxo->txid, utxo->outnum); + *total_satoshi += utxo->amount; tal_free(utxo); num_utxos++; @@ -1660,3 +1763,117 @@ bool wallet_network_check(struct wallet *w, } return true; } + +/** + * wallet_utxoset_prune -- Remove spent UTXO entries that are old + */ +static void wallet_utxoset_prune(struct wallet *w, const u32 blockheight) +{ + sqlite3_stmt *stmt; + stmt = db_prepare(w->db, "DELETE FROM utxoset WHERE spendheight < ?"); + sqlite3_bind_int(stmt, 1, blockheight - UTXO_PRUNE_DEPTH); + db_exec_prepared(w->db, stmt); +} + +void wallet_block_add(struct wallet *w, struct block *b) +{ + sqlite3_stmt *stmt = db_prepare(w->db, + "INSERT INTO blocks " + "(height, hash, prev_hash) " + "VALUES (?, ?, ?);"); + sqlite3_bind_int(stmt, 1, b->height); + sqlite3_bind_sha256_double(stmt, 2, &b->blkid.shad); + if (b->prev) { + sqlite3_bind_sha256_double(stmt, 3, &b->prev->blkid.shad); + }else { + sqlite3_bind_null(stmt, 3); + } + db_exec_prepared(w->db, stmt); + + /* Now cleanup UTXOs that we don't care about anymore */ + wallet_utxoset_prune(w, b->height); +} + +void wallet_block_remove(struct wallet *w, struct block *b) +{ + sqlite3_stmt *stmt = db_prepare(w->db, + "DELETE FROM blocks WHERE hash = ?"); + sqlite3_bind_sha256_double(stmt, 1, &b->blkid.shad); + db_exec_prepared(w->db, stmt); + + stmt = db_prepare(w->db, "SELECT * FROM blocks WHERE height >= ?;"); + sqlite3_bind_int(stmt, 1, b->height); + assert(sqlite3_step(stmt) == SQLITE_DONE); + sqlite3_finalize(stmt); +} + +void wallet_blocks_rollback(struct wallet *w, u32 height) +{ + sqlite3_stmt *stmt = db_prepare(w->db, "DELETE FROM blocks " + "WHERE height >= ?"); + sqlite3_bind_int(stmt, 1, height); + db_exec_prepared(w->db, stmt); +} + +void wallet_outpoint_spend(struct wallet *w, const u32 blockheight, + const struct bitcoin_txid *txid, const u32 outnum) +{ + sqlite3_stmt *stmt; + if (outpointfilter_matches(w->owned_outpoints, txid, outnum)) { + stmt = db_prepare(w->db, + "UPDATE outputs " + "SET spend_height = ? " + "WHERE prev_out_tx = ?" + " AND prev_out_index = ?"); + + sqlite3_bind_int(stmt, 1, blockheight); + sqlite3_bind_sha256_double(stmt, 2, &txid->shad); + sqlite3_bind_int(stmt, 3, outnum); + + db_exec_prepared(w->db, stmt); + } + + if (outpointfilter_matches(w->utxoset_outpoints, txid, outnum)) { + stmt = db_prepare(w->db, + "UPDATE utxoset " + "SET spendheight = ? " + "WHERE txid = ?" + " AND outnum = ?"); + + sqlite3_bind_int(stmt, 1, blockheight); + sqlite3_bind_sha256_double(stmt, 2, &txid->shad); + sqlite3_bind_int(stmt, 3, outnum); + + db_exec_prepared(w->db, stmt); + } +} + +void wallet_utxoset_add(struct wallet *w, const struct bitcoin_tx *tx, + const u32 outnum, const u32 blockheight, + const u32 txindex, const u8 *scriptpubkey, + const u64 satoshis) +{ + sqlite3_stmt *stmt; + struct bitcoin_txid txid; + bitcoin_txid(tx, &txid); + + stmt = db_prepare(w->db, "INSERT INTO utxoset (" + " txid," + " outnum," + " blockheight," + " spendheight," + " txindex," + " scriptpubkey," + " satoshis" + ") VALUES(?, ?, ?, ?, ?, ?, ?);"); + sqlite3_bind_sha256_double(stmt, 1, &txid.shad); + sqlite3_bind_int(stmt, 2, outnum); + sqlite3_bind_int(stmt, 3, blockheight); + sqlite3_bind_null(stmt, 4); + sqlite3_bind_int(stmt, 5, txindex); + sqlite3_bind_blob(stmt, 6, scriptpubkey, tal_len(scriptpubkey), SQLITE_TRANSIENT); + sqlite3_bind_int64(stmt, 7, satoshis); + db_exec_prepared(w->db, stmt); + + outpointfilter_add(w->utxoset_outpoints, &txid, outnum); +} diff --git a/wallet/wallet.h b/wallet/wallet.h index 68f2090be08e..68a48571c3f4 100644 --- a/wallet/wallet.h +++ b/wallet/wallet.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -31,6 +32,14 @@ struct wallet { struct invoices *invoices; struct list_head unstored_payments; u64 max_channel_dbid; + + /* Filter matching all outpoints corresponding to our owned outputs, + * including all spent ones */ + struct outpointfilter *owned_outpoints; + + /* Filter matching all outpoints that might be a funding transaction on + * the blockchain. This is currently all P2WSH outputs */ + struct outpointfilter *utxoset_outpoints; }; /* Possible states for tracked outputs in the database. Not sure yet @@ -278,7 +287,7 @@ u32 wallet_first_blocknum(struct wallet *w, u32 first_possible); * wallet_extract_owned_outputs - given a tx, extract all of our outputs */ int wallet_extract_owned_outputs(struct wallet *w, const struct bitcoin_tx *tx, - u64 *total_satoshi); + const struct block *block, u64 *total_satoshi); /** * wallet_htlc_save_in - store an htlc_in in the database @@ -684,4 +693,26 @@ void wallet_htlc_sigs_save(struct wallet *w, u64 channel_id, bool wallet_network_check(struct wallet *w, const struct chainparams *chainparams); +/** + * wallet_block_add - Add a block to the blockchain tracked by this wallet + */ +void wallet_block_add(struct wallet *w, struct block *b); + +/** + * wallet_block_remove - Remove a block (and all its descendants) from the tracked blockchain + */ +void wallet_block_remove(struct wallet *w, struct block *b); + +/** + * wallet_blocks_rollback - Roll the blockchain back to the given height + */ +void wallet_blocks_rollback(struct wallet *w, u32 height); + +void wallet_outpoint_spend(struct wallet *w, const u32 blockheight, + const struct bitcoin_txid *txid, const u32 outnum); + +void wallet_utxoset_add(struct wallet *w, const struct bitcoin_tx *tx, + const u32 outnum, const u32 blockheight, + const u32 txindex, const u8 *scriptpubkey, + const u64 satoshis); #endif /* WALLET_WALLET_H */ diff --git a/wallet/walletrpc.c b/wallet/walletrpc.c index e3b231ba8471..53a8369cae09 100644 --- a/wallet/walletrpc.c +++ b/wallet/walletrpc.c @@ -58,7 +58,7 @@ static void wallet_withdrawal_broadcast(struct bitcoind *bitcoind UNUSED, * generated the hex tx, so this should always work */ tx = bitcoin_tx_from_hex(withdraw, withdraw->hextx, strlen(withdraw->hextx)); assert(tx != NULL); - wallet_extract_owned_outputs(ld->wallet, tx, &change_satoshi); + wallet_extract_owned_outputs(ld->wallet, tx, NULL, &change_satoshi); /* Note normally, change_satoshi == withdraw->changesatoshi, but * not if we're actually making a payment to ourselves! */ @@ -383,6 +383,14 @@ static void json_listfunds(struct command *cmd, const char *buffer UNUSED, json_add_txid(response, "txid", &utxos[i]->txid); json_add_num(response, "output", utxos[i]->outnum); json_add_u64(response, "value", utxos[i]->amount); + + if (utxos[i]->spendheight) + json_add_string(response, "status", "spent"); + else if (utxos[i]->blockheight) + json_add_string(response, "status", "confirmed"); + else + json_add_string(response, "status", "unconfirmed"); + json_object_end(response); } json_array_end(response);