Fixes

CHANGELOG.md

@@ -4,8 +4,36 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.9.0.1] - 2020-08-10: "(Still) Rat Poison Squared on Steroids"
 
-## [0.9.0rc4] - 2020-07-28
+This is a point release, mainly including some omissions in the `listpays` result, and a fix for routehints from invoices being skipped for payments larger than 10,000 satoshis.
+
+### Added
+
+ - JSON-RPC: `listpays` now lists the `destination` if it was provided (e.g., via the `pay` plugin or `keysend` plugin) ([None](https://github.com/ElementsProject/lightning/pull/None))
+ - JSON-RPC: `listpays` can be used to query payments using the `payment_hash` ([3888](https://github.com/ElementsProject/lightning/pull/3888))
+ - JSON-RPC: `listpays` now includes the `payment_hash` ([3888](https://github.com/ElementsProject/lightning/pull/3888))
+ - JSON-RPC: The result returned by `listpays` now includes the timestamp of the first part of the payment ([3909](https://github.com/ElementsProject/lightning/pull/3909))
+
+### Changed
+
+
+### Deprecated
+
+Note: You should always set `allow-deprecated-apis=false` to test for changes.
+
+
+### Removed
+
+
+### Fixed
+
+ - pay: Fixed a bug where routehints would be ignored if the payment exceeded 10,000 satoshi. This is particularly bad if the payee is only reachable via routehints in an invoice. ([3908](https://github.com/ElementsProject/lightning/pull/3908))
+
+### Security
+
+
+## [0.9.0] - 2020-07-31: "Rat Poison Squared on Steroids"
 
 ### Added
 
@@ -785,7 +813,8 @@ There predate the BOLT specifications, and are only of vague historic interest:
 6. [0.5.1] - 2016-10-21
 7. [0.5.2] - 2016-11-21: "Bitcoin Savings & Trust Daily Interest II"
 
-[0.9.0rc4]: https://github.com/ElementsProject/lightning/releases/tag/v0.9.0rc4
+[0.9.0.1]: https://github.com/ElementsProject/lightning/releases/tag/v0.9.0.1
+[0.9.0]: https://github.com/ElementsProject/lightning/releases/tag/v0.9.0
 [0.8.2]: https://github.com/ElementsProject/lightning/releases/tag/v0.8.2
 [0.8.1]: https://github.com/ElementsProject/lightning/releases/tag/v0.8.1
 [0.8.0]: https://github.com/ElementsProject/lightning/releases/tag/v0.8.0

doc/lightning-listpays.7.md

@@ -4,30 +4,33 @@ lightning-listpays -- Command for querying payment status
 SYNOPSIS
 --------
 
-**listpays** \[bolt11\]
+**listpays** \[bolt11\] \[payment_hash\]
 
 DESCRIPTION
 -----------
 
 The **listpay** RPC command gets the status of all *pay* commands, or a
-single one if *bolt11* is specified.
+single one if either *bolt11* or *payment_hash* was specified.
 
 RETURN VALUE
 ------------
 
 On success, an array of objects is returned. Each object contains:
 
  *bolt11*
-the *bolt11* argument given to *pay* (see below for exceptions).
+the *bolt11* invoice if provided to `pay`.
+
+ *payment_hash*
+the *payment_hash* of the payment.
 
  *status*
 one of *complete*, *failed* or *pending*.
 
  *payment\_preimage*
-(if *status* is *complete*) proves payment was received.
+if *status* is *complete*.
 
  *label*
-optional *label*, if provided to *pay*.
+optional *label*, if provided to *pay* or *sendonion*.
 
  *amount\_sent\_msat*
 total amount sent, in "NNNmsat" format.

doc/lightning-sendonion.7.md

@@ -4,7 +4,8 @@ lightning-sendonion -- Send a payment with a custom onion packet
 SYNOPSIS
 --------
 
-**sendonion** *onion* *first_hop* *payment_hash* \[*label*\] \[*shared_secrets*\] \[*partid*\] \[*bolt11*\] \[*msatoshi*\]
+**sendonion** *onion* *first_hop* *payment_hash* \[*label*\] \[*shared_secrets*\] \[*partid*\] \[*bolt11*\]
+\[*msatoshi*\] \[*destination*\]
 
 DESCRIPTION
 -----------
@@ -78,6 +79,8 @@ partial payments with the same *payment_hash*.
 The *bolt11* parameter, if provided, will be returned in
 *waitsendpay* and *listsendpays* results.
 
+The *destination* parameter, if provided, will be returned in **listpays** result.
+
 The *msatoshi* parameter is used to annotate the payment, and is returned by
 *waitsendpay* and *listsendpays*.
 

lightningd/pay.c

@@ -1178,6 +1178,7 @@ static struct command_result *json_sendonion(struct command *cmd,
 	struct sha256 *payment_hash;
 	struct lightningd *ld = cmd->ld;
 	const char *label, *b11str;
+	struct node_id *destination;
 	struct secret *path_secrets;
 	struct amount_msat *msat;
 	u64 *partid;
@@ -1191,6 +1192,7 @@ static struct command_result *json_sendonion(struct command *cmd,
 		   p_opt_def("partid", param_u64, &partid, 0),
 		   p_opt("bolt11", param_string, &b11str),
 		   p_opt_def("msatoshi", param_msat, &msat, AMOUNT_MSAT(0)),
+		   p_opt("destination", param_node_id, &destination),
 		   NULL))
 		return command_param_failed();
 
@@ -1204,7 +1206,7 @@ static struct command_result *json_sendonion(struct command *cmd,
 
 	return send_payment_core(ld, cmd, payment_hash, *partid,
 				 first_hop, *msat, AMOUNT_MSAT(0),
-				 label, b11str, &packet, NULL, NULL, NULL,
+				 label, b11str, &packet, destination, NULL, NULL,
 				 path_secrets);
 }
 

plugins/libplugin-pay.c

@@ -1045,6 +1045,9 @@ static struct command_result *payment_createonion_success(struct command *cmd,
 	if (p->bolt11)
 		json_add_string(req->js, "bolt11", p->bolt11);
 
+	if (p->destination)
+		json_add_node_id(req->js, "destination", p->destination);
+
 	send_outreq(p->plugin, req);
 	return command_still_pending(cmd);
 }

plugins/pay.c

@@ -1666,6 +1666,12 @@ struct pay_mpp {
 	 * only). Null if we have any part for which we didn't know the
 	 * amount. */
 	struct amount_msat *amount;
+
+	/* Timestamp of the first part */
+	u32 timestamp;
+
+	/* The destination of the payment, if specified. */
+	struct node_id *destination;
 };
 
 static const struct sha256 *pay_mpp_key(const struct pay_mpp *pm)
@@ -1733,8 +1739,16 @@ static void add_new_entry(struct json_stream *ret,
 			  const struct pay_mpp *pm)
 {
 	json_object_start(ret, NULL);
-	json_add_string(ret, "bolt11", pm->b11);
+	if (pm->b11)
+		json_add_string(ret, "bolt11", pm->b11);
+
+	if (pm->destination)
+		json_add_node_id(ret, "destination", pm->destination);
+
+	json_add_sha256(ret, "payment_hash", pm->payment_hash);
 	json_add_string(ret, "status", pm->status);
+	json_add_u32(ret, "created_at", pm->timestamp);
+
 	if (pm->label)
 		json_add_tok(ret, "label", pm->label, buf);
 	if (pm->preimage)
@@ -1777,29 +1791,40 @@ static struct command_result *listsendpays_done(struct command *cmd,
 	ret = jsonrpc_stream_success(cmd);
 	json_array_start(ret, "pays");
 	json_for_each_arr(i, t, arr) {
-		const jsmntok_t *status, *b11tok, *hashtok;
+		const jsmntok_t *status, *b11tok, *hashtok, *destinationtok, *createdtok;
 		const char *b11 = b11str;
 		struct sha256 payment_hash;
+		struct node_id destination;
+		u32 created_at;
 
 		b11tok = json_get_member(buf, t, "bolt11");
 		hashtok = json_get_member(buf, t, "payment_hash");
+		destinationtok = json_get_member(buf, t, "destination");
+		createdtok = json_get_member(buf, t, "created_at");
 		assert(hashtok != NULL);
+		assert(createdtok != NULL);
 
 		json_to_sha256(buf, hashtok, &payment_hash);
+		json_to_u32(buf, createdtok, &created_at);
 		if (b11tok)
 			b11 = json_strdup(cmd, buf, b11tok);
 
+		if (destinationtok)
+			json_to_node_id(buf, destinationtok, &destination);
+
 		pm = pay_map_get(&pay_map, &payment_hash);
 		if (!pm) {
 			pm = tal(cmd, struct pay_mpp);
 			pm->payment_hash = tal_dup(pm, struct sha256, &payment_hash);
 			pm->b11 = tal_steal(pm, b11);
+			pm->destination = tal_dup(pm,struct node_id, &destination);
 			pm->label = json_get_member(buf, t, "label");
 			pm->preimage = NULL;
 			pm->amount_sent = AMOUNT_MSAT(0);
 			pm->amount = talz(pm, struct amount_msat);
 			pm->num_nonfailed_parts = 0;
 			pm->status = NULL;
+			pm->timestamp = created_at;
 			pay_map_add(&pay_map, pm);
 		}
 
@@ -1844,11 +1869,13 @@ static struct command_result *json_listpays(struct command *cmd,
 					    const jsmntok_t *params)
 {
 	const char *b11str;
+	struct sha256 *payment_hash;
 	struct out_req *req;
 
 	/* FIXME: would be nice to parse as a bolt11 so check worked in future */
 	if (!param(cmd, buf, params,
 		   p_opt("bolt11", param_string, &b11str),
+		   p_opt("payment_hash", param_sha256, &payment_hash),
 		   NULL))
 		return command_param_failed();
 
@@ -1857,6 +1884,10 @@ static struct command_result *json_listpays(struct command *cmd,
 				    cast_const(char *, b11str));
 	if (b11str)
 		json_add_string(req->js, "bolt11", b11str);
+
+	if (payment_hash)
+		json_add_sha256(req->js, "payment_hash", payment_hash);
+
 	return send_outreq(cmd->plugin, req);
 }
 
@@ -1881,9 +1912,25 @@ struct payment_modifier *paymod_mods[] = {
 	&local_channel_hints_pay_mod,
 	&exemptfee_pay_mod,
 	&directpay_pay_mod,
-	&presplit_pay_mod,
 	&shadowroute_pay_mod,
+	/* NOTE: The order in which these two paymods are executed is
+	 * significant!
+	 * routehints *must* execute first before presplit.
+	 *
+	 * FIXME: Giving an ordered list of paymods to the paymod
+	 * system is the wrong interface, given that the order in
+	 * which paymods execute is significant.
+	 * (This is typical of Entity-Component-System pattern.)
+	 * What should be done is that libplugin-pay should have a
+	 * canonical list of paymods in the order they execute
+	 * correctly, and whether they are default-enabled/default-disabled,
+	 * and then clients like `pay` and `keysend` will disable/enable
+	 * paymods that do not help them, instead of the current interface
+	 * where clients provide an *ordered* list of paymods they want to
+	 * use.
+	 */
 	&routehints_pay_mod,
+	&presplit_pay_mod,
 	&waitblockheight_pay_mod,
 	&retry_pay_mod,
 	&adaptive_splitter_pay_mod,
@@ -2028,7 +2075,7 @@ static const struct plugin_command commands[] = {
 	}, {
 		"listpays",
 		"payment",
-		"List result of payment {bolt11}, or all",
+		"List result of payment {bolt11} or {payment_hash}, or all",
 		"Covers old payments (failed and succeeded) and current ones.",
 		json_listpays
 	},

tests/test_pay.py

@@ -3017,3 +3017,58 @@ def test_bolt11_null_after_pay(node_factory, bitcoind):
     pays = l2.rpc.listpays()["pays"]
     assert(pays[0]["bolt11"] == invl1)
     assert('amount_msat' in pays[0] and pays[0]['amount_msat'] == amt)
+    assert('created_at' in pays[0])
+
+
+def test_mpp_presplit_routehint_conflict(node_factory, bitcoind):
+    '''
+    We had a bug where pre-splitting the payment prevents *any*
+    routehints from being taken.
+    We tickle that bug here by building l1->l2->l3, but with
+    l2->l3 as an unpublished channel.
+    If the payment is large enough to trigger pre-splitting, the
+    routehints are not applied in any of the splits.
+    '''
+    l1, l2, l3 = node_factory.get_nodes(3)
+
+    l1.rpc.connect(l2.info['id'], 'localhost', l2.port)
+    l1l2 = l1.fund_channel(l2, 10**7, announce_channel=True)
+    l2.rpc.connect(l3.info['id'], 'localhost', l3.port)
+    l2.fund_channel(l3, 10**7, announce_channel=False)
+
+    bitcoind.generate_block(6)
+    sync_blockheight(bitcoind, [l1, l2, l3])
+
+    # Wait for l3 to learn about l1->l2, otherwise it will think
+    # l2 is a deadend and not add it to the routehint.
+    wait_for(lambda: len(l3.rpc.listchannels(l1l2)['channels']) >= 2)
+
+    inv = l3.rpc.invoice(Millisatoshi(2 * 10000 * 1000), 'i', 'i', exposeprivatechannels=True)['bolt11']
+
+    l1.rpc.pay(inv)
+
+
+def test_listpay_result_with_paymod(node_factory, bitcoind):
+    """
+    The object of this test is to verify the correct behavior
+    of the RPC command listpay e with two different type of
+    payment, such as: keysend (without invoice) and pay (with invoice).
+    l1 -> keysend -> l2
+    l2 -> pay invoice -> l3
+    """
+
+    amount_sat = 10 ** 6
+
+    l1, l2, l3 = node_factory.line_graph(3)
+
+    invl2 = l2.rpc.invoice(amount_sat * 2, "inv_l2", "inv_l2")
+    l1.rpc.pay(invl2['bolt11'])
+
+    l2.rpc.keysend(l3.info['id'], amount_sat * 2, "keysend_l3")
+
+    assert 'bolt11' in l1.rpc.listpays()['pays'][0]
+    assert 'bolt11' not in l2.rpc.listpays()['pays'][0]
+    assert 'payment_hash' in l2.rpc.listpays()['pays'][0]
+    assert 'payment_hash' in l1.rpc.listpays()['pays'][0]
+    assert 'destination' in l1.rpc.listpays()['pays'][0]
+    assert 'destination' in l2.rpc.listpays()['pays'][0]