From 09a1b042f06056a16b8ef182b135f7fca3065949 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 15 Oct 2021 12:05:21 +1030 Subject: [PATCH 1/5] CCAN: add base64 module. Signed-off-by: Rusty Russell --- Makefile | 1 + ccan/ccan/base64/LICENSE | 1 + ccan/ccan/base64/_info | 41 ++++ ccan/ccan/base64/base64.c | 253 ++++++++++++++++++++++ ccan/ccan/base64/base64.h | 241 +++++++++++++++++++++ ccan/ccan/base64/test/moretap.h | 96 +++++++++ ccan/ccan/base64/test/run.c | 359 ++++++++++++++++++++++++++++++++ 7 files changed, 992 insertions(+) create mode 120000 ccan/ccan/base64/LICENSE create mode 100644 ccan/ccan/base64/_info create mode 100644 ccan/ccan/base64/base64.c create mode 100644 ccan/ccan/base64/base64.h create mode 100644 ccan/ccan/base64/test/moretap.h create mode 100644 ccan/ccan/base64/test/run.c diff --git a/Makefile b/Makefile index a3c852fee323..75560150ed29 100644 --- a/Makefile +++ b/Makefile @@ -146,6 +146,7 @@ CCAN_HEADERS := \ $(CCANDIR)/ccan/alignof/alignof.h \ $(CCANDIR)/ccan/array_size/array_size.h \ $(CCANDIR)/ccan/asort/asort.h \ + $(CCANDIR)/ccan/base64/base64.h \ $(CCANDIR)/ccan/bitmap/bitmap.h \ $(CCANDIR)/ccan/bitops/bitops.h \ $(CCANDIR)/ccan/breakpoint/breakpoint.h \ diff --git a/ccan/ccan/base64/LICENSE b/ccan/ccan/base64/LICENSE new file mode 120000 index 000000000000..2354d12945d3 --- /dev/null +++ b/ccan/ccan/base64/LICENSE @@ -0,0 +1 @@ +../../licenses/BSD-MIT \ No newline at end of file diff --git a/ccan/ccan/base64/_info b/ccan/ccan/base64/_info new file mode 100644 index 000000000000..229819902eab --- /dev/null +++ b/ccan/ccan/base64/_info @@ -0,0 +1,41 @@ +#include "config.h" + +/** + * base64 - base64 encoding and decoding (rfc4648). + * + * base64 encoding is used to encode data in a 7-bit clean manner. + * Commonly used for escaping data before encapsulation or transfer + * + * Example: + * #include + * #include + * #include + * + * int main(int argc, char *argv[]) + * { + * char *base64_encoded_string; + * int i; + * + * // print the base64-encoded form of the program arguments + * for(i=1;i +#include +#include +#include + +/** + * sixbit_to_b64 - maps a 6-bit value to the base64 alphabet + * @param map A base 64 map (see base64_init_map) + * @param sixbit Six-bit value to map + * @return a base 64 character + */ +static char sixbit_to_b64(const base64_maps_t *maps, const uint8_t sixbit) +{ + assert(sixbit <= 63); + + return maps->encode_map[(unsigned char)sixbit]; +} + +/** + * sixbit_from_b64 - maps a base64-alphabet character to its 6-bit value + * @param maps A base 64 maps structure (see base64_init_maps) + * @param sixbit Six-bit value to map + * @return a six-bit value + */ +static int8_t sixbit_from_b64(const base64_maps_t *maps, + const unsigned char b64letter) +{ + int8_t ret; + + ret = maps->decode_map[(unsigned char)b64letter]; + if (ret == (char)0xff) { + errno = EDOM; + return -1; + } + + return ret; +} + +bool base64_char_in_alphabet(const base64_maps_t *maps, const char b64char) +{ + return (maps->decode_map[(const unsigned char)b64char] != (char)0xff); +} + +void base64_init_maps(base64_maps_t *dest, const char src[64]) +{ + unsigned char i; + + memcpy(dest->encode_map,src,64); + memset(dest->decode_map,0xff,256); + for (i=0; i<64; i++) { + dest->decode_map[(unsigned char)src[i]] = i; + } +} + +size_t base64_encoded_length(size_t srclen) +{ + return ((srclen + 2) / 3) * 4; +} + +void base64_encode_triplet_using_maps(const base64_maps_t *maps, + char dest[4], const char src[3]) +{ + char a = src[0]; + char b = src[1]; + char c = src[2]; + + dest[0] = sixbit_to_b64(maps, (a & 0xfc) >> 2); + dest[1] = sixbit_to_b64(maps, ((a & 0x3) << 4) | ((b & 0xf0) >> 4)); + dest[2] = sixbit_to_b64(maps, ((c & 0xc0) >> 6) | ((b & 0xf) << 2)); + dest[3] = sixbit_to_b64(maps, c & 0x3f); +} + +void base64_encode_tail_using_maps(const base64_maps_t *maps, char dest[4], + const char *src, const size_t srclen) +{ + char longsrc[3] = { 0 }; + + assert(srclen <= 3); + + memcpy(longsrc, src, srclen); + base64_encode_triplet_using_maps(maps, dest, longsrc); + memset(dest+1+srclen, '=', 3-srclen); +} + +ssize_t base64_encode_using_maps(const base64_maps_t *maps, + char *dest, const size_t destlen, + const char *src, const size_t srclen) +{ + size_t src_offset = 0; + size_t dest_offset = 0; + + if (destlen < base64_encoded_length(srclen)) { + errno = EOVERFLOW; + return -1; + } + + while (srclen - src_offset >= 3) { + base64_encode_triplet_using_maps(maps, &dest[dest_offset], &src[src_offset]); + src_offset += 3; + dest_offset += 4; + } + + if (src_offset < srclen) { + base64_encode_tail_using_maps(maps, &dest[dest_offset], &src[src_offset], srclen-src_offset); + dest_offset += 4; + } + + memset(&dest[dest_offset], '\0', destlen-dest_offset); + + return dest_offset; +} + +size_t base64_decoded_length(size_t srclen) +{ + return ((srclen+3)/4*3); +} + +int base64_decode_quartet_using_maps(const base64_maps_t *maps, char dest[3], + const char src[4]) +{ + signed char a; + signed char b; + signed char c; + signed char d; + + a = sixbit_from_b64(maps, src[0]); + b = sixbit_from_b64(maps, src[1]); + c = sixbit_from_b64(maps, src[2]); + d = sixbit_from_b64(maps, src[3]); + + if ((a == -1) || (b == -1) || (c == -1) || (d == -1)) { + return -1; + } + + dest[0] = (a << 2) | (b >> 4); + dest[1] = ((b & 0xf) << 4) | (c >> 2); + dest[2] = ((c & 0x3) << 6) | d; + + return 0; +} + + +int base64_decode_tail_using_maps(const base64_maps_t *maps, char dest[3], + const char * src, const size_t srclen) +{ + char longsrc[4]; + int quartet_result; + size_t insize = srclen; + + while (insize != 0 && + src[insize-1] == '=') { /* throw away padding symbols */ + insize--; + } + if (insize == 0) { + return 0; + } + if (insize == 1) { + /* the input is malformed.... */ + errno = EINVAL; + return -1; + } + memcpy(longsrc, src, insize); + memset(longsrc+insize, 'A', 4-insize); + quartet_result = base64_decode_quartet_using_maps(maps, dest, longsrc); + if (quartet_result == -1) { + return -1; + } + + return insize - 1; +} + +ssize_t base64_decode_using_maps(const base64_maps_t *maps, + char *dest, const size_t destlen, + const char *src, const size_t srclen) +{ + ssize_t dest_offset = 0; + ssize_t i; + size_t more; + + if (destlen < base64_decoded_length(srclen)) { + errno = EOVERFLOW; + return -1; + } + + for(i=0; srclen - i > 4; i+=4) { + if (base64_decode_quartet_using_maps(maps, &dest[dest_offset], &src[i]) == -1) { + return -1; + } + dest_offset += 3; + } + + more = base64_decode_tail_using_maps(maps, &dest[dest_offset], &src[i], srclen - i); + if (more == -1) { + return -1; + } + dest_offset += more; + + memset(&dest[dest_offset], '\0', destlen-dest_offset); + + return dest_offset; +} + + + + +/** + * base64_maps_rfc4648 - pregenerated maps struct for rfc4648 + */ +const base64_maps_t base64_maps_rfc4648 = { + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/", + + "\xff\xff\xff\xff\xff" /* 0 */ \ + "\xff\xff\xff\xff\xff" /* 5 */ \ + "\xff\xff\xff\xff\xff" /* 10 */ \ + "\xff\xff\xff\xff\xff" /* 15 */ \ + "\xff\xff\xff\xff\xff" /* 20 */ \ + "\xff\xff\xff\xff\xff" /* 25 */ \ + "\xff\xff\xff\xff\xff" /* 30 */ \ + "\xff\xff\xff\xff\xff" /* 35 */ \ + "\xff\xff\xff\x3e\xff" /* 40 */ \ + "\xff\xff\x3f\x34\x35" /* 45 */ \ + "\x36\x37\x38\x39\x3a" /* 50 */ \ + "\x3b\x3c\x3d\xff\xff" /* 55 */ \ + "\xff\xff\xff\xff\xff" /* 60 */ \ + "\x00\x01\x02\x03\x04" /* 65 A */ \ + "\x05\x06\x07\x08\x09" /* 70 */ \ + "\x0a\x0b\x0c\x0d\x0e" /* 75 */ \ + "\x0f\x10\x11\x12\x13" /* 80 */ \ + "\x14\x15\x16\x17\x18" /* 85 */ \ + "\x19\xff\xff\xff\xff" /* 90 */ \ + "\xff\xff\x1a\x1b\x1c" /* 95 */ \ + "\x1d\x1e\x1f\x20\x21" /* 100 */ \ + "\x22\x23\x24\x25\x26" /* 105 */ \ + "\x27\x28\x29\x2a\x2b" /* 110 */ \ + "\x2c\x2d\x2e\x2f\x30" /* 115 */ \ + "\x31\x32\x33\xff\xff" /* 120 */ \ + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 125 */ \ + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" \ + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" \ + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 155 */ \ + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" \ + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" \ + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 185 */ \ + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" \ + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" \ + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 215 */ \ + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" \ + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" \ + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 245 */ +}; diff --git a/ccan/ccan/base64/base64.h b/ccan/ccan/base64/base64.h new file mode 100644 index 000000000000..405dc63fd445 --- /dev/null +++ b/ccan/ccan/base64/base64.h @@ -0,0 +1,241 @@ +/* Licensed under BSD-MIT - see LICENSE file for details */ +#ifndef CCAN_BASE64_H +#define CCAN_BASE64_H + +#include +#include +#include + +/** + * base64_maps_t - structure to hold maps for encode/decode + */ +typedef struct { + char encode_map[64]; + signed char decode_map[256]; +} base64_maps_t; + +/** + * base64_encoded_length - Calculate encode buffer length + * @param srclen the size of the data to be encoded + * @note add 1 to this to get null-termination + * @return Buffer length required for encode + */ +size_t base64_encoded_length(size_t srclen); + +/** + * base64_decoded_length - Calculate decode buffer length + * @param srclen Length of the data to be decoded + * @note This does not return the size of the decoded data! see base64_decode + * @return Minimum buffer length for safe decode + */ +size_t base64_decoded_length(size_t srclen); + +/** + * base64_init_maps - populate a base64_maps_t based on a supplied alphabet + * @param dest A base64 maps object + * @param src Alphabet to populate the maps from (e.g. base64_alphabet_rfc4648) + */ +void base64_init_maps(base64_maps_t *dest, const char src[64]); + + +/** + * base64_encode_triplet_using_maps - encode 3 bytes into base64 using a specific alphabet + * @param maps Maps to use for encoding (see base64_init_maps) + * @param dest Buffer containing 3 bytes + * @param src Buffer containing 4 characters + */ +void base64_encode_triplet_using_maps(const base64_maps_t *maps, + char dest[4], const char src[3]); + +/** + * base64_encode_tail_using_maps - encode the final bytes of a source using a specific alphabet + * @param maps Maps to use for encoding (see base64_init_maps) + * @param dest Buffer containing 4 bytes + * @param src Buffer containing srclen bytes + * @param srclen Number of bytes (<= 3) to encode in src + */ +void base64_encode_tail_using_maps(const base64_maps_t *maps, char dest[4], + const char *src, size_t srclen); + +/** + * base64_encode_using_maps - encode a buffer into base64 using a specific alphabet + * @param maps Maps to use for encoding (see base64_init_maps) + * @param dest Buffer to encode into + * @param destlen Length of dest + * @param src Buffer to encode + * @param srclen Length of the data to encode + * @return Number of encoded bytes set in dest. -1 on error (and errno set) + * @note dest will be nul-padded to destlen (past any required padding) + * @note sets errno = EOVERFLOW if destlen is too small + */ +ssize_t base64_encode_using_maps(const base64_maps_t *maps, + char *dest, size_t destlen, + const char *src, size_t srclen); + +/* + * base64_char_in_alphabet - returns true if character can be part of an encoded string + * @param maps A base64 maps object (see base64_init_maps) + * @param b64char Character to check + */ +bool base64_char_in_alphabet(const base64_maps_t *maps, char b64char); + +/** + * base64_decode_using_maps - decode a base64-encoded string using a specific alphabet + * @param maps A base64 maps object (see base64_init_maps) + * @param dest Buffer to decode into + * @param destlen length of dest + * @param src the buffer to decode + * @param srclen the length of the data to decode + * @return Number of decoded bytes set in dest. -1 on error (and errno set) + * @note dest will be nul-padded to destlen + * @note sets errno = EOVERFLOW if destlen is too small + * @note sets errno = EDOM if src contains invalid characters + */ +ssize_t base64_decode_using_maps(const base64_maps_t *maps, + char *dest, size_t destlen, + const char *src, size_t srclen); + +/** + * base64_decode_quartet_using_maps - decode 4 bytes from base64 using a specific alphabet + * @param maps A base64 maps object (see base64_init_maps) + * @param dest Buffer containing 3 bytes + * @param src Buffer containing 4 bytes + * @return Number of decoded bytes set in dest. -1 on error (and errno set) + * @note sets errno = EDOM if src contains invalid characters + */ +int base64_decode_quartet_using_maps(const base64_maps_t *maps, + char dest[3], const char src[4]); + +/** + * base64_decode_tail_using_maps - decode the final bytes of a base64 string using a specific alphabet + * @param maps A base64 maps object (see base64_init_maps) + * @param dest Buffer containing 3 bytes + * @param src Buffer containing 4 bytes - padded with '=' as required + * @param srclen Number of bytes to decode in src + * @return Number of decoded bytes set in dest. -1 on error (and errno set) + * @note sets errno = EDOM if src contains invalid characters + * @note sets errno = EINVAL if src is an invalid base64 tail + */ +int base64_decode_tail_using_maps(const base64_maps_t *maps, char *dest, + const char *src, size_t srclen); + + +/* the rfc4648 functions: */ + +extern const base64_maps_t base64_maps_rfc4648; + +/** + * base64_encode - Encode a buffer into base64 according to rfc4648 + * @param dest Buffer to encode into + * @param destlen Length of the destination buffer + * @param src Buffer to encode + * @param srclen Length of the data to encode + * @return Number of encoded bytes set in dest. -1 on error (and errno set) + * @note dest will be nul-padded to destlen (past any required padding) + * @note sets errno = EOVERFLOW if destlen is too small + * + * This function encodes src according to http://tools.ietf.org/html/rfc4648 + * + * Example: + * size_t encoded_length; + * char dest[100]; + * const char *src = "This string gets encoded"; + * encoded_length = base64_encode(dest, sizeof(dest), src, strlen(src)); + * printf("Returned data of length %zd @%p\n", encoded_length, &dest); + */ +static inline +ssize_t base64_encode(char *dest, size_t destlen, + const char *src, size_t srclen) +{ + return base64_encode_using_maps(&base64_maps_rfc4648, + dest, destlen, src, srclen); +} + +/** + * base64_encode_triplet - encode 3 bytes into base64 according to rfc4648 + * @param dest Buffer containing 4 bytes + * @param src Buffer containing 3 bytes + */ +static inline +void base64_encode_triplet(char dest[4], const char src[3]) +{ + base64_encode_triplet_using_maps(&base64_maps_rfc4648, dest, src); +} + +/** + * base64_encode_tail - encode the final bytes of a source according to rfc4648 + * @param dest Buffer containing 4 bytes + * @param src Buffer containing srclen bytes + * @param srclen Number of bytes (<= 3) to encode in src + */ +static inline +void base64_encode_tail(char dest[4], const char *src, size_t srclen) +{ + base64_encode_tail_using_maps(&base64_maps_rfc4648, dest, src, srclen); +} + + +/** + * base64_decode - decode An rfc4648 base64-encoded string + * @param dest Buffer to decode into + * @param destlen Length of the destination buffer + * @param src Buffer to decode + * @param srclen Length of the data to decode + * @return Number of decoded bytes set in dest. -1 on error (and errno set) + * @note dest will be nul-padded to destlen + * @note sets errno = EOVERFLOW if destlen is too small + * @note sets errno = EDOM if src contains invalid characters + * + * This function decodes the buffer according to + * http://tools.ietf.org/html/rfc4648 + * + * Example: + * size_t decoded_length; + * char ret[100]; + * const char *src = "Zm9vYmFyYmF6"; + * decoded_length = base64_decode(ret, sizeof(ret), src, strlen(src)); + * printf("Returned data of length %zd @%p\n", decoded_length, &ret); + */ +static inline +ssize_t base64_decode(char *dest, size_t destlen, + const char *src, size_t srclen) +{ + return base64_decode_using_maps(&base64_maps_rfc4648, + dest, destlen, src, srclen); +} + +/** + * base64_decode_quartet - decode the first 4 characters in src into dest + * @param dest Buffer containing 3 bytes + * @param src Buffer containing 4 characters + * @return Number of decoded bytes set in dest. -1 on error (and errno set) + * @note sets errno = EDOM if src contains invalid characters + */ +static inline +int base64_decode_quartet(char dest[3], const char src[4]) +{ + return base64_decode_quartet_using_maps(&base64_maps_rfc4648, + dest, src); +} + +/** + * @brief decode the final bytes of a base64 string from src into dest + * @param dest Buffer containing 3 bytes + * @param src Buffer containing 4 bytes - padded with '=' as required + * @param srclen Number of bytes to decode in src + * @return Number of decoded bytes set in dest. -1 on error (and errno set) + * @note sets errno = EDOM if src contains invalid characters + * @note sets errno = EINVAL if src is an invalid base64 tail + */ +static inline +ssize_t base64_decode_tail(char dest[3], const char *src, size_t srclen) +{ + return base64_decode_tail_using_maps(&base64_maps_rfc4648, + dest, src, srclen); +} + +/* end rfc4648 functions */ + + + +#endif /* CCAN_BASE64_H */ diff --git a/ccan/ccan/base64/test/moretap.h b/ccan/ccan/base64/test/moretap.h new file mode 100644 index 000000000000..114445c3c4b5 --- /dev/null +++ b/ccan/ccan/base64/test/moretap.h @@ -0,0 +1,96 @@ +#ifndef _BASE64_MORETAP_H +#define _BASE64_MORETAP_H + +#include + +/** + * is_str - OK if strings are equal + * @e1: expression for the variable string + * @e2: expression for the expected string + * + * If the strings are equal, the test passes. + * + * Example: + * is_str(give_me_a_fred(),"fred"); + */ +static void _is_str(char *got,const char *expected, const char *got_string, const char *expected_string, const char *func, const char *file, int line) { + if (streq(expected,got)) { + _gen_result(1, func, file, line,"%s eq %s", + got_string,expected_string); + } else { + _gen_result(0, func, file, line,"%s eq %s", + got_string,expected_string); + diag("Expected: %s",expected); + diag(" Got: %s",got); + } +} +# define is_str(got,expected) _is_str(got,expected,#got,#expected,__func__, __FILE__, __LINE__) + + +/** + * is_int - OK if arguments are equal when cast to integers + * @e1: expression for the number + * @e2: expression for the expected number + * + * If the numbers are equal, the test passes. + * + * Example: + * is_int(give_me_17(),17); + */ +# define is_int(e1,e2 ...) \ + (((int)e1)==((int)e2) ? \ + _gen_result(1, __func__, __FILE__, __LINE__,"%s == %s",#e1,#e2) : \ + (_gen_result(0, __func__, __FILE__, __LINE__,"%s == %s",#e1,#e2)) || (diag("Expected: %d",e2),diag(" Got: %d",e1),0)) /* diag is void; note commas. */ + + + +/** + * is_mem - OK if arguments are identical up to length @e3 + * @e1: expression for the buffer + * @e2: expression for the expected buffer + * @e2: length to compare in buffers + * + * If the buffers are equal up to @e2, the test passes. + * + * Example: + * is_mem(give_me_foo(),"foo",3); + */ +static void _is_mem(const char *got, const char *expected, const size_t len, + const char *got_string, const char *expected_string, const char *len_string, + const char *func, const char *file, int line) { + size_t offset = 0; + + for (offset=0; offset +#include +#include + +#include +#include + +#include +#include "moretap.h" + +static void * xmalloc(size_t size); + +/* not defined in terms of test_encode_using_maps so we cross + appropriate paths in library */ +#define test_encode(src,srclen,expected) \ + do { \ + size_t destlen; \ + char * dest; \ + destlen = base64_encoded_length(srclen); \ + destlen++; /* null termination */ \ + dest = xmalloc(destlen); \ + ok1(base64_encode(dest,destlen,src,srclen) != -1); \ + is_str(dest,expected); \ + free(dest); \ + } while (0) + +#define test_encode_using_alphabet(alphastring,src,srclen,expected) \ + do { \ + size_t destlen; \ + char * dest; \ + base64_maps_t maps; \ + base64_init_maps(&maps,alphastring); \ + destlen = base64_encoded_length(srclen); \ + destlen++; /* null termination */ \ + dest = xmalloc(destlen); \ + ok1(base64_encode_using_maps(&maps,dest,destlen,src,srclen) != -1); \ + is_str(dest,expected); \ + free(dest); \ + } while (0) + +/* not defined in terms of test_decode_using_alphabet so we cross + appropriate paths in library */ +#define test_decode(src,srclen,expected,expectedlen) \ + do { \ + size_t destlen; \ + size_t bytes_used; \ + char * dest; \ + destlen = base64_decoded_length(srclen); \ + dest = xmalloc(destlen); \ + ok1((bytes_used = base64_decode(dest,destlen,src,srclen)) != -1); \ + is_size_t(bytes_used,expectedlen); \ + is_mem(dest,expected,bytes_used); \ + free(dest); \ + } while (0) + +#define test_decode_using_alphabet(alphastring,src,srclen,expected,expectedlen) \ + do { \ + size_t destlen; \ + size_t bytes_used; \ + char * dest; \ + base64_maps_t maps; \ + \ + base64_init_maps(&maps,alphastring); \ + destlen = base64_decoded_length(srclen); \ + dest = xmalloc(destlen); \ + ok1((bytes_used = base64_decode_using_maps(&maps,dest,destlen,src,srclen)) != -1); \ + is_size_t(bytes_used,expectedlen); \ + is_mem(dest,expected,bytes_used); \ + free(dest); \ + } while (0) + +#define check_bad_range_decode(stuff_to_test,stufflen) \ +do { \ + char dest[10]; \ + errno = 0; \ + is_size_t(base64_decode(dest,sizeof(dest),stuff_to_test,(size_t)stufflen), \ + (size_t)-1); \ + is_int(errno,EDOM); \ +} while (0) + +int +main(int argc, char *argv[]) +{ + plan_tests(131); + + is_size_t(base64_encoded_length(0),(size_t)0); + is_size_t(base64_encoded_length(1),(size_t)4); + is_size_t(base64_encoded_length(2),(size_t)4); + is_size_t(base64_encoded_length(3),(size_t)4); + is_size_t(base64_encoded_length(512),(size_t)684); + + /* straight from page 11 of http://tools.ietf.org/html/rfc4648 */ + test_encode("",0,""); + test_encode("f",1,"Zg=="); + test_encode("fo",2,"Zm8="); + + test_encode("foo",3,"Zm9v"); + test_encode("foob",4,"Zm9vYg=="); + test_encode("fooba",5,"Zm9vYmE="); + test_encode("foobar",6,"Zm9vYmFy"); + + /* a few more */ + test_encode("foobarb",7,"Zm9vYmFyYg=="); + test_encode("foobarba",8,"Zm9vYmFyYmE="); + test_encode("foobarbaz",9,"Zm9vYmFyYmF6"); + + test_encode("foobart",7,"Zm9vYmFydA=="); + + test_encode("abcdefghijklmnopqrstuvwxyz",26,"YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXo="); + test_encode("\x05\x05\x01\x00\x07",5,"BQUBAAc="); + + test_encode("FOO",3,"Rk9P"); + test_encode("Z",1,"Wg=="); + + /* decode testing */ + + test_decode("",0,"",0); + test_decode("Zg==",4,"f",1); + test_decode("Zm8=",4,"fo",2); + test_decode("Zm9v",4,"foo",3); + test_decode("Zm9vYg==",8,"foob",4); + test_decode("Zm9vYmE=",8,"fooba",5); + test_decode("Zm9vYmFy",8,"foobar",6); + test_decode("Zm9vYmFyYg==",12,"foobarb",7); + test_decode("Zm9vYmFyYmE=",12,"foobarba",8); + test_decode("Zm9vYmFyYmF6",12,"foobarbaz",9); + + test_decode("Rk9P",4,"FOO",3); + + test_decode("Wg==",4,"Z",1); + test_decode("AA==",4,"\0",1); + test_decode("AAA=",4,"\0\0",2); + + { + const char *binary = "\x01\x00\x03"; + const size_t binarylen = 3; + + char * decoded; + char * encoded; + size_t encoded_len; + size_t decoded_len; + size_t decoded_space_required; + + size_t encoded_space_required = base64_encoded_length(binarylen); + encoded_space_required++; /* null termination */ + encoded = xmalloc(encoded_space_required); + encoded_len = base64_encode(encoded,encoded_space_required,binary,binarylen); + is_mem(encoded,"AQAD",encoded_len); + + decoded_space_required = base64_decoded_length(encoded_len); + decoded = xmalloc(decoded_space_required); + decoded_len = base64_decode(decoded,decoded_space_required,encoded,encoded_len); + is_size_t(decoded_len,binarylen); + is_mem(binary,decoded,decoded_len); + } + + /* some expected encode failures: */ + { + size_t destlen = 1; + char dest[destlen]; + errno = 0; + is_size_t(base64_encode(dest,destlen,"A",1),(size_t)-1); + is_int(errno,EOVERFLOW); + } + + /* some expected decode failures: */ + { + base64_maps_t maps; + const char * src = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + base64_init_maps(&maps,src); + + is_int(sixbit_from_b64(&maps,'\xfe'),(signed char)-1); + is_int(errno,EDOM); + } + { + size_t destlen = 10; + char dest[destlen]; + errno = 0; + is_size_t(base64_decode(dest,destlen,"A",1),(size_t)-1); + is_int(errno,EINVAL); + } + { + size_t destlen = 1; + char dest[destlen]; + errno = 0; + is_size_t(base64_decode(dest,destlen,"A",1),(size_t)-1); + is_int(errno,EOVERFLOW); + } + { + /* (char)1 is not a valid base64 character: */ + check_bad_range_decode("A\x01",2); + /* (char)255 is not a valid base64 character: (char is signed on most platforms, so this is actually < 0 */ + check_bad_range_decode("\xff""A",2); + check_bad_range_decode("A\xff",2); + check_bad_range_decode("AA\xff",3); + check_bad_range_decode("A\xff""A",3); + check_bad_range_decode("\xff""AA",3); + check_bad_range_decode("AAA\xff",4); + check_bad_range_decode("\xff\x41\x41\x41\x41",5); + check_bad_range_decode("A\xff\x41\x41\x41\x41",6); + check_bad_range_decode("AA\xff\x41\x41\x41\x41",7); + check_bad_range_decode("AAA\xff\x41\x41\x41\x41",8); + } + /* trigger some failures in the sixbit-to-b64 encoder: */ + /* this function now aborts rather than returning -1/setting errno */ + /* { */ + /* is_int(sixbit_to_b64(base64_maps_rfc4648,'\x70'),(char)-1); */ + /* is_int(sixbit_to_b64(base64_maps_rfc4648,'\xff'),(char)-1); */ + /* } */ + /* following tests all of the mapping from b64 chars to 6-bit values: */ + test_decode("//+FwHRSRIsFU2IhAEGD+AMPhOA=",28,"\xff\xff\x85\xc0\x74\x52\x44\x8b\x05\x53\x62\x21\x00\x41\x83\xf8\x03\x0f\x84\xe0",20); + test_encode("\xff\xff\x85\xc0\x74\x52\x44\x8b\x05\x53\x62\x21\x00\x41\x83\xf8\x03\x0f\x84\xe0",20,"//+FwHRSRIsFU2IhAEGD+AMPhOA="); + + + /* check the null-padding stuff */ + { + size_t destlen = 8; + char dest[destlen]; + memset(dest,'\1',sizeof(dest)); + is_size_t(base64_encode(dest,destlen,"A",1),(size_t)4); + is_mem(&dest[4],"\0\0\0\0",4); + } + { + size_t destlen = 3; + char dest[destlen]; + memset(dest,'\1',sizeof(dest)); + is_size_t(base64_decode(dest,destlen,"Wg==",4), 1); + is_mem(&dest[1],"\0",2); + } + + /* test encoding using different alphabets */ + { + char alphabet_fs_safe[64]; + memcpy(alphabet_fs_safe,base64_maps_rfc4648.encode_map,sizeof(alphabet_fs_safe)); + alphabet_fs_safe[62] = '-'; + alphabet_fs_safe[63] = '_'; + test_encode_using_alphabet(alphabet_fs_safe,"\xff\xff\x85\xc0\x74\x52\x44\x8b\x05\x53\x62\x21\x00\x41\x83\xf8\x03\x0f\x84\xe0",20,"__-FwHRSRIsFU2IhAEGD-AMPhOA="); + } + + /* test decoding using different alphabets */ + { + char alphabet_fs_safe[64]; + #define src "__-FwHRSRIsFU2IhAEGD-AMPhOA=" + #define expected "\xff\xff\x85\xc0\x74\x52\x44\x8b\x05\x53\x62\x21\x00\x41\x83\xf8\x03\x0f\x84\xe0" + + memcpy(alphabet_fs_safe,base64_maps_rfc4648.encode_map,sizeof(alphabet_fs_safe)); + alphabet_fs_safe[62] = '-'; + alphabet_fs_safe[63] = '_'; + + test_decode_using_alphabet(alphabet_fs_safe,src,strlen(src),expected,20); + #undef src + #undef expected + } + + /* explicitly test the non-maps encode_triplet and + encode_tail functions */ + { + size_t destlen = 4; + char dest[destlen]; + const char *src = "AB\04"; + memset(dest,'\1',sizeof(dest)); + base64_encode_triplet(dest,src); + is_mem(dest,"QUIE",sizeof(dest)); + } + { + size_t destlen = 4; + char dest[destlen]; + const char *src = "A"; + memset(dest,'\1',sizeof(dest)); + base64_encode_tail(dest,src,strlen(src)); + is_mem(dest,"QQ==",sizeof(dest)); + } + + /* test the alphabet inversion */ + { + base64_maps_t dest; + const char expected_inverse[] = + "\xff\xff\xff\xff\xff" /* 0 */ + "\xff\xff\xff\xff\xff" /* 5 */ + "\xff\xff\xff\xff\xff" /* 10 */ + "\xff\xff\xff\xff\xff" /* 15 */ + "\xff\xff\xff\xff\xff" /* 20 */ + "\xff\xff\xff\xff\xff" /* 25 */ + "\xff\xff\xff\xff\xff" /* 30 */ + "\xff\xff\xff\xff\xff" /* 35 */ + "\xff\xff\xff\x3e\xff" /* 40 */ + "\xff\xff\x3f\x34\x35" /* 45 - */ + "\x36\x37\x38\x39\x3a" /* 50 */ + "\x3b\x3c\x3d\xff\xff" /* 55 */ + "\xff\xff\xff\xff\xff" /* 60 */ + "\x00\x01\x02\x03\x04" /* 65 A */ + "\x05\x06\x07\x08\x09" /* 70 */ + "\x0a\x0b\x0c\x0d\x0e" /* 75 */ + "\x0f\x10\x11\x12\x13" /* 80 */ + "\x14\x15\x16\x17\x18" /* 85 */ + "\x19\xff\xff\xff\xff" /* 90 */ + "\xff\xff\x1a\x1b\x1c" /* 95 _ */ + "\x1d\x1e\x1f\x20\x21" /* 100 */ + "\x22\x23\x24\x25\x26" /* 105 */ + "\x27\x28\x29\x2a\x2b" /* 110 */ + "\x2c\x2d\x2e\x2f\x30" /* 115 */ + "\x31\x32\x33\xff\xff" /* 120 */ + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 125 */ + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 155 */ + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 185 */ + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 215 */ + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 245 */ + ; + const char * src = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + base64_init_maps(&dest, src); + is_mem((const char *)dest.decode_map, expected_inverse, 256); + ok1(base64_char_in_alphabet(&dest,'A')); + ok1(!base64_char_in_alphabet(&dest,'\n')); + } + + /* explicitly test the non-alpha decode_tail and decode_quartet */ + { + char dest[4]; + const char *src = "QQ=="; + const char * expected = "A"; + memset(dest, '%', sizeof(dest)); + base64_decode_tail(dest,src,4); + is_mem(dest, expected, 1); + } + { + char dest[4]; + const char *src = "Zm9v"; + const char * expected = "foo"; + memset(dest, '%', sizeof(dest)); + base64_decode_quartet(dest,src); + is_mem(dest, expected, 1); + } + + exit(exit_status()); +} + +static void * xmalloc(size_t size) +{ + char * ret; + ret = malloc(size); + if (ret == NULL) { + perror("malloc"); + abort(); + } + return ret; +} + +/* End of run.c test */ From 6e33cc56b0e463bd6d5d6c9b3f15ed4ca6492736 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 15 Oct 2021 16:18:44 +1030 Subject: [PATCH 2/5] patch websocket-address-support.patch --- common/json_helpers.c | 3 +++ common/wireaddr.c | 24 ++++++++++++++++++ common/wireaddr.h | 9 ++++++- connectd/connectd.c | 42 ++++++++++++++++++++----------- connectd/netaddress.c | 1 + devtools/gossipwith.c | 3 +++ doc/lightning-getinfo.7.md | 8 +++--- doc/lightning-listnodes.7.md | 8 +++--- doc/schemas/getinfo.schema.json | 38 ++++++++++++++++++++++------ doc/schemas/listnodes.schema.json | 38 ++++++++++++++++++++++------ 10 files changed, 139 insertions(+), 35 deletions(-) diff --git a/common/json_helpers.c b/common/json_helpers.c index 0629b03e7f47..65308d7ba37d 100644 --- a/common/json_helpers.c +++ b/common/json_helpers.c @@ -281,6 +281,9 @@ void json_add_address(struct json_stream *response, const char *fieldname, json_add_string(response, "type", "torv3"); json_add_string(response, "address", fmt_wireaddr_without_port(tmpctx, addr)); json_add_num(response, "port", addr->port); + } else if (addr->type == ADDR_TYPE_WEBSOCKET) { + json_add_string(response, "type", "websocket"); + json_add_num(response, "port", addr->port); } json_object_end(response); } diff --git a/common/wireaddr.c b/common/wireaddr.c index 514b0ba2afd7..6851df8f80eb 100644 --- a/common/wireaddr.c +++ b/common/wireaddr.c @@ -37,6 +37,9 @@ bool fromwire_wireaddr(const u8 **cursor, size_t *max, struct wireaddr *addr) case ADDR_TYPE_TOR_V3: addr->addrlen = TOR_V3_ADDRLEN; break; + case ADDR_TYPE_WEBSOCKET: + addr->addrlen = 0; + break; default: return false; } @@ -160,6 +163,14 @@ void wireaddr_from_ipv6(struct wireaddr *addr, memcpy(&addr->addr, ip6, addr->addrlen); } +void wireaddr_from_websocket(struct wireaddr *addr, const u16 port) +{ + addr->type = ADDR_TYPE_WEBSOCKET; + addr->addrlen = 0; + addr->port = port; + memset(addr->addr, 0, sizeof(addr->addr)); +} + bool wireaddr_to_ipv4(const struct wireaddr *addr, struct sockaddr_in *s4) { if (addr->type != ADDR_TYPE_IPV4) @@ -184,6 +195,14 @@ bool wireaddr_to_ipv6(const struct wireaddr *addr, struct sockaddr_in6 *s6) return true; } +bool wireaddr_to_websocket(const struct wireaddr *addr, u16 *port) +{ + if (addr->type != ADDR_TYPE_WEBSOCKET) + return false; + *port = addr->port; + return true; +} + bool wireaddr_is_wildcard(const struct wireaddr *addr) { switch (addr->type) { @@ -192,6 +211,7 @@ bool wireaddr_is_wildcard(const struct wireaddr *addr) return memeqzero(addr->addr, addr->addrlen); case ADDR_TYPE_TOR_V2: case ADDR_TYPE_TOR_V3: + case ADDR_TYPE_WEBSOCKET: return false; } abort(); @@ -239,6 +259,8 @@ char *fmt_wireaddr_without_port(const tal_t * ctx, const struct wireaddr *a) case ADDR_TYPE_TOR_V3: return tal_fmt(ctx, "%s.onion", b32_encode(tmpctx, a->addr, a->addrlen)); + case ADDR_TYPE_WEBSOCKET: + return tal_strdup(ctx, "websocket"); } hex = tal_hexstr(ctx, a->addr, a->addrlen); @@ -675,6 +697,7 @@ struct addrinfo *wireaddr_to_addrinfo(const tal_t *ctx, return ai; case ADDR_TYPE_TOR_V2: case ADDR_TYPE_TOR_V3: + case ADDR_TYPE_WEBSOCKET: break; } abort(); @@ -729,6 +752,7 @@ bool all_tor_addresses(const struct wireaddr_internal *wireaddr) return false; case ADDR_TYPE_TOR_V2: case ADDR_TYPE_TOR_V3: + case ADDR_TYPE_WEBSOCKET: continue; } } diff --git a/common/wireaddr.h b/common/wireaddr.h index 7ca73b46ae0f..383bcb5195b9 100644 --- a/common/wireaddr.h +++ b/common/wireaddr.h @@ -37,6 +37,10 @@ struct sockaddr_un; * where `checksum = sha3(".onion checksum" | pubkey || version)[:2]` */ +/* BOLT-websockets #7: + * * `5`: WebSocket port; data = `[2:port]` (length 2) + */ + #define TOR_V2_ADDRLEN 10 #define TOR_V3_ADDRLEN 35 #define LARGEST_ADDRLEN TOR_V3_ADDRLEN @@ -47,7 +51,8 @@ enum wire_addr_type { ADDR_TYPE_IPV4 = 1, ADDR_TYPE_IPV6 = 2, ADDR_TYPE_TOR_V2 = 3, - ADDR_TYPE_TOR_V3 = 4 + ADDR_TYPE_TOR_V3 = 4, + ADDR_TYPE_WEBSOCKET = 5, }; /* Structure now fit for tor support */ @@ -98,8 +103,10 @@ void wireaddr_from_ipv4(struct wireaddr *addr, void wireaddr_from_ipv6(struct wireaddr *addr, const struct in6_addr *ip6, const u16 port); +void wireaddr_from_websocket(struct wireaddr *addr, const u16 port); bool wireaddr_to_ipv4(const struct wireaddr *addr, struct sockaddr_in *s4); bool wireaddr_to_ipv6(const struct wireaddr *addr, struct sockaddr_in6 *s6); +bool wireaddr_to_websocket(const struct wireaddr *addr, u16 *port); bool wireaddr_is_wildcard(const struct wireaddr *addr); diff --git a/connectd/connectd.c b/connectd/connectd.c index 3b5f33969ecf..892a531fc55f 100644 --- a/connectd/connectd.c +++ b/connectd/connectd.c @@ -835,36 +835,41 @@ static void try_connect_one_addr(struct connecting *connect) case ADDR_TYPE_IPV6: af = AF_INET6; break; + case ADDR_TYPE_WEBSOCKET: + af = -1; + break; } } /* If we have to use proxy but we don't have one, we fail. */ if (use_proxy) { if (!connect->daemon->proxyaddr) { - status_debug("Need proxy"); - af = -1; - } else - af = connect->daemon->proxyaddr->ai_family; + tal_append_fmt(&connect->errors, + "%s: need a proxy. ", + type_to_string(tmpctx, + struct wireaddr_internal, + addr)); + goto next; + } + af = connect->daemon->proxyaddr->ai_family; } if (af == -1) { - fd = -1; - errno = EPROTONOSUPPORT; - } else - fd = socket(af, SOCK_STREAM, 0); + tal_append_fmt(&connect->errors, + "%s: not supported. ", + type_to_string(tmpctx, struct wireaddr_internal, + addr)); + goto next; + } - /* We might not have eg. IPv6 support, or it might be an onion addr - * and we have no proxy. */ + fd = socket(af, SOCK_STREAM, 0); if (fd < 0) { tal_append_fmt(&connect->errors, "%s: opening %i socket gave %s. ", type_to_string(tmpctx, struct wireaddr_internal, addr), af, strerror(errno)); - /* This causes very limited recursion. */ - connect->addrnum++; - try_connect_one_addr(connect); - return; + goto next; } /* This creates the new connection using our fd, with the initialization @@ -878,6 +883,13 @@ static void try_connect_one_addr(struct connecting *connect) * that frees connect. */ if (conn) connect->conn = conn; + + return; + +next: + /* This causes very limited recursion. */ + connect->addrnum++; + try_connect_one_addr(connect); } /*~ connectd is responsible for incoming connections, but it's the process of @@ -988,6 +1000,8 @@ static bool handle_wireaddr_listen(struct daemon *daemon, return true; } return false; + /* Handle specially by callers. */ + case ADDR_TYPE_WEBSOCKET: case ADDR_TYPE_TOR_V2: case ADDR_TYPE_TOR_V3: break; diff --git a/connectd/netaddress.c b/connectd/netaddress.c index 1acfa6375389..ce55cfa3939d 100644 --- a/connectd/netaddress.c +++ b/connectd/netaddress.c @@ -258,6 +258,7 @@ bool guess_address(struct wireaddr *addr) } case ADDR_TYPE_TOR_V2: case ADDR_TYPE_TOR_V3: + case ADDR_TYPE_WEBSOCKET: status_broken("Cannot guess address type %u", addr->type); break; } diff --git a/devtools/gossipwith.c b/devtools/gossipwith.c index c110c7908d38..2bd0a4778305 100644 --- a/devtools/gossipwith.c +++ b/devtools/gossipwith.c @@ -321,6 +321,9 @@ int main(int argc, char *argv[]) case ADDR_TYPE_TOR_V3: opt_usage_exit_fail("Don't support proxy use"); break; + case ADDR_TYPE_WEBSOCKET: + opt_usage_exit_fail("Don't support websockets"); + break; case ADDR_TYPE_IPV4: af = AF_INET; break; diff --git a/doc/lightning-getinfo.7.md b/doc/lightning-getinfo.7.md index 2ec0ba960206..1d6bd5187b65 100644 --- a/doc/lightning-getinfo.7.md +++ b/doc/lightning-getinfo.7.md @@ -40,9 +40,11 @@ On success, an object is returned, containing: - **network** (string): represents the type of network on the node are working (e.g: `bitcoin`, `testnet`, or `regtest`) - **fees_collected_msat** (msat): Total routing fees collected by this node - **address** (array of objects, optional): The addresses we announce to the world: - - **type** (string): Type of connection (one of "ipv4", "ipv6", "torv2", "torv3") - - **address** (string): address in expected format for **type** + - **type** (string): Type of connection (one of "ipv4", "ipv6", "torv2", "torv3", "websocket") - **port** (u16): port number + + If **type** is "ipv4", "ipv6", "torv2" or "torv3": + - **address** (string): address in expected format for **type** - **binding** (array of objects, optional): The addresses we are listening on: - **type** (string): Type of connection (one of "local socket", "ipv4", "ipv6", "torv2", "torv3") - **address** (string, optional): address in expected format for **type** @@ -115,4 +117,4 @@ RESOURCES --------- Main web site: -[comment]: # ( SHA256STAMP:a41fb9bb8e6e61bec105ff250584ae019dda93d4f97bfff53bc86d57ab6e8607) +[comment]: # ( SHA256STAMP:50c41a77a5f440cc22e5df9e3748e4280cd4508469887382690c580f10bc5af4) diff --git a/doc/lightning-listnodes.7.md b/doc/lightning-listnodes.7.md index bc7f074f3642..59208b5a9062 100644 --- a/doc/lightning-listnodes.7.md +++ b/doc/lightning-listnodes.7.md @@ -36,10 +36,12 @@ If **last_timestamp** is present: - **color** (hex): The favorite RGB color this node advertized (always 6 characters) - **features** (hex): BOLT #9 features bitmap this node advertized - **addresses** (array of objects): The addresses this node advertized: - - **type** (string): Type of connection (one of "ipv4", "ipv6", "torv2", "torv3") - - **address** (string): address in expected format for *type* + - **type** (string): Type of connection (one of "ipv4", "ipv6", "torv2", "torv3", "websocket") - **port** (u16): port number + If **type** is "ipv4", "ipv6", "torv2" or "torv3": + - **address** (string): address in expected format for **type** + If **option_will_fund** is present: - **option_will_fund** (object): - **lease_fee_base_msat** (msat): the fixed fee for a lease (whole number of satoshis) @@ -93,4 +95,4 @@ RESOURCES --------- Main web site: -[comment]: # ( SHA256STAMP:158477348efb51a8cf71a595b3d76dde545ab6824958c8a32d4b3dbbbe2c8121) +[comment]: # ( SHA256STAMP:f9e1f4655b416c5e60279cf11a832bc4c652f503e48095dc3cf39deee5f0c769) diff --git a/doc/schemas/getinfo.schema.json b/doc/schemas/getinfo.schema.json index 3c964fee25f9..02e2c5b430e3 100644 --- a/doc/schemas/getinfo.schema.json +++ b/doc/schemas/getinfo.schema.json @@ -64,22 +64,46 @@ "description": "The addresses we announce to the world", "items": { "type": "object", - "required": [ "type", "address", "port" ], - "additionalProperties": false, + "required": [ "type", "port" ], + "additionalProperties": true, "properties": { "type": { "type": "string", - "enum": [ "ipv4", "ipv6", "torv2", "torv3" ], + "enum": [ "ipv4", "ipv6", "torv2", "torv3", "websocket" ], "description": "Type of connection" }, - "address": { - "type": "string", - "description": "address in expected format for **type**" - }, "port": { "type": "u16", "description": "port number" } + }, + "if": { + "properties": { + "type": { + "type": "string", + "enum": [ "ipv4", "ipv6", "torv2", "torv3" ] + } + } + }, + "then": { + "required": [ "type", "address", "port" ], + "additionalProperties": false, + "properties": { + "type": { }, + "port": { }, + "address": { + "type": "string", + "description": "address in expected format for **type**" + } + } + }, + "else": { + "required": [ "type", "port" ], + "additionalProperties": false, + "properties": { + "type": { }, + "port": { } + } } } }, diff --git a/doc/schemas/listnodes.schema.json b/doc/schemas/listnodes.schema.json index 1672bc4856e1..4437bfbc8777 100644 --- a/doc/schemas/listnodes.schema.json +++ b/doc/schemas/listnodes.schema.json @@ -52,22 +52,46 @@ "description": "The addresses this node advertized", "items": { "type": "object", - "required": [ "type", "address", "port" ], - "additionalProperties": false, + "required": [ "type", "port" ], + "additionalProperties": true, "properties": { "type": { "type": "string", - "enum": [ "ipv4", "ipv6", "torv2", "torv3" ], + "enum": [ "ipv4", "ipv6", "torv2", "torv3", "websocket" ], "description": "Type of connection" }, - "address": { - "type": "string", - "description": "address in expected format for *type*" - }, "port": { "type": "u16", "description": "port number" } + }, + "if": { + "properties": { + "type": { + "type": "string", + "enum": [ "ipv4", "ipv6", "torv2", "torv3" ] + } + } + }, + "then": { + "required": [ "type", "address", "port" ], + "additionalProperties": false, + "properties": { + "type": { }, + "port": { }, + "address": { + "type": "string", + "description": "address in expected format for **type**" + } + } + }, + "else": { + "required": [ "type", "port" ], + "additionalProperties": false, + "properties": { + "type": { }, + "port": { } + } } } } From 32d22a3bbbdcaf799a29d7845f959100f0babf90 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 15 Oct 2021 16:19:05 +1030 Subject: [PATCH 3/5] connectd: listen on ports for which we should spawn a proxy. If the port is set, we spawn it (lightning_websocketd) on any connection to that port. That means websocketd is a per-peer daemon, but it means every other daemon uses the connection normally (it's just actually talking to websocketd instead of the client directly). Signed-off-by: Rusty Russell --- connectd/connectd.c | 238 +++++++++++++++++++++++--- connectd/connectd_wire.csv | 2 + connectd/handshake.c | 1 - connectd/test/run-initiator-success.c | 3 + connectd/test/run-responder-success.c | 3 + lightningd/connect_control.c | 5 +- 6 files changed, 225 insertions(+), 27 deletions(-) diff --git a/connectd/connectd.c b/connectd/connectd.c index 892a531fc55f..68c767813aa4 100644 --- a/connectd/connectd.c +++ b/connectd/connectd.c @@ -33,9 +33,13 @@ #include #include #include +#include #include #include #include +#include +#include +#include #include /*~ We are passed two file descriptors when exec'ed from `lightningd`: the @@ -127,6 +131,12 @@ struct daemon { /* Our features, as lightningd told us */ struct feature_set *our_features; + + /* Subdaemon to proxy websocket requests. */ + char *websocket_helper; + + /* If non-zero, port to listen for websocket connections. */ + u16 websocket_port; }; /* Peers we're trying to reach: we iterate through addrs until we succeed @@ -529,11 +539,10 @@ static void conn_timeout(struct io_conn *conn) io_close(conn); } -/*~ When we get a connection in we set up its network address then call - * handshake.c to set up the crypto state. */ -static struct io_plan *connection_in(struct io_conn *conn, struct daemon *daemon) +/*~ So, where are you from? */ +static bool get_remote_address(struct io_conn *conn, + struct wireaddr_internal *addr) { - struct wireaddr_internal addr; struct sockaddr_storage s = {}; socklen_t len = sizeof(s); @@ -541,28 +550,44 @@ static struct io_plan *connection_in(struct io_conn *conn, struct daemon *daemon if (getpeername(io_conn_fd(conn), (struct sockaddr *)&s, &len) != 0) { status_debug("Failed to get peername for incoming conn: %s", strerror(errno)); - return io_close(conn); + return false; } if (s.ss_family == AF_INET6) { struct sockaddr_in6 *s6 = (void *)&s; - addr.itype = ADDR_INTERNAL_WIREADDR; - wireaddr_from_ipv6(&addr.u.wireaddr, + addr->itype = ADDR_INTERNAL_WIREADDR; + wireaddr_from_ipv6(&addr->u.wireaddr, &s6->sin6_addr, ntohs(s6->sin6_port)); } else if (s.ss_family == AF_INET) { struct sockaddr_in *s4 = (void *)&s; - addr.itype = ADDR_INTERNAL_WIREADDR; - wireaddr_from_ipv4(&addr.u.wireaddr, + addr->itype = ADDR_INTERNAL_WIREADDR; + wireaddr_from_ipv4(&addr->u.wireaddr, &s4->sin_addr, ntohs(s4->sin_port)); } else if (s.ss_family == AF_UNIX) { struct sockaddr_un *sun = (void *)&s; - addr.itype = ADDR_INTERNAL_SOCKNAME; - memcpy(addr.u.sockname, sun->sun_path, sizeof(sun->sun_path)); + addr->itype = ADDR_INTERNAL_SOCKNAME; + memcpy(addr->u.sockname, sun->sun_path, sizeof(sun->sun_path)); } else { status_broken("Unknown socket type %i for incoming conn", s.ss_family); - return io_close(conn); + return false; } + return true; +} + +/*~ As so common in C, we need to bundle two args into a callback, so we + * allocate a temporary structure to hold them: */ +struct conn_in { + struct wireaddr_internal addr; + struct daemon *daemon; +}; + +/*~ Once we've got a connection in, we set it up here (whether it's via the + * websocket proxy, or direct). */ +static struct io_plan *conn_in(struct io_conn *conn, + struct conn_in *conn_in_arg) +{ + struct daemon *daemon = conn_in_arg->daemon; /* If they don't complete handshake in reasonable time, hang up */ notleak(new_reltimer(&daemon->timers, conn, @@ -574,10 +599,122 @@ static struct io_plan *connection_in(struct io_conn *conn, struct daemon *daemon * Note, again, the notleak() to avoid our simplistic leak detection * code from thinking `conn` (which we don't keep a pointer to) is * leaked */ - return responder_handshake(notleak(conn), &daemon->mykey, &addr, + return responder_handshake(notleak(conn), &daemon->mykey, + &conn_in_arg->addr, handshake_in_success, daemon); } +/*~ When we get a direct connection in we set up its network address + * then call handshake.c to set up the crypto state. */ +static struct io_plan *connection_in(struct io_conn *conn, + struct daemon *daemon) +{ + struct conn_in conn_in_arg; + + if (!get_remote_address(conn, &conn_in_arg.addr)) + return io_close(conn); + + conn_in_arg.daemon = daemon; + return conn_in(conn, &conn_in_arg); +} + +/*~ I speak web socket. + * + * Actually that's dumb, websocket (aka rfc6455) looks nothing like that. */ +static struct io_plan *websocket_connection_in(struct io_conn *conn, + struct daemon *daemon) +{ + int childmsg[2], execfail[2]; + pid_t childpid; + int err; + struct conn_in conn_in_arg; + + if (!get_remote_address(conn, &conn_in_arg.addr)) + return io_close(conn); + + status_debug("Websocket connection in from %s", + type_to_string(tmpctx, struct wireaddr_internal, + &conn_in_arg.addr)); + + if (socketpair(AF_LOCAL, SOCK_STREAM, 0, childmsg) != 0) + goto fail; + + if (pipe(execfail) != 0) + goto close_msgfd_fail; + + if (fcntl(execfail[1], F_SETFD, fcntl(execfail[1], F_GETFD) + | FD_CLOEXEC) < 0) + goto close_execfail_fail; + + childpid = fork(); + if (childpid < 0) + goto close_execfail_fail; + + if (childpid == 0) { + size_t max; + close(childmsg[0]); + close(execfail[0]); + + /* Attach remote socket to stdin. */ + if (dup2(io_conn_fd(conn), STDIN_FILENO) == -1) + goto child_errno_fail; + + /* Attach our socket to stdout. */ + if (dup2(childmsg[1], STDOUT_FILENO) == -1) + goto child_errno_fail; + + /* Make (fairly!) sure all other fds are closed. */ + max = sysconf(_SC_OPEN_MAX); + for (size_t i = STDERR_FILENO + 1; i < max; i++) + close(i); + + /* Tell websocket helper what we read so far. */ + execlp(daemon->websocket_helper, daemon->websocket_helper, + NULL); + + child_errno_fail: + err = errno; + /* Gcc's warn-unused-result fail. */ + if (write(execfail[1], &err, sizeof(err))) { + ; + } + exit(127); + } + + close(childmsg[1]); + close(execfail[1]); + + /* Child will close this without writing on successful exec. */ + if (read(execfail[0], &err, sizeof(err)) == sizeof(err)) { + close(execfail[0]); + waitpid(childpid, NULL, 0); + status_broken("Exec of helper %s failed: %s", + daemon->websocket_helper, strerror(err)); + errno = err; + return io_close(conn); + } + + close(execfail[0]); + + /* New connection actually talks to proxy process. */ + conn_in_arg.daemon = daemon; + io_new_conn(tal_parent(conn), childmsg[0], conn_in, &conn_in_arg); + + /* Abandon original (doesn't close since child has dup'd fd) */ + return io_close(conn); + +close_execfail_fail: + close_noerr(execfail[0]); + close_noerr(execfail[1]); +close_msgfd_fail: + close_noerr(childmsg[0]); + close_noerr(childmsg[1]); +fail: + status_broken("Preparation of helper failed: %s", + strerror(errno)); + return io_close(conn); +} + /*~ These are the mirror functions for the connecting-out case. */ static struct io_plan *handshake_out_success(struct io_conn *conn, const struct pubkey *key, @@ -906,9 +1043,14 @@ struct listen_fd { * covers IPv4 too. Normally we'd consider failing to listen on a * port to be fatal, so we note this when setting up addresses. */ bool mayfail; + /* Callback to use for the listening: either connection_in, or for + * our much-derided WebSocket ability, websocket_connection_in! */ + struct io_plan *(*in_cb)(struct io_conn *conn, struct daemon *daemon); }; -static void add_listen_fd(struct daemon *daemon, int fd, bool mayfail) +static void add_listen_fd(struct daemon *daemon, int fd, bool mayfail, + struct io_plan *(*in_cb)(struct io_conn *, + struct daemon *)) { /*~ utils.h contains a convenience macro tal_arr_expand which * reallocates a tal_arr to make it one longer, then returns a pointer @@ -916,6 +1058,7 @@ static void add_listen_fd(struct daemon *daemon, int fd, bool mayfail) struct listen_fd l; l.fd = fd; l.mayfail = mayfail; + l.in_cb = in_cb; tal_arr_expand(&daemon->listen_fds, l); } @@ -970,11 +1113,18 @@ static int make_listen_fd(int domain, void *addr, socklen_t len, bool mayfail) /* Return true if it created socket successfully. */ static bool handle_wireaddr_listen(struct daemon *daemon, const struct wireaddr *wireaddr, - bool mayfail) + bool mayfail, + bool websocket) { int fd; struct sockaddr_in addr; struct sockaddr_in6 addr6; + struct io_plan *(*in_cb)(struct io_conn *, struct daemon *); + + if (websocket) + in_cb = websocket_connection_in; + else + in_cb = connection_in; /* Note the use of a switch() over enum here, even though it must be * IPv4 or IPv6 here; that will catch future changes. */ @@ -984,9 +1134,10 @@ static bool handle_wireaddr_listen(struct daemon *daemon, /* We might fail if IPv6 bound to port first */ fd = make_listen_fd(AF_INET, &addr, sizeof(addr), mayfail); if (fd >= 0) { - status_debug("Created IPv4 listener on port %u", + status_debug("Created IPv4 %slistener on port %u", + websocket ? "websocket ": "", wireaddr->port); - add_listen_fd(daemon, fd, mayfail); + add_listen_fd(daemon, fd, mayfail, in_cb); return true; } return false; @@ -994,9 +1145,10 @@ static bool handle_wireaddr_listen(struct daemon *daemon, wireaddr_to_ipv6(wireaddr, &addr6); fd = make_listen_fd(AF_INET6, &addr6, sizeof(addr6), mayfail); if (fd >= 0) { - status_debug("Created IPv6 listener on port %u", + status_debug("Created IPv6 %slistener on port %u", + websocket ? "websocket ": "", wireaddr->port); - add_listen_fd(daemon, fd, mayfail); + add_listen_fd(daemon, fd, mayfail, in_cb); return true; } return false; @@ -1122,7 +1274,7 @@ static struct wireaddr_internal *setup_listeners(const tal_t *ctx, false); status_debug("Created socket listener on file %s", addrun.sun_path); - add_listen_fd(daemon, fd, false); + add_listen_fd(daemon, fd, false, connection_in); /* We don't announce socket names, though we allow * them to lazily specify --addr=/socket. */ add_binding(&binding, &wa); @@ -1147,7 +1299,7 @@ static struct wireaddr_internal *setup_listeners(const tal_t *ctx, sizeof(wa.u.wireaddr.addr)); ipv6_ok = handle_wireaddr_listen(daemon, &wa.u.wireaddr, - true); + true, false); if (ipv6_ok) { add_binding(&binding, &wa); if (announce @@ -1163,7 +1315,7 @@ static struct wireaddr_internal *setup_listeners(const tal_t *ctx, sizeof(wa.u.wireaddr.addr)); /* OK if this fails, as long as one succeeds! */ if (handle_wireaddr_listen(daemon, &wa.u.wireaddr, - ipv6_ok)) { + ipv6_ok, false)) { add_binding(&binding, &wa); if (announce && public_address(daemon, &wa.u.wireaddr)) @@ -1174,7 +1326,8 @@ static struct wireaddr_internal *setup_listeners(const tal_t *ctx, } /* This is a vanilla wireaddr as per BOLT #7 */ case ADDR_INTERNAL_WIREADDR: - handle_wireaddr_listen(daemon, &wa.u.wireaddr, false); + handle_wireaddr_listen(daemon, &wa.u.wireaddr, + false, false); add_binding(&binding, &wa); if (announce && public_address(daemon, &wa.u.wireaddr)) add_announcable(announcable, &wa.u.wireaddr); @@ -1188,6 +1341,38 @@ static struct wireaddr_internal *setup_listeners(const tal_t *ctx, proposed_wireaddr[i].itype); } + /* If we want websockets to match IPv4/v6, set it up now. */ + if (daemon->websocket_port) { + bool announced_some = false; + struct wireaddr addr; + + for (size_t i = 0; i < tal_count(binding); i++) { + /* Ignore UNIX sockets */ + if (binding[i].itype != ADDR_INTERNAL_WIREADDR) + continue; + + /* Override with websocket port */ + addr = binding[i].u.wireaddr; + addr.port = daemon->websocket_port; + handle_wireaddr_listen(daemon, &addr, false, true); + announced_some = true; + /* FIXME: We don't report these bindings to + * lightningd, so they don't appear in + * getinfo. */ + } + + + /* We add the websocket port to the announcement if it + * applies to any */ + if (announced_some) { + wireaddr_from_websocket(&addr, daemon->websocket_port); + add_announcable(announcable, &addr); + } + } + + /* FIXME: Websocket over Tor (difficult for autotor, since we need + * to use the same onion addr!) */ + /* Now we have bindings, set up any Tor auto addresses: we will point * it at the first bound IPv4 or IPv6 address we have. */ for (size_t i = 0; i < tal_count(proposed_wireaddr); i++) { @@ -1294,7 +1479,9 @@ static struct io_plan *connect_init(struct io_conn *conn, &daemon->dev_allow_localhost, &daemon->use_dns, &tor_password, &daemon->use_v3_autotor, - &daemon->timeout_secs)) { + &daemon->timeout_secs, + &daemon->websocket_helper, + &daemon->websocket_port)) { /* This is a helper which prints the type expected and the actual * message, then exits (it should never be called!). */ master_badmsg(WIRE_CONNECTD_INIT, msg); @@ -1367,7 +1554,8 @@ static struct io_plan *connect_activate(struct io_conn *conn, } notleak(io_new_listener(daemon, daemon->listen_fds[i].fd, - connection_in, daemon)); + daemon->listen_fds[i].in_cb, + daemon)); } } /* Free, with NULL assignment just as an extra sanity check. */ diff --git a/connectd/connectd_wire.csv b/connectd/connectd_wire.csv index 093f7d33f2d7..0719838d380c 100644 --- a/connectd/connectd_wire.csv +++ b/connectd/connectd_wire.csv @@ -18,6 +18,8 @@ msgdata,connectd_init,use_dns,bool, msgdata,connectd_init,tor_password,wirestring, msgdata,connectd_init,use_v3_autotor,bool, msgdata,connectd_init,timeout_secs,u32, +msgdata,connectd_init,websocket_helper,wirestring, +msgdata,connectd_init,websocket_port,u16, # Connectd->master, here are the addresses I bound, can announce. msgtype,connectd_init_reply,2100 diff --git a/connectd/handshake.c b/connectd/handshake.c index cc1fb92b4db7..768b4a2a6a6b 100644 --- a/connectd/handshake.c +++ b/connectd/handshake.c @@ -857,7 +857,6 @@ static struct io_plan *act_two_responder(struct io_conn *conn, return io_write(conn, &h->act2, ACT_TWO_SIZE, act_three_responder, h); } - static struct io_plan *act_one_responder2(struct io_conn *conn, struct handshake *h) { diff --git a/connectd/test/run-initiator-success.c b/connectd/test/run-initiator-success.c index 627e0e2414dd..926fa7c8a7de 100644 --- a/connectd/test/run-initiator-success.c +++ b/connectd/test/run-initiator-success.c @@ -83,6 +83,9 @@ u8 fromwire_u8(const u8 **cursor UNNEEDED, size_t *max UNNEEDED) /* Generated stub for fromwire_u8_array */ void fromwire_u8_array(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, u8 *arr UNNEEDED, size_t num UNNEEDED) { fprintf(stderr, "fromwire_u8_array called!\n"); abort(); } +/* Generated stub for notleak_ */ +void *notleak_(const void *ptr UNNEEDED, bool plus_children UNNEEDED) +{ fprintf(stderr, "notleak_ called!\n"); abort(); } /* Generated stub for towire */ void towire(u8 **pptr UNNEEDED, const void *data UNNEEDED, size_t len UNNEEDED) { fprintf(stderr, "towire called!\n"); abort(); } diff --git a/connectd/test/run-responder-success.c b/connectd/test/run-responder-success.c index a531f19ca35a..0a24a1009922 100644 --- a/connectd/test/run-responder-success.c +++ b/connectd/test/run-responder-success.c @@ -83,6 +83,9 @@ u8 fromwire_u8(const u8 **cursor UNNEEDED, size_t *max UNNEEDED) /* Generated stub for fromwire_u8_array */ void fromwire_u8_array(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, u8 *arr UNNEEDED, size_t num UNNEEDED) { fprintf(stderr, "fromwire_u8_array called!\n"); abort(); } +/* Generated stub for notleak_ */ +void *notleak_(const void *ptr UNNEEDED, bool plus_children UNNEEDED) +{ fprintf(stderr, "notleak_ called!\n"); abort(); } /* Generated stub for towire */ void towire(u8 **pptr UNNEEDED, const void *data UNNEEDED, size_t len UNNEEDED) { fprintf(stderr, "towire called!\n"); abort(); } diff --git a/lightningd/connect_control.c b/lightningd/connect_control.c index f26d392474b7..314efccb4599 100644 --- a/lightningd/connect_control.c +++ b/lightningd/connect_control.c @@ -350,6 +350,7 @@ int connectd_init(struct lightningd *ld) int hsmfd; struct wireaddr_internal *wireaddrs = ld->proposed_wireaddr; enum addr_listen_announce *listen_announce = ld->proposed_listen_announce; + const char *websocket_helper_path = ""; if (socketpair(AF_LOCAL, SOCK_STREAM, 0, fds) != 0) fatal("Could not socketpair for connectd<->gossipd"); @@ -381,7 +382,9 @@ int connectd_init(struct lightningd *ld) IFDEV(ld->dev_allow_localhost, false), ld->config.use_dns, ld->tor_service_password ? ld->tor_service_password : "", ld->config.use_v3_autotor, - ld->config.connection_timeout_secs); + ld->config.connection_timeout_secs, + websocket_helper_path, + 0); subd_req(ld->connectd, ld->connectd, take(msg), -1, 0, connect_init_done, NULL); From 3678006f76340fef36dfe44ed7c848530a723b0e Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 15 Oct 2021 16:19:05 +1030 Subject: [PATCH 4/5] lightning_websocketd: simple proxy for websockets. WebSocket is a bit weird: 1. It starts like an HTTP connection, but they send special headers. 2. We reply with special headers, one of which involves SHA1 of one of theirs. 3. We are then in WebSocket mode, where each frame starts with a 2-20 byte header. We relay data in a simplistic way: if either side sends something, we read it and relay it synchronously. That avoids any gratuitous buffering. Signed-off-by: Rusty Russell --- Makefile | 3 +- connectd/Makefile | 13 +- connectd/sha1.c | 190 +++++++++++++++++++ connectd/sha1.h | 9 + connectd/test/Makefile | 2 +- connectd/test/run-websocket.c | 181 ++++++++++++++++++ connectd/websocketd.c | 347 ++++++++++++++++++++++++++++++++++ 7 files changed, 741 insertions(+), 4 deletions(-) create mode 100644 connectd/sha1.c create mode 100644 connectd/sha1.h create mode 100644 connectd/test/run-websocket.c create mode 100644 connectd/websocketd.c diff --git a/Makefile b/Makefile index 75560150ed29..5903b1baec2b 100644 --- a/Makefile +++ b/Makefile @@ -363,7 +363,8 @@ PKGLIBEXEC_PROGRAMS = \ lightningd/lightning_gossipd \ lightningd/lightning_hsmd \ lightningd/lightning_onchaind \ - lightningd/lightning_openingd + lightningd/lightning_openingd \ + lightningd/lightning_websocketd # Don't delete these intermediaries. .PRECIOUS: $(ALL_GEN_HEADERS) $(ALL_GEN_SOURCES) diff --git a/connectd/Makefile b/connectd/Makefile index 0a32022d7739..8d30894c8964 100644 --- a/connectd/Makefile +++ b/connectd/Makefile @@ -14,10 +14,17 @@ CONNECTD_SRC := $(CONNECTD_HEADERS:.h=.c) connectd/connectd.c CONNECTD_OBJS := $(CONNECTD_SRC:.c=.o) $(CONNECTD_OBJS): $(CONNECTD_HEADERS) +WEBSOCKETD_HEADERS := connectd/sha1.h +WEBSOCKETD_SRC := $(WEBSOCKETD_HEADERS:.h=.c) connectd/websocketd.c + +WEBSOCKETD_OBJS := $(WEBSOCKETD_SRC:.c=.o) +$(WEBSOCKETD_OBJS): $(WEBSOCKETD_HEADERS) + # Make sure these depend on everything. -ALL_C_SOURCES += $(CONNECTD_SRC) -ALL_C_HEADERS += $(CONNECTD_HEADERS) +ALL_C_SOURCES += $(CONNECTD_SRC) $(WEBSOCKETD_SRC) +ALL_C_HEADERS += $(CONNECTD_HEADERS) $(WEBSOCKETD_HEADERS) ALL_PROGRAMS += lightningd/lightning_connectd +ALL_PROGRAMS += lightningd/lightning_websocketd # Here's what lightningd depends on LIGHTNINGD_CONTROL_HEADERS += connectd/connectd_wiregen.h @@ -69,4 +76,6 @@ CONNECTD_COMMON_OBJS := \ lightningd/lightning_connectd: $(CONNECTD_OBJS) $(CONNECTD_COMMON_OBJS) $(BITCOIN_OBJS) $(WIRE_OBJS) $(HSMD_CLIENT_OBJS) +lightningd/lightning_websocketd: $(WEBSOCKETD_OBJS) common/setup.o common/utils.o + include connectd/test/Makefile diff --git a/connectd/sha1.c b/connectd/sha1.c new file mode 100644 index 000000000000..d9e4951978a6 --- /dev/null +++ b/connectd/sha1.c @@ -0,0 +1,190 @@ +/* hex variants removed -- RR */ +#include + +/******************************************************************************* + * Teeny SHA-1 + * + * The below sha1digest() calculates a SHA-1 hash value for a + * specified data buffer and generates a hex representation of the + * result. This implementation is a re-forming of the SHA-1 code at + * https://github.com/jinqiangshou/EncryptionLibrary. + * + * Copyright (c) 2017 CTrabant + * + * License: MIT, see included LICENSE file for details. + * + * To use the sha1digest() function either copy it into an existing + * project source code file or include this file in a project and put + * the declaration (example below) in the sources files where needed. + ******************************************************************************/ + +#include + +/* Declaration: +extern int sha1digest(uint8_t *digest, const uint8_t *data, size_t databytes); +*/ + +/******************************************************************************* + * sha1digest: https://github.com/CTrabant/teeny-sha1 + * + * Calculate the SHA-1 value for supplied data buffer and generate a + * text representation in hexadecimal. + * + * Based on https://github.com/jinqiangshou/EncryptionLibrary, credit + * goes to @jinqiangshou, all new bugs are mine. + * + * @input: + * data -- data to be hashed + * databytes -- bytes in data buffer to be hashed + * + * @output: + * digest -- the result, MUST be at least 20 bytes + * + * @return: 0 on success and non-zero on error. + ******************************************************************************/ +int +sha1digest(uint8_t *digest, const uint8_t *data, size_t databytes) +{ +#define SHA1ROTATELEFT(value, bits) (((value) << (bits)) | ((value) >> (32 - (bits)))) + + uint32_t W[80]; + uint32_t H[] = {0x67452301, + 0xEFCDAB89, + 0x98BADCFE, + 0x10325476, + 0xC3D2E1F0}; + uint32_t a; + uint32_t b; + uint32_t c; + uint32_t d; + uint32_t e; + uint32_t f = 0; + uint32_t k = 0; + + uint32_t idx; + uint32_t lidx; + uint32_t widx; + uint32_t didx = 0; + + int32_t wcount; + uint32_t temp; + uint64_t databits = ((uint64_t)databytes) * 8; + uint32_t loopcount = (databytes + 8) / 64 + 1; + uint32_t tailbytes = 64 * loopcount - databytes; + uint8_t datatail[128] = {0}; + + if (!digest) + return -1; + + if (!data) + return -1; + + /* Pre-processing of data tail (includes padding to fill out 512-bit chunk): + Add bit '1' to end of message (big-endian) + Add 64-bit message length in bits at very end (big-endian) */ + datatail[0] = 0x80; + datatail[tailbytes - 8] = (uint8_t) (databits >> 56 & 0xFF); + datatail[tailbytes - 7] = (uint8_t) (databits >> 48 & 0xFF); + datatail[tailbytes - 6] = (uint8_t) (databits >> 40 & 0xFF); + datatail[tailbytes - 5] = (uint8_t) (databits >> 32 & 0xFF); + datatail[tailbytes - 4] = (uint8_t) (databits >> 24 & 0xFF); + datatail[tailbytes - 3] = (uint8_t) (databits >> 16 & 0xFF); + datatail[tailbytes - 2] = (uint8_t) (databits >> 8 & 0xFF); + datatail[tailbytes - 1] = (uint8_t) (databits >> 0 & 0xFF); + + /* Process each 512-bit chunk */ + for (lidx = 0; lidx < loopcount; lidx++) + { + /* Compute all elements in W */ + memset (W, 0, 80 * sizeof (uint32_t)); + + /* Break 512-bit chunk into sixteen 32-bit, big endian words */ + for (widx = 0; widx <= 15; widx++) + { + wcount = 24; + + /* Copy byte-per byte from specified buffer */ + while (didx < databytes && wcount >= 0) + { + W[widx] += (((uint32_t)data[didx]) << wcount); + didx++; + wcount -= 8; + } + /* Fill out W with padding as needed */ + while (wcount >= 0) + { + W[widx] += (((uint32_t)datatail[didx - databytes]) << wcount); + didx++; + wcount -= 8; + } + } + + /* Extend the sixteen 32-bit words into eighty 32-bit words, with potential optimization from: + "Improving the Performance of the Secure Hash Algorithm (SHA-1)" by Max Locktyukhin */ + for (widx = 16; widx <= 31; widx++) + { + W[widx] = SHA1ROTATELEFT ((W[widx - 3] ^ W[widx - 8] ^ W[widx - 14] ^ W[widx - 16]), 1); + } + for (widx = 32; widx <= 79; widx++) + { + W[widx] = SHA1ROTATELEFT ((W[widx - 6] ^ W[widx - 16] ^ W[widx - 28] ^ W[widx - 32]), 2); + } + + /* Main loop */ + a = H[0]; + b = H[1]; + c = H[2]; + d = H[3]; + e = H[4]; + + for (idx = 0; idx <= 79; idx++) + { + if (idx <= 19) + { + f = (b & c) | ((~b) & d); + k = 0x5A827999; + } + else if (idx >= 20 && idx <= 39) + { + f = b ^ c ^ d; + k = 0x6ED9EBA1; + } + else if (idx >= 40 && idx <= 59) + { + f = (b & c) | (b & d) | (c & d); + k = 0x8F1BBCDC; + } + else if (idx >= 60 && idx <= 79) + { + f = b ^ c ^ d; + k = 0xCA62C1D6; + } + temp = SHA1ROTATELEFT (a, 5) + f + e + k + W[idx]; + e = d; + d = c; + c = SHA1ROTATELEFT (b, 30); + b = a; + a = temp; + } + + H[0] += a; + H[1] += b; + H[2] += c; + H[3] += d; + H[4] += e; + } + + /* Store binary digest in supplied buffer */ + if (digest) + { + for (idx = 0; idx < 5; idx++) + { + digest[idx * 4 + 0] = (uint8_t) (H[idx] >> 24); + digest[idx * 4 + 1] = (uint8_t) (H[idx] >> 16); + digest[idx * 4 + 2] = (uint8_t) (H[idx] >> 8); + digest[idx * 4 + 3] = (uint8_t) (H[idx]); + } + } + + return 0; +} /* End of sha1digest() */ diff --git a/connectd/sha1.h b/connectd/sha1.h new file mode 100644 index 000000000000..45e8ba5bc12c --- /dev/null +++ b/connectd/sha1.h @@ -0,0 +1,9 @@ +#ifndef LIGHTNING_CONNECTD_SHA1_H +#define LIGHTNING_CONNECTD_SHA1_H +#include "config.h" +#include +#include + +extern int sha1digest(uint8_t *digest, const uint8_t *data, size_t databytes); + +#endif /* LIGHTNING_CONNECTD_SHA1_H */ diff --git a/connectd/test/Makefile b/connectd/test/Makefile index fa3ace59106f..4ac2df0d445e 100644 --- a/connectd/test/Makefile +++ b/connectd/test/Makefile @@ -18,7 +18,7 @@ ALL_TEST_PROGRAMS += $(CONNECTD_TEST_PROGRAMS) $(CONNECTD_TEST_PROGRAMS): $(CONNECTD_TEST_COMMON_OBJS) $(BITCOIN_OBJS) # Test objects depend on ../ src and headers. -$(CONNECTD_TEST_OBJS): $(CONNECTD_HEADERS) $(CONNECTD_SRC) +$(CONNECTD_TEST_OBJS): $(CONNECTD_HEADERS) $(CONNECTD_SRC) $(WEBSOCKETD_HEADERS) $(WEBSOCKETD_SRC) check-units: $(CONNECTD_TEST_PROGRAMS:%=unittest/%) diff --git a/connectd/test/run-websocket.c b/connectd/test/run-websocket.c new file mode 100644 index 000000000000..8d50415bc24f --- /dev/null +++ b/connectd/test/run-websocket.c @@ -0,0 +1,181 @@ +#include "config.h" +#include +#include +#include +#include +#include + +/* We don't want to actually do io! */ +#define write my_write +#define read my_read +#define write_all my_write_all +#define read_all my_read_all + +static char *my_rbuf, *my_wbuf; +static size_t my_rbuf_off; + +static ssize_t my_read(int fd, void *buf, size_t count) +{ + if (strlen(my_rbuf + my_rbuf_off) < count) + count = strlen(my_rbuf + my_rbuf_off); + memcpy(buf, my_rbuf + my_rbuf_off, count); + my_rbuf_off += count; + return count; +} +static bool my_read_all(int fd, void *buf, size_t count) +{ + my_read(fd, buf, count); + return true; +} + +static ssize_t my_write(int fd, const void *buf, size_t count) +{ + size_t buflen = tal_bytelen(my_wbuf); + tal_resize(&my_wbuf, buflen + count); + memcpy(my_wbuf + buflen, buf, count); + return count; +} +static bool my_write_all(int fd, const void *buf, size_t count) +{ + my_write(fd, buf, count); + return true; +} + +int websocket_main(int argc, char *argv[]); + +#define main websocket_main + #include "../websocketd.c" + #include "../sha1.c" +#undef main + +/* AUTOGENERATED MOCKS START */ +/* Generated stub for amount_asset_is_main */ +bool amount_asset_is_main(struct amount_asset *asset UNNEEDED) +{ fprintf(stderr, "amount_asset_is_main called!\n"); abort(); } +/* Generated stub for amount_asset_to_sat */ +struct amount_sat amount_asset_to_sat(struct amount_asset *asset UNNEEDED) +{ fprintf(stderr, "amount_asset_to_sat called!\n"); abort(); } +/* Generated stub for amount_sat */ +struct amount_sat amount_sat(u64 satoshis UNNEEDED) +{ fprintf(stderr, "amount_sat called!\n"); abort(); } +/* Generated stub for amount_sat_add */ + bool amount_sat_add(struct amount_sat *val UNNEEDED, + struct amount_sat a UNNEEDED, + struct amount_sat b UNNEEDED) +{ fprintf(stderr, "amount_sat_add called!\n"); abort(); } +/* Generated stub for amount_sat_eq */ +bool amount_sat_eq(struct amount_sat a UNNEEDED, struct amount_sat b UNNEEDED) +{ fprintf(stderr, "amount_sat_eq called!\n"); abort(); } +/* Generated stub for amount_sat_greater_eq */ +bool amount_sat_greater_eq(struct amount_sat a UNNEEDED, struct amount_sat b UNNEEDED) +{ fprintf(stderr, "amount_sat_greater_eq called!\n"); abort(); } +/* Generated stub for amount_sat_sub */ + bool amount_sat_sub(struct amount_sat *val UNNEEDED, + struct amount_sat a UNNEEDED, + struct amount_sat b UNNEEDED) +{ fprintf(stderr, "amount_sat_sub called!\n"); abort(); } +/* Generated stub for amount_sat_to_asset */ +struct amount_asset amount_sat_to_asset(struct amount_sat *sat UNNEEDED, const u8 *asset UNNEEDED) +{ fprintf(stderr, "amount_sat_to_asset called!\n"); abort(); } +/* Generated stub for amount_tx_fee */ +struct amount_sat amount_tx_fee(u32 fee_per_kw UNNEEDED, size_t weight UNNEEDED) +{ fprintf(stderr, "amount_tx_fee called!\n"); abort(); } +/* Generated stub for fromwire */ +const u8 *fromwire(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, void *copy UNNEEDED, size_t n UNNEEDED) +{ fprintf(stderr, "fromwire called!\n"); abort(); } +/* Generated stub for fromwire_amount_sat */ +struct amount_sat fromwire_amount_sat(const u8 **cursor UNNEEDED, size_t *max UNNEEDED) +{ fprintf(stderr, "fromwire_amount_sat called!\n"); abort(); } +/* Generated stub for fromwire_bool */ +bool fromwire_bool(const u8 **cursor UNNEEDED, size_t *max UNNEEDED) +{ fprintf(stderr, "fromwire_bool called!\n"); abort(); } +/* Generated stub for fromwire_fail */ +void *fromwire_fail(const u8 **cursor UNNEEDED, size_t *max UNNEEDED) +{ fprintf(stderr, "fromwire_fail called!\n"); abort(); } +/* Generated stub for fromwire_secp256k1_ecdsa_signature */ +void fromwire_secp256k1_ecdsa_signature(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, + secp256k1_ecdsa_signature *signature UNNEEDED) +{ fprintf(stderr, "fromwire_secp256k1_ecdsa_signature called!\n"); abort(); } +/* Generated stub for fromwire_sha256 */ +void fromwire_sha256(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct sha256 *sha256 UNNEEDED) +{ fprintf(stderr, "fromwire_sha256 called!\n"); abort(); } +/* Generated stub for fromwire_tal_arrn */ +u8 *fromwire_tal_arrn(const tal_t *ctx UNNEEDED, + const u8 **cursor UNNEEDED, size_t *max UNNEEDED, size_t num UNNEEDED) +{ fprintf(stderr, "fromwire_tal_arrn called!\n"); abort(); } +/* Generated stub for fromwire_u16 */ +u16 fromwire_u16(const u8 **cursor UNNEEDED, size_t *max UNNEEDED) +{ fprintf(stderr, "fromwire_u16 called!\n"); abort(); } +/* Generated stub for fromwire_u32 */ +u32 fromwire_u32(const u8 **cursor UNNEEDED, size_t *max UNNEEDED) +{ fprintf(stderr, "fromwire_u32 called!\n"); abort(); } +/* Generated stub for fromwire_u64 */ +u64 fromwire_u64(const u8 **cursor UNNEEDED, size_t *max UNNEEDED) +{ fprintf(stderr, "fromwire_u64 called!\n"); abort(); } +/* Generated stub for fromwire_u8 */ +u8 fromwire_u8(const u8 **cursor UNNEEDED, size_t *max UNNEEDED) +{ fprintf(stderr, "fromwire_u8 called!\n"); abort(); } +/* Generated stub for fromwire_u8_array */ +void fromwire_u8_array(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, u8 *arr UNNEEDED, size_t num UNNEEDED) +{ fprintf(stderr, "fromwire_u8_array called!\n"); abort(); } +/* Generated stub for towire */ +void towire(u8 **pptr UNNEEDED, const void *data UNNEEDED, size_t len UNNEEDED) +{ fprintf(stderr, "towire called!\n"); abort(); } +/* Generated stub for towire_amount_sat */ +void towire_amount_sat(u8 **pptr UNNEEDED, const struct amount_sat sat UNNEEDED) +{ fprintf(stderr, "towire_amount_sat called!\n"); abort(); } +/* Generated stub for towire_bool */ +void towire_bool(u8 **pptr UNNEEDED, bool v UNNEEDED) +{ fprintf(stderr, "towire_bool called!\n"); abort(); } +/* Generated stub for towire_secp256k1_ecdsa_signature */ +void towire_secp256k1_ecdsa_signature(u8 **pptr UNNEEDED, + const secp256k1_ecdsa_signature *signature UNNEEDED) +{ fprintf(stderr, "towire_secp256k1_ecdsa_signature called!\n"); abort(); } +/* Generated stub for towire_sha256 */ +void towire_sha256(u8 **pptr UNNEEDED, const struct sha256 *sha256 UNNEEDED) +{ fprintf(stderr, "towire_sha256 called!\n"); abort(); } +/* Generated stub for towire_u16 */ +void towire_u16(u8 **pptr UNNEEDED, u16 v UNNEEDED) +{ fprintf(stderr, "towire_u16 called!\n"); abort(); } +/* Generated stub for towire_u32 */ +void towire_u32(u8 **pptr UNNEEDED, u32 v UNNEEDED) +{ fprintf(stderr, "towire_u32 called!\n"); abort(); } +/* Generated stub for towire_u64 */ +void towire_u64(u8 **pptr UNNEEDED, u64 v UNNEEDED) +{ fprintf(stderr, "towire_u64 called!\n"); abort(); } +/* Generated stub for towire_u8 */ +void towire_u8(u8 **pptr UNNEEDED, u8 v UNNEEDED) +{ fprintf(stderr, "towire_u8 called!\n"); abort(); } +/* Generated stub for towire_u8_array */ +void towire_u8_array(u8 **pptr UNNEEDED, const u8 *arr UNNEEDED, size_t num UNNEEDED) +{ fprintf(stderr, "towire_u8_array called!\n"); abort(); } +/* AUTOGENERATED MOCKS END */ + +int main(int argc, char *argv[]) +{ + const char *hdr; + + common_setup(argv[0]); + + hdr = "GET /chat HTTP/1.1\r\n" + "Host: server.example.com\r\n" + "Upgrade: websocket\r\n" + "Connection: Upgrade\r\n" + "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n" + "Origin: http://example.com\r\n" + "Sec-WebSocket-Protocol: chat, superchat\r\n" + "Sec-WebSocket-Version: 13\r\n\r\n"; + + my_rbuf = tal_strdup(tmpctx, hdr); + my_wbuf = tal_arr(tmpctx, char, 0); + + http_upgrade(STDIN_FILENO); + assert(streq(tal_strndup(tmpctx, my_wbuf, tal_bytelen(my_wbuf)), + "HTTP/1.1 101 Switching Protocols\r\n" + "Upgrade: websocket\r\n" + "Connection: Upgrade\r\n" + "Sec-WebSocket-Accept: s3pPLMBiTxaQ9kYGzzhZRbK+xOo=\r\n" + "\r\n")); + + common_shutdown(); +} diff --git a/connectd/websocketd.c b/connectd/websocketd.c new file mode 100644 index 000000000000..324a17f7922b --- /dev/null +++ b/connectd/websocketd.c @@ -0,0 +1,347 @@ +/* A simple standalone websocket <-> binary proxy. + * See https://datatracker.ietf.org/doc/html/rfc6455 + */ +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + +-+-+-+-+-------+-+-------------+-------------------------------+ + |F|R|R|R| opcode|M| Payload len | Extended payload length | + |I|S|S|S| (4) |A| (7) | (16/64) | + |N|V|V|V| |S| | (if payload len==126/127) | + | |1|2|3| |K| | | + +-+-+-+-+-------+-+-------------+ - - - - - - - - - - - - - - - + + | Extended payload length continued, if payload len == 127 | + + - - - - - - - - - - - - - - - +-------------------------------+ + | |Masking-key, if MASK set to 1 | + +-------------------------------+-------------------------------+ + | Masking-key (continued) | Payload Data | + +-------------------------------- - - - - - - - - - - - - - - - + + : Payload Data continued ... : + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + | Payload Data continued ... | + +---------------------------------------------------------------+ +*/ + +/* RFC-6455: + + A |Sec-WebSocket-Accept| header field. The value of this header field + is constructed by concatenating /key/, defined above in step 4 in + Section 4.2.2, with the string "258EAFA5- + E914-47DA-95CA-C5AB0DC85B11", taking the SHA-1 hash of this + concatenated value to obtain a 20-byte value and base64- encoding (see + Section 4 of [RFC4648]) this 20-byte hash. + +... + + NOTE: As an example, if the value of the |Sec-WebSocket-Key| header + field in the client's handshake were "dGhlIHNhbXBsZSBub25jZQ==", the + server would append the string "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" + to form the string "dGhlIHNhbXBsZSBub25jZQ==258EAFA5-E914-47DA-95CA- + C5AB0DC85B11". The server would then take the SHA-1 hash of this + string, giving the value 0xb3 0x7a 0x4f 0x2c 0xc0 0x62 0x4f 0x16 0x90 + 0xf6 0x46 0x06 0xcf 0x38 0x59 0x45 0xb2 0xbe 0xc4 0xea. This value + is then base64-encoded, to give the value + "s3pPLMBiTxaQ9kYGzzhZRbK+xOo=", which would be returned in the + |Sec-WebSocket-Accept| header field. +*/ +static const char *websocket_accept_str(const tal_t *ctx, const char *key) +{ + u8 sha1[20]; + const char *concat; + char base64[100]; + + concat = tal_fmt(tmpctx, "%s258EAFA5-E914-47DA-95CA-C5AB0DC85B11", + key); + sha1digest(sha1, (const u8 *)concat, strlen(concat)); + if (base64_encode(base64, sizeof(base64), (const char *)sha1, sizeof(sha1)) == -1) + abort(); + + return tal_strdup(ctx, base64); +} + +static void NORETURN PRINTF_FMT(2,3) +bad_http(int fd, const char *fmt, ...) +{ + va_list ap; + char *resp; + + resp = tal_strdup(tmpctx, "HTTP/1.1 400 I only speak websocket\r\n\r\n"); + va_start(ap, fmt); + tal_append_vfmt(&resp, fmt, ap); + va_end(ap); + + write_all(fd, resp, strlen(resp)); + exit(1); +} + +/* We know headers are terminated by \r\n\r\n at this point */ +static const char *get_http_hdr(const tal_t *ctx, const u8 *buf, size_t buflen, + const char *hdrname) +{ + size_t hdrlen; + + for (;;) { + const u8 *end = memmem(buf, buflen, "\r\n", 2); + hdrlen = end - buf; + + /* Empty line? End of headers. */ + if (hdrlen == 0) + return NULL; + /* header name followed by : */ + if (memstarts(buf, hdrlen, hdrname, strlen(hdrname)) + && buf[strlen(hdrname)] == ':') + break; + buf = end + 2; + } + + buf += strlen(hdrname) + 1; + hdrlen -= strlen(hdrname) + 1; + + /* Ignore leading whitespace (technically, they can split + * fields over multiple lines, but that's silly for the fields + * we're dealing with, so Naah). */ + while (hdrlen && cisspace(*buf)) { + buf++; + hdrlen--; + } + + return tal_strndup(ctx, (const char *)buf, hdrlen); +} + +static bool http_headers_complete(const u8 *buf, size_t len) +{ + return memmem(buf, len, "\r\n\r\n", 4) != NULL; +} + +static void http_respond(int fd, const u8 *buf, size_t len) +{ + const char *hdr; + char *resp; + + /* RFC-6455: + + The client's opening handshake consists of the following + parts. If the server, while reading the handshake, finds + that the client did not send a handshake that matches the + description below ... the server MUST stop processing the + client's handshake and return an HTTP response with an + appropriate error code (such as 400 Bad Request). + + 1. An HTTP/1.1 or higher GET request, including a "Request-URI" + [RFC2616] that should be interpreted as a /resource name/ + defined in Section 3 (or an absolute HTTP/HTTPS URI containing + the /resource name/). + + 2. A |Host| header field containing the server's authority. + + 3. An |Upgrade| header field containing the value "websocket", + treated as an ASCII case-insensitive value. + + 4. A |Connection| header field that includes the token "Upgrade", + treated as an ASCII case-insensitive value. + + 5. A |Sec-WebSocket-Key| header field with a base64-encoded (see + Section 4 of [RFC4648]) value that, when decoded, is 16 bytes in + length. + + 6. A |Sec-WebSocket-Version| header field, with a value of 13. + */ + hdr = get_http_hdr(tmpctx, buf, len, "Upgrade"); + if (!hdr || !strstr(hdr, "websocket")) + bad_http(fd, "Upgrade: websocket missing"); + hdr = get_http_hdr(tmpctx, buf, len, "Connection"); + if (!hdr || !strstr(hdr, "Upgrade")) + bad_http(fd, "Connection: Upgrade missing"); + hdr = get_http_hdr(tmpctx, buf, len, "Sec-WebSocket-Version"); + if (!hdr || !streq(hdr, "13")) + bad_http(fd, "Sec-WebSocket-Version: must be 13"); + hdr = get_http_hdr(tmpctx, buf, len, "Sec-WebSocket-Key"); + if (!hdr) + bad_http(fd, "Sec-WebSocket-Key missing"); + + resp = tal_fmt(tmpctx, + "HTTP/1.1 101 Switching Protocols\r\n" + "Upgrade: websocket\r\n" + "Connection: Upgrade\r\n" + "Sec-WebSocket-Accept: %s\r\n\r\n", + websocket_accept_str(tmpctx, hdr)); + + if (!write_all(fd, resp, strlen(resp))) + exit(0); +} + +static void http_upgrade(int fd) +{ + u8 buf[65536]; + size_t len = 0; + + alarm(60); + while (!http_headers_complete(buf, len)) { + int r; + r = read(STDIN_FILENO, buf + len, sizeof(buf) - len); + if (r <= 0) + bad_http(STDIN_FILENO, "No header end after %zu bytes", + len); + len += r; + } + http_respond(STDIN_FILENO, buf, len); + alarm(0); +} + +static void lightningd_to_websocket(int lightningfd, int wsfd) +{ + /* We prepend ws header */ + u8 buf[4 + 65535]; + int len; + /* Not continued frame (0x80), opcode = 2 (binary) */ + const u8 firstbyte = 0x82; + size_t off; + + len = read(lightningfd, 4 + buf, sizeof(buf) - 4); + if (len <= 0) + exit(0); + + if (len > 125) { + buf[0] = firstbyte; + buf[1] = 126; + buf[2] = (len >> 8); + buf[3] = len; + off = 0; + len += 4; + } else { + buf[2] = firstbyte; + buf[3] = len; + off = 2; + len += 2; + } + if (!write_all(wsfd, buf + off, len)) + exit(0); +} + +/* Returns payload size, sets inmask, is_binframe */ +static size_t read_payload_header(int fd, u8 inmask[4], bool *is_binframe) +{ + /* Worst case header. */ + u8 frame_hdr[20]; + bool mask_set; + size_t hdrsize = 2, len; + + /* First two bytes define hdr size. */ + if (!read_all(fd, frame_hdr, 2)) + exit(0); + + /* RFC-6455: + * %x2 denotes a binary frame + */ + *is_binframe = ((frame_hdr[0] & 0x0F) == 2); + mask_set = (frame_hdr[1] & 0x80); + len = (frame_hdr[1] & 0x7f); + + if (len == 126) + hdrsize += 2; + else if (len == 127) + hdrsize += 8; + + if (mask_set) + hdrsize += 4; + + /* Read rest of hdr if necessary */ + if (hdrsize > 2 && !read_all(fd, frame_hdr + 2, hdrsize - 2)) + exit(0); + + if (len == 126) { + be16 be16len; + memcpy(&be16len, frame_hdr + 2, 2); + len = be16_to_cpu(be16len); + } else if (len == 127) { + be64 be64len; + memcpy(&be64len, frame_hdr + 2, 8); + len = be64_to_cpu(be64len); + } + + if (mask_set) { + memcpy(inmask, frame_hdr + hdrsize - 4, 4); + hdrsize += 4; + } else + memset(inmask, 0, 4); + + return len; +} + +static void apply_mask(u8 *buf, size_t len, const u8 inmask[4]) +{ + for (size_t i = 0; i < len; i++) + buf[i] ^= inmask[i % 4]; +} + +static void websocket_to_lightningd(int wsfd, int lightningfd) +{ + size_t len; + u8 inmask[4]; + bool is_binframe; + + len = read_payload_header(wsfd, inmask, &is_binframe); + while (len > 0) { + u8 buf[65536]; + int rlen = len; + + if (rlen > sizeof(buf)) + rlen = sizeof(buf); + + rlen = read(wsfd, buf, rlen); + if (rlen <= 0) + exit(0); + apply_mask(buf, rlen, inmask); + len -= rlen; + /* We ignore non binary frames (FIXME: Send error!) */ + if (is_binframe && !write_all(lightningfd, buf, rlen)) + exit(0); + } +} + +/* stdin goes to the client, stdout goes to lightningd */ +int main(int argc, char *argv[]) +{ + struct pollfd pfds[2]; + + common_setup(argv[0]); + + if (argc != 1) + errx(1, "Usage: %s", argv[0]); + + /* Do HTTP-style negotiation to get into websocket frames. */ + http_upgrade(STDIN_FILENO); + + pfds[0].fd = STDIN_FILENO; + pfds[0].events = POLLIN; + pfds[1].fd = STDOUT_FILENO; + pfds[1].events = POLLIN; + + for (;;) { + poll(pfds, 2, -1); + + if (pfds[1].revents & POLLIN) + lightningd_to_websocket(STDOUT_FILENO, STDIN_FILENO); + if (pfds[0].revents & POLLIN) + websocket_to_lightningd(STDIN_FILENO, STDOUT_FILENO); + } + + common_shutdown(); + exit(0); +} From 702d4855276a17879d0f1ead11be9833159fe41c Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 18 Oct 2021 10:43:33 +1030 Subject: [PATCH 5/5] experimental-websocket-port: option to create a WebSocket port. Signed-off-by: Rusty Russell --- doc/lightning-listconfigs.7.md | 3 +- doc/lightningd-config.5.md | 7 ++++ doc/schemas/listconfigs.schema.json | 4 ++ lightningd/connect_control.c | 7 +++- lightningd/lightningd.c | 1 + lightningd/lightningd.h | 3 ++ lightningd/options.c | 25 +++++++++++ requirements.lock | 60 +++++++++++++-------------- requirements.txt | 1 + tests/test_connection.py | 64 +++++++++++++++++++++++++++++ 10 files changed, 142 insertions(+), 33 deletions(-) diff --git a/doc/lightning-listconfigs.7.md b/doc/lightning-listconfigs.7.md index 7c71339d50d4..352bd1c409ae 100644 --- a/doc/lightning-listconfigs.7.md +++ b/doc/lightning-listconfigs.7.md @@ -56,6 +56,7 @@ On success, an object is returned, containing: - **experimental-onion-messages** (boolean, optional): `experimental-onion-messages` field from config or cmdline, or default - **experimental-offers** (boolean, optional): `experimental-offers` field from config or cmdline, or default - **experimental-shutdown-wrong-funding** (boolean, optional): `experimental-shutdown-wrong-funding` field from config or cmdline, or default +- **experimental-websocket-port** (u16, optional): `experimental-websocket-port` field from config or cmdline, or default - **rgb** (hex, optional): `rgb` field from config or cmdline, or default (always 6 characters) - **alias** (string, optional): `alias` field from config or cmdline, or default - **pid-file** (string, optional): `pid-file` field from config or cmdline, or default @@ -205,4 +206,4 @@ RESOURCES --------- Main web site: -[comment]: # ( SHA256STAMP:7bb40fc8fac201b32d9701b02596d0fa59eb14a3baf606439cbf96dc11548ed4) +[comment]: # ( SHA256STAMP:47c067588120e0f9a71206313685cebb2a8c515e9b04b688b202d2772c8f8146) diff --git a/doc/lightningd-config.5.md b/doc/lightningd-config.5.md index 488db84c9294..05c096944f01 100644 --- a/doc/lightningd-config.5.md +++ b/doc/lightningd-config.5.md @@ -517,6 +517,13 @@ about whether to add funds or not to a proposed channel is handled automatically by a plugin that implements the appropriate logic for your needs. The default behavior is to not contribute funds. + **experimental-websocket-port** + +Specifying this enables support for accepting incoming WebSocket +connections on that port, on any IPv4 and IPv6 addresses you listen +to. The normal protocol is expected to be sent over WebSocket binary +frames once the connection is upgraded. + BUGS ---- diff --git a/doc/schemas/listconfigs.schema.json b/doc/schemas/listconfigs.schema.json index 877cea2107a4..9f910292f9f1 100644 --- a/doc/schemas/listconfigs.schema.json +++ b/doc/schemas/listconfigs.schema.json @@ -121,6 +121,10 @@ "type": "boolean", "description": "`experimental-shutdown-wrong-funding` field from config or cmdline, or default" }, + "experimental-websocket-port": { + "type": "u16", + "description": "`experimental-websocket-port` field from config or cmdline, or default" + }, "rgb": { "type": "hex", "description": "`rgb` field from config or cmdline, or default", diff --git a/lightningd/connect_control.c b/lightningd/connect_control.c index 314efccb4599..d17d3e780194 100644 --- a/lightningd/connect_control.c +++ b/lightningd/connect_control.c @@ -350,7 +350,10 @@ int connectd_init(struct lightningd *ld) int hsmfd; struct wireaddr_internal *wireaddrs = ld->proposed_wireaddr; enum addr_listen_announce *listen_announce = ld->proposed_listen_announce; - const char *websocket_helper_path = ""; + const char *websocket_helper_path; + + websocket_helper_path = subdaemon_path(tmpctx, ld, + "lightning_websocketd"); if (socketpair(AF_LOCAL, SOCK_STREAM, 0, fds) != 0) fatal("Could not socketpair for connectd<->gossipd"); @@ -384,7 +387,7 @@ int connectd_init(struct lightningd *ld) ld->config.use_v3_autotor, ld->config.connection_timeout_secs, websocket_helper_path, - 0); + ld->websocket_port); subd_req(ld->connectd, ld->connectd, take(msg), -1, 0, connect_init_done, NULL); diff --git a/lightningd/lightningd.c b/lightningd/lightningd.c index 84f9c9cd5f4d..878856c94c1a 100644 --- a/lightningd/lightningd.c +++ b/lightningd/lightningd.c @@ -216,6 +216,7 @@ static struct lightningd *new_lightningd(const tal_t *ctx) ld->always_use_proxy = false; ld->pure_tor_setup = false; ld->tor_service_password = NULL; + ld->websocket_port = 0; /*~ This is initialized later, but the plugin loop examines this, * so set it to NULL explicitly now. */ diff --git a/lightningd/lightningd.h b/lightningd/lightningd.h index 274995422500..532841dcab7e 100644 --- a/lightningd/lightningd.h +++ b/lightningd/lightningd.h @@ -284,6 +284,9 @@ struct lightningd { /* Array of (even) TLV types that we should allow. This is required * since we otherwise would outright reject them. */ u64 *accept_extra_tlv_types; + + /* EXPERIMENTAL: websocket port if non-zero */ + u16 websocket_port; }; /* Turning this on allows a tal allocation to return NULL, rather than aborting. diff --git a/lightningd/options.c b/lightningd/options.c index b88cc8234c3f..ef65f9dd26b4 100644 --- a/lightningd/options.c +++ b/lightningd/options.c @@ -850,6 +850,21 @@ static char *opt_set_wumbo(struct lightningd *ld) return NULL; } +static char *opt_set_websocket_port(const char *arg, struct lightningd *ld) +{ + u32 port COMPILER_WANTS_INIT("9.3.0 -O2"); + char *err; + + err = opt_set_u32(arg, &port); + if (err) + return err; + + ld->websocket_port = port; + if (ld->websocket_port != port) + return tal_fmt(NULL, "'%s' is out of range", arg); + return NULL; +} + static char *opt_set_dual_fund(struct lightningd *ld) { /* Dual funding implies anchor outputs */ @@ -1051,6 +1066,11 @@ static void register_opts(struct lightningd *ld) "--subdaemon=hsmd:remote_signer " "would use a hypothetical remote signing subdaemon."); + opt_register_arg("--experimental-websocket-port", + opt_set_websocket_port, NULL, + ld, + "experimental: alternate port for peers to connect" + " using WebSockets (RFC6455)"); opt_register_logging(ld); opt_register_version(); @@ -1463,6 +1483,11 @@ static void add_config(struct lightningd *ld, json_add_opt_disable_plugins(response, ld->plugins); } else if (opt->cb_arg == (void *)opt_force_feerates) { answer = fmt_force_feerates(name0, ld->force_feerates); + } else if (opt->cb_arg == (void *)opt_set_websocket_port) { + if (ld->websocket_port) + json_add_u32(response, name0, + ld->websocket_port); + return; } else if (opt->cb_arg == (void *)opt_important_plugin) { /* Do nothing, this is already handled by * opt_add_plugin. */ diff --git a/requirements.lock b/requirements.lock index 6c95dc9e59a8..cc193eea8c3d 100644 --- a/requirements.lock +++ b/requirements.lock @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with python 3.8 # To update, run: # -# pip-compile --output-file=requirements.lock requirements.in +# pip-compile --output-file=requirements.lock requirements.txt # alabaster==0.7.12 # via sphinx @@ -15,9 +15,9 @@ attrs==21.2.0 babel==2.9.1 # via sphinx base58==2.0.1 - # via -r requirements.in + # via pyln.proto bitstring==3.1.9 - # via -r requirements.in + # via pyln.proto certifi==2021.5.30 # via requests cffi==1.14.6 @@ -27,17 +27,17 @@ cffi==1.14.6 charset-normalizer==2.0.6 # via requests cheroot==8.5.2 - # via -r requirements.in + # via pyln-testing click==7.1.2 # via flask coincurve==13.0.0 - # via -r requirements.in + # via pyln.proto commonmark==0.9.1 # via recommonmark crc32c==2.2.post0 - # via -r requirements.in + # via -r requirements.txt cryptography==3.4.8 - # via -r requirements.in + # via pyln.proto docutils==0.17.1 # via # recommonmark @@ -45,15 +45,15 @@ docutils==0.17.1 entrypoints==0.3 # via flake8 ephemeral-port-reserve==1.1.1 - # via -r requirements.in + # via pyln-testing execnet==1.9.0 # via pytest-xdist flake8==3.7.9 - # via -r requirements.in + # via -r requirements.txt flaky==3.7.0 - # via -r requirements.in + # via pyln-testing flask==1.1.4 - # via -r requirements.in + # via pyln-testing idna==3.2 # via requests imagesize==1.2.0 @@ -70,9 +70,9 @@ jinja2==2.11.3 # mrkd # sphinx jsonschema==3.2.0 - # via -r requirements.in + # via pyln-testing mako==1.1.5 - # via -r requirements.in + # via -r requirements.txt markupsafe==2.0.1 # via # jinja2 @@ -88,9 +88,9 @@ more-itertools==8.10.0 # cheroot # jaraco.functools mrkd==0.1.6 - # via -r requirements.in + # via -r requirements.txt mypy==0.910 - # via -r requirements.in + # via pyln.proto mypy-extensions==0.4.3 # via mypy packaging==21.0 @@ -102,9 +102,9 @@ plac==1.3.3 pluggy==0.13.1 # via pytest psutil==5.7.3 - # via -r requirements.in + # via pyln-testing psycopg2-binary==2.8.6 - # via -r requirements.in + # via pyln-testing py==1.10.0 # via # pytest @@ -112,9 +112,7 @@ py==1.10.0 pycodestyle==2.5.0 # via flake8 pycparser==2.20 - # via - # -r requirements.in - # cffi + # via cffi pyflakes==2.1.1 # via flake8 pygments==2.10.0 @@ -126,10 +124,10 @@ pyparsing==2.4.7 pyrsistent==0.18.0 # via jsonschema pysocks==1.7.1 - # via -r requirements.in + # via pyln.proto pytest==6.1.2 # via - # -r requirements.in + # pyln-testing # pytest-forked # pytest-rerunfailures # pytest-timeout @@ -137,17 +135,17 @@ pytest==6.1.2 pytest-forked==1.3.0 # via pytest-xdist pytest-rerunfailures==9.1.1 - # via -r requirements.in + # via pyln-testing pytest-timeout==1.4.2 - # via -r requirements.in + # via pyln-testing pytest-xdist==2.2.1 - # via -r requirements.in + # via pyln-testing python-bitcoinlib==0.11.0 - # via -r requirements.in + # via pyln-testing pytz==2021.1 # via babel recommonmark==0.7.1 - # via -r requirements.in + # via pyln-client requests==2.26.0 # via sphinx six==1.16.0 @@ -171,13 +169,15 @@ sphinxcontrib-qthelp==1.0.3 sphinxcontrib-serializinghtml==1.1.5 # via sphinx toml==0.10.2 - # via - # mypy - # pytest + # via pytest +typed-ast==1.4.3 + # via mypy typing-extensions==3.10.0.2 # via mypy urllib3==1.26.7 # via requests +websocket-client==1.2.1 + # via -r requirements.txt werkzeug==1.0.1 # via flask diff --git a/requirements.txt b/requirements.txt index ef5e2b9dcb5d..4986279a274f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ mrkd ~= 0.1.6 Mako ~= 1.1.3 flake8 ~= 3.7.8 +websocket-client ./contrib/pyln-client ./contrib/pyln-proto diff --git a/tests/test_connection.py b/tests/test_connection.py index 0d9d58ff53f9..005032cc2c7c 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -2,7 +2,9 @@ from fixtures import * # noqa: F401,F403 from fixtures import TEST_NETWORK from flaky import flaky # noqa: F401 +from ephemeral_port_reserve import reserve # type: ignore from pyln.client import RpcError, Millisatoshi +import pyln.proto.wire as wire from utils import ( only_one, wait_for, sync_blockheight, TIMEOUT, expected_peer_features, expected_node_features, @@ -20,6 +22,7 @@ import shutil import time import unittest +import websocket def test_connect(node_factory): @@ -3740,6 +3743,67 @@ def test_old_feerate(node_factory): l1.pay(l2, 1000) +@pytest.mark.developer("needs --dev-allow-localhost") +def test_websocket(node_factory): + ws_port = reserve() + l1, l2 = node_factory.line_graph(2, + opts=[{'experimental-websocket-port': ws_port, + 'dev-allow-localhost': None}, + {'dev-allow-localhost': None}], + wait_for_announce=True) + assert l1.rpc.listconfigs()['experimental-websocket-port'] == ws_port + + # Adapter to turn websocket into a stream "connection" + class BinWebSocket(object): + def __init__(self, hostname, port): + self.ws = websocket.WebSocket() + self.ws.connect("ws://" + hostname + ":" + str(port)) + self.recvbuf = bytes() + + def send(self, data): + self.ws.send(data, websocket.ABNF.OPCODE_BINARY) + + def recv(self, maxlen): + while len(self.recvbuf) < maxlen: + self.recvbuf += self.ws.recv() + + ret = self.recvbuf[:maxlen] + self.recvbuf = self.recvbuf[maxlen:] + return ret + + ws = BinWebSocket('localhost', ws_port) + lconn = wire.LightningConnection(ws, + wire.PublicKey(bytes.fromhex(l1.info['id'])), + wire.PrivateKey(bytes([1] * 32)), + is_initiator=True) + + l1.daemon.wait_for_log('Websocket connection in from') + + # Perform handshake. + lconn.shake() + + # Expect to receive init msg. + msg = lconn.read_message() + assert int.from_bytes(msg[0:2], 'big') == 16 + + # Echo same message back. + lconn.send_message(msg) + + # Now try sending a ping, ask for 50 bytes + msg = bytes((0, 18, 0, 50, 0, 0)) + lconn.send_message(msg) + + # Could actually reply with some gossip msg! + while True: + msg = lconn.read_message() + if int.from_bytes(msg[0:2], 'big') == 19: + break + + # Check node_announcement has websocket + assert (only_one(l2.rpc.listnodes(l1.info['id'])['nodes'])['addresses'] + == [{'type': 'ipv4', 'address': '127.0.0.1', 'port': l1.port}, {'type': 'websocket', 'port': ws_port}]) + + @pytest.mark.developer("dev-disconnect required") def test_ping_timeout(node_factory): # Disconnects after this, but doesn't know it.