From 4ace7c84836a1da75fc66e727b5f012b4ea0290c Mon Sep 17 00:00:00 2001 From: Jake Sylvestre Date: Thu, 31 Oct 2019 11:32:18 -0400 Subject: [PATCH 01/21] allow non local tls disable for compatibility with private network interfaces --- config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.go b/config.go index 3bcc65dacd..facbea6ac6 100644 --- a/config.go +++ b/config.go @@ -911,7 +911,7 @@ func loadConfig() (*config, []string, error) { // Only allow TLS to be disabled if the RPC is bound to localhost // addresses. - if !cfg.DisableRPC && cfg.DisableTLS { + if false && !cfg.DisableRPC && cfg.DisableTLS { allowedTLSListeners := map[string]struct{}{ "localhost": {}, "127.0.0.1": {}, From 22338acc37afd31ecde8d34f5d925f7562dadecc Mon Sep 17 00:00:00 2001 From: Jake Sylvestre Date: Tue, 3 Dec 2019 23:42:06 -0500 Subject: [PATCH 02/21] new flag --- rpcclient/infrastructure.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/rpcclient/infrastructure.go b/rpcclient/infrastructure.go index 7a8f1885d1..ddaa4ccd4b 100644 --- a/rpcclient/infrastructure.go +++ b/rpcclient/infrastructure.go @@ -1136,6 +1136,17 @@ type ConnConfig struct { // flag can be set to true to use basic HTTP POST requests instead. HTTPPostMode bool + + // Bulk RPC mode instructs the client to chunk requests to the server by + // issuing json-rpc requests instead of using btcd websockets + // Websockets are generally preferred when using a btcd server as some of the + // features of the client such notifications only work with websockets, + // however, not all servers support the websocket extensions, so this + // flag can be set to true to use basic HTTP POST requests instead. + JsonRPCServer bool + + + // EnableBCInfoHacks is an option provided to enable compatibility hacks // when connecting to blockchain.info RPC server EnableBCInfoHacks bool From c5ecee441e08002cba29a75adc730abe1725189f Mon Sep 17 00:00:00 2001 From: Jake Sylvestre Date: Wed, 4 Dec 2019 00:36:16 -0500 Subject: [PATCH 03/21] add bulk request changes --- btcjson/btcdextcmds_test.go | 4 +- btcjson/btcwalletextcmds_test.go | 4 +- btcjson/chainsvrcmds_test.go | 4 +- btcjson/chainsvrwscmds_test.go | 4 +- btcjson/chainsvrwsntfns_test.go | 4 +- btcjson/cmdparse.go | 4 +- btcjson/cmdparse_test.go | 2 +- btcjson/example_test.go | 2 +- btcjson/jsonrpc.go | 120 +++++-- btcjson/walletsvrcmds_test.go | 4 +- btcjson/walletsvrwscmds_test.go | 4 +- btcjson/walletsvrwsntfns_test.go | 4 +- cmd/btcctl/btcctl.go | 2 +- rpcclient/infrastructure.go | 6 +- rpcserver.go | 78 ++++- rpcwebsocket.go | 554 +++++++++++++++++++++++-------- 16 files changed, 603 insertions(+), 197 deletions(-) diff --git a/btcjson/btcdextcmds_test.go b/btcjson/btcdextcmds_test.go index 10e6da3896..e6833a6a3e 100644 --- a/btcjson/btcdextcmds_test.go +++ b/btcjson/btcdextcmds_test.go @@ -193,7 +193,7 @@ func TestBtcdExtCmds(t *testing.T) { for i, test := range tests { // Marshal the command as created by the new static command // creation function. - marshalled, err := btcjson.MarshalCmd(testID, test.staticCmd()) + marshalled, err := btcjson.MarshalCmd("1.0", testID, test.staticCmd()) if err != nil { t.Errorf("MarshalCmd #%d (%s) unexpected error: %v", i, test.name, err) @@ -217,7 +217,7 @@ func TestBtcdExtCmds(t *testing.T) { // Marshal the command as created by the generic new command // creation function. - marshalled, err = btcjson.MarshalCmd(testID, cmd) + marshalled, err = btcjson.MarshalCmd("1.0", testID, cmd) if err != nil { t.Errorf("MarshalCmd #%d (%s) unexpected error: %v", i, test.name, err) diff --git a/btcjson/btcwalletextcmds_test.go b/btcjson/btcwalletextcmds_test.go index 58de1c81d0..a05e1b1d88 100644 --- a/btcjson/btcwalletextcmds_test.go +++ b/btcjson/btcwalletextcmds_test.go @@ -145,7 +145,7 @@ func TestBtcWalletExtCmds(t *testing.T) { for i, test := range tests { // Marshal the command as created by the new static command // creation function. - marshalled, err := btcjson.MarshalCmd(testID, test.staticCmd()) + marshalled, err := btcjson.MarshalCmd("1.0", testID, test.staticCmd()) if err != nil { t.Errorf("MarshalCmd #%d (%s) unexpected error: %v", i, test.name, err) @@ -169,7 +169,7 @@ func TestBtcWalletExtCmds(t *testing.T) { // Marshal the command as created by the generic new command // creation function. - marshalled, err = btcjson.MarshalCmd(testID, cmd) + marshalled, err = btcjson.MarshalCmd("1.0", testID, cmd) if err != nil { t.Errorf("MarshalCmd #%d (%s) unexpected error: %v", i, test.name, err) diff --git a/btcjson/chainsvrcmds_test.go b/btcjson/chainsvrcmds_test.go index 8cb4ee765a..bc3ec8f7e5 100644 --- a/btcjson/chainsvrcmds_test.go +++ b/btcjson/chainsvrcmds_test.go @@ -1092,7 +1092,7 @@ func TestChainSvrCmds(t *testing.T) { for i, test := range tests { // Marshal the command as created by the new static command // creation function. - marshalled, err := btcjson.MarshalCmd(testID, test.staticCmd()) + marshalled, err := btcjson.MarshalCmd("1.0", testID, test.staticCmd()) if err != nil { t.Errorf("MarshalCmd #%d (%s) unexpected error: %v", i, test.name, err) @@ -1117,7 +1117,7 @@ func TestChainSvrCmds(t *testing.T) { // Marshal the command as created by the generic new command // creation function. - marshalled, err = btcjson.MarshalCmd(testID, cmd) + marshalled, err = btcjson.MarshalCmd("1.0", testID, cmd) if err != nil { t.Errorf("MarshalCmd #%d (%s) unexpected error: %v", i, test.name, err) diff --git a/btcjson/chainsvrwscmds_test.go b/btcjson/chainsvrwscmds_test.go index b0cd63cc61..5aec0e46ac 100644 --- a/btcjson/chainsvrwscmds_test.go +++ b/btcjson/chainsvrwscmds_test.go @@ -233,7 +233,7 @@ func TestChainSvrWsCmds(t *testing.T) { for i, test := range tests { // Marshal the command as created by the new static command // creation function. - marshalled, err := btcjson.MarshalCmd(testID, test.staticCmd()) + marshalled, err := btcjson.MarshalCmd("1.0", testID, test.staticCmd()) if err != nil { t.Errorf("MarshalCmd #%d (%s) unexpected error: %v", i, test.name, err) @@ -257,7 +257,7 @@ func TestChainSvrWsCmds(t *testing.T) { // Marshal the command as created by the generic new command // creation function. - marshalled, err = btcjson.MarshalCmd(testID, cmd) + marshalled, err = btcjson.MarshalCmd("1.0", testID, cmd) if err != nil { t.Errorf("MarshalCmd #%d (%s) unexpected error: %v", i, test.name, err) diff --git a/btcjson/chainsvrwsntfns_test.go b/btcjson/chainsvrwsntfns_test.go index 2da1e7ad2f..fdd8dd2e2f 100644 --- a/btcjson/chainsvrwsntfns_test.go +++ b/btcjson/chainsvrwsntfns_test.go @@ -231,7 +231,7 @@ func TestChainSvrWsNtfns(t *testing.T) { for i, test := range tests { // Marshal the notification as created by the new static // creation function. The ID is nil for notifications. - marshalled, err := btcjson.MarshalCmd(nil, test.staticNtfn()) + marshalled, err := btcjson.MarshalCmd("1.0", nil, test.staticNtfn()) if err != nil { t.Errorf("MarshalCmd #%d (%s) unexpected error: %v", i, test.name, err) @@ -256,7 +256,7 @@ func TestChainSvrWsNtfns(t *testing.T) { // Marshal the notification as created by the generic new // notification creation function. The ID is nil for // notifications. - marshalled, err = btcjson.MarshalCmd(nil, cmd) + marshalled, err = btcjson.MarshalCmd("1.0", nil, cmd) if err != nil { t.Errorf("MarshalCmd #%d (%s) unexpected error: %v", i, test.name, err) diff --git a/btcjson/cmdparse.go b/btcjson/cmdparse.go index 48c6278a61..57e14a285c 100644 --- a/btcjson/cmdparse.go +++ b/btcjson/cmdparse.go @@ -35,7 +35,7 @@ func makeParams(rt reflect.Type, rv reflect.Value) []interface{} { // is suitable for transmission to an RPC server. The provided command type // must be a registered type. All commands provided by this package are // registered by default. -func MarshalCmd(id interface{}, cmd interface{}) ([]byte, error) { +func MarshalCmd(rpcVersion string, id interface{}, cmd interface{}) ([]byte, error) { // Look up the cmd type and error out if not registered. rt := reflect.TypeOf(cmd) registerLock.RLock() @@ -59,7 +59,7 @@ func MarshalCmd(id interface{}, cmd interface{}) ([]byte, error) { params := makeParams(rt.Elem(), rv.Elem()) // Generate and marshal the final JSON-RPC request. - rawCmd, err := NewRequest(id, method, params) + rawCmd, err := NewRequest(rpcVersion, id, method, params) if err != nil { return nil, err } diff --git a/btcjson/cmdparse_test.go b/btcjson/cmdparse_test.go index 7c13a0bb44..d10bfa413d 100644 --- a/btcjson/cmdparse_test.go +++ b/btcjson/cmdparse_test.go @@ -433,7 +433,7 @@ func TestMarshalCmdErrors(t *testing.T) { t.Logf("Running %d tests", len(tests)) for i, test := range tests { - _, err := btcjson.MarshalCmd(test.id, test.cmd) + _, err := btcjson.MarshalCmd("1.0", test.id, test.cmd) if reflect.TypeOf(err) != reflect.TypeOf(test.err) { t.Errorf("Test #%d (%s) wrong error - got %T (%v), "+ "want %T", i, test.name, err, err, test.err) diff --git a/btcjson/example_test.go b/btcjson/example_test.go index 527252c7fb..29883c3a68 100644 --- a/btcjson/example_test.go +++ b/btcjson/example_test.go @@ -27,7 +27,7 @@ func ExampleMarshalCmd() { // server. Typically the client would increment the id here which is // request so the response can be identified. id := 1 - marshalledBytes, err := btcjson.MarshalCmd(id, gbCmd) + marshalledBytes, err := btcjson.MarshalCmd("1.0", id, gbCmd) if err != nil { fmt.Println(err) return diff --git a/btcjson/jsonrpc.go b/btcjson/jsonrpc.go index e99d9f4265..ecfd4dbac9 100644 --- a/btcjson/jsonrpc.go +++ b/btcjson/jsonrpc.go @@ -73,15 +73,65 @@ type Request struct { ID interface{} `json:"id"` } -// NewRequest returns a new JSON-RPC 1.0 request object given the provided id, -// method, and parameters. The parameters are marshalled into a json.RawMessage -// for the Params field of the returned request object. This function is only -// provided in case the caller wants to construct raw requests for some reason. -// -// Typically callers will instead want to create a registered concrete command -// type with the NewCmd or NewCmd functions and call the MarshalCmd -// function with that command to generate the marshalled JSON-RPC request. -func NewRequest(id interface{}, method string, params []interface{}) (*Request, error) { +// UnmarshalJSON is a custom unmarshal func for the Request struct. The param +// field defaults to an empty json.RawMessage array it is omitted by the request +// or nil if the supplied value is invalid. +func (request *Request) UnmarshalJSON(b []byte) error { + var data map[string]interface{} + err := json.Unmarshal(b, &data) + if err != nil { + return err + } + + request.ID = data["id"] + methodValue, hasMethod := data["method"] + if hasMethod { + request.Method = methodValue.(string) + } + jsonrpcValue, hasJsonrpc := data["jsonrpc"] + if hasJsonrpc { + request.Jsonrpc = jsonrpcValue.(string) + } + paramsValue, hasParams := data["params"] + if !hasParams { + // set the request param to an empty array if it is ommited in the request + request.Params = []json.RawMessage{} + } + if hasParams { + // assert the request params is an array of data + params, paramsOk := paramsValue.([]interface{}) + if paramsOk { + rawParams := make([]json.RawMessage, 0, len(params)) + for _, param := range params { + marshalledParam, err := json.Marshal(param) + if err != nil { + return err + } + rawMessage := json.RawMessage(marshalledParam) + rawParams = append(rawParams, rawMessage) + } + + request.Params = rawParams + } + } + + return nil +} + +// NewRequest returns a new JSON-RPC request object given the provided rpc +// version, id, method, and parameters. The parameters are marshalled into a +// json.RawMessage for the Params field of the returned request object. This +// function is only provided in case the caller wants to construct raw requests +// for some reason. Typically callers will instead want to create a registered +// concrete command type with the NewCmd or NewCmd functions and call the +// MarshalCmd function with that command to generate the marshalled JSON-RPC +// request. +func NewRequest(rpcVersion string, id interface{}, method string, params []interface{}) (*Request, error) { + // default to JSON-RPC 1.0 if RPC type is not specified + if rpcVersion != "2.0" && rpcVersion != "1.0" { + rpcVersion = "1.0" + } + if !IsValidIDType(id) { str := fmt.Sprintf("the id of type '%T' is invalid", id) return nil, makeError(ErrInvalidType, str) @@ -98,30 +148,35 @@ func NewRequest(id interface{}, method string, params []interface{}) (*Request, } return &Request{ - Jsonrpc: "1.0", + Jsonrpc: rpcVersion, ID: id, Method: method, Params: rawParams, }, nil } -// Response is the general form of a JSON-RPC response. The type of the Result -// field varies from one command to the next, so it is implemented as an -// interface. The ID field has to be a pointer for Go to put a null in it when +// Response is the general form of a JSON-RPC response. The type of the +// Result field varies from one command to the next, so it is implemented as an +// interface. The ID field has to be a pointer to allow for a nil value when // empty. type Response struct { - Result json.RawMessage `json:"result"` - Error *RPCError `json:"error"` - ID *interface{} `json:"id"` + Jsonrpc string `json:"jsonrpc"` + Result json.RawMessage `json:"result"` + Error *RPCError `json:"error"` + ID *interface{} `json:"id"` } -// NewResponse returns a new JSON-RPC response object given the provided id, -// marshalled result, and RPC error. This function is only provided in case the -// caller wants to construct raw responses for some reason. -// + +// NewResponse returns a new JSON-RPC response object given the provided rpc +// version, id, marshalled result, and RPC error. This function is only +// provided in case the caller wants to construct raw responses for some reason. // Typically callers will instead want to create the fully marshalled JSON-RPC // response to send over the wire with the MarshalResponse function. -func NewResponse(id interface{}, marshalledResult []byte, rpcErr *RPCError) (*Response, error) { +func NewResponse(rpcVersion string, id interface{}, marshalledResult []byte, rpcErr *RPCError) (*Response, error) { + if rpcVersion != "2.0" && rpcVersion != "1.0" { + rpcVersion = "1.0" + } + if !IsValidIDType(id) { str := fmt.Sprintf("the id of type '%T' is invalid", id) return nil, makeError(ErrInvalidType, str) @@ -129,22 +184,29 @@ func NewResponse(id interface{}, marshalledResult []byte, rpcErr *RPCError) (*Re pid := &id return &Response{ - Result: marshalledResult, - Error: rpcErr, - ID: pid, + Jsonrpc: rpcVersion, + Result: marshalledResult, + Error: rpcErr, + ID: pid, }, nil } -// MarshalResponse marshals the passed id, result, and RPCError to a JSON-RPC -// response byte slice that is suitable for transmission to a JSON-RPC client. -func MarshalResponse(id interface{}, result interface{}, rpcErr *RPCError) ([]byte, error) { + +// MarshalResponse marshals the passed rpc version, id, result, and RPCError to +// a JSON-RPC response byte slice that is suitable for transmission to a +// JSON-RPC client. +func MarshalResponse(rpcVersion string, id interface{}, result interface{}, rpcErr *RPCError) ([]byte, error) { + if rpcVersion != "2.0" && rpcVersion != "1.0" { + rpcVersion = "1.0" + } + marshalledResult, err := json.Marshal(result) if err != nil { return nil, err } - response, err := NewResponse(id, marshalledResult, rpcErr) + response, err := NewResponse(rpcVersion, id, marshalledResult, rpcErr) if err != nil { return nil, err } return json.Marshal(&response) -} +} \ No newline at end of file diff --git a/btcjson/walletsvrcmds_test.go b/btcjson/walletsvrcmds_test.go index efc08cc945..a7508e2e90 100644 --- a/btcjson/walletsvrcmds_test.go +++ b/btcjson/walletsvrcmds_test.go @@ -1210,7 +1210,7 @@ func TestWalletSvrCmds(t *testing.T) { for i, test := range tests { // Marshal the command as created by the new static command // creation function. - marshalled, err := btcjson.MarshalCmd(testID, test.staticCmd()) + marshalled, err := btcjson.MarshalCmd("1.0", testID, test.staticCmd()) if err != nil { t.Errorf("MarshalCmd #%d (%s) unexpected error: %v", i, test.name, err) @@ -1234,7 +1234,7 @@ func TestWalletSvrCmds(t *testing.T) { // Marshal the command as created by the generic new command // creation function. - marshalled, err = btcjson.MarshalCmd(testID, cmd) + marshalled, err = btcjson.MarshalCmd("1.0", testID, cmd) if err != nil { t.Errorf("MarshalCmd #%d (%s) unexpected error: %v", i, test.name, err) diff --git a/btcjson/walletsvrwscmds_test.go b/btcjson/walletsvrwscmds_test.go index 17144b6ea7..566dad93e9 100644 --- a/btcjson/walletsvrwscmds_test.go +++ b/btcjson/walletsvrwscmds_test.go @@ -195,7 +195,7 @@ func TestWalletSvrWsCmds(t *testing.T) { for i, test := range tests { // Marshal the command as created by the new static command // creation function. - marshalled, err := btcjson.MarshalCmd(testID, test.staticCmd()) + marshalled, err := btcjson.MarshalCmd("1.0", testID, test.staticCmd()) if err != nil { t.Errorf("MarshalCmd #%d (%s) unexpected error: %v", i, test.name, err) @@ -219,7 +219,7 @@ func TestWalletSvrWsCmds(t *testing.T) { // Marshal the command as created by the generic new command // creation function. - marshalled, err = btcjson.MarshalCmd(testID, cmd) + marshalled, err = btcjson.MarshalCmd("1.0", testID, cmd) if err != nil { t.Errorf("MarshalCmd #%d (%s) unexpected error: %v", i, test.name, err) diff --git a/btcjson/walletsvrwsntfns_test.go b/btcjson/walletsvrwsntfns_test.go index 7662b3c2a1..01502b4e67 100644 --- a/btcjson/walletsvrwsntfns_test.go +++ b/btcjson/walletsvrwsntfns_test.go @@ -122,7 +122,7 @@ func TestWalletSvrWsNtfns(t *testing.T) { for i, test := range tests { // Marshal the notification as created by the new static // creation function. The ID is nil for notifications. - marshalled, err := btcjson.MarshalCmd(nil, test.staticNtfn()) + marshalled, err := btcjson.MarshalCmd("1.0", nil, test.staticNtfn()) if err != nil { t.Errorf("MarshalCmd #%d (%s) unexpected error: %v", i, test.name, err) @@ -147,7 +147,7 @@ func TestWalletSvrWsNtfns(t *testing.T) { // Marshal the notification as created by the generic new // notification creation function. The ID is nil for // notifications. - marshalled, err = btcjson.MarshalCmd(nil, cmd) + marshalled, err = btcjson.MarshalCmd("1.0", nil, cmd) if err != nil { t.Errorf("MarshalCmd #%d (%s) unexpected error: %v", i, test.name, err) diff --git a/cmd/btcctl/btcctl.go b/cmd/btcctl/btcctl.go index 5c412f867f..2afbfaf37a 100644 --- a/cmd/btcctl/btcctl.go +++ b/cmd/btcctl/btcctl.go @@ -127,7 +127,7 @@ func main() { // Marshal the command into a JSON-RPC byte slice in preparation for // sending it to the RPC server. - marshalledJSON, err := btcjson.MarshalCmd(1, cmd) + marshalledJSON, err := btcjson.MarshalCmd("1.0", 1, cmd) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) diff --git a/rpcclient/infrastructure.go b/rpcclient/infrastructure.go index ddaa4ccd4b..84b5b5d6f8 100644 --- a/rpcclient/infrastructure.go +++ b/rpcclient/infrastructure.go @@ -901,7 +901,7 @@ func (c *Client) sendCmd(cmd interface{}) chan *response { // Marshal the command. id := c.NextID() - marshalledJSON, err := btcjson.MarshalCmd(id, cmd) + marshalledJSON, err := btcjson.MarshalCmd("1.0", id, cmd) if err != nil { return newFutureError(err) } @@ -1143,9 +1143,7 @@ type ConnConfig struct { // features of the client such notifications only work with websockets, // however, not all servers support the websocket extensions, so this // flag can be set to true to use basic HTTP POST requests instead. - JsonRPCServer bool - - + JsonBulkRPC bool // EnableBCInfoHacks is an option provided to enable compatibility hacks // when connecting to blockchain.info RPC server diff --git a/rpcserver.go b/rpcserver.go index e762cc1a9d..3e98ae16db 100644 --- a/rpcserver.go +++ b/rpcserver.go @@ -101,6 +101,9 @@ var ( // declared here to avoid the overhead of creating the slice on every // invocation for constant data. gbtCapabilities = []string{"proposal"} + + // JSON 2.0 batched request prefix + batchedRequestPrefix = []byte("[") ) // Errors @@ -3827,12 +3830,14 @@ func (s *rpcServer) checkAuth(r *http.Request, require bool) (bool, bool, error) // a known concrete command along with any error that might have happened while // parsing it. type parsedRPCCmd struct { - id interface{} - method string - cmd interface{} - err *btcjson.RPCError + jsonrpc string + id interface{} + method string + cmd interface{} + err *btcjson.RPCError } + // standardCmdResult checks that a parsed command is a standard Bitcoin JSON-RPC // command and runs the appropriate handler to reply to the command. Any // commands which are not recognized or not implemented will return an error @@ -3863,7 +3868,12 @@ handled: // is suitable for use in replies if the command is invalid in some way such as // an unregistered command or invalid parameters. func parseCmd(request *btcjson.Request) *parsedRPCCmd { - var parsedCmd parsedRPCCmd + parsedCmd := parsedRPCCmd{ + jsonrpc: request.Jsonrpc, + id: request.ID, + method: request.Method, + } + parsedCmd.id = request.ID parsedCmd.method = request.Method @@ -3892,7 +3902,7 @@ func parseCmd(request *btcjson.Request) *parsedRPCCmd { // createMarshalledReply returns a new marshalled JSON-RPC response given the // passed parameters. It will automatically convert errors that are not of // the type *btcjson.RPCError to the appropriate type as needed. -func createMarshalledReply(id, result interface{}, replyErr error) ([]byte, error) { +func createMarshalledReply(rpcVersion string, id interface{}, result interface{}, replyErr error) ([]byte, error) { var jsonErr *btcjson.RPCError if replyErr != nil { if jErr, ok := replyErr.(*btcjson.RPCError); ok { @@ -3902,7 +3912,61 @@ func createMarshalledReply(id, result interface{}, replyErr error) ([]byte, erro } } - return btcjson.MarshalResponse(id, result, jsonErr) + return btcjson.MarshalResponse(rpcVersion, id, result, jsonErr) +} + + +// processRequest determines the incoming request type (single or batched), +// parses it and returns a marshalled response. +func (s *rpcServer) processRequest(request *btcjson.Request, isAdmin bool, closeChan <-chan struct{}) []byte { + var result interface{} + var jsonErr *btcjson.RPCError + + if !isAdmin { + if _, ok := rpcLimited[request.Method]; !ok { + jsonErr = internalRPCError("limited user not " + + "authorized for this method", "") + } + } + + if jsonErr == nil { + if request.Method == "" || request.Params == nil { + jsonErr = &btcjson.RPCError{ + Code: btcjson.ErrRPCInvalidRequest.Code, + Message: fmt.Sprintf("Invalid request: malformed"), + } + msg, err := createMarshalledReply(request.Jsonrpc, request.ID, result, jsonErr) + if err != nil { + rpcsLog.Errorf("Failed to marshal reply: %v", err) + return nil + } + return msg + } + + // Valid requests with no ID (notifications) must not have a response + // per the JSON-RPC spec. + if request.ID == nil { + return nil + } + + // Attempt to parse the JSON-RPC request into a known + // concrete command. + parsedCmd := parseCmd(request) + if parsedCmd.err != nil { + jsonErr = parsedCmd.err + } else { + result, _ = s.standardCmdResult(parsedCmd, + closeChan) + } + } + + // Marshal the response. + msg, err := createMarshalledReply(request.Jsonrpc, request.ID, result, jsonErr) + if err != nil { + rpcsLog.Errorf("Failed to marshal reply: %v", err) + return nil + } + return msg } // jsonRPCRead handles reading and responding to RPC messages. diff --git a/rpcwebsocket.go b/rpcwebsocket.go index 32e466d115..0964b70331 100644 --- a/rpcwebsocket.go +++ b/rpcwebsocket.go @@ -695,7 +695,7 @@ func (*wsNotificationManager) notifyBlockConnected(clients map[chan struct{}]*ws // Notify interested websocket clients about the connected block. ntfn := btcjson.NewBlockConnectedNtfn(block.Hash().String(), block.Height(), block.MsgBlock().Header.Timestamp.Unix()) - marshalledJSON, err := btcjson.MarshalCmd(nil, ntfn) + marshalledJSON, err := btcjson.MarshalCmd("1.0", nil, ntfn) if err != nil { rpcsLog.Errorf("Failed to marshal block connected notification: "+ "%v", err) @@ -719,7 +719,7 @@ func (*wsNotificationManager) notifyBlockDisconnected(clients map[chan struct{}] // Notify interested websocket clients about the disconnected block. ntfn := btcjson.NewBlockDisconnectedNtfn(block.Hash().String(), block.Height(), block.MsgBlock().Header.Timestamp.Unix()) - marshalledJSON, err := btcjson.MarshalCmd(nil, ntfn) + marshalledJSON, err := btcjson.MarshalCmd("1.0", nil, ntfn) if err != nil { rpcsLog.Errorf("Failed to marshal block disconnected "+ "notification: %v", err) @@ -765,7 +765,7 @@ func (m *wsNotificationManager) notifyFilteredBlockConnected(clients map[chan st ntfn.SubscribedTxs = subscribedTxs[quitChan] // Marshal and queue notification. - marshalledJSON, err := btcjson.MarshalCmd(nil, ntfn) + marshalledJSON, err := btcjson.MarshalCmd("1.0", nil, ntfn) if err != nil { rpcsLog.Errorf("Failed to marshal filtered block "+ "connected notification: %v", err) @@ -796,7 +796,7 @@ func (*wsNotificationManager) notifyFilteredBlockDisconnected(clients map[chan s } ntfn := btcjson.NewFilteredBlockDisconnectedNtfn(block.Height(), hex.EncodeToString(w.Bytes())) - marshalledJSON, err := btcjson.MarshalCmd(nil, ntfn) + marshalledJSON, err := btcjson.MarshalCmd("1.0", nil, ntfn) if err != nil { rpcsLog.Errorf("Failed to marshal filtered block disconnected "+ "notification: %v", err) @@ -831,7 +831,7 @@ func (m *wsNotificationManager) notifyForNewTx(clients map[chan struct{}]*wsClie } ntfn := btcjson.NewTxAcceptedNtfn(txHashStr, btcutil.Amount(amount).ToBTC()) - marshalledJSON, err := btcjson.MarshalCmd(nil, ntfn) + marshalledJSON, err := btcjson.MarshalCmd("1.0", nil, ntfn) if err != nil { rpcsLog.Errorf("Failed to marshal tx notification: %s", err.Error()) return @@ -854,7 +854,7 @@ func (m *wsNotificationManager) notifyForNewTx(clients map[chan struct{}]*wsClie } verboseNtfn = btcjson.NewTxAcceptedVerboseNtfn(*rawTx) - marshalledJSONVerbose, err = btcjson.MarshalCmd(nil, + marshalledJSONVerbose, err = btcjson.MarshalCmd("1.0", nil, verboseNtfn) if err != nil { rpcsLog.Errorf("Failed to marshal verbose tx "+ @@ -980,7 +980,7 @@ func blockDetails(block *btcutil.Block, txIndex int) *btcjson.BlockDetails { func newRedeemingTxNotification(txHex string, index int, block *btcutil.Block) ([]byte, error) { // Create and marshal the notification. ntfn := btcjson.NewRedeemingTxNtfn(txHex, blockDetails(block, index)) - return btcjson.MarshalCmd(nil, ntfn) + return btcjson.MarshalCmd("1.0", nil, ntfn) } // notifyForTxOuts examines each transaction output, notifying interested @@ -1016,7 +1016,7 @@ func (m *wsNotificationManager) notifyForTxOuts(ops map[wire.OutPoint]map[chan s ntfn := btcjson.NewRecvTxNtfn(txHex, blockDetails(block, tx.Index())) - marshalledJSON, err := btcjson.MarshalCmd(nil, ntfn) + marshalledJSON, err := btcjson.MarshalCmd("1.0", nil, ntfn) if err != nil { rpcsLog.Errorf("Failed to marshal processedtx notification: %v", err) continue @@ -1047,7 +1047,7 @@ func (m *wsNotificationManager) notifyRelevantTxAccepted(tx *btcutil.Tx, if len(clientsToNotify) != 0 { n := btcjson.NewRelevantTxAcceptedNtfn(txHexString(tx.MsgTx())) - marshalled, err := btcjson.MarshalCmd(nil, n) + marshalled, err := btcjson.MarshalCmd("1.0", nil, n) if err != nil { rpcsLog.Errorf("Failed to marshal notification: %v", err) return @@ -1323,153 +1323,435 @@ out: break out } - var request btcjson.Request - err = json.Unmarshal(msg, &request) - if err != nil { - if !c.authenticated { - break out - } + var batchedRequest bool - jsonErr := &btcjson.RPCError{ - Code: btcjson.ErrRPCParse.Code, - Message: "Failed to parse request: " + err.Error(), - } - reply, err := createMarshalledReply(nil, nil, jsonErr) + // Determine request type + //if bytes.HasPrefix(msg, "batchedRequestPrefix") { + // batchedRequest = true + //} + + if !batchedRequest { + var req btcjson.Request + var reply json.RawMessage + err = json.Unmarshal(msg, &req) if err != nil { - rpcsLog.Errorf("Failed to marshal parse failure "+ - "reply: %v", err) - continue - } - c.SendMessage(reply, nil) - continue - } + // only process requests from authenticated clients + if !c.authenticated { + break out + } - // The JSON-RPC 1.0 spec defines that notifications must have their "id" - // set to null and states that notifications do not have a response. - // - // A JSON-RPC 2.0 notification is a request with "json-rpc":"2.0", and - // without an "id" member. The specification states that notifications - // must not be responded to. JSON-RPC 2.0 permits the null value as a - // valid request id, therefore such requests are not notifications. - // - // Bitcoin Core serves requests with "id":null or even an absent "id", - // and responds to such requests with "id":null in the response. - // - // Btcd does not respond to any request without and "id" or "id":null, - // regardless the indicated JSON-RPC protocol version unless RPC quirks - // are enabled. With RPC quirks enabled, such requests will be responded - // to if the reqeust does not indicate JSON-RPC version. - // - // RPC quirks can be enabled by the user to avoid compatibility issues - // with software relying on Core's behavior. - if request.ID == nil && !(cfg.RPCQuirks && request.Jsonrpc == "") { - if !c.authenticated { - break out + jsonErr := &btcjson.RPCError{ + Code: btcjson.ErrRPCParse.Code, + Message: "Failed to parse request: " + err.Error(), + } + reply, err = createMarshalledReply("1.0", nil, nil, jsonErr) + if err != nil { + rpcsLog.Errorf("Failed to marshal reply: %v", err) + continue + } + c.SendMessage(reply, nil) + continue } - continue - } - cmd := parseCmd(&request) - if cmd.err != nil { - if !c.authenticated { - break out + if req.Method == "" || req.Params == nil { + jsonErr := &btcjson.RPCError{ + Code: btcjson.ErrRPCInvalidRequest.Code, + Message: fmt.Sprintf("Invalid request: malformed"), + } + reply, err := createMarshalledReply(req.Jsonrpc, req.ID, nil, jsonErr) + if err != nil { + rpcsLog.Errorf("Failed to marshal reply: %v", err) + continue + } + c.SendMessage(reply, nil) + continue } - reply, err := createMarshalledReply(cmd.id, nil, cmd.err) - if err != nil { - rpcsLog.Errorf("Failed to marshal parse failure "+ - "reply: %v", err) + // Valid requests with no ID (notifications) must not have a response + // per the JSON-RPC spec. + if req.ID == nil { + if !c.authenticated { + break out + } continue } - c.SendMessage(reply, nil) - continue - } - rpcsLog.Debugf("Received command <%s> from %s", cmd.method, c.addr) - - // Check auth. The client is immediately disconnected if the - // first request of an unauthentiated websocket client is not - // the authenticate request, an authenticate request is received - // when the client is already authenticated, or incorrect - // authentication credentials are provided in the request. - switch authCmd, ok := cmd.cmd.(*btcjson.AuthenticateCmd); { - case c.authenticated && ok: - rpcsLog.Warnf("Websocket client %s is already authenticated", - c.addr) - break out - case !c.authenticated && !ok: - rpcsLog.Warnf("Unauthenticated websocket message " + - "received") - break out - case !c.authenticated: - // Check credentials. - login := authCmd.Username + ":" + authCmd.Passphrase - auth := "Basic " + base64.StdEncoding.EncodeToString([]byte(login)) - authSha := sha256.Sum256([]byte(auth)) - cmp := subtle.ConstantTimeCompare(authSha[:], c.server.authsha[:]) - limitcmp := subtle.ConstantTimeCompare(authSha[:], c.server.limitauthsha[:]) - if cmp != 1 && limitcmp != 1 { - rpcsLog.Warnf("Auth failure.") - break out - } - c.authenticated = true - c.isAdmin = cmp == 1 - // Marshal and send response. - reply, err := createMarshalledReply(cmd.id, nil, nil) - if err != nil { - rpcsLog.Errorf("Failed to marshal authenticate reply: "+ - "%v", err.Error()) + cmd := parseCmd(&req) + if cmd.err != nil { + // Only process requests from authenticated clients + if !c.authenticated { + break out + } + + reply, err = createMarshalledReply(cmd.jsonrpc, cmd.id, nil, cmd.err) + if err != nil { + rpcsLog.Errorf("Failed to marshal reply: %v", err) + continue + } + c.SendMessage(reply, nil) continue } - c.SendMessage(reply, nil) - continue - } - // Check if the client is using limited RPC credentials and - // error when not authorized to call this RPC. - if !c.isAdmin { - if _, ok := rpcLimited[request.Method]; !ok { - jsonErr := &btcjson.RPCError{ - Code: btcjson.ErrRPCInvalidParams.Code, - Message: "limited user not authorized for this method", + rpcsLog.Debugf("Received command <%s> from %s", cmd.method, c.addr) + + // Check auth. The client is immediately disconnected if the + // first request of an unauthentiated websocket client is not + // the authenticate request, an authenticate request is received + // when the client is already authenticated, or incorrect + // authentication credentials are provided in the request. + switch authCmd, ok := cmd.cmd.(*btcjson.AuthenticateCmd); { + case c.authenticated && ok: + rpcsLog.Warnf("Websocket client %s is already authenticated", + c.addr) + break out + case !c.authenticated && !ok: + rpcsLog.Warnf("Unauthenticated websocket message " + + "received") + break out + case !c.authenticated: + // Check credentials. + login := authCmd.Username + ":" + authCmd.Passphrase + auth := "Basic " + base64.StdEncoding.EncodeToString([]byte(login)) + authSha := sha256.Sum256([]byte(auth)) + cmp := subtle.ConstantTimeCompare(authSha[:], c.server.authsha[:]) + limitcmp := subtle.ConstantTimeCompare(authSha[:], c.server.limitauthsha[:]) + if cmp != 1 && limitcmp != 1 { + rpcsLog.Warnf("Auth failure.") + break out } + c.authenticated = true + c.isAdmin = cmp == 1 + // Marshal and send response. - reply, err := createMarshalledReply(request.ID, nil, jsonErr) + reply, err = createMarshalledReply(cmd.jsonrpc, cmd.id, nil, nil) if err != nil { - rpcsLog.Errorf("Failed to marshal parse failure "+ - "reply: %v", err) + rpcsLog.Errorf("Failed to marshal authenticate reply: "+ + "%v", err.Error()) continue } c.SendMessage(reply, nil) continue } + + // Check if the client is using limited RPC credentials and + // error when not authorized to call the supplied RPC. + if !c.isAdmin { + if _, ok := rpcLimited[req.Method]; !ok { + jsonErr := &btcjson.RPCError{ + Code: btcjson.ErrRPCInvalidParams.Code, + Message: "limited user not authorized for this method", + } + // Marshal and send response. + reply, err = createMarshalledReply("", req.ID, nil, jsonErr) + if err != nil { + rpcsLog.Errorf("Failed to marshal parse failure "+ + "reply: %v", err) + continue + } + c.SendMessage(reply, nil) + continue + } + } + + // Asynchronously handle the request. A semaphore is used to + // limit the number of concurrent requests currently being + // serviced. If the semaphore can not be acquired, simply wait + // until a request finished before reading the next RPC request + // from the websocket client. + // + // This could be a little fancier by timing out and erroring + // when it takes too long to service the request, but if that is + // done, the read of the next request should not be blocked by + // this semaphore, otherwise the next request will be read and + // will probably sit here for another few seconds before timing + // out as well. This will cause the total timeout duration for + // later requests to be much longer than the check here would + // imply. + // + // If a timeout is added, the semaphore acquiring should be + // moved inside of the new goroutine with a select statement + // that also reads a time.After channel. This will unblock the + // read of the next request from the websocket client and allow + // many requests to be waited on concurrently. + c.serviceRequestSem.acquire() + go func() { + c.serviceRequest(cmd) + c.serviceRequestSem.release() + }() } - // Asynchronously handle the request. A semaphore is used to - // limit the number of concurrent requests currently being - // serviced. If the semaphore can not be acquired, simply wait - // until a request finished before reading the next RPC request - // from the websocket client. - // - // This could be a little fancier by timing out and erroring - // when it takes too long to service the request, but if that is - // done, the read of the next request should not be blocked by - // this semaphore, otherwise the next request will be read and - // will probably sit here for another few seconds before timing - // out as well. This will cause the total timeout duration for - // later requests to be much longer than the check here would - // imply. - // - // If a timeout is added, the semaphore acquiring should be - // moved inside of the new goroutine with a select statement - // that also reads a time.After channel. This will unblock the - // read of the next request from the websocket client and allow - // many requests to be waited on concurrently. - c.serviceRequestSem.acquire() - go func() { - c.serviceRequest(cmd) + // Process a batched request + if batchedRequest { + var batchedRequests []interface{} + var results []json.RawMessage + var batchSize int + var reply json.RawMessage + c.serviceRequestSem.acquire() + err = json.Unmarshal(msg, &batchedRequests) + if err != nil { + // Only process requests from authenticated clients + if !c.authenticated { + break out + } + + jsonErr := &btcjson.RPCError{ + Code: btcjson.ErrRPCParse.Code, + Message: fmt.Sprintf("Failed to parse request: %v", + err), + } + reply, err = btcjson.MarshalResponse("2.0", nil, nil, jsonErr) + if err != nil { + rpcsLog.Errorf("Failed to create reply: %v", err) + } + + if reply != nil { + results = append(results, reply) + } + } + + if err == nil { + // Response with an empty batch error if the batch size is zero + if len(batchedRequests) == 0 { + if !c.authenticated { + break out + } + + jsonErr := &btcjson.RPCError{ + Code: btcjson.ErrRPCInvalidRequest.Code, + Message: fmt.Sprint("Invalid request: empty batch"), + } + reply, err = btcjson.MarshalResponse("2.0", nil, nil, jsonErr) + if err != nil { + rpcsLog.Errorf("Failed to marshal reply: %v", err) + } + + if reply != nil { + results = append(results, reply) + } + } + + // Process each batch entry individually + if len(batchedRequests) > 0 { + batchSize = len(batchedRequests) + for _, entry := range batchedRequests { + var reqBytes []byte + reqBytes, err = json.Marshal(entry) + if err != nil { + // Only process requests from authenticated clients + if !c.authenticated { + break out + } + + jsonErr := &btcjson.RPCError{ + Code: btcjson.ErrRPCInvalidRequest.Code, + Message: fmt.Sprintf("Invalid request: %v", + err), + } + reply, err = btcjson.MarshalResponse("2.0", nil, nil, jsonErr) + if err != nil { + rpcsLog.Errorf("Failed to create reply: %v", err) + continue + } + + if reply != nil { + results = append(results, reply) + } + continue + } + + var req btcjson.Request + err := json.Unmarshal(reqBytes, &req) + if err != nil { + // Only process requests from authenticated clients + if !c.authenticated { + break out + } + + jsonErr := &btcjson.RPCError{ + Code: btcjson.ErrRPCInvalidRequest.Code, + Message: fmt.Sprintf("Invalid request: %v", + err), + } + reply, err = btcjson.MarshalResponse("2.0", nil, nil, jsonErr) + if err != nil { + rpcsLog.Errorf("Failed to create reply: %v", err) + continue + } + + if reply != nil { + results = append(results, reply) + } + continue + } + + if req.Method == "" || req.Params == nil { + jsonErr := &btcjson.RPCError{ + Code: btcjson.ErrRPCInvalidRequest.Code, + Message: fmt.Sprintf("Invalid request: malformed"), + } + reply, err := createMarshalledReply(req.Jsonrpc, req.ID, nil, jsonErr) + if err != nil { + rpcsLog.Errorf("Failed to marshal reply: %v", err) + continue + } + + if reply != nil { + results = append(results, reply) + } + continue + } + + // Valid requests with no ID (notifications) must not have a response + // per the JSON-RPC spec. + if req.ID == nil { + if !c.authenticated { + break out + } + continue + } + + cmd := parseCmd(&req) + if cmd.err != nil { + // Only process requests from authenticated clients + if !c.authenticated { + break out + } + + reply, err = createMarshalledReply(cmd.jsonrpc, cmd.id, nil, cmd.err) + if err != nil { + rpcsLog.Errorf("Failed to marshal reply: %v", err) + continue + } + + if reply != nil { + results = append(results, reply) + } + continue + } + + rpcsLog.Debugf("Received command <%s> from %s", cmd.method, c.addr) + + // Check auth. The client is immediately disconnected if the + // first request of an unauthentiated websocket client is not + // the authenticate request, an authenticate request is received + // when the client is already authenticated, or incorrect + // authentication credentials are provided in the request. + switch authCmd, ok := cmd.cmd.(*btcjson.AuthenticateCmd); { + case c.authenticated && ok: + rpcsLog.Warnf("Websocket client %s is already authenticated", + c.addr) + break out + case !c.authenticated && !ok: + rpcsLog.Warnf("Unauthenticated websocket message " + + "received") + break out + case !c.authenticated: + // Check credentials. + login := authCmd.Username + ":" + authCmd.Passphrase + auth := "Basic " + base64.StdEncoding.EncodeToString([]byte(login)) + authSha := sha256.Sum256([]byte(auth)) + cmp := subtle.ConstantTimeCompare(authSha[:], c.server.authsha[:]) + limitcmp := subtle.ConstantTimeCompare(authSha[:], c.server.limitauthsha[:]) + if cmp != 1 && limitcmp != 1 { + rpcsLog.Warnf("Auth failure.") + break out + } + + c.authenticated = true + c.isAdmin = cmp == 1 + + // Marshal and send response. + reply, err = createMarshalledReply(cmd.jsonrpc, cmd.id, nil, nil) + if err != nil { + rpcsLog.Errorf("Failed to marshal authenticate reply: "+ + "%v", err.Error()) + continue + } + + if reply != nil { + results = append(results, reply) + } + continue + } + + // Check if the client is using limited RPC credentials and + // error when not authorized to call the supplied RPC. + if !c.isAdmin { + if _, ok := rpcLimited[req.Method]; !ok { + jsonErr := &btcjson.RPCError{ + Code: btcjson.ErrRPCInvalidParams.Code, + Message: "limited user not authorized for this method", + } + // Marshal and send response. + reply, err = createMarshalledReply(req.Jsonrpc, req.ID, nil, jsonErr) + if err != nil { + rpcsLog.Errorf("Failed to marshal parse failure "+ + "reply: %v", err) + continue + } + + if reply != nil { + results = append(results, reply) + } + continue + } + } + + // Lookup the websocket extension for the command, if it doesn't + // exist fallback to handling the command as a standard command. + var resp interface{} + wsHandler, ok := wsHandlers[cmd.method] + if ok { + resp, err = wsHandler(c, cmd.cmd) + } else { + resp, err = c.server.standardCmdResult(cmd, nil) + } + + // Marshal request output. + reply, err := createMarshalledReply(cmd.jsonrpc, cmd.id, resp, err) + if err != nil { + rpcsLog.Errorf("Failed to marshal reply for <%s> "+ + "command: %v", cmd.method, err) + return + } + + if reply != nil { + results = append(results, reply) + } + } + } + } + + // generate reply + var payload = []byte{} + if batchedRequest && batchSize > 0 { + if len(results) > 0 { + // Form the batched response json + var buffer bytes.Buffer + buffer.WriteByte('[') + for idx, marshalledReply := range results { + if idx == len(results)-1 { + buffer.Write(marshalledReply) + buffer.WriteByte(']') + break + } + buffer.Write(marshalledReply) + buffer.WriteByte(',') + } + payload = buffer.Bytes() + } + } + + if !batchedRequest || batchSize == 0 { + // Respond with the first results entry for single requests + if len(results) > 0 { + payload = results[0] + } + } + + c.SendMessage(payload, nil) c.serviceRequestSem.release() - }() + } } // Ensure the connection is closed. @@ -1495,7 +1777,7 @@ func (c *wsClient) serviceRequest(r *parsedRPCCmd) { } else { result, err = c.server.standardCmdResult(r, nil) } - reply, err := createMarshalledReply(r.id, result, err) + reply, err := createMarshalledReply(r.jsonrpc, r.id, result, err) if err != nil { rpcsLog.Errorf("Failed to marshal reply for <%s> "+ "command: %v", r.method, err) @@ -2125,7 +2407,7 @@ func rescanBlock(wsc *wsClient, lookups *rescanKeys, blk *btcutil.Block) { ntfn := btcjson.NewRecvTxNtfn(txHex, blockDetails(blk, tx.Index())) - marshalledJSON, err := btcjson.MarshalCmd(nil, ntfn) + marshalledJSON, err := btcjson.MarshalCmd("1.0", nil, ntfn) if err != nil { rpcsLog.Errorf("Failed to marshal recvtx notification: %v", err) return @@ -2492,7 +2774,7 @@ fetchRange: hashList[i].String(), blk.Height(), blk.MsgBlock().Header.Timestamp.Unix(), ) - mn, err := btcjson.MarshalCmd(nil, n) + mn, err := btcjson.MarshalCmd("1.0", nil, n) if err != nil { rpcsLog.Errorf("Failed to marshal rescan "+ "progress notification: %v", err) @@ -2637,7 +2919,7 @@ func handleRescan(wsc *wsClient, icmd interface{}) (interface{}, error) { lastBlockHash.String(), lastBlock.Height(), lastBlock.MsgBlock().Header.Timestamp.Unix(), ) - if mn, err := btcjson.MarshalCmd(nil, n); err != nil { + if mn, err := btcjson.MarshalCmd("1.0", nil, n); err != nil { rpcsLog.Errorf("Failed to marshal rescan finished "+ "notification: %v", err) } else { From 2b6b35c253e25c8e983ff0e6ac0af7907da88a93 Mon Sep 17 00:00:00 2001 From: Jake Sylvestre Date: Wed, 4 Dec 2019 00:53:52 -0500 Subject: [PATCH 04/21] server http --- rpcclient/examples/bitcoincorehttp/main.go | 3 +- rpcserver.go | 203 +++++++++++++++------ 2 files changed, 146 insertions(+), 60 deletions(-) diff --git a/rpcclient/examples/bitcoincorehttp/main.go b/rpcclient/examples/bitcoincorehttp/main.go index 489770a25b..113fb8e1ff 100644 --- a/rpcclient/examples/bitcoincorehttp/main.go +++ b/rpcclient/examples/bitcoincorehttp/main.go @@ -13,9 +13,10 @@ import ( func main() { // Connect to local bitcoin core RPC server using HTTP POST mode. connCfg := &rpcclient.ConnConfig{ - Host: "localhost:8332", + Host: "144.202.28.201:8332", User: "yourrpcuser", Pass: "yourrpcpass", + JsonBulkRPC: true, HTTPPostMode: true, // Bitcoin core only supports HTTP POST mode DisableTLS: true, // Bitcoin core does not provide TLS by default } diff --git a/rpcserver.go b/rpcserver.go index 3e98ae16db..1506854767 100644 --- a/rpcserver.go +++ b/rpcserver.go @@ -4011,80 +4011,165 @@ func (s *rpcServer) jsonRPCRead(w http.ResponseWriter, r *http.Request, isAdmin conn.SetReadDeadline(timeZeroVal) // Attempt to parse the raw body into a JSON-RPC request. - var responseID interface{} - var jsonErr error - var result interface{} - var request btcjson.Request - if err := json.Unmarshal(body, &request); err != nil { - jsonErr = &btcjson.RPCError{ - Code: btcjson.ErrRPCParse.Code, - Message: "Failed to parse request: " + err.Error(), + // Setup a close notifier. Since the connection is hijacked, + // the CloseNotifer on the ResponseWriter is not available. + closeChan := make(chan struct{}, 1) + go func() { + _, err = conn.Read(make([]byte, 1)) + if err != nil { + close(closeChan) } + }() + + var results []json.RawMessage + var batchSize int + var batchedRequest bool + + // Determine request type + if bytes.HasPrefix(body, batchedRequestPrefix) { + batchedRequest = true } - if jsonErr == nil { - // The JSON-RPC 1.0 spec defines that notifications must have their "id" - // set to null and states that notifications do not have a response. - // - // A JSON-RPC 2.0 notification is a request with "json-rpc":"2.0", and - // without an "id" member. The specification states that notifications - // must not be responded to. JSON-RPC 2.0 permits the null value as a - // valid request id, therefore such requests are not notifications. - // - // Bitcoin Core serves requests with "id":null or even an absent "id", - // and responds to such requests with "id":null in the response. - // - // Btcd does not respond to any request without and "id" or "id":null, - // regardless the indicated JSON-RPC protocol version unless RPC quirks - // are enabled. With RPC quirks enabled, such requests will be responded - // to if the reqeust does not indicate JSON-RPC version. - // - // RPC quirks can be enabled by the user to avoid compatibility issues - // with software relying on Core's behavior. - if request.ID == nil && !(cfg.RPCQuirks && request.Jsonrpc == "") { - return + + // Process a single request + if !batchedRequest { + var req btcjson.Request + var resp json.RawMessage + err = json.Unmarshal(body, &req) + if err != nil { + jsonErr := &btcjson.RPCError{ + Code: btcjson.ErrRPCParse.Code, + Message: fmt.Sprintf("Failed to parse request: %v", + err), + } + resp, err = btcjson.MarshalResponse("1.0", nil, nil, jsonErr) + if err != nil { + rpcsLog.Errorf("Failed to create reply: %v", err) + } + } + + if err == nil { + resp = s.processRequest(&req, isAdmin, closeChan) } - // The parse was at least successful enough to have an ID so - // set it for the response. - responseID = request.ID + if resp != nil { + results = append(results, resp) + } + } - // Setup a close notifier. Since the connection is hijacked, - // the CloseNotifer on the ResponseWriter is not available. - closeChan := make(chan struct{}, 1) - go func() { - _, err := conn.Read(make([]byte, 1)) + // Process a batched request + if batchedRequest { + var batchedRequests []interface{} + var resp json.RawMessage + err = json.Unmarshal(body, &batchedRequests) + if err != nil { + jsonErr := &btcjson.RPCError{ + Code: btcjson.ErrRPCParse.Code, + Message: fmt.Sprintf("Failed to parse request: %v", + err), + } + resp, err = btcjson.MarshalResponse("2.0", nil, nil, jsonErr) if err != nil { - close(closeChan) + rpcsLog.Errorf("Failed to create reply: %v", err) + } + + if resp != nil { + results = append(results, resp) + } + } + + if err == nil { + // Response with an empty batch error if the batch size is zero + if len(batchedRequests) == 0 { + jsonErr := &btcjson.RPCError{ + Code: btcjson.ErrRPCInvalidRequest.Code, + Message: fmt.Sprint("Invalid request: empty batch"), + } + resp, err = btcjson.MarshalResponse("2.0", nil, nil, jsonErr) + if err != nil { + rpcsLog.Errorf("Failed to marshal reply: %v", err) + } + + if resp != nil { + results = append(results, resp) + } } - }() - - // Check if the user is limited and set error if method unauthorized - if !isAdmin { - if _, ok := rpcLimited[request.Method]; !ok { - jsonErr = &btcjson.RPCError{ - Code: btcjson.ErrRPCInvalidParams.Code, - Message: "limited user not authorized for this method", + + // Process each batch entry individually + if len(batchedRequests) > 0 { + batchSize = len(batchedRequests) + + for _, entry := range batchedRequests { + var reqBytes []byte + reqBytes, err = json.Marshal(entry) + if err != nil { + jsonErr := &btcjson.RPCError{ + Code: btcjson.ErrRPCInvalidRequest.Code, + Message: fmt.Sprintf("Invalid request: %v", + err), + } + resp, err = btcjson.MarshalResponse("2.0", nil, nil, jsonErr) + if err != nil { + rpcsLog.Errorf("Failed to create reply: %v", err) + } + + if resp != nil { + results = append(results, resp) + } + continue + } + + var req btcjson.Request + err := json.Unmarshal(reqBytes, &req) + if err != nil { + jsonErr := &btcjson.RPCError{ + Code: btcjson.ErrRPCInvalidRequest.Code, + Message: fmt.Sprintf("Invalid request: %v", + err), + } + resp, err = btcjson.MarshalResponse("", nil, nil, jsonErr) + if err != nil { + rpcsLog.Errorf("Failed to create reply: %v", err) + } + + if resp != nil { + results = append(results, resp) + } + continue + } + + resp = s.processRequest(&req, isAdmin, closeChan) + if resp != nil { + results = append(results, resp) + } } } } + } - if jsonErr == nil { - // Attempt to parse the JSON-RPC request into a known concrete - // command. - parsedCmd := parseCmd(&request) - if parsedCmd.err != nil { - jsonErr = parsedCmd.err - } else { - result, jsonErr = s.standardCmdResult(parsedCmd, closeChan) + var msg = []byte{} + if batchedRequest && batchSize > 0 { + if len(results) > 0 { + // Form the batched response json + var buffer bytes.Buffer + buffer.WriteByte('[') + for idx, reply := range results { + if idx == len(results)-1 { + buffer.Write(reply) + buffer.WriteByte(']') + break + } + buffer.Write(reply) + buffer.WriteByte(',') } + msg = buffer.Bytes() } } - // Marshal the response. - msg, err := createMarshalledReply(responseID, result, jsonErr) - if err != nil { - rpcsLog.Errorf("Failed to marshal reply: %v", err) - return + if !batchedRequest || batchSize == 0 { + // Respond with the first results entry for single requests + if len(results) > 0 { + msg = results[0] + } } // Write the response. From e4c7407a7fe14ad1e29e05622404565e469948d2 Mon Sep 17 00:00:00 2001 From: Jake Sylvestre Date: Wed, 4 Dec 2019 01:58:10 -0500 Subject: [PATCH 05/21] bulk first pass --- rpcclient/examples/bitcoincorehttp/main.go | 5 +- .../examples/bitcoincorehttpbulk/README.md | 33 ++++++++++ .../examples/bitcoincorehttpbulk/main.go | 65 +++++++++++++++++++ rpcwebsocket.go | 6 +- 4 files changed, 104 insertions(+), 5 deletions(-) create mode 100644 rpcclient/examples/bitcoincorehttpbulk/README.md create mode 100644 rpcclient/examples/bitcoincorehttpbulk/main.go diff --git a/rpcclient/examples/bitcoincorehttp/main.go b/rpcclient/examples/bitcoincorehttp/main.go index 113fb8e1ff..91d4a1096d 100644 --- a/rpcclient/examples/bitcoincorehttp/main.go +++ b/rpcclient/examples/bitcoincorehttp/main.go @@ -13,11 +13,12 @@ import ( func main() { // Connect to local bitcoin core RPC server using HTTP POST mode. connCfg := &rpcclient.ConnConfig{ - Host: "144.202.28.201:8332", + Host: "localhost:8332", User: "yourrpcuser", Pass: "yourrpcpass", + DisableConnectOnNew: true, JsonBulkRPC: true, - HTTPPostMode: true, // Bitcoin core only supports HTTP POST mode + //HTTPPostMode: true, // Bitcoin core only supports HTTP POST mode DisableTLS: true, // Bitcoin core does not provide TLS by default } // Notice the notification parameter is nil since notifications are diff --git a/rpcclient/examples/bitcoincorehttpbulk/README.md b/rpcclient/examples/bitcoincorehttpbulk/README.md new file mode 100644 index 0000000000..4d1f0adfe1 --- /dev/null +++ b/rpcclient/examples/bitcoincorehttpbulk/README.md @@ -0,0 +1,33 @@ +Bitcoin Core HTTP POST Example +============================== + +This example shows how to use the rpcclient package to connect to a Bitcoin +Core RPC server using HTTP POST mode with TLS disabled and gets the current +block count. + +## Running the Example + +The first step is to use `go get` to download and install the rpcclient package: + +```bash +$ go get github.com/btcsuite/btcd/rpcclient +``` + +Next, modify the `main.go` source to specify the correct RPC username and +password for the RPC server: + +```Go + User: "yourrpcuser", + Pass: "yourrpcpass", +``` + +Finally, navigate to the example's directory and run it with: + +```bash +$ cd $GOPATH/src/github.com/btcsuite/btcd/rpcclient/examples/bitcoincorehttp +$ go run *.go +``` + +## License + +This example is licensed under the [copyfree](http://copyfree.org) ISC License. diff --git a/rpcclient/examples/bitcoincorehttpbulk/main.go b/rpcclient/examples/bitcoincorehttpbulk/main.go new file mode 100644 index 0000000000..a524d7b2e6 --- /dev/null +++ b/rpcclient/examples/bitcoincorehttpbulk/main.go @@ -0,0 +1,65 @@ +package main + +import ( + "bytes" + "fmt" + "io/ioutil" + "net/http" +) + +func main() { + const ( + url = "http://localhost:8332" + rpcUser = "yourrpcuser" + rpcPass = "yourrpcpass" + ) + + // populate request set + reqs := []string{ + `{}`, + `[]`, + `[1]`, + `[1,2,3]`, + `{"foo": "boo"}`, // should be an invalid request + `{"jsonrpc": "1.0", "foo": "boo", "id": "1"}`, + `{"jsonrpc": "1.0", "method": "getblockcount", "params": [], "id": "1"}`, + `{"jsonrpc": "1.0", "method": "getblockcount", "params": "a", "id": "1"}`, // should be invalid since params is neither an array nor a json object. + `[ + {"jsonrpc": "2.0", "method": "getblockcount", "params": [], "id": "1"}, + {"jsonrpc": "2.0", "method": "decodescript", "params": ["ac"]}, + {"jsonrpc": "2.0", "method": "getbestblockhash", "params": [], "id": "2"}, + {"foo": "boo"}, + {"jsonrpc": "2.0", "method": "getblockcount", "id": "9"} + ]`, // should produce invalid request for the `{"foo": "boo"}`. + } + + // Connect to local btcd RPC server using websockets. + + client := http.Client{} + + for _, jsonReq := range reqs { + bodyReader := bytes.NewReader([]byte(jsonReq)) + httpReq, err := http.NewRequest("POST", url, bodyReader) + if err != nil { + fmt.Println(err) + return + } + httpReq.Close = true + httpReq.Header.Set("Content-Type", "application/json") + httpReq.SetBasicAuth(rpcUser, rpcPass) + resp, err := client.Do(httpReq) + if err != nil { + fmt.Println("request:", jsonReq, "response:", err) + return + } + + respBytes, err := ioutil.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + fmt.Println(err) + return + } + + fmt.Println("request:", jsonReq, "response:", string(respBytes)) + } +} \ No newline at end of file diff --git a/rpcwebsocket.go b/rpcwebsocket.go index 0964b70331..5cc1ee1d89 100644 --- a/rpcwebsocket.go +++ b/rpcwebsocket.go @@ -1326,9 +1326,9 @@ out: var batchedRequest bool // Determine request type - //if bytes.HasPrefix(msg, "batchedRequestPrefix") { - // batchedRequest = true - //} + if bytes.HasPrefix(msg, batchedRequestPrefix) { + batchedRequest = true + } if !batchedRequest { var req btcjson.Request From c25e1e8afeaef50c2cd9da8289e9afa52e6ccd25 Mon Sep 17 00:00:00 2001 From: Jake Sylvestre Date: Wed, 4 Dec 2019 02:11:48 -0500 Subject: [PATCH 06/21] tls unwrangle --- config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.go b/config.go index facbea6ac6..3bcc65dacd 100644 --- a/config.go +++ b/config.go @@ -911,7 +911,7 @@ func loadConfig() (*config, []string, error) { // Only allow TLS to be disabled if the RPC is bound to localhost // addresses. - if false && !cfg.DisableRPC && cfg.DisableTLS { + if !cfg.DisableRPC && cfg.DisableTLS { allowedTLSListeners := map[string]struct{}{ "localhost": {}, "127.0.0.1": {}, From e74049822dbe179a8afb831eaa16fc4a17b96db7 Mon Sep 17 00:00:00 2001 From: Jake Sylvestre Date: Wed, 4 Dec 2019 02:15:21 -0500 Subject: [PATCH 07/21] Revert "tls unwrangle" This reverts commit c25e1e8afeaef50c2cd9da8289e9afa52e6ccd25. --- config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.go b/config.go index 3bcc65dacd..facbea6ac6 100644 --- a/config.go +++ b/config.go @@ -911,7 +911,7 @@ func loadConfig() (*config, []string, error) { // Only allow TLS to be disabled if the RPC is bound to localhost // addresses. - if !cfg.DisableRPC && cfg.DisableTLS { + if false && !cfg.DisableRPC && cfg.DisableTLS { allowedTLSListeners := map[string]struct{}{ "localhost": {}, "127.0.0.1": {}, From a59d23d0af72e1e481ee377f1c678118d9e7d2a9 Mon Sep 17 00:00:00 2001 From: Jake Sylvestre Date: Wed, 4 Dec 2019 17:29:14 -0500 Subject: [PATCH 08/21] add bulk mode --- rpcclient/examples/bitcoincorehttp/main.go | 7 +++---- rpcclient/infrastructure.go | 16 +++++++--------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/rpcclient/examples/bitcoincorehttp/main.go b/rpcclient/examples/bitcoincorehttp/main.go index 91d4a1096d..f9fa890757 100644 --- a/rpcclient/examples/bitcoincorehttp/main.go +++ b/rpcclient/examples/bitcoincorehttp/main.go @@ -13,12 +13,11 @@ import ( func main() { // Connect to local bitcoin core RPC server using HTTP POST mode. connCfg := &rpcclient.ConnConfig{ - Host: "localhost:8332", + Host: "127.0.0.1:8332", User: "yourrpcuser", Pass: "yourrpcpass", DisableConnectOnNew: true, - JsonBulkRPC: true, - //HTTPPostMode: true, // Bitcoin core only supports HTTP POST mode + HTTPPostMode: true, // Bitcoin core only supports HTTP POST mode DisableTLS: true, // Bitcoin core does not provide TLS by default } // Notice the notification parameter is nil since notifications are @@ -30,7 +29,7 @@ func main() { defer client.Shutdown() // Get the current block count. - blockCount, err := client.GetBlockCount() + blockCount, err := client.Bulk().GetBlockCount() if err != nil { log.Fatal(err) } diff --git a/rpcclient/infrastructure.go b/rpcclient/infrastructure.go index 84b5b5d6f8..533291d10b 100644 --- a/rpcclient/infrastructure.go +++ b/rpcclient/infrastructure.go @@ -157,6 +157,9 @@ type Client struct { // disconnected indicated whether or not the server is disconnected. disconnected bool + // wether or not to batch requests, false unless changed by Bulk() + batch bool + // retryCount holds the number of times the client has tried to // reconnect to the RPC server. retryCount int64 @@ -1136,15 +1139,6 @@ type ConnConfig struct { // flag can be set to true to use basic HTTP POST requests instead. HTTPPostMode bool - - // Bulk RPC mode instructs the client to chunk requests to the server by - // issuing json-rpc requests instead of using btcd websockets - // Websockets are generally preferred when using a btcd server as some of the - // features of the client such notifications only work with websockets, - // however, not all servers support the websocket extensions, so this - // flag can be set to true to use basic HTTP POST requests instead. - JsonBulkRPC bool - // EnableBCInfoHacks is an option provided to enable compatibility hacks // when connecting to blockchain.info RPC server EnableBCInfoHacks bool @@ -1450,3 +1444,7 @@ func (c *Client) BackendVersion() (BackendVersion, error) { return *c.backendVersion, nil } + +func (c *Client) Bulk() *Client { + return c +} From fe2192dd39b57776527f273d2c69a4d949bd13cb Mon Sep 17 00:00:00 2001 From: Jake Sylvestre Date: Wed, 4 Dec 2019 17:53:38 -0500 Subject: [PATCH 09/21] batch chan --- rpcclient/examples/bitcoincorehttp/main.go | 2 +- .../examples/bitcoincorehttpbulk/main.go | 2 +- rpcclient/infrastructure.go | 21 ++++++++++++++++--- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/rpcclient/examples/bitcoincorehttp/main.go b/rpcclient/examples/bitcoincorehttp/main.go index f9fa890757..29a786be50 100644 --- a/rpcclient/examples/bitcoincorehttp/main.go +++ b/rpcclient/examples/bitcoincorehttp/main.go @@ -29,7 +29,7 @@ func main() { defer client.Shutdown() // Get the current block count. - blockCount, err := client.Bulk().GetBlockCount() + blockCount, err := client.Batch().GetBlockCount() if err != nil { log.Fatal(err) } diff --git a/rpcclient/examples/bitcoincorehttpbulk/main.go b/rpcclient/examples/bitcoincorehttpbulk/main.go index a524d7b2e6..c07dc453fb 100644 --- a/rpcclient/examples/bitcoincorehttpbulk/main.go +++ b/rpcclient/examples/bitcoincorehttpbulk/main.go @@ -29,7 +29,7 @@ func main() { {"jsonrpc": "2.0", "method": "decodescript", "params": ["ac"]}, {"jsonrpc": "2.0", "method": "getbestblockhash", "params": [], "id": "2"}, {"foo": "boo"}, - {"jsonrpc": "2.0", "method": "getblockcount", "id": "9"} + {"jsonrpc": "2.0", "method": "getblockcount", "id": "9"} /**/ ]`, // should produce invalid request for the `{"foo": "boo"}`. } diff --git a/rpcclient/infrastructure.go b/rpcclient/infrastructure.go index 533291d10b..0c750e592b 100644 --- a/rpcclient/infrastructure.go +++ b/rpcclient/infrastructure.go @@ -159,6 +159,7 @@ type Client struct { // wether or not to batch requests, false unless changed by Bulk() batch bool + batchChan chan *jsonRequest // retryCount holds the number of times the client has tried to // reconnect to the RPC server. @@ -866,7 +867,11 @@ func (c *Client) sendRequest(jReq *jsonRequest) { // POST mode, the command is issued via an HTTP client. Otherwise, // the command is issued via the asynchronous websocket channels. if c.config.HTTPPostMode { - c.sendPost(jReq) + if c.batch{ + c.batchChan <- jReq + } else { + c.sendPost(jReq) + } return } @@ -896,6 +901,10 @@ func (c *Client) sendRequest(jReq *jsonRequest) { // future. It handles both websocket and HTTP POST mode depending on the // configuration of the client. func (c *Client) sendCmd(cmd interface{}) chan *response { + rpcVersion := "1.0" + if c.batch { + rpcVersion = "2.0" + } // Get the method associated with the command. method, err := btcjson.CmdMethod(cmd) if err != nil { @@ -904,7 +913,7 @@ func (c *Client) sendCmd(cmd interface{}) chan *response { // Marshal the command. id := c.NextID() - marshalledJSON, err := btcjson.MarshalCmd("1.0", id, cmd) + marshalledJSON, err := btcjson.MarshalCmd(rpcVersion, id, cmd) if err != nil { return newFutureError(err) } @@ -918,6 +927,8 @@ func (c *Client) sendCmd(cmd interface{}) chan *response { marshalledJSON: marshalledJSON, responseChan: responseChan, } + + c.sendRequest(jReq) return responseChan @@ -1284,6 +1295,8 @@ func New(config *ConnConfig, ntfnHandlers *NotificationHandlers) (*Client, error httpClient: httpClient, requestMap: make(map[uint64]*list.Element), requestList: list.New(), + batch: false, + batchChan: make(chan *jsonRequest), ntfnHandlers: ntfnHandlers, ntfnState: newNotificationState(), sendChan: make(chan []byte, sendBufferSize), @@ -1445,6 +1458,8 @@ func (c *Client) BackendVersion() (BackendVersion, error) { return *c.backendVersion, nil } -func (c *Client) Bulk() *Client { +// make batch requests +func (c *Client) Batch()*Client { + c.batch = true //copy the client with changed batch setting return c } From a5d1ff5bf3e3f517bc8a27b61b9c3f81407ddece Mon Sep 17 00:00:00 2001 From: Jake Sylvestre Date: Thu, 5 Dec 2019 02:37:08 -0500 Subject: [PATCH 10/21] add batch method --- rpcclient/chain.go | 29 ++++++- rpcclient/examples/bitcoincorehttp/main.go | 25 ++++-- rpcclient/infrastructure.go | 94 +++++++++++++++++++--- 3 files changed, 130 insertions(+), 18 deletions(-) diff --git a/rpcclient/chain.go b/rpcclient/chain.go index 996d80458c..90e3ccc10f 100644 --- a/rpcclient/chain.go +++ b/rpcclient/chain.go @@ -9,7 +9,6 @@ import ( "bytes" "encoding/hex" "encoding/json" - "github.com/btcsuite/btcd/btcjson" "github.com/btcsuite/btcd/chaincfg/chainhash" "github.com/btcsuite/btcd/wire" @@ -982,3 +981,31 @@ func (c *Client) GetCFilterHeader(blockHash *chainhash.Hash, filterType wire.FilterType) (*wire.MsgCFHeaders, error) { return c.GetCFilterHeaderAsync(blockHash, filterType).Receive() } + +type FutureGetBulkResult chan *response + +type IndividualBulkResult struct { + Result interface{} `json:"result"` + Error string `json:"error"` + Id uint64 `json:"id"` +} + +type BulkResult = map[uint64]IndividualBulkResult + +// Receive waits for the response promised by the future and returns the hash of +// the best block in the longest block chain. +func (r FutureGetBulkResult) Receive() (BulkResult, error) { + m := make(BulkResult) + res, err := receiveFuture(r) + var arr []IndividualBulkResult + err = json.Unmarshal(res, &arr) + if err != nil { + return nil, err + } + + for _, results := range arr { + m[results.Id] = results + } + + return m, nil +} \ No newline at end of file diff --git a/rpcclient/examples/bitcoincorehttp/main.go b/rpcclient/examples/bitcoincorehttp/main.go index 29a786be50..fd7a6495c8 100644 --- a/rpcclient/examples/bitcoincorehttp/main.go +++ b/rpcclient/examples/bitcoincorehttp/main.go @@ -5,9 +5,9 @@ package main import ( - "log" - + "fmt" "github.com/btcsuite/btcd/rpcclient" + "log" ) func main() { @@ -29,9 +29,20 @@ func main() { defer client.Shutdown() // Get the current block count. - blockCount, err := client.Batch().GetBlockCount() - if err != nil { - log.Fatal(err) - } - log.Printf("Block count: %d", blockCount) + batchClient := client.Batch() + + // batch mode requires async requests + blockCount := batchClient.GetBlockCountAsync() + block1 := batchClient.GetBlockHashAsync(1) + batchClient.GetBlockHashAsync(2) + batchClient.GetBlockHashAsync(3) + block4 := batchClient.GetBlockHashAsync(4) + difficulty := batchClient.GetDifficultyAsync() + + batchClient.Send() + //result, err + fmt.Println(blockCount.Receive()) + fmt.Println(block1.Receive()) + fmt.Println(block4.Receive()) + fmt.Println(difficulty.Receive()) } diff --git a/rpcclient/infrastructure.go b/rpcclient/infrastructure.go index 0c750e592b..054925a5e2 100644 --- a/rpcclient/infrastructure.go +++ b/rpcclient/infrastructure.go @@ -159,7 +159,7 @@ type Client struct { // wether or not to batch requests, false unless changed by Bulk() batch bool - batchChan chan *jsonRequest + batchList *list.List // retryCount holds the number of times the client has tried to // reconnect to the RPC server. @@ -218,8 +218,13 @@ func (c *Client) addRequest(jReq *jsonRequest) error { default: } - element := c.requestList.PushBack(jReq) - c.requestMap[jReq.id] = element + if !c.batch { + element := c.requestList.PushBack(jReq) + c.requestMap[jReq.id] = element + } else { + element := c.batchList.PushBack(jReq) + c.requestMap[jReq.id] = element + } return nil } @@ -741,7 +746,12 @@ func (c *Client) handleSendPostMessage(details *sendPostDetails) { // Try to unmarshal the response as a regular JSON-RPC response. var resp rawResponse - err = json.Unmarshal(respBytes, &resp) + var batchResponse json.RawMessage + if c.batch { + err = json.Unmarshal(respBytes, &batchResponse) + }else { + err = json.Unmarshal(respBytes, &resp) + } if err != nil { // When the response itself isn't a valid JSON-RPC response // return an error which includes the HTTP status code and raw @@ -751,8 +761,14 @@ func (c *Client) handleSendPostMessage(details *sendPostDetails) { jReq.responseChan <- &response{err: err} return } - - res, err := resp.result() + var res []byte + if c.batch { + // errors must be dealt with downstream since a whole request cannot + // "error out" other than through the status code error handled above + res, err = batchResponse, nil + }else { + res, err = resp.result() + } jReq.responseChan <- &response{result: res, err: err} } @@ -868,7 +884,9 @@ func (c *Client) sendRequest(jReq *jsonRequest) { // the command is issued via the asynchronous websocket channels. if c.config.HTTPPostMode { if c.batch{ - c.batchChan <- jReq + if err := c.addRequest(jReq); err != nil { + log.Warn(err) + } } else { c.sendPost(jReq) } @@ -1296,7 +1314,7 @@ func New(config *ConnConfig, ntfnHandlers *NotificationHandlers) (*Client, error requestMap: make(map[uint64]*list.Element), requestList: list.New(), batch: false, - batchChan: make(chan *jsonRequest), + batchList: list.New(), ntfnHandlers: ntfnHandlers, ntfnState: newNotificationState(), sendChan: make(chan []byte, sendBufferSize), @@ -1459,7 +1477,63 @@ func (c *Client) BackendVersion() (BackendVersion, error) { } // make batch requests -func (c *Client) Batch()*Client { +func (c *Client) Batch() Client { c.batch = true //copy the client with changed batch setting - return c + c.start() + return *c +} + + +func (c *Client) sendAsync() FutureGetBulkResult { + // convert the array of marshalled json requests to a single request we can send + responseChan := make(chan *response, 1) + marshalledRequest := []byte("[") + for iter := c.batchList.Front(); iter != nil; iter = iter.Next() { + request := iter.Value.(*jsonRequest) + marshalledRequest = append(marshalledRequest, request.marshalledJSON...) + marshalledRequest = append(marshalledRequest, []byte(",")...) + } + if len(marshalledRequest) > 0 { + marshalledRequest = marshalledRequest[:len(marshalledRequest)-1] + } + marshalledRequest = append(marshalledRequest, []byte("]")...) + request := jsonRequest{ + id: c.NextID(), + method: "", + cmd: nil, + marshalledJSON: marshalledRequest, + responseChan: responseChan, + } + c.sendPost(&request) + return responseChan +} + +// send batch requests +func (c *Client) Send() { + result, err := c.sendAsync().Receive() + + if err != nil{ + log.Error(err) + } + + for iter := c.batchList.Front(); iter != nil; iter = iter.Next() { + var requestError error + request := iter.Value.(*jsonRequest) + individualResult := result[request.id] + fullResult, err := json.Marshal(individualResult.Result) + if err != nil { + log.Error(err) + } + + if individualResult.Error != "" { + requestError = errors.New(individualResult.Error) + } + + result := response{ + result: fullResult, + err: requestError, + } + request.responseChan <- &result + } + c.batchList = list.New() } From 6278ae4503781af139f7cbf0fed02f06f085936f Mon Sep 17 00:00:00 2001 From: Jake Sylvestre Date: Thu, 5 Dec 2019 04:46:08 -0500 Subject: [PATCH 11/21] (rpcclient) add batch requests --- btcjson/jsonrpc.go | 4 +- config.go | 2 +- mempool/estimatefee.go | 2 +- rpcclient/chain.go | 8 +- rpcclient/examples/bitcoincorehttp/main.go | 28 ++---- .../examples/bitcoincorehttpbulk/README.md | 7 +- .../examples/bitcoincorehttpbulk/main.go | 95 ++++++++----------- rpcclient/infrastructure.go | 20 ++-- rpcserver.go | 4 +- 9 files changed, 67 insertions(+), 103 deletions(-) diff --git a/btcjson/jsonrpc.go b/btcjson/jsonrpc.go index ecfd4dbac9..6189f6d215 100644 --- a/btcjson/jsonrpc.go +++ b/btcjson/jsonrpc.go @@ -166,7 +166,6 @@ type Response struct { ID *interface{} `json:"id"` } - // NewResponse returns a new JSON-RPC response object given the provided rpc // version, id, marshalled result, and RPC error. This function is only // provided in case the caller wants to construct raw responses for some reason. @@ -191,7 +190,6 @@ func NewResponse(rpcVersion string, id interface{}, marshalledResult []byte, rpc }, nil } - // MarshalResponse marshals the passed rpc version, id, result, and RPCError to // a JSON-RPC response byte slice that is suitable for transmission to a // JSON-RPC client. @@ -209,4 +207,4 @@ func MarshalResponse(rpcVersion string, id interface{}, result interface{}, rpcE return nil, err } return json.Marshal(&response) -} \ No newline at end of file +} diff --git a/config.go b/config.go index facbea6ac6..3bcc65dacd 100644 --- a/config.go +++ b/config.go @@ -911,7 +911,7 @@ func loadConfig() (*config, []string, error) { // Only allow TLS to be disabled if the RPC is bound to localhost // addresses. - if false && !cfg.DisableRPC && cfg.DisableTLS { + if !cfg.DisableRPC && cfg.DisableTLS { allowedTLSListeners := map[string]struct{}{ "localhost": {}, "127.0.0.1": {}, diff --git a/mempool/estimatefee.go b/mempool/estimatefee.go index 53933469b6..3546f6b356 100644 --- a/mempool/estimatefee.go +++ b/mempool/estimatefee.go @@ -47,7 +47,7 @@ const ( bytePerKb = 1000 - btcPerSatoshi = 1E-8 + btcPerSatoshi = 1e-8 ) var ( diff --git a/rpcclient/chain.go b/rpcclient/chain.go index 90e3ccc10f..3d217c2f67 100644 --- a/rpcclient/chain.go +++ b/rpcclient/chain.go @@ -985,9 +985,9 @@ func (c *Client) GetCFilterHeader(blockHash *chainhash.Hash, type FutureGetBulkResult chan *response type IndividualBulkResult struct { - Result interface{} `json:"result"` - Error string `json:"error"` - Id uint64 `json:"id"` + Result interface{} `json:"result"` + Error string `json:"error"` + Id uint64 `json:"id"` } type BulkResult = map[uint64]IndividualBulkResult @@ -1008,4 +1008,4 @@ func (r FutureGetBulkResult) Receive() (BulkResult, error) { } return m, nil -} \ No newline at end of file +} diff --git a/rpcclient/examples/bitcoincorehttp/main.go b/rpcclient/examples/bitcoincorehttp/main.go index fd7a6495c8..489770a25b 100644 --- a/rpcclient/examples/bitcoincorehttp/main.go +++ b/rpcclient/examples/bitcoincorehttp/main.go @@ -5,18 +5,17 @@ package main import ( - "fmt" - "github.com/btcsuite/btcd/rpcclient" "log" + + "github.com/btcsuite/btcd/rpcclient" ) func main() { // Connect to local bitcoin core RPC server using HTTP POST mode. connCfg := &rpcclient.ConnConfig{ - Host: "127.0.0.1:8332", + Host: "localhost:8332", User: "yourrpcuser", Pass: "yourrpcpass", - DisableConnectOnNew: true, HTTPPostMode: true, // Bitcoin core only supports HTTP POST mode DisableTLS: true, // Bitcoin core does not provide TLS by default } @@ -29,20 +28,9 @@ func main() { defer client.Shutdown() // Get the current block count. - batchClient := client.Batch() - - // batch mode requires async requests - blockCount := batchClient.GetBlockCountAsync() - block1 := batchClient.GetBlockHashAsync(1) - batchClient.GetBlockHashAsync(2) - batchClient.GetBlockHashAsync(3) - block4 := batchClient.GetBlockHashAsync(4) - difficulty := batchClient.GetDifficultyAsync() - - batchClient.Send() - //result, err - fmt.Println(blockCount.Receive()) - fmt.Println(block1.Receive()) - fmt.Println(block4.Receive()) - fmt.Println(difficulty.Receive()) + blockCount, err := client.GetBlockCount() + if err != nil { + log.Fatal(err) + } + log.Printf("Block count: %d", blockCount) } diff --git a/rpcclient/examples/bitcoincorehttpbulk/README.md b/rpcclient/examples/bitcoincorehttpbulk/README.md index 4d1f0adfe1..44e65f7864 100644 --- a/rpcclient/examples/bitcoincorehttpbulk/README.md +++ b/rpcclient/examples/bitcoincorehttpbulk/README.md @@ -1,9 +1,8 @@ -Bitcoin Core HTTP POST Example +Bitcoin Core Batch HTTP POST Example ============================== This example shows how to use the rpcclient package to connect to a Bitcoin -Core RPC server using HTTP POST mode with TLS disabled and gets the current -block count. +Core RPC server using HTTP POST mode with TLS disabled and uses the batch json rpc mode ## Running the Example @@ -30,4 +29,4 @@ $ go run *.go ## License -This example is licensed under the [copyfree](http://copyfree.org) ISC License. +This example is licensed under the [copyfree](http://copyfree.org) ISC License. \ No newline at end of file diff --git a/rpcclient/examples/bitcoincorehttpbulk/main.go b/rpcclient/examples/bitcoincorehttpbulk/main.go index c07dc453fb..e77d8cf3a2 100644 --- a/rpcclient/examples/bitcoincorehttpbulk/main.go +++ b/rpcclient/examples/bitcoincorehttpbulk/main.go @@ -1,65 +1,48 @@ +// Copyright (c) 2014-2017 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + package main import ( - "bytes" "fmt" - "io/ioutil" - "net/http" + "github.com/btcsuite/btcd/rpcclient" + "log" ) func main() { - const ( - url = "http://localhost:8332" - rpcUser = "yourrpcuser" - rpcPass = "yourrpcpass" - ) - - // populate request set - reqs := []string{ - `{}`, - `[]`, - `[1]`, - `[1,2,3]`, - `{"foo": "boo"}`, // should be an invalid request - `{"jsonrpc": "1.0", "foo": "boo", "id": "1"}`, - `{"jsonrpc": "1.0", "method": "getblockcount", "params": [], "id": "1"}`, - `{"jsonrpc": "1.0", "method": "getblockcount", "params": "a", "id": "1"}`, // should be invalid since params is neither an array nor a json object. - `[ - {"jsonrpc": "2.0", "method": "getblockcount", "params": [], "id": "1"}, - {"jsonrpc": "2.0", "method": "decodescript", "params": ["ac"]}, - {"jsonrpc": "2.0", "method": "getbestblockhash", "params": [], "id": "2"}, - {"foo": "boo"}, - {"jsonrpc": "2.0", "method": "getblockcount", "id": "9"} /**/ - ]`, // should produce invalid request for the `{"foo": "boo"}`. + // Connect to local bitcoin core RPC server using HTTP POST mode. + connCfg := &rpcclient.ConnConfig{ + Host: "localhost:8332", + User: "yourrpcuser", + Pass: "yourrpcpass", + DisableConnectOnNew: true, + HTTPPostMode: true, // Bitcoin core only supports HTTP POST mode + DisableTLS: true, // Bitcoin core does not provide TLS by default } - - // Connect to local btcd RPC server using websockets. - - client := http.Client{} - - for _, jsonReq := range reqs { - bodyReader := bytes.NewReader([]byte(jsonReq)) - httpReq, err := http.NewRequest("POST", url, bodyReader) - if err != nil { - fmt.Println(err) - return - } - httpReq.Close = true - httpReq.Header.Set("Content-Type", "application/json") - httpReq.SetBasicAuth(rpcUser, rpcPass) - resp, err := client.Do(httpReq) - if err != nil { - fmt.Println("request:", jsonReq, "response:", err) - return - } - - respBytes, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - fmt.Println(err) - return - } - - fmt.Println("request:", jsonReq, "response:", string(respBytes)) + // Notice the notification parameter is nil since notifications are + // not supported in HTTP POST mode. + client, err := rpcclient.New(connCfg, nil) + if err != nil { + log.Fatal(err) } -} \ No newline at end of file + defer client.Shutdown() + + // Get the current block count. + batchClient := client.Batch() + + // batch mode requires async requests + blockCount := batchClient.GetBlockCountAsync() + block1 := batchClient.GetBlockHashAsync(1) + batchClient.GetBlockHashAsync(2) + batchClient.GetBlockHashAsync(3) + block4 := batchClient.GetBlockHashAsync(4) + difficulty := batchClient.GetDifficultyAsync() + + batchClient.Send() + //result, err + fmt.Println(blockCount.Receive()) + fmt.Println(block1.Receive()) + fmt.Println(block4.Receive()) + fmt.Println(difficulty.Receive()) +} diff --git a/rpcclient/infrastructure.go b/rpcclient/infrastructure.go index 054925a5e2..14bfc997e5 100644 --- a/rpcclient/infrastructure.go +++ b/rpcclient/infrastructure.go @@ -158,7 +158,7 @@ type Client struct { disconnected bool // wether or not to batch requests, false unless changed by Bulk() - batch bool + batch bool batchList *list.List // retryCount holds the number of times the client has tried to @@ -749,7 +749,7 @@ func (c *Client) handleSendPostMessage(details *sendPostDetails) { var batchResponse json.RawMessage if c.batch { err = json.Unmarshal(respBytes, &batchResponse) - }else { + } else { err = json.Unmarshal(respBytes, &resp) } if err != nil { @@ -766,7 +766,7 @@ func (c *Client) handleSendPostMessage(details *sendPostDetails) { // errors must be dealt with downstream since a whole request cannot // "error out" other than through the status code error handled above res, err = batchResponse, nil - }else { + } else { res, err = resp.result() } jReq.responseChan <- &response{result: res, err: err} @@ -883,7 +883,7 @@ func (c *Client) sendRequest(jReq *jsonRequest) { // POST mode, the command is issued via an HTTP client. Otherwise, // the command is issued via the asynchronous websocket channels. if c.config.HTTPPostMode { - if c.batch{ + if c.batch { if err := c.addRequest(jReq); err != nil { log.Warn(err) } @@ -946,7 +946,6 @@ func (c *Client) sendCmd(cmd interface{}) chan *response { responseChan: responseChan, } - c.sendRequest(jReq) return responseChan @@ -1313,8 +1312,8 @@ func New(config *ConnConfig, ntfnHandlers *NotificationHandlers) (*Client, error httpClient: httpClient, requestMap: make(map[uint64]*list.Element), requestList: list.New(), - batch: false, - batchList: list.New(), + batch: false, + batchList: list.New(), ntfnHandlers: ntfnHandlers, ntfnState: newNotificationState(), sendChan: make(chan []byte, sendBufferSize), @@ -1483,7 +1482,6 @@ func (c *Client) Batch() Client { return *c } - func (c *Client) sendAsync() FutureGetBulkResult { // convert the array of marshalled json requests to a single request we can send responseChan := make(chan *response, 1) @@ -1509,10 +1507,10 @@ func (c *Client) sendAsync() FutureGetBulkResult { } // send batch requests -func (c *Client) Send() { +func (c *Client) Send() { result, err := c.sendAsync().Receive() - if err != nil{ + if err != nil { log.Error(err) } @@ -1529,7 +1527,7 @@ func (c *Client) Send() { requestError = errors.New(individualResult.Error) } - result := response{ + result := response{ result: fullResult, err: requestError, } diff --git a/rpcserver.go b/rpcserver.go index 1506854767..0b9e246c15 100644 --- a/rpcserver.go +++ b/rpcserver.go @@ -3837,7 +3837,6 @@ type parsedRPCCmd struct { err *btcjson.RPCError } - // standardCmdResult checks that a parsed command is a standard Bitcoin JSON-RPC // command and runs the appropriate handler to reply to the command. Any // commands which are not recognized or not implemented will return an error @@ -3915,7 +3914,6 @@ func createMarshalledReply(rpcVersion string, id interface{}, result interface{} return btcjson.MarshalResponse(rpcVersion, id, result, jsonErr) } - // processRequest determines the incoming request type (single or batched), // parses it and returns a marshalled response. func (s *rpcServer) processRequest(request *btcjson.Request, isAdmin bool, closeChan <-chan struct{}) []byte { @@ -3924,7 +3922,7 @@ func (s *rpcServer) processRequest(request *btcjson.Request, isAdmin bool, close if !isAdmin { if _, ok := rpcLimited[request.Method]; !ok { - jsonErr = internalRPCError("limited user not " + + jsonErr = internalRPCError("limited user not "+ "authorized for this method", "") } } From bd91ca2b4f21eefb23a25c4d33e6ea30221da776 Mon Sep 17 00:00:00 2001 From: Jake Sylvestre Date: Thu, 5 Dec 2019 04:53:29 -0500 Subject: [PATCH 12/21] (rpcclient) dereference fix --- rpcclient/infrastructure.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rpcclient/infrastructure.go b/rpcclient/infrastructure.go index 14bfc997e5..65ced018c1 100644 --- a/rpcclient/infrastructure.go +++ b/rpcclient/infrastructure.go @@ -1476,10 +1476,10 @@ func (c *Client) BackendVersion() (BackendVersion, error) { } // make batch requests -func (c *Client) Batch() Client { +func (c *Client) Batch() *Client { c.batch = true //copy the client with changed batch setting c.start() - return *c + return c } func (c *Client) sendAsync() FutureGetBulkResult { From 301229fbdf1fa5d9892642860f03291ca978a7f5 Mon Sep 17 00:00:00 2001 From: Jake Sylvestre Date: Mon, 30 Dec 2019 03:11:15 -0500 Subject: [PATCH 13/21] Revert "tls unwrangle" This reverts commit c25e1e8afeaef50c2cd9da8289e9afa52e6ccd25. --- config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.go b/config.go index 3bcc65dacd..facbea6ac6 100644 --- a/config.go +++ b/config.go @@ -911,7 +911,7 @@ func loadConfig() (*config, []string, error) { // Only allow TLS to be disabled if the RPC is bound to localhost // addresses. - if !cfg.DisableRPC && cfg.DisableTLS { + if false && !cfg.DisableRPC && cfg.DisableTLS { allowedTLSListeners := map[string]struct{}{ "localhost": {}, "127.0.0.1": {}, From 935dfb7d2da8cb7f2c5d9ac333d04be730c2685d Mon Sep 17 00:00:00 2001 From: Jake Sylvestre Date: Mon, 30 Dec 2019 21:45:14 -0500 Subject: [PATCH 14/21] fix issues reviewed by @torkelrogstad --- rpcclient/chain.go | 3 +++ rpcclient/examples/bitcoincorehttpbulk/main.go | 8 +++++--- rpcclient/infrastructure.go | 11 ++++++----- rpcserver.go | 2 +- rpcwebsocket.go | 4 ++-- 5 files changed, 17 insertions(+), 11 deletions(-) diff --git a/rpcclient/chain.go b/rpcclient/chain.go index 3d217c2f67..dab029dd7e 100644 --- a/rpcclient/chain.go +++ b/rpcclient/chain.go @@ -997,6 +997,9 @@ type BulkResult = map[uint64]IndividualBulkResult func (r FutureGetBulkResult) Receive() (BulkResult, error) { m := make(BulkResult) res, err := receiveFuture(r) + if err != nil { + return nil, err + } var arr []IndividualBulkResult err = json.Unmarshal(res, &arr) if err != nil { diff --git a/rpcclient/examples/bitcoincorehttpbulk/main.go b/rpcclient/examples/bitcoincorehttpbulk/main.go index e77d8cf3a2..e3291d5ec6 100644 --- a/rpcclient/examples/bitcoincorehttpbulk/main.go +++ b/rpcclient/examples/bitcoincorehttpbulk/main.go @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2017 The btcsuite developers +// Copyright (c) 2014-2019 The btcsuite developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -23,10 +23,11 @@ func main() { // Notice the notification parameter is nil since notifications are // not supported in HTTP POST mode. client, err := rpcclient.New(connCfg, nil) + defer client.Shutdown() + if err != nil { log.Fatal(err) } - defer client.Shutdown() // Get the current block count. batchClient := client.Batch() @@ -39,8 +40,9 @@ func main() { block4 := batchClient.GetBlockHashAsync(4) difficulty := batchClient.GetDifficultyAsync() + // sends all queued batch requests batchClient.Send() - //result, err + fmt.Println(blockCount.Receive()) fmt.Println(block1.Receive()) fmt.Println(block4.Receive()) diff --git a/rpcclient/infrastructure.go b/rpcclient/infrastructure.go index 65ced018c1..fe754873f0 100644 --- a/rpcclient/infrastructure.go +++ b/rpcclient/infrastructure.go @@ -1475,7 +1475,7 @@ func (c *Client) BackendVersion() (BackendVersion, error) { return *c.backendVersion, nil } -// make batch requests +// Batch makes batch requests func (c *Client) Batch() *Client { c.batch = true //copy the client with changed batch setting c.start() @@ -1506,12 +1506,12 @@ func (c *Client) sendAsync() FutureGetBulkResult { return responseChan } -// send batch requests -func (c *Client) Send() { +// Send sends batch requests +func (c *Client) Send() error { result, err := c.sendAsync().Receive() if err != nil { - log.Error(err) + return err } for iter := c.batchList.Front(); iter != nil; iter = iter.Next() { @@ -1520,7 +1520,7 @@ func (c *Client) Send() { individualResult := result[request.id] fullResult, err := json.Marshal(individualResult.Result) if err != nil { - log.Error(err) + return err } if individualResult.Error != "" { @@ -1534,4 +1534,5 @@ func (c *Client) Send() { request.responseChan <- &result } c.batchList = list.New() + return nil } diff --git a/rpcserver.go b/rpcserver.go index 0b9e246c15..d02f15cb3f 100644 --- a/rpcserver.go +++ b/rpcserver.go @@ -3931,7 +3931,7 @@ func (s *rpcServer) processRequest(request *btcjson.Request, isAdmin bool, close if request.Method == "" || request.Params == nil { jsonErr = &btcjson.RPCError{ Code: btcjson.ErrRPCInvalidRequest.Code, - Message: fmt.Sprintf("Invalid request: malformed"), + Message: "Invalid request: malformed", } msg, err := createMarshalledReply(request.Jsonrpc, request.ID, result, jsonErr) if err != nil { diff --git a/rpcwebsocket.go b/rpcwebsocket.go index 5cc1ee1d89..4853eb4586 100644 --- a/rpcwebsocket.go +++ b/rpcwebsocket.go @@ -1356,7 +1356,7 @@ out: if req.Method == "" || req.Params == nil { jsonErr := &btcjson.RPCError{ Code: btcjson.ErrRPCInvalidRequest.Code, - Message: fmt.Sprintf("Invalid request: malformed"), + Message: "Invalid request: malformed", } reply, err := createMarshalledReply(req.Jsonrpc, req.ID, nil, jsonErr) if err != nil { @@ -1587,7 +1587,7 @@ out: if req.Method == "" || req.Params == nil { jsonErr := &btcjson.RPCError{ Code: btcjson.ErrRPCInvalidRequest.Code, - Message: fmt.Sprintf("Invalid request: malformed"), + Message: "Invalid request: malformed", } reply, err := createMarshalledReply(req.Jsonrpc, req.ID, nil, jsonErr) if err != nil { From ef60e3084059388d41a1f2ab1a00aab9f62232b1 Mon Sep 17 00:00:00 2001 From: Jake Sylvestre Date: Mon, 30 Dec 2019 21:53:31 -0500 Subject: [PATCH 15/21] fix nits --- btcjson/jsonrpc.go | 35 ++++++++++--------- mempool/estimatefee.go | 2 +- rpcclient/chain.go | 3 +- .../examples/bitcoincorehttpbulk/README.md | 3 +- 4 files changed, 21 insertions(+), 22 deletions(-) diff --git a/btcjson/jsonrpc.go b/btcjson/jsonrpc.go index 6189f6d215..8b9ba33f84 100644 --- a/btcjson/jsonrpc.go +++ b/btcjson/jsonrpc.go @@ -96,23 +96,21 @@ func (request *Request) UnmarshalJSON(b []byte) error { if !hasParams { // set the request param to an empty array if it is ommited in the request request.Params = []json.RawMessage{} - } - if hasParams { // assert the request params is an array of data - params, paramsOk := paramsValue.([]interface{}) - if paramsOk { - rawParams := make([]json.RawMessage, 0, len(params)) - for _, param := range params { - marshalledParam, err := json.Marshal(param) - if err != nil { - return err - } - rawMessage := json.RawMessage(marshalledParam) - rawParams = append(rawParams, rawMessage) + } else if params, paramsOk := paramsValue.([]interface{}); paramsOk { + rawParams := make([]json.RawMessage, 0, len(params)) + for _, param := range params { + marshalledParam, err := json.Marshal(param) + if err != nil { + return err } - - request.Params = rawParams + rawMessage := json.RawMessage(marshalledParam) + rawParams = append(rawParams, rawMessage) } + + request.Params = rawParams + } else { + return Error{Description: "No response received"} } return nil @@ -129,7 +127,8 @@ func (request *Request) UnmarshalJSON(b []byte) error { func NewRequest(rpcVersion string, id interface{}, method string, params []interface{}) (*Request, error) { // default to JSON-RPC 1.0 if RPC type is not specified if rpcVersion != "2.0" && rpcVersion != "1.0" { - rpcVersion = "1.0" + str := fmt.Sprintf("rpcversion '%s' is invalid", rpcVersion) + return nil, makeError(ErrInvalidType, str) } if !IsValidIDType(id) { @@ -173,7 +172,8 @@ type Response struct { // response to send over the wire with the MarshalResponse function. func NewResponse(rpcVersion string, id interface{}, marshalledResult []byte, rpcErr *RPCError) (*Response, error) { if rpcVersion != "2.0" && rpcVersion != "1.0" { - rpcVersion = "1.0" + str := fmt.Sprintf("rpcversion '%s' is invalid", rpcVersion) + return nil, makeError(ErrInvalidType, str) } if !IsValidIDType(id) { @@ -195,7 +195,8 @@ func NewResponse(rpcVersion string, id interface{}, marshalledResult []byte, rpc // JSON-RPC client. func MarshalResponse(rpcVersion string, id interface{}, result interface{}, rpcErr *RPCError) ([]byte, error) { if rpcVersion != "2.0" && rpcVersion != "1.0" { - rpcVersion = "1.0" + str := fmt.Sprintf("rpcversion '%s' is invalid", rpcVersion) + return nil, makeError(ErrInvalidType, str) } marshalledResult, err := json.Marshal(result) diff --git a/mempool/estimatefee.go b/mempool/estimatefee.go index 3546f6b356..53933469b6 100644 --- a/mempool/estimatefee.go +++ b/mempool/estimatefee.go @@ -47,7 +47,7 @@ const ( bytePerKb = 1000 - btcPerSatoshi = 1e-8 + btcPerSatoshi = 1E-8 ) var ( diff --git a/rpcclient/chain.go b/rpcclient/chain.go index dab029dd7e..49a02912a9 100644 --- a/rpcclient/chain.go +++ b/rpcclient/chain.go @@ -18,8 +18,7 @@ import ( // GetBestBlockAsync RPC invocation (or an applicable error). type FutureGetBestBlockHashResult chan *response -// Receive waits for the response promised by the future and returns the hash of -// the best block in the longest block chain. +// Receive waits for the response promised by the future and returns error if present func (r FutureGetBestBlockHashResult) Receive() (*chainhash.Hash, error) { res, err := receiveFuture(r) if err != nil { diff --git a/rpcclient/examples/bitcoincorehttpbulk/README.md b/rpcclient/examples/bitcoincorehttpbulk/README.md index 44e65f7864..df420e2c57 100644 --- a/rpcclient/examples/bitcoincorehttpbulk/README.md +++ b/rpcclient/examples/bitcoincorehttpbulk/README.md @@ -1,8 +1,7 @@ Bitcoin Core Batch HTTP POST Example ============================== -This example shows how to use the rpcclient package to connect to a Bitcoin -Core RPC server using HTTP POST mode with TLS disabled and uses the batch json rpc mode +This example shows how to use the rpclient package to connect to a Bitcoin Core RPC server using HTTP POST mode with TLS disabled and uses the batch JSON RPC mode ## Running the Example From b1f9e419bdb40f705752cec19d74a953853c29ab Mon Sep 17 00:00:00 2001 From: Jake Sylvestre Date: Tue, 21 Jan 2020 21:49:57 -0500 Subject: [PATCH 16/21] unfalse --- config.go | 2 +- mempool/estimatefee.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config.go b/config.go index facbea6ac6..3bcc65dacd 100644 --- a/config.go +++ b/config.go @@ -911,7 +911,7 @@ func loadConfig() (*config, []string, error) { // Only allow TLS to be disabled if the RPC is bound to localhost // addresses. - if false && !cfg.DisableRPC && cfg.DisableTLS { + if !cfg.DisableRPC && cfg.DisableTLS { allowedTLSListeners := map[string]struct{}{ "localhost": {}, "127.0.0.1": {}, diff --git a/mempool/estimatefee.go b/mempool/estimatefee.go index 53933469b6..3546f6b356 100644 --- a/mempool/estimatefee.go +++ b/mempool/estimatefee.go @@ -47,7 +47,7 @@ const ( bytePerKb = 1000 - btcPerSatoshi = 1E-8 + btcPerSatoshi = 1e-8 ) var ( From 1d228c1a921b37ecd59f3ded15d267463fe94893 Mon Sep 17 00:00:00 2001 From: Jake Sylvestre Date: Sun, 2 Feb 2020 00:07:35 -0500 Subject: [PATCH 17/21] ci fixes --- btcjson/example_test.go | 6 +++--- btcjson/jsonrpc_test.go | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/btcjson/example_test.go b/btcjson/example_test.go index 29883c3a68..10191c6dfb 100644 --- a/btcjson/example_test.go +++ b/btcjson/example_test.go @@ -97,7 +97,7 @@ func ExampleUnmarshalCmd() { func ExampleMarshalResponse() { // Marshal a new JSON-RPC response. For example, this is a response // to a getblockheight request. - marshalledBytes, err := btcjson.MarshalResponse(1, 350001, nil) + marshalledBytes, err := btcjson.MarshalResponse("1.0",1, 350001, nil) if err != nil { fmt.Println(err) return @@ -109,7 +109,7 @@ func ExampleMarshalResponse() { fmt.Printf("%s\n", marshalledBytes) // Output: - // {"result":350001,"error":null,"id":1} + // {"jsonrpc":"1.0","result":350001,"error":null,"id":1} } // This example demonstrates how to unmarshal a JSON-RPC response and then @@ -118,7 +118,7 @@ func Example_unmarshalResponse() { // Ordinarily this would be read from the wire, but for this example, // it is hard coded here for clarity. This is an example response to a // getblockheight request. - data := []byte(`{"result":350001,"error":null,"id":1}`) + data := []byte(`{"jsonrpc":"1.0","result":350001,"error":null,"id":1}`) // Unmarshal the raw bytes from the wire into a JSON-RPC response. var response btcjson.Response diff --git a/btcjson/jsonrpc_test.go b/btcjson/jsonrpc_test.go index 7a5d75618c..eb8a094aa9 100644 --- a/btcjson/jsonrpc_test.go +++ b/btcjson/jsonrpc_test.go @@ -68,7 +68,7 @@ func TestMarshalResponse(t *testing.T) { name: "ordinary bool result with no error", result: true, jsonErr: nil, - expected: []byte(`{"result":true,"error":null,"id":1}`), + expected: []byte(`{"jsonrpc":"1.0","result":true,"error":null,"id":1}`), }, { name: "result with error", @@ -76,14 +76,14 @@ func TestMarshalResponse(t *testing.T) { jsonErr: func() *btcjson.RPCError { return btcjson.NewRPCError(btcjson.ErrRPCBlockNotFound, "123 not found") }(), - expected: []byte(`{"result":null,"error":{"code":-5,"message":"123 not found"},"id":1}`), + expected: []byte(`{"jsonrpc":"1.0","result":null,"error":{"code":-5,"message":"123 not found"},"id":1}`), }, } t.Logf("Running %d tests", len(tests)) for i, test := range tests { _, _ = i, test - marshalled, err := btcjson.MarshalResponse(testID, test.result, test.jsonErr) + marshalled, err := btcjson.MarshalResponse("1.0", testID, test.result, test.jsonErr) if err != nil { t.Errorf("Test #%d (%s) unexpected error: %v", i, test.name, err) @@ -104,7 +104,7 @@ func TestMiscErrors(t *testing.T) { // Force an error in NewRequest by giving it a parameter type that is // not supported. - _, err := btcjson.NewRequest(nil, "test", []interface{}{make(chan int)}) + _, err := btcjson.NewRequest("1.0", nil, "test", []interface{}{make(chan int)}) if err == nil { t.Error("NewRequest: did not receive error") return @@ -113,7 +113,7 @@ func TestMiscErrors(t *testing.T) { // Force an error in MarshalResponse by giving it an id type that is not // supported. wantErr := btcjson.Error{ErrorCode: btcjson.ErrInvalidType} - _, err = btcjson.MarshalResponse(make(chan int), nil, nil) + _, err = btcjson.MarshalResponse("1.0", make(chan int), nil, nil) if jerr, ok := err.(btcjson.Error); !ok || jerr.ErrorCode != wantErr.ErrorCode { t.Errorf("MarshalResult: did not receive expected error - got "+ "%v (%[1]T), want %v (%[2]T)", err, wantErr) @@ -122,7 +122,7 @@ func TestMiscErrors(t *testing.T) { // Force an error in MarshalResponse by giving it a result type that // can't be marshalled. - _, err = btcjson.MarshalResponse(1, make(chan int), nil) + _, err = btcjson.MarshalResponse("1.0",1, make(chan int), nil) if _, ok := err.(*json.UnsupportedTypeError); !ok { wantErr := &json.UnsupportedTypeError{} t.Errorf("MarshalResult: did not receive expected error - got "+ From 6521f5e63e1a45471f918acf88fc854ff61881db Mon Sep 17 00:00:00 2001 From: Jake Sylvestre Date: Sun, 2 Feb 2020 00:13:15 -0500 Subject: [PATCH 18/21] uncache --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5063496735..f2bcae343a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: go cache: directories: - - $GOCACHE +# - $GOCACHE - $GOPATH - $GOPATH/pkg/mod - $GOPATH/github.com/golang From eba3b3f6065ae11142196741227a62cacccaaeec Mon Sep 17 00:00:00 2001 From: Jake Sylvestre Date: Sun, 2 Feb 2020 00:21:59 -0500 Subject: [PATCH 19/21] Revert "uncache" This reverts commit 6521f5e63e1a45471f918acf88fc854ff61881db. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f2bcae343a..5063496735 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: go cache: directories: -# - $GOCACHE + - $GOCACHE - $GOPATH - $GOPATH/pkg/mod - $GOPATH/github.com/golang From 7855b3068950c0dbf6c18fa8eec65b26f3587bb0 Mon Sep 17 00:00:00 2001 From: Jake Sylvestre Date: Sun, 2 Feb 2020 00:23:02 -0500 Subject: [PATCH 20/21] bust cache --- goclean.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/goclean.sh b/goclean.sh index 6d0e0ca104..95f81881f2 100755 --- a/goclean.sh +++ b/goclean.sh @@ -28,4 +28,4 @@ test -z "$(gometalinter.v2 -j 4 --disable-all \ --enable=gosimple \ --enable=unconvert \ --deadline=10m $linter_targets 2>&1 | grep -v 'ALL_CAPS\|OP_' 2>&1 | tee /dev/stderr)" -GO111MODULE=on go test -tags="rpctest" $linter_targets +GO111MODULE=on go test -count=1 -tags="rpctest" $linter_targets From 4012d26a609b7051e5d92f88a803980e017876d2 Mon Sep 17 00:00:00 2001 From: Jake Sylvestre Date: Sat, 8 Feb 2020 19:50:40 -0500 Subject: [PATCH 21/21] remove gocaching as per @pavel in fe003e236663978c5c792e0ed077d5e83b4db6be --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5063496735..5a611c3960 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,6 @@ language: go cache: directories: - $GOCACHE - - $GOPATH - $GOPATH/pkg/mod - $GOPATH/github.com/golang - $GOPATH/gopkg.in/alecthomas