From 37551535712c267141f425fcb56ec6c474e46123 Mon Sep 17 00:00:00 2001 From: Stefan Eissing Date: Thu, 15 Feb 2024 16:22:53 +0100 Subject: [PATCH] lib: Curl_read/Curl_write clarifications - replace `Curl_read()`, `Curl_write()` and `Curl_nwrite()` to clarify when and at what level they operate - send/recv of transfer related data is now done via `Curl_xfer_send()/Curl_xfer_recv()` which no longer has socket/socketindex as parameter. It decides on the transfer setup of `conn->sockfd` and `conn->writesockfd` on which connection filter chain to operate. - send/recv on a specific connection filter chain is done via `Curl_conn_send()/Curl_conn_recv()` which get the socket index as parameter. - rename `Curl_setup_transfer()` to `Curl_xfer_setup()` for naming consistency - clarify that the special CURLE_AGAIN hangling to return `CURLE_OK` with length 0 only applies to `Curl_xfer_send()` and CURLE_AGAIN is returned by all other send() variants. - fix a bug in websocket `curl_ws_recv()` that mixed up data when it arrived in more than a single chunk The method for sending not just raw bytes, but bytes that are either "headers" or "body". The send abstraction stack, to to bottom, now is: * `Curl_req_send()`: has parameter to indicate amount of header bytes, buffers all data. * `Curl_xfer_send()`: knows on which socket index to send, returns amount of bytes sent. * `Curl_conn_send()`: called with socket index, returns amount of bytes sent. In addition there is `Curl_req_flush()` for writing out all buffered bytes. `Curl_req_send()` is active for requests without body, `Curl_buffer_send()` still being used for others. This is because the special quirks need to be addressed in future parts: * `expect-100` handling * `Curl_fillreadbuffer()` needs to add directly to the new `data->req.sendbuf` * special body handlings, like `chunked` encodings and line end conversions will be moved into something like a Client Reader. In functions of the pattern `CURLcode xxx_send(..., ssize_t *written)`, replace the `ssize_t` with a `size_t`. It makes no sense to allow for negative values as the returned `CURLcode` already specifies error conditions. This allows easier handling of lengths without casting. Closes #12964 --- lib/c-hyper.c | 9 +-- lib/cf-h1-proxy.c | 7 +- lib/cf-haproxy.c | 5 +- lib/cfilters.c | 4 +- lib/cfilters.h | 2 +- lib/dict.c | 2 +- lib/easy.c | 6 +- lib/easyif.h | 2 +- lib/gopher.c | 4 +- lib/http.c | 90 +++++++++++------------ lib/http.h | 8 +-- lib/krb5.c | 8 +-- lib/mqtt.c | 4 +- lib/pingpong.c | 10 +-- lib/request.c | 173 +++++++++++++++++++++++++++++++++++++++++---- lib/request.h | 36 +++++++++- lib/rtsp.c | 5 +- lib/smb.c | 8 +-- lib/smtp.c | 3 +- lib/telnet.c | 12 ++-- lib/transfer.c | 34 ++++----- lib/transfer.h | 2 +- lib/url.c | 3 - lib/vssh/libssh2.c | 6 +- lib/ws.c | 13 ++-- 25 files changed, 314 insertions(+), 142 deletions(-) diff --git a/lib/c-hyper.c b/lib/c-hyper.c index f8e36e909d07ca..ae51c5ca403862 100644 --- a/lib/c-hyper.c +++ b/lib/c-hyper.c @@ -112,11 +112,13 @@ size_t Curl_hyper_send(void *userp, hyper_context *ctx, struct hyp_io_ctx *io_ctx = userp; struct Curl_easy *data = io_ctx->data; CURLcode result; - ssize_t nwrote; + size_t nwrote; DEBUGF(infof(data, "Curl_hyper_send(%zu)", buflen)); result = Curl_conn_send(data, io_ctx->sockindex, (void *)buf, buflen, &nwrote); + if(!result && !nwrote) + result = CURLE_AGAIN; if(result == CURLE_AGAIN) { DEBUGF(infof(data, "Curl_hyper_send(%zu) -> EAGAIN", buflen)); /* would block, register interest */ @@ -759,7 +761,6 @@ static int uploadstreamed(void *userdata, hyper_context *ctx, */ static CURLcode bodysend(struct Curl_easy *data, - struct connectdata *conn, hyper_headers *headers, hyper_request *hyperreq, Curl_HttpReq httpreq) @@ -772,7 +773,7 @@ static CURLcode bodysend(struct Curl_easy *data, else { hyper_body *body; Curl_dyn_init(&req, DYN_HTTP_REQUEST); - result = Curl_http_bodysend(data, conn, &req, httpreq); + result = Curl_http_req_send(data, &req, httpreq); if(!result) result = Curl_hyper_header(data, headers, Curl_dyn_ptr(&req)); @@ -1171,7 +1172,7 @@ CURLcode Curl_http(struct Curl_easy *data, bool *done) if(result) goto error; - result = bodysend(data, conn, headers, req, httpreq); + result = bodysend(data, headers, req, httpreq); if(result) goto error; diff --git a/lib/cf-h1-proxy.c b/lib/cf-h1-proxy.c index b7afe27a3fe623..6ca7babf8e6b3a 100644 --- a/lib/cf-h1-proxy.c +++ b/lib/cf-h1-proxy.c @@ -212,6 +212,11 @@ static void tunnel_free(struct Curl_cfilter *cf, } } +static bool tunnel_want_send(struct h1_tunnel_state *ts) +{ + return (ts->tunnel_state == H1_TUNNEL_CONNECT); +} + #ifndef USE_HYPER static CURLcode start_CONNECT(struct Curl_cfilter *cf, struct Curl_easy *data, @@ -1032,7 +1037,7 @@ static void cf_h1_proxy_adjust_pollset(struct Curl_cfilter *cf, wait for the socket to become readable to be able to get the response headers or if we're still sending the request, wait for write. */ - if(ts->CONNECT.sending == HTTPSEND_REQUEST) + if(tunnel_want_send(ts)) Curl_pollset_set_out_only(data, ps, sock); else Curl_pollset_set_in_only(data, ps, sock); diff --git a/lib/cf-haproxy.c b/lib/cf-haproxy.c index 652070e12dda09..90532952b1813f 100644 --- a/lib/cf-haproxy.c +++ b/lib/cf-haproxy.c @@ -129,7 +129,7 @@ static CURLcode cf_haproxy_connect(struct Curl_cfilter *cf, case HAPROXY_SEND: len = Curl_dyn_len(&ctx->data_out); if(len > 0) { - ssize_t written; + size_t written; result = Curl_conn_send(data, cf->sockindex, Curl_dyn_ptr(&ctx->data_out), len, &written); @@ -139,8 +139,7 @@ static CURLcode cf_haproxy_connect(struct Curl_cfilter *cf, } else if(result) goto out; - DEBUGASSERT(written >= 0); - Curl_dyn_tail(&ctx->data_out, len - (size_t)written); + Curl_dyn_tail(&ctx->data_out, len - written); if(Curl_dyn_len(&ctx->data_out) > 0) { result = CURLE_OK; goto out; diff --git a/lib/cfilters.c b/lib/cfilters.c index 2bf1dd844f206e..4e4c8e569c0150 100644 --- a/lib/cfilters.c +++ b/lib/cfilters.c @@ -694,7 +694,7 @@ CURLcode Curl_conn_recv(struct Curl_easy *data, int sockindex, CURLcode Curl_conn_send(struct Curl_easy *data, int sockindex, const void *buf, size_t blen, - ssize_t *pnwritten) + size_t *pnwritten) { ssize_t nwritten; CURLcode result = CURLE_OK; @@ -719,7 +719,7 @@ CURLcode Curl_conn_send(struct Curl_easy *data, int sockindex, #endif nwritten = conn->send[sockindex](data, sockindex, buf, blen, &result); DEBUGASSERT((nwritten >= 0) || result); - *pnwritten = nwritten; + *pnwritten = (nwritten < 0)? 0 : (size_t)nwritten; return result; } diff --git a/lib/cfilters.h b/lib/cfilters.h index 65ae3d4cbb3f07..92a2132fe0e1dc 100644 --- a/lib/cfilters.h +++ b/lib/cfilters.h @@ -518,7 +518,7 @@ CURLcode Curl_conn_recv(struct Curl_easy *data, int sockindex, */ CURLcode Curl_conn_send(struct Curl_easy *data, int sockindex, const void *buf, size_t blen, - ssize_t *pnwritten); + size_t *pnwritten); void Curl_pollset_reset(struct Curl_easy *data, diff --git a/lib/dict.c b/lib/dict.c index 955290f4da5741..f37767882e7c61 100644 --- a/lib/dict.c +++ b/lib/dict.c @@ -127,7 +127,7 @@ static CURLcode sendf(struct Curl_easy *data, static CURLcode sendf(struct Curl_easy *data, const char *fmt, ...) { - ssize_t bytes_written; + size_t bytes_written; size_t write_len; CURLcode result = CURLE_OK; char *s; diff --git a/lib/easy.c b/lib/easy.c index e6bda9bc8b163f..6c932ad823c6ec 100644 --- a/lib/easy.c +++ b/lib/easy.c @@ -1251,7 +1251,7 @@ CURLcode Curl_connect_only_attach(struct Curl_easy *data) * This is the private internal version of curl_easy_send() */ CURLcode Curl_senddata(struct Curl_easy *data, const void *buffer, - size_t buflen, ssize_t *n) + size_t buflen, size_t *n) { CURLcode result; struct connectdata *c = NULL; @@ -1283,13 +1283,13 @@ CURLcode Curl_senddata(struct Curl_easy *data, const void *buffer, CURLcode curl_easy_send(struct Curl_easy *data, const void *buffer, size_t buflen, size_t *n) { - ssize_t written = 0; + size_t written = 0; CURLcode result; if(Curl_is_in_callback(data)) return CURLE_RECURSIVE_API_CALL; result = Curl_senddata(data, buffer, buflen, &written); - *n = (size_t)written; + *n = written; return result; } diff --git a/lib/easyif.h b/lib/easyif.h index 64489529660cbe..6ce3483c643bc3 100644 --- a/lib/easyif.h +++ b/lib/easyif.h @@ -28,7 +28,7 @@ * Prototypes for library-wide functions provided by easy.c */ CURLcode Curl_senddata(struct Curl_easy *data, const void *buffer, - size_t buflen, ssize_t *n); + size_t buflen, size_t *n); #ifdef USE_WEBSOCKETS CURLcode Curl_connect_only_attach(struct Curl_easy *data); diff --git a/lib/gopher.c b/lib/gopher.c index e49da8147c99b6..e1a1ba64886288 100644 --- a/lib/gopher.c +++ b/lib/gopher.c @@ -139,8 +139,8 @@ static CURLcode gopher_do(struct Curl_easy *data, bool *done) char *sel = NULL; char *sel_org = NULL; timediff_t timeout_ms; - ssize_t amount, k; - size_t len; + ssize_t k; + size_t amount, len; int what; *done = TRUE; /* unconditionally */ diff --git a/lib/http.c b/lib/http.c index 97ef4550f5eb59..77832d2a466ada 100644 --- a/lib/http.c +++ b/lib/http.c @@ -1174,6 +1174,7 @@ static bool http_should_fail(struct Curl_easy *data) return data->state.authproblem; } +#ifndef USE_HYPER /* * readmoredata() is a "fread() emulation" to provide POST and/or request * data. It is used when a huge POST is to be made and the entire chunk wasn't @@ -1238,17 +1239,16 @@ static size_t readmoredata(char *buffer, * * Returns CURLcode */ -CURLcode Curl_buffer_send(struct dynbuf *in, - struct Curl_easy *data, - struct HTTP *http, - /* add the number of sent bytes to this - counter */ - curl_off_t *bytes_written, - /* how much of the buffer contains body data */ - curl_off_t included_body_bytes, - int sockindex) +static CURLcode buffer_send(struct dynbuf *in, + struct Curl_easy *data, + struct HTTP *http, + /* add the number of sent bytes to this + counter */ + curl_off_t *bytes_written, + /* how much of the buffer contains body data */ + curl_off_t included_body_bytes) { - ssize_t amount; + size_t amount; CURLcode result; char *ptr; size_t size; @@ -1256,8 +1256,6 @@ CURLcode Curl_buffer_send(struct dynbuf *in, size_t sendsize; size_t headersize; - DEBUGASSERT(sockindex <= SECONDARYSOCKET && sockindex >= 0); - /* The looping below is required since we use non-blocking sockets, but due to the circumstances we will just loop and try again and again etc */ @@ -1356,11 +1354,7 @@ CURLcode Curl_buffer_send(struct dynbuf *in, sendsize = (size_t)data->set.upload_buffer_size; } - result = Curl_conn_send(data, sockindex, ptr, sendsize, &amount); - if(result == CURLE_AGAIN) { - result = CURLE_OK; - amount = 0; - } + result = Curl_xfer_send(data, ptr, sendsize, &amount); if(!result) { /* @@ -1443,6 +1437,11 @@ CURLcode Curl_buffer_send(struct dynbuf *in, /* end of the add_buffer functions */ /* ------------------------------------------------------------------------- */ +#else /* !USE_HYPER */ + /* In hyper, this is an ugly NOP */ +#define buffer_send(a,b,c,d,e) CURLE_OK + +#endif /* !USE_HYPER(else) */ @@ -1619,13 +1618,12 @@ static const char *get_http_string(const struct Curl_easy *data, #endif /* check and possibly add an Expect: header */ -static CURLcode expect100(struct Curl_easy *data, - struct connectdata *conn, - struct dynbuf *req) +static CURLcode expect100(struct Curl_easy *data, struct dynbuf *req) { CURLcode result = CURLE_OK; - if(!data->state.disableexpect && Curl_use_http_1_1plus(data, conn) && - (conn->httpversion < 20)) { + if(!data->state.disableexpect && + Curl_use_http_1_1plus(data, data->conn) && + (data->conn->httpversion < 20)) { /* if not doing HTTP 1.0 or version 2, or disabled explicitly, we add an Expect: 100-continue to the headers which actually speeds up post operations (as there is one packet coming back from the web server) */ @@ -2441,8 +2439,7 @@ CURLcode Curl_http_body(struct Curl_easy *data, struct connectdata *conn, return result; } -static CURLcode addexpect(struct Curl_easy *data, struct connectdata *conn, - struct dynbuf *r) +static CURLcode addexpect(struct Curl_easy *data, struct dynbuf *r) { data->state.expect100header = FALSE; /* Avoid Expect: 100-continue if Upgrade: is used */ @@ -2459,24 +2456,22 @@ static CURLcode addexpect(struct Curl_easy *data, struct connectdata *conn, STRCONST("100-continue")); } else if(http->postsize > EXPECT_100_THRESHOLD || http->postsize < 0) - return expect100(data, conn, r); + return expect100(data, r); } return CURLE_OK; } -CURLcode Curl_http_bodysend(struct Curl_easy *data, struct connectdata *conn, +CURLcode Curl_http_req_send(struct Curl_easy *data, struct dynbuf *r, Curl_HttpReq httpreq) { #ifndef USE_HYPER /* Hyper always handles the body separately */ curl_off_t included_body = 0; -#else - /* from this point down, this function should not be used */ -#define Curl_buffer_send(a,b,c,d,e,f) CURLE_OK #endif CURLcode result = CURLE_OK; struct HTTP *http = data->req.p.http; + DEBUGASSERT(data->conn); switch(httpreq) { case HTTPREQ_PUT: /* Let's PUT the data to the server! */ @@ -2495,7 +2490,7 @@ CURLcode Curl_http_bodysend(struct Curl_easy *data, struct connectdata *conn, return result; } - result = addexpect(data, conn, r); + result = addexpect(data, r); if(result) return result; @@ -2508,9 +2503,8 @@ CURLcode Curl_http_bodysend(struct Curl_easy *data, struct connectdata *conn, Curl_pgrsSetUploadSize(data, http->postsize); /* this sends the buffer and frees all the buffer resources */ - result = Curl_buffer_send(r, data, data->req.p.http, - &data->info.request_size, 0, - FIRSTSOCKET); + result = buffer_send(r, data, data->req.p.http, + &data->info.request_size, 0); if(result) failf(data, "Failed sending PUT request"); else @@ -2531,9 +2525,8 @@ CURLcode Curl_http_bodysend(struct Curl_easy *data, struct connectdata *conn, if(result) return result; - result = Curl_buffer_send(r, data, data->req.p.http, - &data->info.request_size, 0, - FIRSTSOCKET); + result = buffer_send(r, data, data->req.p.http, + &data->info.request_size, 0); if(result) failf(data, "Failed sending POST request"); else @@ -2571,7 +2564,7 @@ CURLcode Curl_http_bodysend(struct Curl_easy *data, struct connectdata *conn, } #endif - result = addexpect(data, conn, r); + result = addexpect(data, r); if(result) return result; @@ -2589,9 +2582,8 @@ CURLcode Curl_http_bodysend(struct Curl_easy *data, struct connectdata *conn, http->sending = HTTPSEND_BODY; /* this sends the buffer and frees all the buffer resources */ - result = Curl_buffer_send(r, data, data->req.p.http, - &data->info.request_size, 0, - FIRSTSOCKET); + result = buffer_send(r, data, data->req.p.http, + &data->info.request_size, 0); if(result) failf(data, "Failed sending POST request"); else @@ -2633,7 +2625,7 @@ CURLcode Curl_http_bodysend(struct Curl_easy *data, struct connectdata *conn, return result; } - result = addexpect(data, conn, r); + result = addexpect(data, r); if(result) return result; @@ -2732,9 +2724,8 @@ CURLcode Curl_http_bodysend(struct Curl_easy *data, struct connectdata *conn, } } /* issue the request */ - result = Curl_buffer_send(r, data, data->req.p.http, - &data->info.request_size, included_body, - FIRSTSOCKET); + result = buffer_send(r, data, data->req.p.http, + &data->info.request_size, included_body); if(result) failf(data, "Failed sending HTTP POST request"); @@ -2749,13 +2740,12 @@ CURLcode Curl_http_bodysend(struct Curl_easy *data, struct connectdata *conn, return result; /* issue the request */ - result = Curl_buffer_send(r, data, data->req.p.http, - &data->info.request_size, 0, - FIRSTSOCKET); + result = Curl_req_send_hds(data, Curl_dyn_ptr(r), Curl_dyn_len(r)); + Curl_dyn_free(r); if(result) failf(data, "Failed sending HTTP request"); #ifdef USE_WEBSOCKETS - else if((conn->handler->protocol & (CURLPROTO_WS|CURLPROTO_WSS)) && + else if((data->conn->handler->protocol & (CURLPROTO_WS|CURLPROTO_WSS)) && !(data->set.connect_only)) /* Set up the transfer for two-way since without CONNECT_ONLY set, this request probably wants to send data too post upgrade */ @@ -3329,8 +3319,8 @@ CURLcode Curl_http(struct Curl_easy *data, bool *done) (httpreq == HTTPREQ_HEAD)) Curl_pgrsSetUploadSize(data, 0); /* nothing */ - /* bodysend takes ownership of the 'req' memory on success */ - result = Curl_http_bodysend(data, conn, &req, httpreq); + /* req_send takes ownership of the 'req' memory on success */ + result = Curl_http_req_send(data, &req, httpreq); } if(result) { Curl_dyn_free(&req); diff --git a/lib/http.h b/lib/http.h index ad2697c9e733ea..7991f938038a2a 100644 --- a/lib/http.h +++ b/lib/http.h @@ -74,12 +74,6 @@ char *Curl_checkProxyheaders(struct Curl_easy *data, const char *thisheader, const size_t thislen); struct HTTP; /* see below */ -CURLcode Curl_buffer_send(struct dynbuf *in, - struct Curl_easy *data, - struct HTTP *http, - curl_off_t *bytes_written, - curl_off_t included_body_bytes, - int socketindex); CURLcode Curl_add_timecondition(struct Curl_easy *data, #ifndef USE_HYPER @@ -118,7 +112,7 @@ CURLcode Curl_transferencode(struct Curl_easy *data); CURLcode Curl_http_body(struct Curl_easy *data, struct connectdata *conn, Curl_HttpReq httpreq, const char **teep); -CURLcode Curl_http_bodysend(struct Curl_easy *data, struct connectdata *conn, +CURLcode Curl_http_req_send(struct Curl_easy *data, struct dynbuf *r, Curl_HttpReq httpreq); bool Curl_use_http_1_1plus(const struct Curl_easy *data, const struct connectdata *conn); diff --git a/lib/krb5.c b/lib/krb5.c index 885319e00feff3..309e12a58aeb00 100644 --- a/lib/krb5.c +++ b/lib/krb5.c @@ -66,7 +66,7 @@ static CURLcode ftpsend(struct Curl_easy *data, struct connectdata *conn, const char *cmd) { - ssize_t bytes_written; + size_t bytes_written; #define SBUF_SIZE 1024 char s[SBUF_SIZE]; size_t write_len; @@ -100,9 +100,9 @@ static CURLcode ftpsend(struct Curl_easy *data, struct connectdata *conn, if(result) break; - Curl_debug(data, CURLINFO_HEADER_OUT, sptr, (size_t)bytes_written); + Curl_debug(data, CURLINFO_HEADER_OUT, sptr, bytes_written); - if(bytes_written != (ssize_t)write_len) { + if(bytes_written != write_len) { write_len -= bytes_written; sptr += bytes_written; } @@ -494,7 +494,7 @@ socket_write(struct Curl_easy *data, int sockindex, const void *to, { const char *to_p = to; CURLcode result; - ssize_t written; + size_t written; while(len > 0) { result = Curl_conn_send(data, sockindex, to_p, len, &written); diff --git a/lib/mqtt.c b/lib/mqtt.c index b0aafc79707e57..9290da031ba3fd 100644 --- a/lib/mqtt.c +++ b/lib/mqtt.c @@ -119,12 +119,12 @@ static CURLcode mqtt_send(struct Curl_easy *data, { CURLcode result = CURLE_OK; struct MQTT *mq = data->req.p.mqtt; - ssize_t n; + size_t n; result = Curl_xfer_send(data, buf, len, &n); if(result) return result; Curl_debug(data, CURLINFO_HEADER_OUT, buf, (size_t)n); - if(len != (size_t)n) { + if(len != n) { size_t nsend = len - n; char *sendleftovers = Curl_memdup(&buf[n], nsend); if(!sendleftovers) diff --git a/lib/pingpong.c b/lib/pingpong.c index c12f7cd7b6351e..7f240be9500d44 100644 --- a/lib/pingpong.c +++ b/lib/pingpong.c @@ -164,7 +164,7 @@ CURLcode Curl_pp_vsendf(struct Curl_easy *data, const char *fmt, va_list args) { - ssize_t bytes_written = 0; + size_t bytes_written = 0; size_t write_len; char *s; CURLcode result; @@ -211,9 +211,9 @@ CURLcode Curl_pp_vsendf(struct Curl_easy *data, conn->data_prot = (unsigned char)data_sec; #endif - Curl_debug(data, CURLINFO_HEADER_OUT, s, (size_t)bytes_written); + Curl_debug(data, CURLINFO_HEADER_OUT, s, bytes_written); - if(bytes_written != (ssize_t)write_len) { + if(bytes_written != write_len) { /* the whole chunk was not sent, keep it around and adjust sizes */ pp->sendthis = s; pp->sendsize = write_len; @@ -398,7 +398,7 @@ CURLcode Curl_pp_flushsend(struct Curl_easy *data, struct pingpong *pp) { /* we have a piece of a command still left to send */ - ssize_t written; + size_t written; CURLcode result; result = Curl_conn_send(data, FIRSTSOCKET, @@ -411,7 +411,7 @@ CURLcode Curl_pp_flushsend(struct Curl_easy *data, if(result) return result; - if(written != (ssize_t)pp->sendleft) { + if(written != pp->sendleft) { /* only a fraction was sent */ pp->sendleft -= written; } diff --git a/lib/request.c b/lib/request.c index 933f8850c366b6..334a0567398929 100644 --- a/lib/request.c +++ b/lib/request.c @@ -27,8 +27,10 @@ #include "urldata.h" #include "dynbuf.h" #include "doh.h" +#include "progress.h" #include "request.h" #include "sendf.h" +#include "transfer.h" #include "url.h" /* The last 3 #include files should be in this order */ @@ -39,8 +41,6 @@ CURLcode Curl_req_init(struct SingleRequest *req) { memset(req, 0, sizeof(*req)); - Curl_bufq_init2(&req->sendbuf, UPLOADBUFFER_DEFAULT, 1, - BUFQ_OPT_SOFT_LIMIT); return CURLE_OK; } @@ -49,6 +49,20 @@ CURLcode Curl_req_start(struct SingleRequest *req, { req->start = Curl_now(); Curl_cw_reset(data); + if(!req->sendbuf_init) { + Curl_bufq_init2(&req->sendbuf, data->set.upload_buffer_size, 1, + BUFQ_OPT_SOFT_LIMIT); + req->sendbuf_init = TRUE; + } + else { + Curl_bufq_reset(&req->sendbuf); + if(data->set.upload_buffer_size != req->sendbuf.chunk_size) { + Curl_bufq_free(&req->sendbuf); + Curl_bufq_init2(&req->sendbuf, data->set.upload_buffer_size, 1, + BUFQ_OPT_SOFT_LIMIT); + } + } + return CURLE_OK; } @@ -56,33 +70,40 @@ CURLcode Curl_req_done(struct SingleRequest *req, struct Curl_easy *data, bool aborted) { (void)req; - /* TODO: add flush handling for client output */ - (void)aborted; + if(!aborted) + (void)Curl_req_flush(data); Curl_cw_reset(data); return CURLE_OK; } void Curl_req_reset(struct SingleRequest *req, struct Curl_easy *data) { + struct bufq savebuf; + bool save_init; + /* This is a bit ugly. `req->p` is a union and we assume we can * free this safely without leaks. */ Curl_safefree(req->p.http); Curl_safefree(req->newurl); Curl_cw_reset(data); - Curl_bufq_reset(&req->sendbuf); - if(data->set.upload_buffer_size != req->sendbuf.chunk_size) { - Curl_bufq_free(&req->sendbuf); - Curl_bufq_init2(&req->sendbuf, data->set.upload_buffer_size, 1, - BUFQ_OPT_SOFT_LIMIT); - } - #ifndef CURL_DISABLE_DOH if(req->doh) { Curl_close(&req->doh->probe[0].easy); Curl_close(&req->doh->probe[1].easy); } #endif + + savebuf = req->sendbuf; + save_init = req->sendbuf_init; + + memset(req, 0, sizeof(*req)); + data->req.size = data->req.maxdownload = -1; + data->req.no_body = data->set.opt_no_body; + if(save_init) { + req->sendbuf = savebuf; + req->sendbuf_init = save_init; + } } void Curl_req_free(struct SingleRequest *req, struct Curl_easy *data) @@ -91,7 +112,8 @@ void Curl_req_free(struct SingleRequest *req, struct Curl_easy *data) * free this safely without leaks. */ Curl_safefree(req->p.http); Curl_safefree(req->newurl); - Curl_bufq_free(&req->sendbuf); + if(req->sendbuf_init) + Curl_bufq_free(&req->sendbuf); Curl_cw_reset(data); #ifndef CURL_DISABLE_DOH @@ -106,3 +128,130 @@ void Curl_req_free(struct SingleRequest *req, struct Curl_easy *data) #endif } +static CURLcode req_send(struct Curl_easy *data, + const char *buf, size_t blen, + size_t hds_len, size_t *pnwritten) +{ + CURLcode result = CURLE_OK; + + *pnwritten = 0; +#ifdef CURLDEBUG + { + /* Allow debug builds to override this logic to force short initial + sends + */ + char *p = getenv("CURL_SMALLREQSEND"); + if(p) { + size_t altsize = (size_t)strtoul(p, NULL, 10); + if(altsize && altsize < blen) + blen = altsize; + } + } +#endif + /* Make sure this doesn't send more body bytes than what the max send + speed says. The headers do not count to the max speed. */ + if(data->set.max_send_speed) { + size_t body_bytes = blen - hds_len; + if((curl_off_t)body_bytes > data->set.max_send_speed) + blen = hds_len + (size_t)data->set.max_send_speed; + } + + result = Curl_xfer_send(data, buf, blen, pnwritten); + if(!result && *pnwritten) { + if(hds_len) + Curl_debug(data, CURLINFO_HEADER_OUT, (char *)buf, + CURLMIN(hds_len, *pnwritten)); + if(*pnwritten > hds_len) { + size_t body_len = *pnwritten - hds_len; + Curl_debug(data, CURLINFO_DATA_OUT, (char *)buf + hds_len, body_len); + data->req.writebytecount += body_len; + Curl_pgrsSetUploadCounter(data, data->req.writebytecount); + } + } + return result; +} + +static CURLcode req_send_buffer_add(struct Curl_easy *data, + const char *buf, size_t blen, + size_t hds_len) +{ + CURLcode result = CURLE_OK; + ssize_t n; + n = Curl_bufq_write(&data->req.sendbuf, + (const unsigned char *)buf, blen, &result); + if(n < 0) + return result; + /* We rely on a SOFTLIMIT on sendbuf, so it can take all data in */ + DEBUGASSERT((size_t)n == blen); + data->req.sendbuf_hds_len += hds_len; + return CURLE_OK; +} + +static CURLcode req_send_buffer_flush(struct Curl_easy *data) +{ + CURLcode result = CURLE_OK; + const unsigned char *buf; + size_t blen; + + while(Curl_bufq_peek(&data->req.sendbuf, &buf, &blen)) { + size_t nwritten, hds_len = CURLMIN(data->req.sendbuf_hds_len, blen); + result = req_send(data, (const char *)buf, blen, hds_len, &nwritten); + if(result) + break; + + Curl_bufq_skip(&data->req.sendbuf, nwritten); + if(hds_len) + data->req.sendbuf_hds_len -= CURLMIN(hds_len, nwritten); + /* leave if we could not send all. Maybe network blocking or + * speed limits on transfer */ + if(nwritten < blen) + break; + } + return result; +} + +CURLcode Curl_req_flush(struct Curl_easy *data) +{ + CURLcode result; + + if(!data || !data->conn) + return CURLE_FAILED_INIT; + + if(!Curl_bufq_is_empty(&data->req.sendbuf)) { + result = req_send_buffer_flush(data); + if(result) + return result; + if(!Curl_bufq_is_empty(&data->req.sendbuf)) { + return CURLE_AGAIN; + } + } + return CURLE_OK; +} + +CURLcode Curl_req_send(struct Curl_easy *data, + const char *buf, size_t blen, + size_t hds_len) +{ + CURLcode result; + + if(!data || !data->conn) + return CURLE_FAILED_INIT; + + /* We always buffer and send from there. The reason is that on + * blocking, we can retry using the same memory address. This is + * important for TLS libraries that expect this. + * We *could* optimized for non-TLS transfers, but that would mean + * separate code paths and seems not worth it. */ + result = req_send_buffer_add(data, buf, blen, hds_len); + if(result) + return result; + result = req_send_buffer_flush(data); + if(result == CURLE_AGAIN) + result = CURLE_OK; + return result; +} + +bool Curl_req_want_send(struct Curl_easy *data) +{ + return data->req.sendbuf_init && !Curl_bufq_is_empty(&data->req.sendbuf); +} diff --git a/lib/request.h b/lib/request.h index c769e0e1ff6f5b..54c784f6a88bd5 100644 --- a/lib/request.h +++ b/lib/request.h @@ -64,7 +64,7 @@ struct SingleRequest { curl_off_t bytecount; /* total number of bytes read */ curl_off_t writebytecount; /* number of bytes written */ - curl_off_t pendingheader; /* this many bytes left to send is actually + size_t pendingheader; /* this many bytes left to send is actually header and not body */ struct curltime start; /* transfer started at this time */ unsigned int headerbytecount; /* received server headers (not CONNECT @@ -91,6 +91,7 @@ struct SingleRequest { * checks, pausing by client callbacks. */ struct Curl_cwriter *writer_stack; struct bufq sendbuf; /* data which needs to be send to the server */ + size_t sendbuf_hds_len; /* amount of header bytes in sendbuf */ time_t timeofdoc; long bodywrites; char *location; /* This points to an allocated version of the Location: @@ -100,7 +101,7 @@ struct SingleRequest { /* 'upload_present' is used to keep a byte counter of how much data there is still left in the buffer, aimed for upload. */ - ssize_t upload_present; + size_t upload_present; /* 'upload_fromhere' is used as a read-pointer when we uploaded parts of a buffer, so the next read should read from where this pointer points to, @@ -154,6 +155,7 @@ struct SingleRequest { that we are creating a request with an auth header, but it is not the final request in the auth negotiation. */ + BIT(sendbuf_init); /* sendbuf is initialized */ }; /** @@ -189,4 +191,34 @@ void Curl_req_free(struct SingleRequest *req, struct Curl_easy *data); void Curl_req_reset(struct SingleRequest *req, struct Curl_easy *data); +/** + * Send request bytes for transfer. If not all could be sent + * they will be buffered. Use `Curl_req_flush()` to make sure + * bytes are really send. + * @param data the transfer making the request + * @param buf the bytes to send + * @param blen the number of bytes to send + * @param hds_len the number of bytes from the start that are headers + * @return CURLE_OK (on blocking with *pnwritten == 0) or error. + */ +CURLcode Curl_req_send(struct Curl_easy *data, + const char *buf, size_t blen, + size_t hds_len); + +/* Convenience for sending only header bytes */ +#define Curl_req_send_hds(data, buf, blen) \ + Curl_req_send((data), (buf), (blen), (blen)) + +/** + * Flush all buffered request bytes. + * @return CURLE_OK on success, CURLE_AGAIN if sending was blocked, + * or the error on the sending. + */ +CURLcode Curl_req_flush(struct Curl_easy *data); + +/** + * TRUE iff the request wants to send, e.g. has buffered bytes. + */ +bool Curl_req_want_send(struct Curl_easy *data); + #endif /* HEADER_CURL_REQUEST_H */ diff --git a/lib/rtsp.c b/lib/rtsp.c index b4290246ac9a4d..a4c70a6b448e60 100644 --- a/lib/rtsp.c +++ b/lib/rtsp.c @@ -566,8 +566,9 @@ static CURLcode rtsp_do(struct Curl_easy *data, bool *done) } /* issue the request */ - result = Curl_buffer_send(&req_buffer, data, data->req.p.http, - &data->info.request_size, 0, FIRSTSOCKET); + result = Curl_req_send_hds(data, Curl_dyn_ptr(&req_buffer), + Curl_dyn_len(&req_buffer)); + Curl_dyn_free(&req_buffer); if(result) { failf(data, "Failed sending RTSP request"); return result; diff --git a/lib/smb.c b/lib/smb.c index ae585b4c1aeb33..72991f955ec162 100644 --- a/lib/smb.c +++ b/lib/smb.c @@ -559,12 +559,12 @@ static void smb_format_message(struct Curl_easy *data, struct smb_header *h, h->pid = smb_swap16((unsigned short) pid); } -static CURLcode smb_send(struct Curl_easy *data, ssize_t len, +static CURLcode smb_send(struct Curl_easy *data, size_t len, size_t upload_size) { struct connectdata *conn = data->conn; struct smb_conn *smbc = &conn->proto.smbc; - ssize_t bytes_written; + size_t bytes_written; CURLcode result; result = Curl_xfer_send(data, data->state.ulbuf, len, &bytes_written); @@ -585,8 +585,8 @@ static CURLcode smb_flush(struct Curl_easy *data) { struct connectdata *conn = data->conn; struct smb_conn *smbc = &conn->proto.smbc; - ssize_t bytes_written; - ssize_t len = smbc->send_size - smbc->sent; + size_t bytes_written; + size_t len = smbc->send_size - smbc->sent; CURLcode result; if(!smbc->send_size) diff --git a/lib/smtp.c b/lib/smtp.c index e10a00477b105e..ddee2223ac48ca 100644 --- a/lib/smtp.c +++ b/lib/smtp.c @@ -1395,8 +1395,7 @@ static CURLcode smtp_done(struct Curl_easy *data, CURLcode status, struct SMTP *smtp = data->req.p.smtp; struct pingpong *pp = &conn->proto.smtpc.pp; char *eob; - ssize_t len; - ssize_t bytes_written; + size_t len, bytes_written; (void)premature; diff --git a/lib/telnet.c b/lib/telnet.c index 9b6ae3c611a6fb..56ee0855f0f875 100644 --- a/lib/telnet.c +++ b/lib/telnet.c @@ -1231,20 +1231,24 @@ CURLcode telrcv(struct Curl_easy *data, static CURLcode send_telnet_data(struct Curl_easy *data, char *buffer, ssize_t nread) { - ssize_t i, outlen; + size_t i, outlen; unsigned char *outbuf; CURLcode result = CURLE_OK; - ssize_t bytes_written, total_written = 0; + size_t bytes_written; + size_t total_written = 0; struct connectdata *conn = data->conn; struct TELNET *tn = data->req.p.telnet; DEBUGASSERT(tn); + DEBUGASSERT(nread > 0); + if(nread < 0) + return CURLE_TOO_LARGE; if(memchr(buffer, CURL_IAC, nread)) { /* only use the escape buffer when necessary */ Curl_dyn_reset(&tn->out); - for(i = 0; i < nread && !result; i++) { + for(i = 0; i < (size_t)nread && !result; i++) { result = Curl_dyn_addn(&tn->out, &buffer[i], 1); if(!result && ((unsigned char)buffer[i] == CURL_IAC)) /* IAC is FF in hex */ @@ -1255,7 +1259,7 @@ static CURLcode send_telnet_data(struct Curl_easy *data, outbuf = Curl_dyn_uptr(&tn->out); } else { - outlen = nread; + outlen = (size_t)nread; outbuf = (unsigned char *)buffer; } while(!result && total_written < outlen) { diff --git a/lib/transfer.c b/lib/transfer.c index 1935909b91a76d..43d6f6f4b4cdaa 100644 --- a/lib/transfer.c +++ b/lib/transfer.c @@ -607,7 +607,7 @@ static void win_update_buffer_size(curl_socket_t sockfd) #endif #define curl_upload_refill_watermark(data) \ - ((ssize_t)((data)->set.upload_buffer_size >> 5)) + ((size_t)((data)->set.upload_buffer_size >> 5)) /* * Send data to upload to the server, when the socket is writable. @@ -617,7 +617,7 @@ static CURLcode readwrite_upload(struct Curl_easy *data, int *didwhat) { ssize_t i, si; - ssize_t bytes_written; + size_t bytes_written; CURLcode result; ssize_t nread; /* number of bytes read */ bool sending_http_headers = FALSE; @@ -625,6 +625,14 @@ static CURLcode readwrite_upload(struct Curl_easy *data, *didwhat |= KEEP_SEND; + if(!(k->keepon & KEEP_SEND_PAUSE)) { + result = Curl_req_flush(data); + if(result == CURLE_AGAIN) /* unable to send all we have */ + return CURLE_OK; + else if(result) + return result; + } + do { curl_off_t nbody; ssize_t offset = 0; @@ -633,8 +641,8 @@ static CURLcode readwrite_upload(struct Curl_easy *data, k->upload_present < curl_upload_refill_watermark(data) && !k->upload_chunky &&/*(variable sized chunked header; append not safe)*/ !k->upload_done && /*!(k->upload_done once k->upload_present sent)*/ - !(k->writebytecount + k->upload_present - k->pendingheader == - data->state.infilesize)) { + !(k->writebytecount + (curl_off_t)k->upload_present - + (curl_off_t)k->pendingheader == data->state.infilesize)) { offset = k->upload_present; } @@ -770,9 +778,6 @@ static CURLcode readwrite_upload(struct Curl_easy *data, that instead of reading more data */ } - if(!Curl_bufq_is_empty(&k->sendbuf)) { - DEBUGASSERT(0); - } /* write to socket (send away data) */ result = Curl_xfer_send(data, k->upload_fromhere, /* buffer pointer */ @@ -793,9 +798,9 @@ static CURLcode readwrite_upload(struct Curl_easy *data, if(k->pendingheader) { /* parts of what was sent was header */ - curl_off_t n = CURLMIN(k->pendingheader, bytes_written); + size_t n = CURLMIN(k->pendingheader, bytes_written); /* show the data before we change the pointer upload_fromhere */ - Curl_debug(data, CURLINFO_HEADER_OUT, k->upload_fromhere, (size_t)n); + Curl_debug(data, CURLINFO_HEADER_OUT, k->upload_fromhere, n); k->pendingheader -= n; nbody = bytes_written - n; /* size of the written body part */ } @@ -1607,22 +1612,19 @@ void Curl_xfer_setup( struct SingleRequest *k = &data->req; struct connectdata *conn = data->conn; struct HTTP *http = data->req.p.http; - bool httpsending; + bool want_send = Curl_req_want_send(data); DEBUGASSERT(conn != NULL); DEBUGASSERT((sockindex <= 1) && (sockindex >= -1)); DEBUGASSERT((writesockindex <= 1) && (writesockindex >= -1)); - httpsending = ((conn->handler->protocol&PROTO_FAMILY_HTTP) && - (http->sending == HTTPSEND_REQUEST)); - - if(conn->bits.multiplex || conn->httpversion >= 20 || httpsending) { + if(conn->bits.multiplex || conn->httpversion >= 20 || want_send) { /* when multiplexing, the read/write sockets need to be the same! */ conn->sockfd = sockindex == -1 ? ((writesockindex == -1 ? CURL_SOCKET_BAD : conn->sock[writesockindex])) : conn->sock[sockindex]; conn->writesockfd = conn->sockfd; - if(httpsending) + if(want_send) /* special and very HTTP-specific */ writesockindex = FIRSTSOCKET; } @@ -1732,7 +1734,7 @@ CURLcode Curl_xfer_write_done(struct Curl_easy *data, bool premature) CURLcode Curl_xfer_send(struct Curl_easy *data, const void *buf, size_t blen, - ssize_t *pnwritten) + size_t *pnwritten) { CURLcode result; int sockindex; diff --git a/lib/transfer.h b/lib/transfer.h index 917a3d23e4a4e4..81c70718077ac5 100644 --- a/lib/transfer.h +++ b/lib/transfer.h @@ -97,7 +97,7 @@ CURLcode Curl_xfer_write_done(struct Curl_easy *data, bool premature); */ CURLcode Curl_xfer_send(struct Curl_easy *data, const void *buf, size_t blen, - ssize_t *pnwritten); + size_t *pnwritten); /** * Receive data on the socket/connection filter designated diff --git a/lib/url.c b/lib/url.c index ae976d7b11dc2a..d72dfc773bbd1e 100644 --- a/lib/url.c +++ b/lib/url.c @@ -3845,9 +3845,6 @@ CURLcode Curl_connect(struct Curl_easy *data, /* init the single-transfer specific data */ Curl_req_reset(&data->req, data); - memset(&data->req, 0, sizeof(struct SingleRequest)); - data->req.size = data->req.maxdownload = -1; - data->req.no_body = data->set.opt_no_body; /* call the stuff that needs to be called */ result = create_conn(data, &conn, asyncp); diff --git a/lib/vssh/libssh2.c b/lib/vssh/libssh2.c index 33dbe1c9200d91..36975420bc3348 100644 --- a/lib/vssh/libssh2.c +++ b/lib/vssh/libssh2.c @@ -3213,7 +3213,7 @@ static ssize_t ssh_tls_send(libssh2_socket_t sock, const void *buffer, size_t length, int flags, void **abstract) { struct Curl_easy *data = (struct Curl_easy *)*abstract; - ssize_t nwrite; + size_t nwrite; CURLcode result; struct connectdata *conn = data->conn; Curl_send *backup = conn->send[0]; @@ -3230,8 +3230,8 @@ static ssize_t ssh_tls_send(libssh2_socket_t sock, const void *buffer, return -EAGAIN; /* magic return code for libssh2 */ else if(result) return -1; /* error */ - Curl_debug(data, CURLINFO_DATA_OUT, (char *)buffer, (size_t)nwrite); - return nwrite; + Curl_debug(data, CURLINFO_DATA_OUT, (char *)buffer, nwrite); + return (ssize_t)nwrite; } #endif diff --git a/lib/ws.c b/lib/ws.c index b2305932accf4c..37108946d41156 100644 --- a/lib/ws.c +++ b/lib/ws.c @@ -1014,8 +1014,7 @@ static CURLcode ws_flush(struct Curl_easy *data, struct websocket *ws, if(!Curl_bufq_is_empty(&ws->sendbuf)) { CURLcode result; const unsigned char *out; - size_t outlen; - ssize_t n; + size_t outlen, n; while(Curl_bufq_peek(&ws->sendbuf, &out, &outlen)) { if(data->set.connect_only) @@ -1044,8 +1043,8 @@ static CURLcode ws_flush(struct Curl_easy *data, struct websocket *ws, } } else { - infof(data, "WS: flushed %zu bytes", (size_t)n); - Curl_bufq_skip(&ws->sendbuf, (size_t)n); + infof(data, "WS: flushed %zu bytes", n); + Curl_bufq_skip(&ws->sendbuf, n); } } } @@ -1058,8 +1057,8 @@ CURL_EXTERN CURLcode curl_ws_send(CURL *data, const void *buffer, unsigned int flags) { struct websocket *ws; - ssize_t nwritten, n; - size_t space; + ssize_t n; + size_t nwritten, space; CURLcode result; *sent = 0; @@ -1097,7 +1096,7 @@ CURL_EXTERN CURLcode curl_ws_send(CURL *data, const void *buffer, infof(data, "WS: wanted to send %zu bytes, sent %zu bytes", buflen, nwritten); - *sent = (nwritten >= 0)? (size_t)nwritten : 0; + *sent = nwritten; return result; }