test: fix flaky test-http2-client-upload by Flarna · Pull Request #29889 · nodejs/node

Maybe, I found the difference between TCP with windows and TCP with Linux that causes the error on windows, when calls the sendPendingData().
I just walked through source code and there could be some misunderstands.

  • Difference
    With Linux, if fails to write data to a socket, libuv retries it
    On windows doesn't retry.

  • Linux
    SendPendingData eventually calls uv__write(). If uv__write() fails to write data with temporal error(ENOBUF, EAGAIN, EINTR...), it queues the write request and it will call uv__write() again.

    static void uv__write(uv_stream_t* stream) {
    struct iovec* iov;
    QUEUE* q;
    uv_write_t* req;
    int iovmax;
    int iovcnt;
    ssize_t n;
    int err;
    start:
    assert(uv__stream_fd(stream) >= 0);
    if (QUEUE_EMPTY(&stream->write_queue))
    return;
    q = QUEUE_HEAD(&stream->write_queue);
    req = QUEUE_DATA(q, uv_write_t, queue);
    assert(req->handle == stream);
    /*
    * Cast to iovec. We had to have our own uv_buf_t instead of iovec
    * because Windows's WSABUF is not an iovec.
    */
    assert(sizeof(uv_buf_t) == sizeof(struct iovec));
    iov = (struct iovec*) &(req->bufs[req->write_index]);
    iovcnt = req->nbufs - req->write_index;
    iovmax = uv__getiovmax();
    /* Limit iov count to avoid EINVALs from writev() */
    if (iovcnt > iovmax)
    iovcnt = iovmax;
    /*
    * Now do the actual writev. Note that we've been updating the pointers
    * inside the iov each time we write. So there is no need to offset it.
    */
    if (req->send_handle) {
    int fd_to_send;
    struct msghdr msg;
    struct cmsghdr *cmsg;
    union {
    char data[64];
    struct cmsghdr alias;
    } scratch;
    if (uv__is_closing(req->send_handle)) {
    err = UV_EBADF;
    goto error;
    }
    fd_to_send = uv__handle_fd((uv_handle_t*) req->send_handle);
    memset(&scratch, 0, sizeof(scratch));
    assert(fd_to_send >= 0);
    msg.msg_name = NULL;
    msg.msg_namelen = 0;
    msg.msg_iov = iov;
    msg.msg_iovlen = iovcnt;
    msg.msg_flags = 0;
    msg.msg_control = &scratch.alias;
    msg.msg_controllen = CMSG_SPACE(sizeof(fd_to_send));
    cmsg = CMSG_FIRSTHDR(&msg);
    cmsg->cmsg_level = SOL_SOCKET;
    cmsg->cmsg_type = SCM_RIGHTS;
    cmsg->cmsg_len = CMSG_LEN(sizeof(fd_to_send));
    /* silence aliasing warning */
    {
    void* pv = CMSG_DATA(cmsg);
    int* pi = pv;
    *pi = fd_to_send;
    }
    do
    n = sendmsg(uv__stream_fd(stream), &msg, 0);
    while (n == -1 && RETRY_ON_WRITE_ERROR(errno));
    /* Ensure the handle isn't sent again in case this is a partial write. */
    if (n >= 0)
    req->send_handle = NULL;
    } else {
    do
    n = uv__writev(uv__stream_fd(stream), iov, iovcnt);
    while (n == -1 && RETRY_ON_WRITE_ERROR(errno));
    }
    if (n == -1 && !IS_TRANSIENT_WRITE_ERROR(errno, req->send_handle)) {
    err = UV__ERR(errno);
    goto error;
    }
    if (n >= 0 && uv__write_req_update(stream, req, n)) {
    uv__write_req_finish(req);
    return; /* TODO(bnoordhuis) Start trying to write the next request. */
    }
    /* If this is a blocking stream, try again. */
    if (stream->flags & UV_HANDLE_BLOCKING_WRITES)
    goto start;
    /* We're not done. */
    uv__io_start(stream->loop, &stream->io_watcher, POLLOUT);
    /* Notify select() thread about state change */
    uv__stream_osx_interrupt_select(stream);
    return;
    error:
    req->error = err;
    uv__write_req_finish(req);
    uv__io_stop(stream->loop, &stream->io_watcher, POLLOUT);
    if (!uv__io_active(&stream->io_watcher, POLLIN))
    uv__handle_stop(stream);
    uv__stream_osx_interrupt_select(stream);
    }
  • Windows
    SendPendingData() seems to expect that uv__tcp_try_write() returns UV_EAGAIN or UV_ENOSYS here, when it will be able to retry writing data. But uv__tcp_try_write() seemingly could return other retryable errors(WSAEINTR map to UV_ECANCELED, WSAENOBUFS map to UV_ENOBUFS).

    int uv__tcp_try_write(uv_tcp_t* handle,
    const uv_buf_t bufs[],
    unsigned int nbufs) {
    int result;
    DWORD bytes;
    if (handle->stream.conn.write_reqs_pending > 0)
    return UV_EAGAIN;
    result = WSASend(handle->socket,
    (WSABUF*) bufs,
    nbufs,
    &bytes,
    0,
    NULL,
    NULL);
    if (result == SOCKET_ERROR)
    return uv_translate_sys_error(WSAGetLastError());
    else
    return bytes;
    }

    If uv__tcp_try_write() failed and returns UV_EAGAIN or UV_ENOSYS, SendPendingData() calls uv_tcp_write() through uv_write2. uv_tcp_write() does not have retry argorizme, althought libuv has it on Linux.
    int uv_tcp_write(uv_loop_t* loop,
    uv_write_t* req,
    uv_tcp_t* handle,
    const uv_buf_t bufs[],
    unsigned int nbufs,
    uv_write_cb cb) {
    int result;
    DWORD bytes;
    UV_REQ_INIT(req, UV_WRITE);
    req->handle = (uv_stream_t*) handle;
    req->cb = cb;
    /* Prepare the overlapped structure. */
    memset(&(req->u.io.overlapped), 0, sizeof(req->u.io.overlapped));
    if (handle->flags & UV_HANDLE_EMULATE_IOCP) {
    req->event_handle = CreateEvent(NULL, 0, 0, NULL);
    if (!req->event_handle) {
    uv_fatal_error(GetLastError(), "CreateEvent");
    }
    req->u.io.overlapped.hEvent = (HANDLE) ((ULONG_PTR) req->event_handle | 1);
    req->wait_handle = INVALID_HANDLE_VALUE;
    }
    result = WSASend(handle->socket,
    (WSABUF*) bufs,
    nbufs,
    &bytes,
    0,
    &req->u.io.overlapped,
    NULL);
    if (UV_SUCCEEDED_WITHOUT_IOCP(result == 0)) {
    /* Request completed immediately. */
    req->u.io.queued_bytes = 0;
    handle->reqs_pending++;
    handle->stream.conn.write_reqs_pending++;
    REGISTER_HANDLE_REQ(loop, handle, req);
    uv_insert_pending_req(loop, (uv_req_t*) req);
    } else if (UV_SUCCEEDED_WITH_IOCP(result == 0)) {
    /* Request queued by the kernel. */
    req->u.io.queued_bytes = uv__count_bufs(bufs, nbufs);
    handle->reqs_pending++;
    handle->stream.conn.write_reqs_pending++;
    REGISTER_HANDLE_REQ(loop, handle, req);
    handle->write_queue_size += req->u.io.queued_bytes;
    if (handle->flags & UV_HANDLE_EMULATE_IOCP &&
    !RegisterWaitForSingleObject(&req->wait_handle,
    req->event_handle, post_write_completion, (void*) req,
    INFINITE, WT_EXECUTEINWAITTHREAD | WT_EXECUTEONLYONCE)) {
    SET_REQ_ERROR(req, GetLastError());
    uv_insert_pending_req(loop, (uv_req_t*)req);
    }
    } else {
    /* Send failed due to an error, report it later */
    req->u.io.queued_bytes = 0;
    handle->reqs_pending++;
    handle->stream.conn.write_reqs_pending++;
    REGISTER_HANDLE_REQ(loop, handle, req);
    SET_REQ_ERROR(req, WSAGetLastError());
    uv_insert_pending_req(loop, (uv_req_t*) req);
    }
    return 0;
    }

So, on windows, SendPendingData() could fail when 1 or 2 error happens. however on Linux, SendPendingData() retries writing data till success if there is no critical error.