High memory usage for upgraded http requests
- Version:
v7.7.3 - Platform:
macOS/Linux - Subsystem:
net/http
I noticed that n socket connections obtained with the HTTP upgrade mechanism use a lot more memory than n socket connections obtained with a plain net server. Consider for example the following test.
net server
'use strict'; const net = require('net'); const headers = [ 'HTTP/1.1 101 Switching Protocols', 'Connection: Upgrade', 'Upgrade: foo', '', '' ].join('\r\n'); let count = 0; const handler = (socket) => { socket.resume(); socket.write(headers); if (++count === 150000) { gc(); const rss = process.memoryUsage().rss; console.log(rss / 1024 / 1024); } }; const server = net.createServer({ allowHalfOpen: true }); server.on('connection', handler); server.listen(3000, () => console.log('listening on *:3000'));
net client
'use strict'; const net = require('net'); const headers = [ 'GET / HTTP/1.1', 'Connection: Upgrade', 'Upgrade: foo', 'Host: localhost:3000', '', '' ].join('\r\n'); let i = 0; (function createClient() { const socket = net.connect({ localAddress: `127.0.0.${i % 100 + 1}`, port: 3000 }); socket.on('connect', () => { socket.resume(); socket.write(headers); if (++i === 150000) return; createClient(); }); })();
http server
'use strict'; const http = require('http'); const headers = [ 'HTTP/1.1 101 Switching Protocols', 'Connection: Upgrade', 'Upgrade: foo', '', '' ].join('\r\n'); let count = 0; const handler = (req, socket, head) => { socket.resume(); socket.write(headers); if (++count === 150000) { gc(); const rss = process.memoryUsage().rss; console.log(rss / 1024 / 1024); } }; const server = http.createServer(); server.setTimeout(0); server.on('upgrade', handler); server.listen(3000, () => console.log('listening on *:3000'));
http client
'use strict'; const http = require('http'); let i = 0; (function createClient() { const req = http.get({ localAddress: `127.0.0.${i % 100 + 1}`, port: 3000, headers: { 'Connection': 'Upgrade', 'Upgrade': 'foo' } }); req.on('upgrade', (res, socket, head) => { socket.resume(); if (++i === 150000) return; createClient(); }); })();
The first (net) server uses ~295 MiB of memory while the second (http) ~525 MiB. Shouldn't they use more or less the same amount of memory?
It seems that, in part, the difference is caused by the additional event listeners. If I add
socket.removeAllListeners('drain'); socket.removeAllListeners('error'); socket.removeAllListeners('timeout');
in the upgrade event handler, memory usage drops to ~420 MiB.