I ran nginx via ldp and vcl on top of vpp's host stack, and use apache ab to measures the performance of nginx. But nginx crashed after running for some time. By analyzing the generated core file, I found that the Nginx crash was probably caused by VCL memory leak. And through several tests,I found that the larger the parameter of vcl's heapsize is set, the more http connections will be processed before nginx crash.

operating environment

root@dj:/usr/local/nginx/logs# uname -a
Linux dj 5.4.0-89-generic #100-Ubuntu SMP Fri Sep 24 14:50:10 UTC 2021 x86_64 x86_64 x86_64 GNU/Linux
root@dj:/usr/local/nginx/logs# cat /etc/issue
Ubuntu 20.04.1 LTS \n \l

nginx version

root@dj:/usr/local/nginx/logs# /usr/local/nginx/sbin/nginx -v
nginx version: nginx/1.8.0

HTTP request test command

proxy:

ab -c 500 -n 500000 http://10.1.1.2/bWAPP/login.php

web server:

ab -c 500 -n 5000000 http://10.1.1.2/1KB.json

vpp data

root@dj:/usr/local/nginx/logs# vppctl show version verbose cmdline
Version: v21.06-release
Compiled by: root
Compile host: dj
Compile date: 2021-11-10T03:35:09
Compile location: /usr/local/src/vpp
Compiler: GCC 9.3.0
Current PID: 45805
Command line arguments:
/usr/bin/vpp
unix

{ nodaemon log /var/log/vpp/vpp.log full-coredump cli-listen /run/vpp/cli.sock exec /usr/local/src/vsap/configs/vppenvset }

api-trace

{ on }

session

{ evt_qs_memfd_seg }

socksvr

{ socket-name /usr/local/src/vsap/vpp-api.sock }

cpu

{ main-core 0 corelist-workers 1 }

tcp

{cc-algo cubic}

dpdk
{
dev
0000:02:06.0

{ name eth0 num-rx-queues 1 num-rx-desc 512 num-tx-desc 512 }

dev
0000:02:07.0

{ name eth1 num-rx-queues 1 num-rx-desc 512 num-tx-desc 512 }

}

 

root@dj:/usr/local/nginx/logs# vppctl show hardware
Name Idx Link Hardware
eth0 1 up eth0
Link speed: 1 Gbps
RX Queues:
queue thread mode
0 vpp_wk_0 (1) polling
Ethernet address 00:0c:29:4e:64:73
Intel 82540EM (e1000)
carrier up full duplex mtu 9206
flags: admin-up pmd maybe-multiseg tx-offload intel-phdr-cksum rx-ip4-cksum int-supported
Devargs:
rx: queues 1 (max 1), desc 512 (min 32 max 4096 align 8)
tx: queues 1 (max 1), desc 512 (min 32 max 4096 align 8)
pci: device 8086:100f subsystem 15ad:0750 address 0000:02:06.00 numa 0
max rx packet len: 16128
promiscuous: unicast off all-multicast on
vlan offload: strip off filter off qinq off
rx offload avail: vlan-strip ipv4-cksum udp-cksum tcp-cksum vlan-filter
jumbo-frame scatter keep-crc
rx offload active: ipv4-cksum jumbo-frame scatter
tx offload avail: vlan-insert ipv4-cksum udp-cksum tcp-cksum multi-segs
tx offload active: udp-cksum tcp-cksum multi-segs
rss avail: none
rss active: none
tx burst function: eth_em_xmit_pkts
rx burst function: eth_em_recv_scattered_pkts

tx frames ok 708125
tx bytes ok 220353243
rx frames ok 737130
rx bytes ok 65333902
extended stats:
rx_good_packets 737130
tx_good_packets 708125
rx_good_bytes 65333902
tx_good_bytes 220353243
eth1 2 up eth1
Link speed: 1 Gbps
RX Queues:
queue thread mode
0 vpp_wk_0 (1) polling
Ethernet address 00:0c:29:4e:64:7d
Intel 82540EM (e1000)
carrier up full duplex mtu 9206
flags: admin-up pmd maybe-multiseg tx-offload intel-phdr-cksum rx-ip4-cksum int-supported
Devargs:
rx: queues 1 (max 1), desc 512 (min 32 max 4096 align 8)
tx: queues 1 (max 1), desc 512 (min 32 max 4096 align 8)
pci: device 8086:100f subsystem 15ad:0750 address 0000:02:07.00 numa 0
max rx packet len: 16128
promiscuous: unicast off all-multicast on
vlan offload: strip off filter off qinq off
rx offload avail: vlan-strip ipv4-cksum udp-cksum tcp-cksum vlan-filter
jumbo-frame scatter keep-crc
rx offload active: ipv4-cksum jumbo-frame scatter
tx offload avail: vlan-insert ipv4-cksum udp-cksum tcp-cksum multi-segs
tx offload active: udp-cksum tcp-cksum multi-segs
rss avail: none
rss active: none
tx burst function: eth_em_xmit_pkts
rx burst function: eth_em_recv_scattered_pkts

local0 0 down local0
Link speed: unknown
local

 

root@dj:/usr/local/nginx/logs# vppctl show interface address
eth0 (up):
L3 10.1.1.2/24
eth1 (up):
L3 10.2.1.1/24
local0 (dn):

 

root@dj:/usr/local/nginx/logs# vppctl show interface
Name Idx State MTU (L3/IP4/IP6/MPLS) Counter Count
eth0 1 up 9000/0/0/0 rx packets 737130
rx bytes 62385606
tx packets 708125
tx bytes 220353243
drops 15
punt 14
ip4 737127
eth1 2 up 9000/0/0/0
local0 0 down 0/0/0/0

 

root@dj:/usr/local/nginx/logs# vppctl show run
Thread 0 vpp_main (lcore 0)
Time 223.5, 10 sec internal node vector rate 0.00 loops/sec 446200.12
vector rates in 0.0000e0, out 0.0000e0, drop 0.0000e0, punt 0.0000e0
Name State Calls Vectors Suspends Clocks Vectors/Call
acl-plugin-fa-cleaner-process event wait 0 0 1 1.34e4 0.00
admin-up-down-process event wait 0 0 1 2.54e4 0.00
api-rx-from-ring any wait 0 0 42 1.56e7 0.00
avf-process event wait 0 0 1 2.53e4 0.00
bfd-process event wait 0 0 1 7.38e3 0.00
bond-process event wait 0 0 1 4.79e4 0.00
cnat-scanner-process event wait 0 0 1 1.29e4 0.00
dhcp-client-process any wait 0 0 1 4.75e3 0.00
dhcp6-client-cp-process any wait 0 0 1 6.19e3 0.00
dhcp6-pd-client-cp-process any wait 0 0 1 5.31e3 0.00
dhcp6-pd-reply-publisher-proce event wait 0 0 1 4.55e3 0.00
dhcp6-reply-publisher-process event wait 0 0 1 3.64e3 0.00
dpdk-process any wait 0 0 65 6.80e7 0.00
fib-walk any wait 0 0 98 4.95e4 0.00
flow-report-process any wait 0 0 1 4.71e3 0.00
flowprobe-timer-process any wait 0 0 1 7.11e3 0.00
gbp-scanner event wait 0 0 1 1.02e4 0.00
igmp-timer-process event wait 0 0 1 1.03e4 0.00
ikev2-manager-process any wait 0 0 98 9.87e3 0.00
ioam-export-process any wait 0 0 1 5.28e3 0.00
ip-neighbor-event event wait 0 0 1 3.68e3 0.00
ip4-full-reassembly-expire-wal any wait 0 0 20 4.23e5 0.00
ip4-neighbor-age-process event wait 0 0 1 3.48e3 0.00
ip4-sv-reassembly-expire-walk any wait 0 0 20 7.49e3 0.00
ip6-full-reassembly-expire-wal any wait 0 0 20 3.96e3 0.00
ip6-mld-process any wait 0 0 194 6.80e3 0.00
ip6-neighbor-age-process event wait 0 0 1 5.98e4 0.00
ip6-ra-process any wait 0 0 194 5.99e3 0.00
ip6-rs-process any wait 0 0 1 5.50e3 0.00
ip6-sv-reassembly-expire-walk any wait 0 0 20 4.78e3 0.00
l2-arp-term-publisher event wait 0 0 1 4.06e3 0.00
l2fib-mac-age-scanner-process event wait 0 0 1 5.10e3 0.00
lldp-process event wait 0 0 1 1.25e6 0.00
memif-process event wait 0 0 1 1.17e4 0.00
nat44-ei-ha-process event wait 0 0 1 9.52e3 0.00
nsh-md2-ioam-export-process any wait 0 0 1 1.48e4 0.00
rd-cp-process any wait 0 0 1 1.01e4 0.00
send-dhcp6-client-message-proc any wait 0 0 1 5.03e3 0.00
send-dhcp6-pd-client-message-p any wait 0 0 1 1.13e4 0.00
session-queue interrupt wa 1 0 0 7.43e4 0.00
session-queue-process any wait 0 0 192 1.54e7 0.00
startup-config-process done 1 0 1 1.85e9 0.00
statseg-collector-process time wait 0 0 20 3.11e6 0.00
udp-ping-process any wait 0 0 1 1.37e4 0.00
unix-cli-local:4 active 4 0 25 6.95e13 0.00
unix-cli-new-session any wait 0 0 16 1.83e8 0.00
unix-epoll-input polling 1106008 0 0 4.48e5 0.00
vhost-user-process any wait 0 0 1 4.55e3 0.00
vhost-user-send-interrupt-proc any wait 0 0 1 4.45e3 0.00
virtio-send-interrupt-process any wait 0 0 1 4.33e3 0.00
vpe-link-state-process event wait 0 0 2 4.26e3 0.00
vrrp-periodic-process event wait 0 0 1 1.87e4 0.00
vxlan-gpe-ioam-export-process any wait 0 0 1 4.62e3 0.00
wg-timer-manager event wait 0 0 1 2.69e6 0.00
---------------
Thread 1 vpp_wk_0 (lcore 1)
Time 223.5, 10 sec internal node vector rate 0.00 loops/sec 8741716.53
vector rates in 6.4665e3, out 3.1687e3, drop 6.7121e-2, punt 6.2646e-2
Name State Calls Vectors Suspends Clocks Vectors/Call
arp-input active 3 3 0 6.46e3 1.00
arp-reply active 3 3 0 5.05e4 1.00
dpdk-input polling 1376665504 737130 0 2.28e5 0.00
drop active 2 15 0 2.33e2 7.50
error-drop active 2 15 0 7.85e2 7.50
error-punt active 1 14 0 1.46e2 14.00
eth0-output active 126210 708125 0 1.18e2 5.61
eth0-tx active 126210 708125 0 7.43e3 5.61
ethernet-input active 147419 737130 0 2.27e2 5.00
interface-output active 2 2 0 3.03e3 1.00
ip4-drop active 1 14 0 7.19e2 14.00
ip4-glean active 1 14 0 6.90e2 14.00
ip4-input-no-checksum active 147416 737127 0 1.92e2 5.00
ip4-local active 147416 737127 0 1.41e2 5.00
ip4-lookup active 191641 1445263 0 1.12e2 7.54
ip4-punt active 1 14 0 1.74e2 14.00
ip4-rewrite active 126207 708122 0 1.19e2 5.61
punt active 1 14 0 4.68e2 14.00
session-queue polling 1328745342 707982 0 3.28e5 0.00
tcp4-established active 23790 144357 0 2.42e3 6.07
tcp4-input active 147415 737113 0 5.76e2 5.00
tcp4-listen active 62045 146763 0 2.37e4 2.37
tcp4-output active 126170 708026 0 2.09e2 5.61
tcp4-rcv-process active 119696 445883 0 2.73e3 3.73
tcp4-reset active 38 110 0 2.34e3 2.89
unix-epoll-input polling 1343094 0 0 1.10e3 0.00
root@dj:/usr/local/nginx/logs#

 

 

 

root@dj:/usr/local/nginx/logs# vppctl show error
Count Node Reason Severity
1 dpdk-input no error error
2 arp-reply ARP replies sent error
707982 session-queue Packets transmitted error
146763 tcp4-listen SYNs received error
5663 tcp4-rcv-process Packets pushed into rx fifo error
293957 tcp4-rcv-process Pure ACKs received error
146263 tcp4-rcv-process FINs received error
141100 tcp4-established Packets pushed into rx fifo error
2712 tcp4-established Old segment error
45 tcp4-established Pure ACKs received error
500 tcp4-established FINs received error
708026 tcp4-output Packets sent error
13 ip4-glean ARP requests throttled error
1 ip4-glean ARP requests sent error
14 ip4-local unknown ip protocol error