Skip to content

Tuning Nginx to handle more than 50.000 reqs/s

Published by tex on December 9, 2016

Hardware used in my test

CPU: E3-1231 v3
CPU SPECS: 4 Core, 8 Threads - 3.40GHz
Memory: 32 GB
Storage: 4 x 240 GB SSD Force 3
Raid adapter: 3Ware 9750

Let’s beginning with os tuning. We will tweak sysctl.conf.

#
# file: '/etc/sysctl.conf'
#

vm.swappiness = 0
vm.max_map_count = 262144

net.ipv4.tcp_wmem = 4096 65536 33554432
net.ipv4.tcp_syn_retries = 3
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_synack_retries = 3
net.ipv4.tcp_slow_start_after_idle = 0
net.ipv4.tcp_rmem = 4096 87380 33554432
net.ipv4.tcp_max_tw_buckets = 5880000
net.ipv4.tcp_max_syn_backlog = 3240000
net.ipv4.tcp_max_orphans = 262144
net.ipv4.tcp_keepalive_probes = 5
net.ipv4.tcp_keepalive_intvl = 30
net.ipv4.tcp_fin_timeout = 10
net.ipv4.tcp_congestion_control = cubic

net.ipv4.neigh.default.gc_thresh3 = 450560
net.ipv4.neigh.default.gc_thresh2 = 450560
net.ipv4.neigh.default.gc_thresh1 = 225280
net.ipv4.neigh.default.gc_stale_time = 7200

net.ipv4.ip_local_port_range = 1024 65535
net.ipv4.ip_forward = 1

net.ipv4.conf.default.send_redirects = 0
net.ipv4.conf.default.secure_redirects = 0
net.ipv4.conf.default.accept_source_route = 0
net.ipv4.conf.default.accept_source_route = 0
net.ipv4.conf.default.accept_redirects = 0
net.ipv4.conf.all.send_redirects = 0
net.ipv4.conf.all.secure_redirects = 0
net.ipv4.conf.all.rp_filter = 1
net.ipv4.conf.all.log_martians = 0
net.ipv4.conf.all.accept_source_route = 0
net.ipv4.conf.all.accept_redirects = 0

net.core.wmem_max = 67108864
net.core.rmem_max = 67108864
net.core.rmem_default = 67108864
net.core.wmem_default = 67108864

net.ipv4.tcp_sack = 0
net.ipv4.tcp_dsack = 0
net.ipv4.tcp_fack = 0

net.core.netdev_max_backlog = 64000

net.core.default_qdisc = fq

kernel.randomize_va_space = 1
kernel.pid_max = 65536

kernel.msgmnb = 65536
kernel.msgmax = 65536

fs.nr_open = 4000000
fs.file-max = 4000000

Nginx configuration file

#
# file: '/etc/nginx/nginx.conf'
#
user www-data;
worker_processes 8;
worker_priority -15;
pid /var/run/nginx.pid;
worker_rlimit_nofile 400000;
timer_resolution 10000ms;

events {
	worker_connections 20000;
	use epoll;
	multi_accept on;
}

http {
	sendfile on;
	tcp_nopush on;
	tcp_nodelay on;
	keepalive_timeout 120;
	keepalive_requests 100000;
	client_max_body_size 128M;
	client_body_timeout   40;
	client_header_timeout 40;
	client_body_buffer_size 128k;
        client_header_buffer_size 8k;
	connection_pool_size 8192;
	request_pool_size 16k;
	server_names_hash_max_size 2048;
	server_names_hash_bucket_size 2048;
	types_hash_max_size 4086;
	server_tokens off;
	resolver 127.0.0.1;
	resolver_timeout 3s;
	reset_timedout_connection on;
	send_timeout 60;

 	open_file_cache max=400000 inactive=30s; 
 	open_file_cache_valid    60s; 
 	open_file_cache_min_uses 2;
 	open_file_cache_errors   on;

	include /etc/nginx/mime.types;
	default_type application/octet-stream;

	access_log /dev/null;
        error_log /dev/null;
	include /etc/nginx/conf.d/*;
}

Virtual host example (SSL)

#
# file '/etc/nginx/sites-enabled/example.conf'
#
server {
     listen 0.0.0.0:443 rcvbuf=64000 sndbuf=120000 backlog=20000 ssl http2;
     server_name example.com www.example.com;
     keepalive_timeout         60;
     ssl                       on;
     ssl_protocols             TLSv1.2 TLSv1.1 TLSv1;
     ssl_ciphers               'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS:!RC4';
     ssl_prefer_server_ciphers on;
     ssl_session_cache         shared:TLSSL:30m;
     ssl_session_timeout       10m;
     ssl_buffer_size 	       32k;
     ssl_certificate           /etc/letsencrypt/live/example.com/fullchain.pem;
     ssl_certificate_key       /etc/letsencrypt/live/example.com/privkey.pem;
     ssl_dhparam 	       /etc/ssl/certs/dhparam.pem;
     more_set_headers          "X-Secure-Connection: true";
     add_header                Strict-Transport-Security max-age=315360000;
     root		       /var/www;

location / {
     root /var/www;
     index index.php index.html;
 }

location ~ .php$ {
     fastcgi_keep_conn on;
     fastcgi_pass   unix:/run/php5.6-fpm.sock;
     fastcgi_index  index.php;
     fastcgi_param  SCRIPT_FILENAME /var/www$fastcgi_script_name;
     include fastcgi_params;
     fastcgi_intercept_errors off;
     fastcgi_buffer_size 32k;
     fastcgi_buffers 32 32k;
     fastcgi_connect_timeout 5;
 }

location ~* ^.+.(jpg|jpeg|gif|png|svg|ico|css|less|xml|html?|swf|js|ttf)$ {
     root /var/www;
     expires 10y;
 }
}

PHP-FPM configuration file

#
# file: '/etc/php5/fpm/pool.d/www.conf'
#

[www]
user = www-data
group = www-data

listen = /run/php5.6-fpm.sock

listen.owner = www-data
listen.group = www-data
process.priority = -10

pm = ondemand
pm.max_children = 2000
pm.process_idle_timeout = 10s
pm.max_requests = 500

Results

Static content:

Server Port:            443
SSL/TLS Protocol:       TLSv1.2,ECDHE-RSA-AES256-GCM-SHA384,2048,256
Concurrency Level:      100
Time taken for tests:   1.216 seconds
Complete requests:      100000
Failed requests:        0
Keep-Alive requests:    100000
Total transferred:      238800000 bytes
HTML transferred:       183800000 bytes
Requests per second:    82207.57 [#/sec] (mean)
Time per request:       1.216 [ms] (mean)
Time per request:       0.012 [ms] (mean, across all concurrent requests)
Transfer rate:          191710.62 [Kbytes/sec] received

Dynamic content:

Server Port:            443
SSL/TLS Protocol:       TLSv1.2,ECDHE-RSA-AES256-GCM-SHA384,2048,256
Concurrency Level:      100
Time taken for tests:   1.342 seconds
Complete requests:      100000
Failed requests:        0
Keep-Alive requests:    100000
Total transferred:      50700000 bytes
HTML transferred:       15400000 bytes
Requests per second:    74527.98 [#/sec] (mean)
Time per request:       1.342 [ms] (mean)
Time per request:       0.013 [ms] (mean, across all concurrent requests)
Transfer rate:          36900.08 [Kbytes/sec] received
Published inBenchmarksLinuxWebservers