emqx4.3/docs/source/tune/linux_kernel_tuning.rst

Linux Kernel Tuning for C1000K concurrent connections::

    # system-wide limit on max opened files for all processes
    # sysctl -n fs.nr_open
    # 1M
    sysctl -w fs.file-max=1048576
    sysctl -w fs.nr_open=1048576
    ulimit -n 1048576
    # Increase number of incoming connections
    net.core.somaxconn = 65536

    sysctl -w net.ipv4.tcp_mem='10000000 10000000 10000000'
    sysctl -w net.ipv4.tcp_rmem='1024 4096 16384'
    sysctl -w net.ipv4.tcp_wmem='1024 4096 16384'
    sysctl -w net.core.rmem_max=16384
    sysctl -w net.core.wmem_max=16384

    net.core.rmem_max = 16777216
    net.core.wmem_max = 16777216
    net.core.rmem_default = 16777216
    net.core.wmem_default = 16777216
    net.core.optmem_max = 2048000
    net.core.netdev_max_backlog = 50000

    net.ipv4.tcp_rmem = 4096 4096 16777216
    net.ipv4.tcp_wmem = 4096 4096 16777216
    net.ipv4.tcp_max_syn_backlog = 30000
    net.ipv4.tcp_max_tw_buckets = 2000000
    net.ipv4.tcp_tw_reuse = 1
    net.ipv4.tcp_fin_timeout = 10
    net.ipv4.conf.all.send_redirects = 0
    net.ipv4.conf.all.accept_redirects = 0
    net.ipv4.conf.all.accept_source_route = 0
    net.ipv4.tcp_slow_start_after_idle = 0
    net.ipv4.ip_local_port_range = 8000 65535

    net.netfilter.nf_conntrack_max = 1000000
    net.netfilter.nf_conntrack_tcp_timeout_time_wait = 30

    vm.min_free_kbytes = 65536
    vm.swappiness = 0
    vm.overcommit_memory = 1


/etc/sysctl.conf
----------------

::

  fs.file-max = 1048576


/etc/security/limits.conf
-------------------------

::

    *      soft   nofile      1048576
    *      hard   nofile      1048576


  ## Kernel/Network Tunings from @galvezlj

  Server side:
  ```
  fs.file-max = 1000000
  net.core.somaxconn = 65536
  net.ipv4.ip_local_port_range = 500 65535
  net.nf_conntrack_max = 1000000
  net.netfilter.nf_conntrack_max = 1000000
  net.ipv4.tcp_rmem = 4096 4096 16777216
  net.ipv4.tcp_wmem = 4096 4096 16777216
  ```

  Client side:

  ```
  sysctl -w net.ipv4.ip_local_port_range="500 65535"
  echo 1000000 > /proc/sys/fs/nr_open
  ```

  ## Kernel/Network Tunings from my benchmark server

  ```
  fs.file-max = 1000000

  net.core.somaxconn = 65536
  net.core.wmem_max = 124928
  net.core.rmem_max = 124928
  net.core.wmem_default = 124928
  net.core.rmem_default = 124928
  net.core.dev_weight = 64
  net.core.netdev_max_backlog = 1000
  net.core.message_cost = 5
  net.core.message_burst = 10
  net.core.optmem_max = 20480
  net.core.busy_poll = 0
  net.core.busy_read = 0
  net.core.netdev_budget = 300

  net.ipv4.tcp_timestamps = 1
  net.ipv4.tcp_window_scaling = 1
  net.ipv4.tcp_sack = 1
  net.ipv4.tcp_retrans_collapse = 1
  net.ipv4.ip_default_ttl = 64
  net.ipv4.tcp_syn_retries = 5
  net.ipv4.tcp_synack_retries = 5
  net.ipv4.tcp_max_orphans = 262144
  net.ipv4.tcp_max_tw_buckets = 262144
  net.ipv4.ip_dynaddr = 0
  net.ipv4.tcp_keepalive_time = 7200
  net.ipv4.tcp_keepalive_probes = 9
  net.ipv4.tcp_keepalive_intvl = 75
  net.ipv4.tcp_retries1 = 3
  net.ipv4.tcp_retries2 = 15
  net.ipv4.tcp_fin_timeout = 60
  net.ipv4.tcp_syncookies = 1
  net.ipv4.tcp_tw_recycle = 0
  net.ipv4.tcp_abort_on_overflow = 0
  net.ipv4.tcp_max_syn_backlog = 2048
  net.ipv4.ip_local_port_range = 32768  61000
  net.ipv4.inet_peer_threshold = 65664
  net.ipv4.inet_peer_minttl = 120
  net.ipv4.inet_peer_maxttl = 600
  net.ipv4.inet_peer_gc_mintime = 10
  net.ipv4.inet_peer_gc_maxtime = 120
  net.ipv4.tcp_mem = 3080640    4107520 6161280
  net.ipv4.tcp_wmem = 4096  16384   4194304
  net.ipv4.tcp_rmem = 4096  87380   4194304
  net.ipv4.tcp_app_win = 31
  net.ipv4.tcp_adv_win_scale = 2
  net.ipv4.tcp_tw_reuse = 0
  net.ipv4.tcp_frto = 2
  net.ipv4.tcp_frto_response = 0
  net.ipv4.tcp_low_latency = 0
  net.ipv4.tcp_no_metrics_save = 0
  net.ipv4.tcp_moderate_rcvbuf = 1
  net.ipv4.tcp_tso_win_divisor = 3
  net.ipv4.tcp_congestion_control = cubic
  net.ipv4.tcp_abc = 0
  net.ipv4.tcp_mtu_probing = 0
  net.ipv4.tcp_base_mss = 512
  net.ipv4.tcp_workaround_signed_windows = 0
  net.ipv4.tcp_challenge_ack_limit = 100
  net.ipv4.tcp_limit_output_bytes = 131072
  net.ipv4.tcp_dma_copybreak = 4096
  net.ipv4.tcp_slow_start_after_idle = 1
  net.ipv4.tcp_available_congestion_control = cubic reno
  net.ipv4.tcp_allowed_congestion_control = cubic reno
  net.ipv4.tcp_max_ssthresh = 0
  net.ipv4.tcp_thin_linear_timeouts = 0
  net.ipv4.tcp_thin_dupack = 0
  net.ipv4.tcp_min_tso_segs = 2
  net.ipv4.udp_mem = 3080640    4107520 6161280
  net.ipv4.udp_rmem_min = 4096
  net.ipv4.udp_wmem_min = 4096
  net.ipv4.conf.all.forwarding = 0
  net.ipv4.conf.all.mc_forwarding = 0
  net.ipv4.conf.all.accept_redirects = 1
  net.ipv4.conf.all.secure_redirects = 1
  net.ipv4.conf.all.shared_media = 1
  net.ipv4.conf.all.rp_filter = 0
  net.ipv4.conf.all.send_redirects = 1
  net.ipv4.conf.all.src_valid_mark = 0
  net.ipv4.conf.all.medium_id = 0
  net.ipv4.conf.all.bootp_relay = 0
  net.ipv4.conf.all.log_martians = 0
  net.ipv4.conf.all.tag = 0

  vm.min_free_kbytes = 67584
  vm.swappiness = 60
  vm.overcommit_memory = 0
  ```