kubernetes v1.20项目之二进制部署Nginx(四层)负载均衡器出现unknown directive “stream“ in /etc/nginx/nginx.conf问题的解决

[root@k8s-master2 ~]# systemctl start nginx
Job for nginx.service failed because the control process exited with error code. See "systemctl status nginx.service" and "journalctl -xe" for details.
[root@k8s-master2 ~]# systemctl status nginx
● nginx.service - The nginx HTTP and reverse proxy server
   Loaded: loaded (/usr/lib/systemd/system/nginx.service; disabled; vendor preset: disabled)
   Active: failed (Result: exit-code) since 三 2021-11-17 22:20:32 CST; 19s ago
  Process: 51050 ExecStartPre=/usr/sbin/nginx -t (code=exited, status=1/FAILURE)
  Process: 51048 ExecStartPre=/usr/bin/rm -f /run/nginx.pid (code=exited, status=0/SUCCESS)

11月 17 22:20:32 k8s-master2 systemd[1]: Starting The nginx HTTP and reverse proxy server...
11月 17 22:20:32 k8s-master2 nginx[51050]: nginx: [emerg] unknown directive "stream" in /etc/nginx/nginx.conf:13
11月 17 22:20:32 k8s-master2 nginx[51050]: nginx: configuration file /etc/nginx/nginx.conf test failed
11月 17 22:20:32 k8s-master2 systemd[1]: nginx.service: control process exited, code=exited status=1
11月 17 22:20:32 k8s-master2 systemd[1]: Failed to start The nginx HTTP and reverse proxy server.
11月 17 22:20:32 k8s-master2 systemd[1]: Unit nginx.service entered failed state.
11月 17 22:20:32 k8s-master2 systemd[1]: nginx.service failed.
[root@k8s-master2 ~]# nginx -t
nginx: [emerg] unknown directive "stream" in /etc/nginx/nginx.conf:13
nginx: configuration file /etc/nginx/nginx.conf test failed

[root@k8s-master2 ~]# nginx -V
nginx version: nginx/1.20.1
built by gcc 4.8.5 20150623 (Red Hat 4.8.5-44) (GCC) 
built with OpenSSL 1.1.1g FIPS  21 Apr 2020
TLS SNI support enabled
configure arguments: --prefix=/usr/share/nginx --sbin-path=/usr/sbin/nginx --modules-path=/usr/lib64/nginx/modules --conf-path=/etc/nginx/nginx.conf --error-log-path=/var/log/nginx/error.log --http-log-path=/var/log/nginx/access.log --http-client-body-temp-path=/var/lib/nginx/tmp/client_body --http-proxy-temp-path=/var/lib/nginx/tmp/proxy --http-fastcgi-temp-path=/var/lib/nginx/tmp/fastcgi --http-uwsgi-temp-path=/var/lib/nginx/tmp/uwsgi --http-scgi-temp-path=/var/lib/nginx/tmp/scgi --pid-path=/run/nginx.pid --lock-path=/run/lock/subsys/nginx --user=nginx --group=nginx --with-compat --with-debug --with-file-aio --with-google_perftools_module --with-http_addition_module --with-http_auth_request_module --with-http_dav_module --with-http_degradation_module --with-http_flv_module --with-http_gunzip_module --with-http_gzip_static_module --with-http_image_filter_module=dynamic --with-http_mp4_module --with-http_perl_module=dynamic --with-http_random_index_module --with-http_realip_module --with-http_secure_link_module --with-http_slice_module --with-http_ssl_module --with-http_stub_status_module --with-http_sub_module --with-http_v2_module --with-http_xslt_module=dynamic --with-mail=dynamic --with-mail_ssl_module --with-pcre --with-pcre-jit --with-stream=dynamic --with-stream_ssl_module --with-stream_ssl_preread_module --with-threads --with-cc-opt='-O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector-strong --param=ssp-buffer-size=4 -grecord-gcc-switches -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -m64 -mtune=generic' --with-ld-opt='-Wl,-z,relro -specs=/usr/lib/rpm/redhat/redhat-hardened-ld -Wl,-E'
###确实缺少了--with-stream模块

[root@k8s-master2 ~]#  rpm -qa nginx
nginx-1.20.1-9.el7.x86_64

[root@k8s-master2 ~]# wget http://nginx.org/download/nginx-1.20.1.tar.gz
--2021-11-17 22:31:09--  http://nginx.org/download/nginx-1.20.1.tar.gz
正在解析主机 nginx.org (nginx.org)... 52.58.199.22, 3.125.197.172, 2a05:d014:edb:5702::6, ...
正在连接 nginx.org (nginx.org)|52.58.199.22|:80... 已连接。
已发出 HTTP 请求，正在等待回应... 200 OK
长度：1061461 (1.0M) [application/octet-stream]
正在保存至: “nginx-1.20.1.tar.gz”

100%[=======================================================================================================>] 1,061,461    684KB/s 用时 1.5s   

2021-11-17 22:31:11 (684 KB/s) - 已保存 “nginx-1.20.1.tar.gz” [1061461/1061461])

[root@k8s-master2 ~]# ls
anaconda-ks.cfg  nginx-1.20.1.tar.gz

[root@k8s-master2 ~]# tar xf nginx-1.20.1.tar.gz && cd nginx-1.20.1

[root@k8s-master2 nginx-1.20.1]# mv /usr/sbin/nginx /usr/sbin/nginx.bak
[root@k8s-master2 nginx-1.20.1]#  cp -r /etc/nginx /etc/nginx.bak

[root@k8s-master2 nginx-1.20.1]# yum -y install libxml2 libxml2-dev libxslt-devel gd-devel perl-devel perl-ExtUtils-Embed GeoIP GeoIP-devel GeoIP-data

[root@k8s-master2 nginx-1.20.1]# yum -y install --skip-broken gcc gcc-c++ autoconf automake gperftools

[root@k8s-master2 nginx-1.20.1]#  yum -y install --skip-broken zlib zlib-devel openssl-devel pcre-devel

[root@k8s-master2 nginx-1.20.1]#  yum -y install redhat-rpm-config.noarch

[root@k8s-master2 nginx-1.20.1]# ls
auto  CHANGES  CHANGES.ru  conf  configure  contrib  html  LICENSE  man  README  src
[root@k8s-master2 nginx-1.20.1]#  ./configure --prefix=/usr/share/nginx --sbin-path=/usr/sbin/nginx --modules-path=/usr/lib64/nginx/modules --conf-path=/etc/nginx/nginx.conf --error-log-path=/var/log/nginx/error.log --http-log-path=/var/log/nginx/access.log --http-client-body-temp-path=/var/lib/nginx/tmp/client_body --http-proxy-temp-path=/var/lib/nginx/tmp/proxy --http-fastcgi-temp-path=/var/lib/nginx/tmp/fastcgi --http-uwsgi-temp-path=/var/lib/nginx/tmp/uwsgi --http-scgi-temp-path=/var/lib/nginx/tmp/scgi --pid-path=/run/nginx.pid --lock-path=/run/lock/subsys/nginx --user=nginx --group=nginx --with-compat --with-debug --with-file-aio --with-google_perftools_module --with-http_addition_module --with-http_auth_request_module --with-http_dav_module --with-http_degradation_module --with-http_flv_module --with-http_gunzip_module --with-http_gzip_static_module --with-http_image_filter_module=dynamic --with-http_mp4_module --with-http_perl_module=dynamic --with-http_random_index_module --with-http_realip_module --with-http_secure_link_module --with-http_slice_module --with-http_ssl_module --with-http_stub_status_module --with-http_sub_module --with-http_v2_module --with-http_xslt_module=dynamic --with-mail=dynamic --with-mail_ssl_module --with-pcre --with-pcre-jit --with-stream=dynamic --with-stream_ssl_module --with-stream_ssl_preread_module --with-threads --with-cc-opt='-O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector-strong --param=ssp-buffer-size=4 -grecord-gcc-switches -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -m64 -mtune=generic' --with-ld-opt='-Wl,-z,relro -specs=/usr/lib/rpm/redhat/redhat-hardened-ld -Wl,-E' --with-stream

[root@k8s-master2 nginx-1.20.1]# make

[root@k8s-master2 nginx-1.20.1]# make install

[root@k8s-master2 nginx-1.20.1]# which nginx
/usr/sbin/nginx
[root@k8s-master2 nginx-1.20.1]# nginx -V
nginx version: nginx/1.20.1
built by gcc 4.8.5 20150623 (Red Hat 4.8.5-44) (GCC) 
built with OpenSSL 1.0.2k-fips  26 Jan 2017
TLS SNI support enabled
configure arguments: --prefix=/usr/share/nginx --sbin-path=/usr/sbin/nginx --modules-path=/usr/lib64/nginx/modules --conf-path=/etc/nginx/nginx.conf --error-log-path=/var/log/nginx/error.log --http-log-path=/var/log/nginx/access.log --http-client-body-temp-path=/var/lib/nginx/tmp/client_body --http-proxy-temp-path=/var/lib/nginx/tmp/proxy --http-fastcgi-temp-path=/var/lib/nginx/tmp/fastcgi --http-uwsgi-temp-path=/var/lib/nginx/tmp/uwsgi --http-scgi-temp-path=/var/lib/nginx/tmp/scgi --pid-path=/run/nginx.pid --lock-path=/run/lock/subsys/nginx --user=nginx --group=nginx --with-compat --with-debug --with-file-aio --with-google_perftools_module --with-http_addition_module --with-http_auth_request_module --with-http_dav_module --with-http_degradation_module --with-http_flv_module --with-http_gunzip_module --with-http_gzip_static_module --with-http_image_filter_module=dynamic --with-http_mp4_module --with-http_perl_module=dynamic --with-http_random_index_module --with-http_realip_module --with-http_secure_link_module --with-http_slice_module --with-http_ssl_module --with-http_stub_status_module --with-http_sub_module --with-http_v2_module --with-http_xslt_module=dynamic --with-mail=dynamic --with-mail_ssl_module --with-pcre --with-pcre-jit --with-stream=dynamic --with-stream_ssl_module --with-stream_ssl_preread_module --with-threads --with-cc-opt='-O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector-strong --param=ssp-buffer-size=4 -grecord-gcc-switches -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -m64 -mtune=generic' --with-ld-opt='-Wl,-z,relro -specs=/usr/lib/rpm/redhat/redhat-hardened-ld -Wl,-E' --with-stream
[root@k8s-master2 nginx-1.20.1]# vim /etc/nginx/nginx.conf
[root@k8s-master2 nginx-1.20.1]# cat /etc/nginx/nginx.conf
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;

include /usr/share/nginx/modules/*.conf;

events {
    worker_connections 1024;
}

# 四层负载均衡，为两台Master apiserver组件提供负载均衡
stream {

    log_format  main  '$remote_addr $upstream_addr - [$time_local] $status $upstream_bytes_sent';

    access_log  /var/log/nginx/k8s-access.log  main;

    upstream k8s-apiserver {
       server 192.168.100.13:6443;   # Master1 APISERVER IP:PORT
       server 192.168.100.16:6443;   # Master2 APISERVER IP:PORT
    }
    
    server {
       listen 16443; # 由于nginx与master节点复用，这个监听端口不能是6443，否则会冲突
       proxy_pass k8s-apiserver;
    }
}

http {
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /var/log/nginx/access.log  main;

    sendfile            on;
    tcp_nopush          on;
    tcp_nodelay         on;
    keepalive_timeout   65;
    types_hash_max_size 2048;

    include             /etc/nginx/mime.types;
    default_type        application/octet-stream;

    server {
        listen       80 default_server;
        server_name  _;

        location / {
        }
    }
}
[root@k8s-master2 nginx-1.20.1]# nginx -t
nginx: the configuration file /etc/nginx/nginx.conf syntax is ok
nginx: configuration file /etc/nginx/nginx.conf test is successful
[root@k8s-master2 nginx-1.20.1]# systemctl start nginx
[root@k8s-master2 nginx-1.20.1]# ps -ef | grep nginx
root      64958      1  0 22:43 ?        00:00:00 nginx: master process /usr/sbin/nginx
nginx     64959  64958  0 22:43 ?        00:00:00 nginx: worker process
nginx     64960  64958  0 22:43 ?        00:00:00 nginx: worker process
root      65022   6661  0 22:43 pts/0    00:00:00 grep --color=auto nginx