基于 Kubernetes v1.14.0 之 使用dns+nginx对外提供服务部署

说明：所有操作在k8s-vip 及 k8s-ingress 服务器，下面以k8s-vip-01 服务器为例 k8s-vip 主要提供公司内部访问k8s集群接口服务

2、修改dns

关闭NetworkManager 同时禁止开机启动不然它会修改resolv.conf  重启服务就会还原resolv.conf
systemctl disable NetworkManager
systemctl stop  NetworkManager
service network restart
修改网卡
vi /etc/sysconfig/network-scripts/ifcfg-eth0 
添加PEERDNS=no

# Generated by parse-kickstart
DEVICE="eth0"
IPV6INIT="yes"
BOOTPROTO="dhcp"
UU
ONBOOT="yes"
PEERDNS=no
修改 resolv.conf
vi /etc/resolv.conf 
nameserver 10.64.0.2
search monitoring.svc.niuke.local svc.niuke.local niuke.local
options ndots:5
### 说明search monitoring.svc.niuke.local svc.niuke.local niuke.local 是在monitoring  命名空间搜索
访问monitoring 命名空间的服务可以使用短services 访问不用添加命名空间如果访问其它空间的名字
就要添加命名空间名字例子
访问monitoring 空间服务
http://nginx
访问kube-system 空间的服务
http://nginx.kube-system.svc.niuke.local
修改完成验证dns 配置是否生效 查找已经安装的服务
kubectl get service -A 
[root@jenkins ~]# kubectl get service -A
NAMESPACE     NAME                        TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)                  AGE
default       myip                        NodePort    10.64.160.236   <none>        8080:39569/TCP           14d
kube-system   kube-dns                    ClusterIP   10.64.0.2       <none>        53/UDP,53/TCP,9153/TCP   61d
monitoring    grafana                     NodePort    10.64.76.72     <none>        3000:30091/TCP           58d
测试default  命名空间
ping myip 
[root@nginx-1 ~]# ping myip 
ping: myip: Name or service not known
ping myip.default
[root@nginx-1 ~]# ping myip.default
PING myip.default.svc.niuke.local (10.64.160.236) 56(84) bytes of data.
64 bytes from myip.default.svc.niuke.local (10.64.160.236): icmp_seq=1 ttl=64 time=0.067 ms
测试kube-system 空间
ping kube-dns.kube-system
[root@nginx-1 ~]# ping kube-dns.kube-system
PING kube-dns.kube-system.svc.niuke.local (10.64.0.2) 56(84) bytes of data.
64 bytes from kube-dns.kube-system.svc.niuke.local (10.64.0.2): icmp_seq=1 ttl=64 time=0.060 ms
测试monitoring  也是 dns 默认搜索空间
ping grafana 
[root@nginx-1 ~]# ping grafana 
PING grafana.monitoring.svc.niuke.local (10.64.76.72) 56(84) bytes of data.
64 bytes from grafana.monitoring.svc.niuke.local (10.64.76.72): icmp_seq=1 ttl=64 time=0.065 ms
64 bytes from grafana.monitoring.svc.niuke.local (10.64.76.72): icmp_seq=2 ttl=64 time=0.087 ms
正常返回

3、安装nginx 准备

3.1 安装依赖

yum -y install epel-release
yum -y install  git patch gcc gcc-c++                  readline-devel zlib-devel                 libffi-devel openssl                 openssl-devel make autoconf                 automake libtool bison libxml2                 libxml2-devel libxslt-devel                 libyaml-devel  python  python-docutils                 cmake imake expat-devel libaio libaio-devel                 bzr ncurses-devel wget libjpeg libjpeg-devel                 libpng libpng-devel freetype freetype-devel                 pcre-devel curl-devel libmcrypt libmcrypt-devel                 libunwind libunwind-devel lua-devel

3.2、安装nginx LUA 支持 LuaJIT-2.0.5

cd /usr/local/src/
git clone https://github.com/openresty/luajit2.git
cd luajit2
make -j$(nproc) && make -j$(nproc) install
ln -sf /usr/local/lib/libluajit-5.1.so.2 /lib64/libluajit-5.1.so.2

3.3、安装nginx tcmalloc 依赖

git clone https://github.com/gperftools/gperftools.git
cd gperftools
./autogen.sh
./configure
make -j$(nproc) && make -j$(nproc) install
ln -sf /usr/local/lib/libtcmalloc.so /usr/lib/
ln -sf /usr/local/lib/libtcmalloc.so /usr/lib64/

3.4、安装nginx jemalloc 支持

cd ../
git clone https://github.com/jemalloc/jemalloc.git
cd jemalloc
./autogen.sh
./configure
make -j$(nproc) && make -j$(nproc) install_bin install_include install_lib
echo ‘/usr/local/lib‘ > /etc/ld.so.conf.d/local.conf
ldconfig
ln -sf /usr/local/lib/libjemalloc.so /usr/lib/
ln -sf /usr/local/lib/libjemalloc.so /usr/lib64/

3.5 安装nginx brotli 压缩支持

git clone https://github.com/bagder/libbrotli
cd libbrotli/
./autogen.sh
./configure
make -j$(nproc) && make -j$(nproc) install

3.6、安装nginx lua 支持依赖

wget https://www.kyne.com.au/~mark/software/download/lua-cjson-2.1.0.tar.gz
tar -xzvf lua-cjson-2.1.0.tar.gz
cd lua-cjson-2.1.0
make -j$(nproc) && make -j$(nproc) install
cd ../
git clone https://github.com/diegonehab/luasocket.git
cd luasocket
make -j$(nproc) && make -j$(nproc) install

3.7 安装 nginx tls1.3 支持

### 说明如果安装haproxy 或者httpd 等请先安装这些最后安装openssl 不然haproxy等会编译出错
wget https://www.openssl.org/source/openssl-1.1.1c.tar.gz
tar -xvf openssl-1.1.1c.tar.gz
cd openssl-1.1.1c
./config   --openssldir=/usr/local/ssl
make -j$(nproc) && make -j$(nproc) install

3.8、下载nginx 支持模块

wget https://people.freebsd.org/~osa/ngx_http_redis-0.3.9.tar.gz
tar -xvf ngx_http_redis-0.3.9.tar.gz
git clone https://github.com/FRiCKLE/ngx_cache_purge.git
git clone https://github.com/weibocom/nginx-upsync-module.git
git clone https://github.com/xiaokai-wang/nginx_upstream_check_module.git
git clone https://github.com/openresty/echo-nginx-module.git
git clone --branch v0.10.14 https://github.com/openresty/lua-nginx-module.git
git clone https://github.com/evanmiller/mod_zip.git
git clone https://github.com/simplresty/ngx_devel_kit.git
git clone https://github.com/wdaike/ngx_upstream_jdomain.git
git clone https://github.com/GUI/nginx-upstream-dynamic-servers.git
git clone https://github.com/vozlt/nginx-module-vts.git
git clone https://github.com/google/ngx_brotli
cd ngx_brotli
git submodule update --init 

3.9、下载nginx 源码

wget https://nginx.org/download/nginx-1.17.0.tar.gz
tar -xvf nginx-1.17.0.tar.gz

3.10 编译nginx

cd nginx-1.17.0
export LUAJIT_LIB=/usr/local/lib
export LUAJIT_INC=/usr/local/include/luajit-2.1
patch -p1 < ../nginx_upstream_check_module/check_1.12.1+.patch
./configure --prefix=/apps/nginx --sbin-path=/usr/sbin/nginx --conf-path=/etc/nginx/nginx.conf --error-log-path=/apps/nginx/log/error.log --http-log-path=/apps/nginx/log/access.log --pid-path=/apps/nginx/run/nginx.pid --lock-path=/apps/nginx/run/nginx.lock --http-client-body-temp-path=/apps/nginx/cache/client_temp --http-proxy-temp-path=/apps/nginx/cache/proxy_temp --http-fastcgi-temp-path=/apps/nginx/cache/fastcgi_temp --http-uwsgi-temp-path=/apps/nginx/cache/uwsgi_temp --http-scgi-temp-path=/apps/nginx/cache/scgi_temp --user=nginx --group=nginx --with-compat --with-pcre --with-http_ssl_module --with-http_realip_module --with-http_addition_module --with-http_sub_module --with-http_dav_module --with-http_flv_module --with-http_mp4_module --with-http_gunzip_module --with-google_perftools_module --with-http_gzip_static_module --with-http_random_index_module --with-http_secure_link_module --with-http_stub_status_module --with-http_auth_request_module --with-threads --with-stream --with-stream_ssl_module --with-stream_realip_module --with-stream_ssl_preread_module --with-openssl=../openssl-1.1.1c --with-http_slice_module --with-mail --with-mail_ssl_module --with-file-aio --with-http_v2_module --with-ipv6 --with-openssl-opt=enable-tls1_3 --add-module=../nginx-upsync-module --add-module=../echo-nginx-module --add-module=../nginx_upstream_check_module --add-module=../lua-nginx-module --add-module=../ngx_devel_kit --add-module=../ngx_brotli --add-module=../mod_zip --add-module=../ngx_cache_purge --add-module=../ngx_http_redis-0.3.9 --add-module=../ngx_upstream_jdomain --add-module=../nginx-upstream-dynamic-servers --add-module=../nginx-module-vts --with-ld-opt="-Wl,-rpath,$LUAJIT_LIB,-ljemalloc"

mkdir -pv /apps/nginx/cache/{client_temp,proxy_temp,fastcgi_temp,uwsgi_temp,scgi_temp,proxy_cache}
mkdir /apps/nginx/tcmalloc
make -j$(nproc) && make -j$(nproc) install
chown -R nginx:nginx /apps/nginx
mkdir -p /etc/nginx/servers

3.11、创建nginx启动文件

vi /usr/lib/systemd/system/nginx.service 
[Unit]
Description=nginx - high performance web server
Documentation=http://nginx.org/en/docs/
After=network.target remote-fs.target nss-lookup.target

[Service]
Type=forking
LimitNOFILE=1024000
LimitNPROC=1024000
LimitCORE=infinity
LimitMEMLOCK=infinity
PIDFile=/apps/nginx/run/nginx.pid
ExecStartPre=/usr/sbin/nginx -t -c /etc/nginx/nginx.conf
ExecStart=/usr/sbin/nginx -c /etc/nginx/nginx.conf
ExecReload=/bin/kill -s HUP $MAINPID
ExecStop=/bin/kill -s QUIT $MAINPID
PrivateTmp=true

[Install]
WantedBy=multi-user.target

### 启动nginx 
systemctl daemon-reload
systemctl enable nginx
systemctl start nginx
systemctl status nginx

4、优化nginx

cd /etc/nginx
mv nginx.conf nginx.conf.old
vi nginx.conf
user  nginx;
worker_processes  8;

worker_rlimit_nofile 1024000;

worker_cpu_affinity 00000001 00000010 00000100 00001000 00010000 00100000 0100000 10000000;

worker_priority 1;
worker_shutdown_timeout 10s;
#生产使用
error_log  /apps/nginx/log/error.log crit;
#调试使用
#error_log  /apps/nginx/log/error.log;
pid        /apps/nginx/run/nginx.pid;
lock_file  /apps/nginx/run/nginx.lock;
#打开谷歌tcmalloc支持
#google_perftools_profiles /apps/nginx/tcmalloc/tcmalloc;
daemon on;
master_process on;

events {
    worker_connections  1024000;
    multi_accept on;  
    use epoll;
}

http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;

    map $http_x_forwarded_for  $clientRealIp {
        ""      $remote_addr;
        ~^(?P<firstAddr>[0-9\.]+),?.*$  $firstAddr;
        } 

    map $request_uri $loggable {

        default 1;
    }
    map $http_upgrade $connection_upgrade {
        default          upgrade;
    ‘‘
        close;
    }
    map $http_x_forwarded_proto $pass_access_scheme {
        default          $http_x_forwarded_proto;
    ‘‘
        $scheme;
    }
    log_format  main  escape=json ‘$clientRealIp - $remote_user [$time_local] $http_host "$request" ‘
                      ‘$status $body_bytes_sent "$http_referer" ‘
                      ‘"$http_user_agent" "$http_x_forwarded_for" $remote_addr ‘
                      ‘ups_add: $upstream_addr ups_resp_time: $upstream_response_time ‘  
                      ‘request_time: $request_time  ups_status: $upstream_status request_body: $request_body‘
                      ‘ upstream_response_length [$proxy_upstream_name]‘;

    access_log  /apps/nginx/log/access.$server_name.log  main;

    vhost_traffic_status_zone shared:vhost_traffic_status:10m;

    #lua_need_request_body on; #开启lua支持 需要编译lua模块 
    #lua_package_path "/apps/nginx/conf/waf/?.lua";
    #lua_shared_dict limit 100m;
    #lua_shared_dict badGuys 100m;
    #lua_code_cache on;
    #init_by_lua_file "/apps/nginx/conf/waf/init.lua";
    #access_by_lua_file "/apps/nginx/conf/waf/access.lua";
    client_header_buffer_size 1024k;
    large_client_header_buffers 4 128k;
    client_header_timeout 900;
    client_body_timeout 900;
    client_max_body_size   0;
    client_body_buffer_size 1024k;
    server_tokens off;
    sendfile on;
    sendfile_max_chunk 512k;
    tcp_nopush on;
    tcp_nodelay on;
    keepalive_timeout 75s;
    keepalive_requests 100;    
    aio       threads;
    aio_write   on;
    http2_max_field_size 4k;
    http2_max_header_size 16k;

    types_hash_max_size             2048;
    server_names_hash_max_size      1024;
    server_names_hash_bucket_size 128;
    #map_hash_bucket_size            64;

    proxy_headers_hash_max_size     512;
    proxy_headers_hash_bucket_size  64;

    variables_hash_bucket_size      128;
    variables_hash_max_size         2048;

    underscores_in_headers          off;
    ignore_invalid_headers          on;
    #反向代理配置
    proxy_ignore_client_abort on;
    proxy_connect_timeout 900;
    proxy_read_timeout 900;
    proxy_send_timeout 900;
    proxy_buffer_size 64k;
    proxy_buffers 4 64k;
    proxy_busy_buffers_size 128k;
    proxy_temp_file_write_size 128k;
    proxy_temp_path /apps/nginx/cache/client_temp; 
    proxy_cache_path /apps/nginx/cache/proxy_temp levels=1:2 keys_zone=cache_one:50m inactive=7d max_size=2g;

    #fastcgi 配置
    #fastcgi_connect_timeout 300;
    #fastcgi_send_timeout 300;
    #fastcgi_read_timeout 300;
    #fastcgi_buffer_size 16k;
    #fastcgi_buffers 16 16k;
    #fastcgi_busy_buffers_size 16k;
    #fastcgi_temp_file_write_size 16k;
    #fastcgi_cache_valid 200 302 1h;
    #fastcgi_cache_valid 301 1d;
    #fastcgi_cache_valid any 1m;
    #fastcgi_cache_min_uses 1;
    #fastcgi_cache_use_stale error timeout invalid_header http_500;
    #fastcgi_intercept_errors on;
    #fastcgi_temp_path /apps/nginx/cache/client_temp;
    #fastcgi_cache_path /apps/nginx/cache/fastcgi_temp levels=1:2 keys_zone=cache_fastcgi:128m inactive=1d max_size=10g;
    log_subrequest      on;    
    uninitialized_variable_warn off;
    reset_timedout_connection on;
    send_timeout 900;
    limit_conn_zone $binary_remote_addr zone=addr:5m;
    limit_conn addr 100;
    charset UTF-8;
    gzip on;
    gzip_http_version 1.0;
    gzip_buffers 4 16k;
    gzip_disable "msie6";
    gzip_proxied any;
    gzip_min_length 1000;
    gzip_comp_level 6;
    gzip_types text/plain text/css text/xml text/javascript application/json application/x-javascript application/javascript application/xml application/xml+rss;
    gzip_vary on;
    open_file_cache max=204800 inactive=20s;
    open_file_cache_valid 30s;
    open_file_cache_min_uses 2;
    open_file_cache_errors on;

    brotli on;
    brotli_comp_level 6;     
    brotli_buffers 16 8k;
    brotli_min_length 20;
    brotli_static off;
    brotli_window 512k;
    brotli_types *;
    #nginx 跨域设置
    #add_header ‘Access-Control-Allow-Origin‘ ‘*‘;
    #add_header ‘Access-Control-Max-Age‘ 1728000;
    #add_header ‘Access-Control-Allow-Headers‘ ‘DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range‘;
    #add_header ‘Access-Control-Expose-Headers‘ ‘Content-Length,Content-Range‘;
    #add_header Access-Control-Allow-Methods "GET, POST, OPTIONS";
    #add_header Access-Control-Allow-Credentials true;   
    # dns 配置
    resolver 10.64.0.2 valid=3s;
    resolver_timeout 5s;
    include /etc/nginx/conf.d/*.conf;
}
cd conf.d
## 生成nginx 默认配置
 vi default.conf
 server {

        listen 80 default_server;
        server_name _;

        #charset koi8-r;
        # add_header X-Real-IP $http_x_real_ip;
        #access_log  logs/host.access.log  main;
        #access_log logs/log/access.log  cdn;
        #check interval=3000 rise=2 fall=5 timeout=1000;
        #location / {
        #    root   /apps/nginx/html;
        #    index  index.php index.html index.htm;
        #}

        #error_page  404              /404.html;

        # redirect server error pages to the static page /50x.html
        #
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }

        # proxy the PHP scripts to Apache listening on 127.0.0.1:80
        #
        #location ~ \.php$ {
        #    proxy_pass   http://127.0.0.1;
        #}

        # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
        #
        location ~ \.php$ {
            root           /apps/nginx/html;
            fastcgi_pass   127.0.0.1:9000;
            fastcgi_index  index.php;
            fastcgi_param  SCRIPT_FILENAME  $document_root$fastcgi_script_name;
            include        fastcgi_params;
        }
        location ~* ^.+\.(jpg|jpeg|gif|png|bmp)$ {
            access_log  off;
            root        /apps/nginx/html;
            expires     30d;
                        break;
        }
        location = /realip {
             default_type ‘text/html‘;
             echo $clientRealIp;
           }
        location = /upstream_show {
            upstream_show;
            access_log off;
        }
        location /nstatus {
        check_status;
        access_log off;
        #allow SOME.IP.ADD.RESS;
        #deny all;
    }
        location /status{
            stub_status on;
            access_log  off;
        }
        location /healthz {
            access_log off;
            return 200;
        }   
 location /{
            access_log  off;
            return       444;
        }

        location /nginx_status {
            set $proxy_upstream_name "internal";
            vhost_traffic_status_display;
            vhost_traffic_status_display_format html;

        }

}
重启nginx
systemctl restart nginx

5、配置nginx 对外服务

k8s 集群创建对外测试服务
kubectl run myip --image=cloudnativelabs/whats-my-ip --replicas=3 --port=8080
第一种模式使用ClusterIP
kubectl expose deployment myip --port=8080 --target-port=8080
创建nginx 配置
cd /etc/nginx/conf.d
使用nginx-upstream-dynamic-servers 模块dns 转发
vi myip.test.com.conf
upstream myip {
        zone myip 64k;
        server myip.default.svc.niuke.local:8080 resolve;
                check interval=3000 rise=2 fall=5 timeout=1000;
        keepalive 64;
}

server {
    listen 80;
    server_name myip.test.com;
    location / {
        client_body_buffer_size 1024k;
        client_max_body_size    0;
        proxy_connect_timeout 300s;
        proxy_send_timeout   900;
        proxy_read_timeout   900;
        proxy_buffer_size    32k;
        proxy_buffers      4 32k;
        proxy_busy_buffers_size 64k;
        proxy_redirect     off;
        proxy_set_header   Accept-Encoding ‘‘;
        proxy_set_header   Host   $host;
        proxy_set_header   Referer $http_referer;
        proxy_set_header   Cookie $http_cookie;
        proxy_set_header   X-Real-IP  $http_x_real_ip;
        proxy_set_header   X-Forwarded-For  $proxy_add_x_forwarded_for;
        proxy_buffering    off;
        proxy_headers_hash_max_size 51200;
        proxy_headers_hash_bucket_size 6400;
        proxy_pass          http://myip/;
        proxy_redirect default;
    }
}
访问 
本地host
192.168.4.1  myip.test.com

查看nginx 状态
http://192.168.4.1/nginx_status

第二种 clusterIP=None 直接访问容器ip
kubectl delete service myip
kubectl expose deployment myip --port=8080 --target-port=8080 --cluster-ip=None
[root@jenkins ansible]# kubectl get service myip      
NAME   TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)    AGE
myip   ClusterIP   None         <none>        8080/TCP   26s

建议使用clusterIP=None 不做转发自己访问

使用ngx_upstream_jdomain 模块做dns 服务发现
vi myip.test.com.conf
upstream myip {
        jdomain  myip.default port=8080 interval=20;
        keepalive 64;
}

server {
    listen 80;
     server_name  myip.test.com;
    location / {
        client_body_buffer_size 1024k;
        client_max_body_size    0;
        proxy_connect_timeout 300s;
        proxy_send_timeout   900;
        proxy_read_timeout   900;
        proxy_buffer_size    32k;
        proxy_buffers      4 32k;
        proxy_busy_buffers_size 64k;
        proxy_redirect     off;
        proxy_set_header   Accept-Encoding ‘‘;
        proxy_set_header   Host   $host;
        proxy_set_header   Referer $http_referer;
        proxy_set_header   Cookie $http_cookie;
        proxy_set_header   X-Real-IP  $http_x_real_ip;
        proxy_set_header   X-Forwarded-For  $proxy_add_x_forwarded_for;
        proxy_buffering    off;
        proxy_headers_hash_max_size 51200;
        proxy_headers_hash_bucket_size 6400;
        proxy_pass          http://myip/;
        proxy_redirect default;
    }
}

查看nginx 状态
http://192.168.4.1/nginx_status

已经没有upstreams显示。

网站依然能正常打开

6、配置grafana 对外访问

vi grafana.monitoring.conf
upstream grafana.monitoring {
        #zone grafana.monitoring 64k;
        jdomain  grafana.monitoring port=3000 interval=20;
        #server grafana.monitoring.svc.niuke.local:3000 resolve;
        keepalive 64;
}

server {
    listen 80;
    server_name monitor.xxxx.com;
    location / {
        client_body_buffer_size 1024k;
        client_max_body_size    0;
        proxy_connect_timeout 300s;
        proxy_send_timeout   900;
        proxy_read_timeout   900;
        proxy_buffer_size    32k;
        proxy_buffers      4 32k;
        proxy_busy_buffers_size 64k;
        proxy_redirect     off;
        proxy_set_header   Accept-Encoding ‘‘;
        proxy_set_header   Host   $host;
        proxy_set_header   Referer $http_referer;
        proxy_set_header   Cookie $http_cookie;
        proxy_set_header   X-Real-IP  $http_x_real_ip;
        proxy_set_header   X-Forwarded-For  $proxy_add_x_forwarded_for;
        proxy_buffering    off;
        proxy_headers_hash_max_size 51200;
        proxy_headers_hash_bucket_size 6400;
        proxy_pass          http://grafana.monitoring/;
        proxy_redirect default;
    }
}
nginx -t
nginx -s reload
外部dns 解析
访问域名
http://monitor.xxxx.com

7、安装nginx-vts-exporter 监控nginx

wget https://github.com/hnlq715/nginx-vts-exporter/releases/download/v0.10.3/nginx-vts-exporter-0.10.3.linux-amd64.tar.gz
tar -xvf nginx-vts-exporter-0.10.3.linux-amd64.tar.gz
mv nginx-vts-exporter-0.10.3.linux-amd64 nginx-vts-exporter
cd nginx-vts-exporter
mkdir bin
mv nginx-vts-exporter ./bin
rm -rf LICENSE
创建启动文件
vi /usr/lib/systemd/system/nginx-vts-exporter.service 
[Unit]
Description=Prometheus‘s nginx metrics exporter
[Service]
LimitNOFILE=1024000
LimitNPROC=1024000
LimitCORE=infinity
LimitMEMLOCK=infinity
ExecStart=/apps/nginx-vts-exporter/bin/nginx-vts-exporter -nginx.scrape_uri=http://localhost/traffic_status/format/json
Restart=on-failure
[Install]
WantedBy=multi-user.target
## 启动nginx-vts-exporter
systemctl daemon-reload
systemctl enable nginx-vts-exporter
systemctl start nginx-vts-exporter
systemctl status nginx-vts-exporter
## 创建prometheus  nginx-vts-exporter 服务发现
vi prometheus-serviceMonitornginx.yaml
apiVersion: v1
kind: Service
metadata:
  labels:
    k8s-app: nginx-vts-exporter
  name: nginx-vts-exporter
  namespace: monitoring
spec:
  clusterIP: None
  ports:
  - name: api
    port: 9913
    protocol: TCP
    targetPort: 9913
  sessionAffinity: None
  type: ClusterIP
---
apiVersion: v1
kind: Endpoints
metadata:
  labels:
    k8s-app: nginx-vts-exporter
  name: nginx-vts-exporter
  namespace: monitoring
subsets:
- addresses:
  - ip: 192.168.4.1
  ports:
  - name: api
    port: 9913
    protocol: TCP
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  labels:
    app: nginx-vts-exporter
  name: nginx-vts-exporter
  namespace: monitoring
spec:
  endpoints:
  - interval: 10s
    port: api
  namespaceSelector:
    matchNames:
    - monitoring
  selector:
    matchLabels:
      k8s-app: nginx-vts-exporter

    kubectl apply -f  prometheus-serviceMonitornginx.yaml
    打开 grafana
    选择 nginx-vts-stats 看是否有数据

写在最后
nginx-upstream-dynamic-servers 模块支持nginx_upstream_check_module 后端检查
ngx_upstream_jdomain 不支持nginx_upstream_check_module 后端检查

下一篇： Kubernetes 生产环境安装部署 基于 Kubernetes v1.14.0 之 prometheus-adapter自定义hpa 部署

原文：https://blog.51cto.com/juestnow/2413119