Home | 简体中文 | 繁体中文 | 杂文 | Github | 知乎专栏 | 51CTO学院 | CSDN程序员研修院 | OSChina 博客 | 腾讯云社区 | 阿里云栖社区 | Facebook | Linkedin | Youtube | 打赏(Donations) | About
知乎专栏多维度架构

10.11. Spring Cloud Alibaba

10.11.1. 安装 Nacos

10.11.1.1. Docker 安装 Nacos

安装 netkiller-devops 库

		
pip install netkiller-devops		
		
			

创建 docker.py 编排文件

		
#!/usr/bin/env python3
from netkiller.docker import *

volume = Volumes()
volume.create('mysql')

mysql = Services('mysql')
mysql.image('mysql:5.7').container_name('mysql').restart('always').hostname('db.netkiller.cn').env_file(os.getcwd()+'/nacos/env/mysql.env')
mysql.ports(['3306:3306']).volumes([
	'mysql:/var/lib/mysql'
]).command([
	'--socket=/var/lib/mysql/mysql.sock',
	'--default-authentication-plugin=mysql_native_password',
    '--character-set-server=utf8mb4',
    '--collation-server=utf8mb4_general_ci',
    '--explicit_defaults_for_timestamp=true',
    '--lower_case_table_names=1',
    '--max_execution_time=0'
])

nacos = Services('nacos')
nacos.container_name('nacos').env_file(os.getcwd()+'/nacos/env/nacos-mysql.env')
# .environment([
# 	'PREFER_HOST_MODE=hostname',
# 	'MODE=standalone'
# ])
nacos.image('nacos/nacos-server').volumes([
	'../nacos/logs/:/home/nacos/logs',
	'../nacos/init.d/custom.properties:/home/nacos/init.d/custom.properties'
]).ports([
	"8848:8848",
    "9848:9848",
	'9555:9555'
]).depends_on('mysql').restart('on-failure')

experiment = Composes('experiment')
experiment.version('3.9')
experiment.volumes(volume)
experiment.services(mysql)
experiment.services(nacos)

if __name__ == '__main__':
	try:
		docker = Docker()
		docker.sysctl([{'vm.max_map_count':'262144'}])
		docker.environment(experiment)
		docker.main()
	except KeyboardInterrupt:
		print ("Crtl+C Pressed. Shutting down.")
		
			

查看帮助信息

		
[root@localhost ~]# python3 docker.py 
Python controls the docker manager.
Usage: docker.py [options] up|rm|start|stop|restart|logs|top|images|exec <service>

Options:
  -h, --help            show this help message and exit
  --debug               debug mode
  -e development|testing|production, --environment=development|testing|production
                        environment
  -d, --daemon          run as daemon
  --logfile=LOGFILE     logs file.
  -l, --list            print service of environment
  -f, --follow          following logging
  -c, --compose         show docker compose
  --export              export docker compose

Homepage: http://www.netkiller.cn	Author: Neo <netkiller@msn.com>		
		
			

启动 nacos

		
[root@localhost ~]# python3 docker.py -e experiment up nacos
mysql is up-to-date
Starting nacos ... done

[root@localhost ~]# python3 docker.py -e experiment ps 
    Name                   Command               State                                                         Ports                                                      
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------                            
mysql           docker-entrypoint.sh --soc ...   Up      0.0.0.0:3306->3306/tcp,:::3306->3306/tcp, 33060/tcp                                                              
nacos           bin/docker-startup.sh            Up      0.0.0.0:8848->8848/tcp,:::8848->8848/tcp, 0.0.0.0:9555->9555/tcp,:::9555->9555/tcp,                              
                                                         0.0.0.0:9848->9848/tcp,:::9848->9848/tcp   		
		
			

查看启动端口

		
[root@localhost ~]# ss -lnt | grep -E "(8848|9848)"
LISTEN 0      1024         0.0.0.0:8848       0.0.0.0:*          
LISTEN 0      1024         0.0.0.0:9848       0.0.0.0:*          
LISTEN 0      1024            [::]:8848          [::]:*          
LISTEN 0      1024            [::]:9848          [::]:*   		
		
			

测试配置中心

		
[root@localhost ~]# curl -X POST "http://127.0.0.1:8848/nacos/v1/cs/configs?dataId=nacos.cfg.dataId&group=test&content=helloWorld"
true

[root@localhost ~]# curl -X GET "http://127.0.0.1:8848/nacos/v1/cs/configs?dataId=nacos.cfg.dataId&group=test"
helloWorld
		
			

登陆 Web 界面 http://192.168.30.12:8848/nacos/ 默认的账号密码是:nacos/nacos

10.11.1.2. Kubernetes 安装 Nacos

创建 nacos 数据库用户

			
CREATE USER 'nacos'@'%' IDENTIFIED BY 'nacos';

GRANT ALL PRIVILEGES ON nacos.* TO 'nacos'@'%';

SHOW GRANTS FOR 'nacos'@'%';			
			
			

前往 https://github.com/alibaba/nacos/blob/master/distribution/conf/nacos-mysql.sql 下来SQL文件,恢复到 nacos 数据中。

			
import sys
sys.path.insert(0, '/Users/neo/workspace/devops')
from netkiller.kubernetes import *
namespace = 'default'

config = ConfigMap('nacos')
config.apiVersion('v1')
config.metadata().name('nacos').namespace(namespace)
config.data({
    'mysql.host': "rm-bp1g441na9an26wsb.mysql.rds.aliyuncs.com",
    'mysql.port': "3306",
    'mysql.dbname': "nacos",
    'mysql.user': "nacos",
    'mysql.password': "nacos"
})
# config.debug()

statefulSet = StatefulSet()
statefulSet = StatefulSet()
statefulSet.apiVersion('apps/v1')
statefulSet.metadata().name('nacos').labels(
    {'app': 'nacos'}).namespace(namespace)
statefulSet.spec().replicas(3)
statefulSet.spec().serviceName('nacos')
statefulSet.spec().selector({'matchLabels': {'app': 'nacos'}})
statefulSet.spec().template().metadata().labels({'app': 'nacos'})
statefulSet.spec().template().metadata().annotations(
    {'pod.alpha.kubernetes.io/initialized': "true"})
# statefulSet.spec().template().spec().affinity().nodeAffinity({
#     'requiredDuringSchedulingIgnoredDuringExecution': [
#         {'labelSelector': {
#             'matchExpressions': [
#                 {'key': 'app',
#                  'operator': 'In',
#                  'values': ['nacos']
#                  }]
#             },
#         'topologyKey': "kubernetes.io/hostname"
#         }
#     ]
# })
statefulSet.spec().template().spec().containers().name('nacos').imagePullPolicy(Define.containers.imagePullPolicy.IfNotPresent).image(
    'nacos/nacos-server:latest').resources(
        # {'requests': {
        # # 'cpu':'200m',
        # 'memory': "2Gi"}}
        ).ports([
        {'name':'client','containerPort': 8848},
        {'name':'client-rpc','containerPort': 9848},
        {'name':'raft-rpc','containerPort': 9849}
    ]).env([
        {'name': 'TZ', 'value': 'Asia/Shanghai'},
        {'name': 'LANG', 'value': 'en_US.UTF-8'},
        {'name': 'NACOS_REPLICAS', 'value': '1'},

        # {'name': 'SPRING_DATASOURCE_PLATFORM', 'value': 'mysql'},
        # {'name': 'MYSQL_SERVICE_HOST', 'value': 'mysql-0.mysql.default.svc.cluster.local'},
        {'name': 'MYSQL_SERVICE_HOST', 'valueFrom':{'configMapKeyRef':{'name': 'nacos','key': 'mysql.host'}}},
        {'name': 'MYSQL_SERVICE_PORT', 'valueFrom':{'configMapKeyRef':{'name': 'nacos','key': 'mysql.port'}}},
        {'name': 'MYSQL_SERVICE_DB_NAME', 'valueFrom':{'configMapKeyRef':{'name': 'nacos','key': 'mysql.dbname'}}},
        {'name': 'MYSQL_SERVICE_USER', 'valueFrom':{'configMapKeyRef':{'name': 'nacos','key': 'mysql.user'}}},
        {'name': 'MYSQL_SERVICE_PASSWORD', 'valueFrom':{'configMapKeyRef':{'name': 'nacos','key': 'mysql.password'}}},
        # {'name': 'MYSQL_SERVICE_DB_PARAM', 'value': 'characterEncoding=utf8&connectTimeout=1000&socketTimeout=3000&autoReconnect=true&useSSL=false&serverTimezone=GMT%2B8'},
        
        {'name': 'NACOS_SERVER_PORT', 'value': '8848'},
        {'name': 'NACOS_APPLICATION_PORT', 'value': '8848'},
        {'name': 'PREFER_HOST_MODE', 'value': 'hostname'},
        {'name': 'NACOS_SERVERS', 'value': 'nacos-0.nacos.default.svc.cluster.local:8848 nacos-1.nacos.default.svc.cluster.local:8848 nacos-2.nacos.default.svc.cluster.local:8848'},

        # {'name': 'JVM_XMX', 'value': '4g'},
        # {'name': 'NACOS_DEBUG', 'value': 'true'},
        # {'name': 'TOMCAT_ACCESSLOG_ENABLED', 'value': 'true'},
    ])

# statefulSet.debug()

service = Service()
service.metadata().name('nacos')
service.metadata().namespace(namespace)
service.metadata().labels({'app':'nacos'})
service.spec().selector({'app': 'nacos'})
service.spec().type('ClusterIP')
service.spec().ports([
    {'name': 'server', 'protocol': 'TCP', 'port': 8848, 'targetPort': 8848},
    {'name': 'client-rpc', 'protocol': 'TCP', 'port': 9848, 'targetPort': 9848},
    {'name': 'raft-rpc', 'protocol': 'TCP', 'port': 9555, 'targetPort': 9555}
])

# service.debug()

ingress = Ingress()
ingress.apiVersion('networking.k8s.io/v1')
ingress.metadata().name('nacos')
ingress.metadata().namespace(namespace)
# ingress.metadata().annotations({'kubernetes.io/ingress.class': 'nginx'})
ingress.spec().rules([{
    'host': 'nacos.netkiller.com',
    'http':{
        'paths': [{
            'pathType': Define.Ingress.pathType.Prefix,
            'path': '/nacos', 
            'backend':{
                'service':{
                    'name':'nacos', 
                    'port':{'number': 8848}
                }
            }}]
}}])
# ingress.debug()

kubernetes = Kubernetes('/Volumes/Data/kubeconfig')
compose = Compose('nacos')
compose.add(config)
compose.add(statefulSet)
compose.add(service)
compose.add(ingress)
kubernetes.compose(compose)
kubernetes.main()

			
			
			
			

10.11.1.3. IP限制,白名单

			 
	location /nacos {
        allow 192.168.0.0/24;
        allow 172.18.0.0/16;
        allow 202.104.66.10;        
        deny all;

        proxy_set_header Host $host;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;

        proxy_pass http://192.168.0.10:8848;
    }
			
			

10.11.1.4. 防火墙配置

配置防火墙,限制 8848 端口的访问策略,防止本地或其他服务注册到 Nacos 中。

			 
$ iptables -A INPUT -s 172.18.5.0/24 -p tcp --dport 8848 -j REJECT
			
			

172.18.5.0/24 是办公网络,添加上面IP规则,可以防止开发人的电脑注册到测试环境。

删除规则

删除方法一

			 
$ iptables -L -n --line-number | grep 8848
8    REJECT     tcp  --  172.18.5.0/24        0.0.0.0/0            tcp dpt:8848 reject-with icmp-port-unreachable
134  ACCEPT     tcp  --  0.0.0.0/0            172.17.0.119         tcp dpt:8848

$ iptables -D INPUT 8
			
			

删除方法二

			 
$ iptables -D INPUT -s 172.18.5.0/24 -p tcp --dport 8848 -j REJECT
			
			

10.11.2. Kubernetes 部署微服务

10.11.2.1. pom.xml 中加入 docker 插件

			
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>

	<groupId>com.example</groupId>
	<artifactId>demo</artifactId>
	<version>0.0.1-SNAPSHOT</version>
	<packaging>jar</packaging>

	<name>demo</name>
	<description>Demo project for Spring Boot</description>

	<parent>
		<groupId>org.springframework.boot</groupId>
		<artifactId>spring-boot-starter-parent</artifactId>
		<version>2.6.3</version>
		<relativePath /> <!-- lookup parent from repository -->
	</parent>

	<properties>
		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
		<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
		<java.version>1.8</java.version>

		<sonar.projectKey>netkiller.cn_java_AX0HsoVkT19KeT2iVgUT</sonar.projectKey>
		<sonar.qualitygate.wait>true</sonar.qualitygate.wait>

		<docker.registry>registry.netkiller.cn/netkiller.cn</docker.registry>

	</properties>

	<repositories>
		<repository>
			<id>gitlab-maven</id>
			<url>${env.CI_API_V4_URL}/projects/${env.CI_PROJECT_ID}/packages/maven</url>
		</repository>
	</repositories>
	<distributionManagement>
		<repository>
			<id>gitlab-maven</id>
			<url>${CI_API_V4_URL}/projects/${env.CI_PROJECT_ID}/packages/maven</url>
		</repository>
		<snapshotRepository>
			<id>gitlab-maven</id>
			<url>${CI_API_V4_URL}/projects/${env.CI_PROJECT_ID}/packages/maven</url>
		</snapshotRepository>
	</distributionManagement>

	<dependencies>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-web</artifactId>
		</dependency>

		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-test</artifactId>
			<scope>test</scope>
		</dependency>

		<dependency>
			<groupId>junit</groupId>
			<artifactId>junit</artifactId>
			<!-- <version>4.13.2</version> -->
			<scope>test</scope>
		</dependency>

	</dependencies>

	<build>
		<plugins>
			<plugin>
				<groupId>org.springframework.boot</groupId>
				<artifactId>spring-boot-maven-plugin</artifactId>
			</plugin>
			<plugin>
				<groupId>com.spotify</groupId>
				<artifactId>docker-maven-plugin</artifactId>
				<version>1.2.2</version>
				<configuration>
					<imageName>${docker.registry}/${project.artifactId}</imageName>
					<baseImage>openjdk:8-alpine</baseImage>
					<maintainer>netkiller@msn.com</maintainer>
					<volumes>/srv</volumes>
					<workdir>/srv</workdir>
					<env>
						<JAVA_OPTS>-server -Xms512m -Xmx4096m -Djava.security.egd=file:/dev/./urandom</JAVA_OPTS>
					</env>
					<exposes>8080</exposes>
					<entryPoint>["sh", "-c", "/srv/docker-entrypoint.sh"]</entryPoint>
					<resources>
						<resource>
							<targetPath>/srv</targetPath>
							<directory>${project.build.directory}</directory>
							<include>${project.build.finalName}.jar</include>
						</resource>
						<resource>
							<targetPath>/srv</targetPath>
							<directory>.</directory>
							<include>docker-entrypoint.sh</include>
						</resource>
					</resources>
					<registryUrl>http://${docker.registry}/v2/</registryUrl>
					<imageTags>
						<imageTag>${project.version}</imageTag>
						<imageTag>latest</imageTag>
					</imageTags>
				</configuration>
			</plugin>
		</plugins>
	</build>
</project>			
			
			

10.11.2.2. 容器启动脚本

在项目目录创建 docker-entrypoint.sh 文件

			
#!/bin/sh

if [ ! -z $1 ]; then
    MODULE=$1
    shift
fi

if [ -z $JAVA_OPTS ]; then
    JAVA_OPTS='-Xms1024m -Xmx4096m -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=512m -Djava.security.egd=file:/dev/./urandom -Duser.timezone=GMT+8 -Dfile.encoding=utf-8'
fi

if [ -z $MODULE ]; then
    echo "MODULE environment is not set"
    exit 127
else
    PACKAGE=/srv/$MODULE.jar
fi

DEBUG='-Xdebug -Xrunjdwp:transport=dt_socket,suspend=n,server=y,address=5555'
SKYWALKING="-javaagent:/srv/skywalking/agent/skywalking-agent.jar -Dskywalking.collector.backend_service=oap.netkiller.cn:11800 -Dskywalking.agent.service_name=${MODULE}"

exec java ${JAVA_OPTS}  -jar ${PACKAGE} $@			
			
			

暂时 DEBUG,SKYWALKING 没有使用,放在一遍不碍事。脚本的用法

			
./docker-entrypoint.sh your_module --server.port=8080
			
			

10.11.2.3. 构建 docker 镜像

运行 mvn 命令构建 docker 镜像

			
neo@Netkiller-iMac ~/w/java.netkiller.cn (master)> mvn package docker:build docker:push			
			
			

不出预料,你会看到下面输出

			
[INFO] Building image registry.netkiller.cn/netkiller.cn/demo
Step 1/9 : FROM openjdk:8-alpine

 ---> a3562aa0b991
Step 2/9 : MAINTAINER netkiller@msn.com

 ---> Using cache
 ---> b4a79be602ae
Step 3/9 : ENV JAVA_OPTS -server -Xms512m -Xmx4096m -Djava.security.egd=file:/dev/./urandom

 ---> Using cache
 ---> 9d685ea4a0d3
Step 4/9 : WORKDIR /srv

 ---> Using cache
 ---> e2feea451bb1
Step 5/9 : ADD /srv/demo-0.0.1-SNAPSHOT.jar /srv/

 ---> 7ad53fb991b8
Step 6/9 : ADD /srv/docker-entrypoint.sh /srv/

 ---> 39def6507064
Step 7/9 : EXPOSE 8080

 ---> Running in 338a99e6ec36
Removing intermediate container 338a99e6ec36
 ---> f192b73ab3b9
Step 8/9 : ENTRYPOINT ["sh", "-c", "/srv/docker-entrypoint.sh"]

 ---> Running in 5bda82acd305
Removing intermediate container 5bda82acd305
 ---> 85c1b2615a97
Step 9/9 : VOLUME /srv

 ---> Running in 27d71c55bf7e
Removing intermediate container 27d71c55bf7e
 ---> 64e0d8992fdd
ProgressMessage{id=null, status=null, stream=null, error=null, progress=null, progressDetail=null}
Successfully built 64e0d8992fdd
Successfully tagged registry.netkiller.cn/netkiller.cn/demo:latest
[INFO] Built registry.netkiller.cn/netkiller.cn/demo
[INFO] Tagging registry.netkiller.cn/netkiller.cn/demo with 0.0.1-SNAPSHOT
[INFO] Tagging registry.netkiller.cn/netkiller.cn/demo with latest
			
			

查看镜像

			
neo@Netkiller-iMac ~/w/java.netkiller.cn (master)> docker image ls | grep netkiller
registry.netkiller.cn/netkiller.cn/demo                       0.0.1-SNAPSHOT   64e0d8992fdd   3 minutes ago    122MB
registry.netkiller.cn/netkiller.cn/demo                       latest           64e0d8992fdd   3 minutes ago    122MB			
			
			

10.11.2.4. 编排 kubernetes 容器

			
from netkiller.kubernetes import *
namespace = 'default'

compose = Compose('development')

module = 'demo'
# version = '0.0.1-SNAPSHOT'
version = 'latest'

deployment = Deployment()
deployment.apiVersion('apps/v1')

deployment.metadata().name(module).labels({'app': module}).namespace(namespace)
deployment.spec().replicas(1)
deployment.spec().selector({'matchLabels': {'app': module}})
deployment.spec().template().metadata().labels({'app': module})
deployment.spec().template().spec().containers().name(module).image(
    'registry.netkiller.cn/netkiller.cn/cloud.netkiller.cn:%s' % version).ports([{
        'containerPort': 8080
    }]).env([
        {'name': 'TZ', 'value': 'Asia/Shanghai'},
        {'name': 'LANG', 'value': 'en_US.UTF-8'},
    ]).args([module,'--server.port=8080'])

# deployment.debug()
# deployment.json()

service = Service()
service.metadata().name(module)
service.metadata().namespace(namespace)
service.spec().selector({'app': module})
service.spec().type('NodePort')
service.spec().ports([{
    'name': 'http',
    'protocol': 'TCP',
    'port': 8080,
    'targetPort': 8080
}])

compose.add(deployment)
compose.add(service)

print("=" * 40, "Compose", "=" * 40)
compose.debug()
compose.delete()
compose.create()⏎  			
			
			

查看容器运行状态

			
neo@Netkiller-iMac ~/w/java.netkiller.cn (master)> kubectl get pods
NAME                       READY   STATUS             RESTARTS   AGE
nginx-88c84c4d8-8pmzp      1/1     Running            1          3d20h
demo-76b7598b76-5hstp      1/1     Running            0          5h43m
busybox                    0/1     CrashLoopBackOff   52         4h44m			
			
			

10.11.2.5. 启动指定 nacos

容器中不方便修改配置文件,我们可以使用环境变量覆盖配置

			 
JAVA_OPTS=-Dspring.cloud.nacos.username=nacos \
-Dspring.cloud.nacos.password=nacos \
-Dspring.cloud.nacos.config.server-addr=mse-032dbef0-nacos-ans.mse.aliyuncs.com:8848 \
-Dspring.cloud.nacos.discovery.server-addr=mse-032dbef0-nacos-ans.mse.aliyuncs.com:8848 \
			
			

相当于

			 
java -Dspring.cloud.nacos.username=nacos \
-Dspring.cloud.nacos.password=nacos \
-Dspring.cloud.nacos.config.server-addr=mse-032dbef0-nacos-ans.mse.aliyuncs.com:8848 \
-Dspring.cloud.nacos.discovery.server-addr=mse-032dbef0-nacos-ans.mse.aliyuncs.com:8848 \
-jar netkiller.jar
			
			

10.11.3. Nacos 配置中心/注册中心代码实例

10.11.3.1. Maven

			
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>
	<groupId>cn.netkiller</groupId>
	<artifactId>bottleneck</artifactId>
	<version>0.0.1-SNAPSHOT</version>
	<name>bottleneck</name>
	<description>bottleneck</description>
	<organization>
		<name>Netkiller Spring Cloud 手札</name>
		<url>https://www.netkiller.cn</url>
	</organization>
	<developers>
		<developer>
			<name>Neo</name>
			<email>netkiller@msn.com</email>
			<organization>Netkiller Spring Cloud 手札</organization>
			<organizationUrl>https://www.netkiller.cn</organizationUrl>
			<roles>
				<role>Author</role>
			</roles>
		</developer>
	</developers>
	<parent>
		<groupId>org.springframework.boot</groupId>
		<artifactId>spring-boot-starter-parent</artifactId>
		<version>2.7.3</version>
		<relativePath />
	</parent>
	<properties>
		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
	</properties>
	<dependencies>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-web</artifactId>
			<!-- Exclude the Tomcat dependency -->
			<exclusions>
				<exclusion>
					<groupId>org.springframework.boot</groupId>
					<artifactId>spring-boot-starter-tomcat</artifactId>
				</exclusion>
			</exclusions>
		</dependency>
		<!-- Use Undertow instead Tomcat -->
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-undertow</artifactId>
		</dependency>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-webflux</artifactId>
		</dependency>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-jdbc</artifactId>
		</dependency>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-actuator</artifactId>
		</dependency>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-test</artifactId>
			<scope>test</scope>
		</dependency>
		<dependency>
			<groupId>mysql</groupId>
			<artifactId>mysql-connector-java</artifactId>
		</dependency>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-data-redis</artifactId>
			<exclusions>
				<exclusion>
					<groupId>io.lettuce</groupId>
					<artifactId>lettuce-core</artifactId>
				</exclusion>
			</exclusions>
		</dependency>
		<dependency>
			<groupId>redis.clients</groupId>
			<artifactId>jedis</artifactId>
		</dependency>
		<dependency>
			<groupId>org.springframework.data</groupId>
			<artifactId>spring-data-commons</artifactId>
		</dependency>
		<dependency>
			<groupId>org.projectlombok</groupId>
			<artifactId>lombok</artifactId>
		</dependency>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-data-elasticsearch</artifactId>
		</dependency>
		<dependency>
			<groupId>com.alibaba.cloud</groupId>
			<artifactId>spring-cloud-starter-alibaba-nacos-config</artifactId>
			<version>2.2.9.RELEASE</version>
		</dependency>
		<dependency>
			<groupId>com.alibaba.cloud</groupId>
			<artifactId>spring-cloud-starter-alibaba-nacos-discovery</artifactId>
			<version>2.2.9.RELEASE</version>
		</dependency>

		<!-- https://mvnrepository.com/artifact/org.fluentd/fluent-logger -->
		<dependency>
			<groupId>org.fluentd</groupId>
			<artifactId>fluent-logger</artifactId>
			<version>0.3.4</version>
		</dependency>
		<dependency>
			<groupId>com.sndyuk</groupId>
			<artifactId>logback-more-appenders</artifactId>
			<version>1.8.7</version>
		</dependency>
		<dependency>
			<groupId>org.springframework.cloud</groupId>
			<artifactId>spring-cloud-context</artifactId>
			<version>3.1.4</version>
		</dependency>
		<dependency>
			<groupId>org.springframework.cloud</groupId>
			<artifactId>spring-cloud-commons</artifactId>
			<version>3.1.4</version>
		</dependency>
		<dependency>
			<groupId>org.springframework.cloud</groupId>
			<artifactId>spring-cloud-starter-loadbalancer</artifactId>
			<version>3.1.4</version>
		</dependency>

		<dependency>
			<groupId>org.springframework.cloud</groupId>
			<artifactId>spring-cloud-starter-bootstrap</artifactId>
			<version>3.1.4</version>
		</dependency>
	</dependencies>
	<build>
		<plugins>
			<plugin>
				<groupId>org.springframework.boot</groupId>
				<artifactId>spring-boot-maven-plugin</artifactId>
				<configuration>
					<mainClass>cn.netkiller.Application</mainClass>
				</configuration>
			</plugin>
			<plugin>
				<artifactId>maven-surefire-plugin</artifactId>
				<configuration>
					<skip>true</skip>
				</configuration>
			</plugin>

			<plugin>
				<groupId>com.spotify</groupId>
				<artifactId>docker-maven-plugin</artifactId>
				<version>1.2.2</version>
				<configuration>
					<imageName>netkiller/${project.artifactId}</imageName>
					<baseImage>openjdk:19-alpine</baseImage>
					<maintainer>netkiller@msn.com</maintainer>
					<volumes>/tmp</volumes>
					<workdir>/srv</workdir>
					<exposes>8080</exposes>
					<env>
						<JAVA_OPTS>-server -Xms128m -Xmx2048m</JAVA_OPTS>
					</env>
					<entryPoint>["sh", "-c", "java ${JAVA_OPTS} -jar /srv/${project.build.finalName}.jar ${SPRING_OPTS}"]</entryPoint>
					<resources>
						<resource>
							<targetPath>/srv</targetPath>
							<directory>${project.build.directory}</directory>
							<include>${project.build.finalName}.jar</include>
						</resource>
					</resources>
					<image>netkiller/${project.artifactId}</image>
					<newName>netkiller/${project.artifactId}:${project.version}</newName>
					<!-- <serverId>docker-hub</serverId> -->
					<registryUrl>http://${docker.registry}/v2/</registryUrl>
					<imageTags>
						<imageTag>undertow</imageTag>
						<!-- <imageTag>tomcat</imageTag> <imageTag>${project.version}</imageTag> <imageTag>latest</imageTag> -->
					</imageTags>
				</configuration>
			</plugin>
		</plugins>
	</build>
</project>			
			
			

10.11.3.2. SpringBootApplication

			
package cn.netkiller;

import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cloud.client.discovery.EnableDiscoveryClient;

@EnableDiscoveryClient
@SpringBootApplication
public class Application {

	public static void main(String[] args) {
		System.out.println("Netkiller bottleneck tool!");
		SpringApplication.run(Application.class, args);
	}
}			
			
			

10.11.3.3. ConfigController

			
package cn.netkiller.controller;

import org.springframework.beans.factory.annotation.Value;
import org.springframework.cloud.context.config.annotation.RefreshScope;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RestController;

@RefreshScope
@RestController
public class ConfigController {

	public ConfigController() {
		// TODO Auto-generated constructor stub
	}

	@Value("${key}")
	public String name;

	@GetMapping("/config")
	public String config() {

		String name = this.name;
		return name;
	}
}
			
			

10.11.3.4. 

			
#debug=true
server.port=8080

#server.tomcat.max-connections=10000
#server.tomcat.max-threads=4096
#server.tomcat.accept-count=1000
#server.tomcat.min-spare-threads=100

server.undertow.max-http-post-size=0 
server.undertow.io-threads=16
server.undertow.worker-threads=4096
server.undertow.buffer-size=1024
server.undertow.buffers-per-region=1024
server.undertow.direct-buffers=true

spring.application.name=bottleneck
spring.profiles.active=dev
#spring.profiles.active=test
##spring.profiles.active=prod
#
#logging.file.path=/tmp
##logging.file.name=spring.log
#
endpoints.metrics.enabled=true
management.endpoints.jmx.exposure.include=*
management.endpoints.web.exposure.include=*
management.endpoints.health.show-details=always
#
spring.datasource.driver-class-name=com.mysql.cj.jdbc.Driver
spring.datasource.url=jdbc:mysql://172.18.200.5:3306/test?useUnicode=true&characterEncoding=UTF-8
spring.datasource.username=root
spring.datasource.password=passw0rd

spring.datasource.type=com.zaxxer.hikari.HikariDataSource
spring.datasource.hikari.minimum-idle=5
spring.datasource.hikari.maximum-pool-size=200
spring.datasource.hikari.auto-commit=true
spring.datasource.hikari.idle-timeout=30000
spring.datasource.hikari.pool-name=Hikari
spring.datasource.hikari.max-lifetime=55000
spring.datasource.hikari.connection-timeout=30000
spring.datasource.hikari.connection-test-query=SELECT 1

spring.redis.host=172.18.200.5
spring.redis.port=6379
spring.redis.password=passw0rd
spring.redis.database=0

spring.redis.jedis.pool.max-active=1000
spring.redis.jedis.pool.max-idle=80
spring.redis.jedis.pool.min-idle=20
spring.redis.jedis.pool.max-wait=-1

spring.cloud.nacos.server-addr=nacos.netkiller.cn:8848
spring.cloud.nacos.username=neo
spring.cloud.nacos.password=netkiller

#spring.cloud.nacos.config.enable-remote-sync-config=true
spring.cloud.nacos.config.namespace=${spring.profiles.active}
spring.cloud.nacos.config.group=DEFAULT_GROUP
spring.cloud.nacos.config.prefix=${spring.application.name}
spring.cloud.nacos.config.file-extension=yaml
spring.cloud.nacos.discovery.service=${spring.application.name:DEFAULT-SERVICE-NAME}
spring.cloud.nacos.discovery.namespace=${spring.profiles.active}
			
			

10.11.4. FAQ

10.11.4.1. 禁用 Nacos

当 Maven 引入了 nacos 依赖,启动就会要求配置 Nacos,可以通过下面方法禁用 Nacos

			
spring.cloud.nacos.config.enabled=false  
spring.cloud.nacos.discovery.enabled=false 
spring.cloud.nacos.config.refresh-enabled=false
spring.cloud.nacos.discovery.instance-enabled=false			
			
			

10.11.4.2. Failed to bind properties under 'server.tomcat.basedir' to java.io.File:

环境 Docker 安装 nacos 提示 server.tomcat.basedir 错误

			
nacos  | ***************************
nacos  | APPLICATION FAILED TO START
nacos  | ***************************
nacos  | 
nacos  | Description:
nacos  | 
nacos  | Failed to bind properties under 'server.tomcat.basedir' to java.io.File:
nacos  | 
nacos  |     Property: server.tomcat.basedir
nacos  |     Value: 
nacos  |     Origin: InputStream resource [resource loaded through InputStream] - 34:0
nacos  |     Reason: failed to convert java.lang.String to java.io.File (caused by java.lang.IllegalStateException: Could not retrieve file for class path resource []: class path resource [] cannot be resolved to absolute file path because it does not reside in the file system: jar:file:/home/nacos/target/nacos-server.jar!/BOOT-INF/classes!/)
nacos  | 
nacos  | Action:
nacos  | 
nacos  | Update your application's configuration			
			
			

解决方案

进入容器复制 application.properties 文件到本地,修改 server.tomcat.basedir= 配置,改为 server.tomcat.basedir=. 然后在挂载到容器卷中。

			
[root@cloud ops.sfzito.com]# docker run -it --entrypoint sh  nacos/nacos-server
sh-4.2# cat /home/nacos/conf/application.properties 
# spring
server.servlet.contextPath=${SERVER_SERVLET_CONTEXTPATH:/nacos}
server.contextPath=/nacos
server.port=${NACOS_APPLICATION_PORT:8848}
spring.datasource.platform=${SPRING_DATASOURCE_PLATFORM:""}
nacos.cmdb.dumpTaskInterval=3600
nacos.cmdb.eventTaskInterval=10
nacos.cmdb.labelTaskInterval=300
nacos.cmdb.loadDataAtStart=false
db.num=${MYSQL_DATABASE_NUM:1}
db.url.0=jdbc:mysql://${MYSQL_SERVICE_HOST}:${MYSQL_SERVICE_PORT:3306}/${MYSQL_SERVICE_DB_NAME}?${MYSQL_SERVICE_DB_PARAM:characterEncoding=utf8&connectTimeout=1000&socketTimeout=3000&autoReconnect=true&useSSL=false}
db.url.1=jdbc:mysql://${MYSQL_SERVICE_HOST}:${MYSQL_SERVICE_PORT:3306}/${MYSQL_SERVICE_DB_NAME}?${MYSQL_SERVICE_DB_PARAM:characterEncoding=utf8&connectTimeout=1000&socketTimeout=3000&autoReconnect=true&useSSL=false}
db.user=${MYSQL_SERVICE_USER}
db.password=${MYSQL_SERVICE_PASSWORD}

### The auth system to use, currently only 'nacos' and 'ldap' is supported:
nacos.core.auth.system.type=${NACOS_AUTH_SYSTEM_TYPE:nacos}

### worked when nacos.core.auth.system.type=nacos
### The token expiration in seconds:
nacos.core.auth.plugin.nacos.token.expire.seconds=${NACOS_AUTH_TOKEN_EXPIRE_SECONDS:18000}
### The default token:
nacos.core.auth.plugin.nacos.token.secret.key=${NACOS_AUTH_TOKEN:SecretKey012345678901234567890123456789012345678901234567890123456789}

### Turn on/off caching of auth information. By turning on this switch, the update of auth information would have a 15 seconds delay.
nacos.core.auth.caching.enabled=${NACOS_AUTH_CACHE_ENABLE:false}
nacos.core.auth.enable.userAgentAuthWhite=${NACOS_AUTH_USER_AGENT_AUTH_WHITE_ENABLE:false}
nacos.core.auth.server.identity.key=${NACOS_AUTH_IDENTITY_KEY:serverIdentity}
nacos.core.auth.server.identity.value=${NACOS_AUTH_IDENTITY_VALUE:security}
server.tomcat.accesslog.enabled=${TOMCAT_ACCESSLOG_ENABLED:false}
server.tomcat.accesslog.pattern=%h %l %u %t "%r" %s %b %D
# default current work dir
server.tomcat.basedir=
## spring security config
### turn off security
nacos.security.ignore.urls=${NACOS_SECURITY_IGNORE_URLS:/,/error,/**/*.css,/**/*.js,/**/*.html,/**/*.map,/**/*.svg,/**/*.png,/**/*.ico,/console-fe/public/**,/v1/auth/**,/v1/console/health/**,/actuator/**,/v1/console/server/**}
# metrics for elastic search
management.metrics.export.elastic.enabled=false
management.metrics.export.influx.enabled=false

nacos.naming.distro.taskDispatchThreadCount=10
nacos.naming.distro.taskDispatchPeriod=200
nacos.naming.distro.batchSyncKeyCount=1000
nacos.naming.distro.initDataRatio=0.9
nacos.naming.distro.syncRetryDelay=5000
nacos.naming.data.warmup=true

			
			

10.11.4.3. 不读取 bootstrap.yaml 文件

Nacos 是一个独立项目,并不依赖 Spring Cloud,如果只是想使用配置中心,在 Springboot 中引入 Nacos 依赖,你会发现 Springboot 不读取 bootstrap.yaml。

解决方法

			
		<dependency>
			<groupId>org.springframework.cloud</groupId>
			<artifactId>spring-cloud-starter-bootstrap</artifactId>
			<version>3.1.4</version>
		</dependency>			
			
			

10.11.4.4. WARN [com.alibaba.nacos.client.naming:177] [,] - out of date data received, old-t: 1665711914993, new-t: 1665711902390

出错信息如下,故障是因为每个节点的时间不同步造成的。

			
[2022-10-14 09:44:51.289 [com.alibaba.nacos.naming.push.receiver] WARN [com.alibaba.nacos.client.naming:177] [,] - out of date data received, old-t: 1665711914993, new-t: 1665711902390
			
			

解决方法,安装时间同步软件。例如 NTP

10.11.4.5. User limit of inotify instances reached or too many open files

			
nacos  | 09:22:23.431 [main] ERROR org.springframework.boot.SpringApplication - Application run failed
nacos  | com.alibaba.nacos.api.exception.runtime.NacosRuntimeException: ErrCode:500, ErrMsg:User limit of inotify instances reached or too many open files
nacos  | 	at com.alibaba.nacos.core.listener.StartingApplicationListener.loadPreProperties(StartingApplicationListener.java:161)
nacos  | 	at com.alibaba.nacos.core.listener.StartingApplicationListener.environmentPrepared(StartingApplicationListener.java:100)
nacos  | 	at com.alibaba.nacos.core.code.SpringApplicationRunListener.environmentPrepared(SpringApplicationRunListener.java:60)
nacos  | 	at org.springframework.boot.SpringApplicationRunListeners.lambda$environmentPrepared$2(SpringApplicationRunListeners.java:66)
nacos  | 	at java.util.ArrayList.forEach(ArrayList.java:1259)
nacos  | 	at org.springframework.boot.SpringApplicationRunListeners.doWithListeners(SpringApplicationRunListeners.java:120)
nacos  | 	at org.springframework.boot.SpringApplicationRunListeners.doWithListeners(SpringApplicationRunListeners.java:114)
nacos  | 	at org.springframework.boot.SpringApplicationRunListeners.environmentPrepared(SpringApplicationRunListeners.java:65)
nacos  | 	at org.springframework.boot.SpringApplication.prepareEnvironment(SpringApplication.java:343)
nacos  | 	at org.springframework.boot.SpringApplication.run(SpringApplication.java:301)
nacos  | 	at org.springframework.boot.SpringApplication.run(SpringApplication.java:1317)
nacos  | 	at org.springframework.boot.SpringApplication.run(SpringApplication.java:1306)
nacos  | 	at com.alibaba.nacos.Nacos.main(Nacos.java:35)
nacos  | 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
nacos  | 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
nacos  | 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
nacos  | 	at java.lang.reflect.Method.invoke(Method.java:498)
nacos  | 	at org.springframework.boot.loader.MainMethodRunner.run(MainMethodRunner.java:49)
nacos  | 	at org.springframework.boot.loader.Launcher.launch(Launcher.java:108)
nacos  | 	at org.springframework.boot.loader.Launcher.launch(Launcher.java:58)
nacos  | 	at org.springframework.boot.loader.PropertiesLauncher.main(PropertiesLauncher.java:467)
nacos  | Caused by: com.alibaba.nacos.api.exception.NacosException: java.io.IOException: User limit of inotify instances reached or too many open files
nacos  | 	at com.alibaba.nacos.sys.file.WatchFileCenter$WatchDirJob.<init>(WatchFileCenter.java:184)
nacos  | 	at com.alibaba.nacos.sys.file.WatchFileCenter.registerWatcher(WatchFileCenter.java:92)
nacos  | 	at com.alibaba.nacos.core.listener.StartingApplicationListener.registerWatcher(StartingApplicationListener.java:167)
nacos  | 	at com.alibaba.nacos.core.listener.StartingApplicationListener.loadPreProperties(StartingApplicationListener.java:159)
nacos  | 	... 20 common frames omitted
nacos  | Caused by: java.io.IOException: User limit of inotify instances reached or too many open files
nacos  | 	at sun.nio.fs.LinuxWatchService.<init>(LinuxWatchService.java:64)
nacos  | 	at sun.nio.fs.LinuxFileSystem.newWatchService(LinuxFileSystem.java:47)
nacos  | 	at com.alibaba.nacos.sys.file.WatchFileCenter$WatchDirJob.<init>(WatchFileCenter.java:179)
nacos  | 	... 23 common frames omitted
nacos  | 09:22:23.435 [Thread-4] WARN com.alibaba.nacos.common.executor.ThreadPoolManager - [ThreadPoolManager] Start destroying ThreadPool
nacos  | 09:22:23.435 [Thread-4] WARN com.alibaba.nacos.common.executor.ThreadPoolManager - [ThreadPoolManager] Destruction of the end
			
			

加大 fs.inotify.max_user_instances 配置即可,默认是 128

			
sysctl fs.inotify.max_user_watches
sudo sysctl -w fs.inotify.max_user_watches=50889300
 
sysctl fs.inotify.max_user_instances
sudo sysctl -w fs.inotify.max_user_instances=65535			
			
			

10.11.4.6. 开启权限

			
nacos.core.auth.enabled= true			
			
			

在容器中设置开启验证:NACOS_AUTH_ENABLE=true

10.11.4.7. ERROR Whitelabel

当Nacos开启了认证配置nacos.core.auth.enabled=true时,当前账号没有该命名空间的权限,就会出现下面的错误提示。

			
2022-11-07 18:42:04,370 [,,,] INFO com.alibaba.nacos.client.config.impl.LocalConfigInfoProcessor:212 [main] - LOCAL_SNAPSHOT_PATH:/root/nacos/config
Mon, Nov 7 2022 6:42:04 pm	2022-11-07 18:42:04,393 [,,,] ERROR com.alibaba.nacos.client.config.impl.ClientWorker:304 [main] - [fixed-nacos.default.svc.cluster.local_8848-test] [sub-server-error] no right, dataId=netkiller, group=DEFAULT_GROUP, tenant=test
Mon, Nov 7 2022 6:42:04 pm	2022-11-07 18:42:04,393 [,,,] ERROR com.alibaba.cloud.nacos.client.NacosPropertySourceBuilder:104 [main] - get data from Nacos error,dataId:netkiller
Mon, Nov 7 2022 6:42:04 pm	com.alibaba.nacos.api.exception.NacosException: <html><body><h1>Whitelabel Error Page</h1><p>This application has no explicit mapping for /error, so you are seeing this as a fallback.</p><div id='created'>Mon Nov 07 18:42:04 CST 2022</div><div>There was an unexpected error (type=Forbidden, status=403).</div></body></html>			
			
			

解决方案,在权限控制->权限管理中「添加权限」可以解决。