Home | 简体中文 | 繁体中文 | 杂文 | Github | 知乎专栏 | 51CTO学院 | CSDN程序员研修院 | OSChina 博客 | 腾讯云社区 | 阿里云栖社区 | Facebook | Linkedin | Youtube | 打赏(Donations) | About
知乎专栏多维度架构

2.20. 多维度架构设计之灰度测试方案

A/B测试 vs 灰度测试 vs 蓝绿部署 vs 金丝雀发布

A/B测试 vs 灰度测试 vs 蓝绿部署 vs 金丝雀发布概念类似,手段类似,只是侧重点不同。

它们都需要两套环境,用户被主动或被动地分配到这两套环境中的一套上去。

2.20.1. 什么是灰度测试?

灰度测试是指将产品/产品新功能,少量给一部分目标人群使用,通过用户的反馈结果来决定是否进一步扩大用户群,直到这个新功能覆盖所有用户,灰度测试能用最少的试错成本收集用户反馈,并及时修改产品的一些不足和缺陷,完善产品的功能,使产品的质量得到提高。

2.20.2. 解决方案

灰度测试解决方案有很多,并自由统一的标准,只要最终实现需求,达到效果即可。

这里的解决方案使用的是 Openresty

		
lua_shared_dict cache 128m;	
lua_shared_dict upstream 1m;
upstream production {
	server 192.168.0.2:80;
}
upstream grey {
    server 192.168.0.3:80;
}
upstream balance {
	server 192.168.0.2:80;
    server 192.168.0.3:80;
}
server {
    listen 80;
	listen 443 ssl http2; 
	server_name www.netkiller.cn;

	ssl_certificate /etc/nginx/cert/netkiller.cn.pem; 
	ssl_certificate_key /etc/nginx/cert/netkiller.cn.key; 
	ssl_session_cache shared:SSL:20m;
	ssl_session_timeout 60m;
	ssl_protocols TLSv1 TLSv1.1 TLSv1.2;

	error_log logs/lua.log notice; 

 	location / {
		set_by_lua_file $proxy_pass_url lua/grey.lua;
        proxy_redirect     off;
        #proxy_set_header   Host             $host;
        proxy_set_header   Host             www.netkiller.cn;
        proxy_set_header   X-Real-IP        $remote_addr;
        proxy_set_header   X-Forwarded-For  $proxy_add_x_forwarded_for;
		proxy_pass http://$proxy_pass_url;
	}

	location /grey/get {
		content_by_lua_file lua/get.lua;
	}
	location /grey/set {
		content_by_lua_file lua/set.lua;
	}
	location /grey/del {
		content_by_lua_file lua/del.lua;
	}
	location /grey/switch{
		content_by_lua_file lua/switch.lua;
	}
	location /grey/check {
		content_by_lua_file lua/check.lua;
	}
	location /grey/log {
		content_by_lua_file lua/log.lua;
	}
	location /grey/debug{
		set_by_lua_file $hit lua/grey.lua;
		echo $hit;
    }
	location /nacos {
		proxy_set_header   Host             www.netkiller.cn;
		proxy_pass http://grey;
	}
}
		
		
		
[root@netkiller ~]# cat lua/check.lua 
json = require("cjson")
local cache = ngx.shared.cache
local args = ngx.req.get_uri_args()
local platform = args['platform']
local city = args['city']
local uid = args['uid']
local result = nil 
local status = false
local data = {}
data["data"] = {}

if platform and platform ~= "" then
	result = cache:get("platform" .. platform)
	if result and result ~= "" then
		status = true
	end
else
	if city and city ~= "" then
		result = cache:get("city" .. city)
		if result and result ~= "" then
                	status = true
		end
	end

	if uid and uid ~= "" then
		result = cache:get("uid" .. uid)
		if result and result ~= "" then
                        status = true
                end
	end
end

if status then
	data["code"] = 200
        data["msg"] = "SUCCESS"
	data["data"]["environment"] = 'grey'
else
	data["code"] = 400
        data["msg"] = "FAILURE"
	data["data"]["environment"] = 'production'
end


json_string = json.encode(data)
ngx.log(ngx.NOTICE, "upgrade: ", "\"".. json_string .."\"")
ngx.header['Content-Type'] = 'application/json; charset=utf-8'
ngx.say(json_string)
		
		
		
[root@netkiller ~]# cat lua/del.lua 
local cache = ngx.shared.cache
local args = ngx.req.get_uri_args()
local status, err, forcible = cache:delete(args['key'] .. args['value'])
ngx.header['Content-Type'] = 'application/json; charset=utf-8'

json = require("cjson")
data = {}
data["status"] = status
data["key"] = args['key']
data["value"] = args['value']
json_string = json.encode(data)
ngx.say(json_string)
ngx.log(ngx.NOTICE, "det: ", json_string)

		
		
		
[root@netkiller ~]# cat lua/get.lua 
local cache = ngx.shared.cache
local args = ngx.req.get_uri_args()
local value = cache:get(args['key'] .. args['value'])

json = require("cjson")
data = {}
data["key"] = args['key']
if not value then
        data["status"] = false
        data['value'] = ''
else
        data["status"] = true
        data['value'] = value
end
json_string = json.encode(data)
ngx.header['Content-Type'] = 'application/json; charset=utf-8'
ngx.say(json_string)
ngx.log(ngx.ERR, "get: ", json_string)

		
		
		
[root@netkiller ~]# cat lua/grey.lua 
local cache = ngx.shared.cache
local upstream = ngx.shared.upstream
local args = ngx.req.get_uri_args()
local platform = args['platform']
local city = args['city']
local uid = args['uid']
local result = nil 
local address = "production"
local key = ngx.var.server_addr
local ups = upstream:get(key)

if ups ~= nil then
	ngx.log(ngx.NOTICE, "hit [", ups,"] from ngx.shared.upstream")
	return ups 
-- else
--	ngx.log(ngx.NOTICE, "miss [", ups,"] from ngx.shared.upstream")
end

if platform and platform ~= "" then
	result = cache:get("platform" .. platform)
elseif city and city ~= "" then
	result = cache:get("city" .. city)
elseif uid and uid ~= "" then
	result = cache:get("uid" .. uid)
end

if result and result ~= "" then
        address = 'grey'
	ngx.log(ngx.NOTICE, "hit: ", "\""..address.."\"")
end

args.platform = nil
ngx.req.set_uri_args(args)

return address
		
		
		
[root@netkiller ~]# cat lua/log.lua 
local keyword = ngx.req.get_uri_args()["keyword"]
local logfile = "/usr/local/openresty/nginx/logs/lua.log"
local command = ""
if keyword == nil or keyword == "" then
	command = "/usr/bin/tail -n 100 "..logfile.." 2>&1"
else
	command = "/usr/bin/tail -n 1000 "..logfile.. " | grep " ..keyword.." 2>&1"
end

local file = io.popen(command)

ngx.header.content_type = "text/plain"

if file == nil then
        ngx.say('open file fail! '..logfile)
else
--        ngx.say(command)
        local output = file:read("*a")
        ngx.say(output)
end
-- for line in file:lines() do
--      ngx.say(line)
-- end
file:close()		
		
		
		
[root@netkiller ~]# cat lua/set.lua 
local cache = ngx.shared.cache
local args = ngx.req.get_uri_args()
local exptime = tonumber(args['exptime'])
local key = args['key']
local value = args['value']
if not exptime then
        exptime = 0
end
local status, err, forcible = cache:set(key .. value, value, exptime)

ngx.header['Content-Type'] = 'application/json; charset=utf-8'

json = require("cjson")
data = {}
data["status"] = status
data["key"] = key
data['value'] = value
data["exptime"] = exptime
json_string = json.encode(data)
ngx.say(json_string)
ngx.log(ngx.NOTICE, "set: ", json_string)
		
		
		
[root@netkiller ~]# cat lua/switch.lua 
local to = ngx.req.get_uri_args()["upstream"]
if to == nil or to == "" then
    ngx.say("upstream is nil")
    return nil
end
-- local key = ngx.var.http_host
local key = ngx.var.server_addr
local upstream = ngx.shared.upstream
local from = upstream:get(key)
local status, err, forcible = upstream:set(key, to)

ngx.log(ngx.WARN, key, " Status is: ",status, ", Change upstream from ", from, " to ", to)
ngx.say(key, " Status is ",status, ", Change upstream from ", from, " to ", to)		
		
		

2.20.3. 工作原理

这里有三个缓存,分别是: platform、city、uid 将需要灰度的条件,设置到缓存中,当匹配到key 和 value 后,做出响应,将流量分发到目标集群

		
https://[*.netkiller.cn]{/xxxx/xxxx/xxxx}?[platform|city|uid]=<string>
            域名部分    |     服务部分     |        参数部分		
		
		

三个参数的优先级顺序 platform|city|uid

2.20.3.1. 检查参数的目的地

			
传递参数 ?platform=<string> 或 ?city=<string> 或 ?uid=<string> 三个参数都可以同时使用,platform 优先级最高 |city 其次 |uid 最低。

http://www.netkiller.cn/grey/check?platform=111
http://www.netkiller.cn/grey/check?city=111
http://www.netkiller.cn/grey/check?uid=111

同时传递三个参数
http://www.netkiller.cn/grey/check?platform=XXXX&city=111&uid=111
			
			

调用接口回返信息 code = 200 表示流量去向灰度,code = 400 表示流程不变

			
{"code":200,"msg":"SUCCESS","data":{"environment":"grey"}}
{"code":400,"msg":"FAILURE","data":{"environment":"production"}}
			
			

2.20.3.2. lua 日志

			
http://www.netkiller.cn/grey/log

日志关键字中 hit 表示灰度匹配成功,流量转发到 grey 那边。
日志关键字中 miss 表示没有在灰度列表中匹配到platform/city/uid,流量回到默认的 prodution 原生产环境
			
			

2.20.3.3. 调试接口

默认返回 production 表示生产环境

			

[root@netkiller nginx]# curl http://www.netkiller.cn/grey/debug
production
			
			

添加灰度列表

			

[root@netkiller nginx]# curl 'http://www.netkiller.cn/grey/set?key=city&value=111'
{"exptime":0,"status":true,"value":"111","key":"city"}
			
			

再次测试 返回 grey

			
[root@netkiller nginx]# curl 'http://www.netkiller.cn/grey/debug?city=111'
grey			
			
			

去掉 city 标识,默认回 production

			
[root@netkiller nginx]# curl http://www.netkiller.cn/grey/debug
production						
			
			

2.20.4. 管理接口

2.20.4.1. 添加灰度名单

			
[root@netkiller nginx]# curl 'http://www.netkiller.cn/grey/set?key=platform&value=111'
{"status":true,"key":"platform","exptime":0,"value":"111"}
			
			

只做 30分钟灰度分发 10 秒)加入参数 exptime=10

			
[root@netkiller nginx]# curl 'http://www.netkiller.cn/grey/set?key=platform&value=111&exptime=10'
{"status":true,"key":"platform","exptime":10,"value":"111"}

[root@netkiller nginx]# curl 'http://www.netkiller.cn/grey/get?key=platform&value=111'
{"status":true,"key":"platform","value":"111"}
			
			

10秒钟过去之后,自动从灰度列表中删除

			
[root@netkiller nginx]# curl 'http://www.netkiller.cn/grey/get?key=platform&value=111'
{"status":false,"key":"platform","value":""}				
			
			

2.20.4.2. 查询灰度名单

查询成功,111在灰度列表中

			
[root@netkiller nginx]# curl 'http://www.netkiller.cn/grey/get?key=platform&value=111'
{"status":true,"key":"platform","value":"111"}
			
			

查询失败,灰度列表中未找到222

			
[root@netkiller nginx]# curl 'http://www.netkiller.cn/grey/get?key=platform&value=222'
{"status":false,"key":"platform","value":""}	
			
			

2.20.4.3. 删除灰度名单

停止某用户灰度,将正在分发的灰度用户从列表中移除

			
[root@netkiller nginx]# curl 'http://www.netkiller.cn/grey/get?key=platform&value=111'
{"status":true,"key":"platform","value":"111"}

[root@netkiller nginx]# curl 'http://www.netkiller.cn/grey/del?key=platform&value=111'
{"status":true,"key":"platform","value":"111"}

[root@netkiller nginx]# curl 'http://www.netkiller.cn/grey/get?key=platform&value=111'
{"status":false,"key":"platform","value":""}
			
			

2.20.5. 使用 Redis 做持久化

2.20.5.1. 从 Redis 恢复缓存数据

			
[root@netkiller ~]# cat /srv/openresty/nginx/lua/cache.lua 
local cache = ngx.shared.cache
local init, err = cache:get("initialize")

if init then
	ngx.log(ngx.NOTICE, "redis: miss")
	ngx.exec("@proxy");
else
	ngx.log(ngx.NOTICE, "redis: hit")
end

local host = "127.0.0.1"
local port = 6379
-- local password = "passw0rd"
local password = ""
local redis = require("resty.redis")
local red = redis:new()

red:set_timeout(5000)

local ok, err = red:connect(host, port)

if not ok then
	ngx.log(ngx.ERR, "connect to redis error : ", err)
	return
elseif password and password ~= "" then
	ok, err = red:auth(password)
	if not ok then
		ngx.log(ngx.ERR, "failed to authenticate: ", err)
		return
	end
end

local citys, err = red:smembers("platform")
for i, value in ipairs(citys) do
    cache:set("platform" .. value, value)
    ngx.log(ngx.NOTICE,"platform: ",value)
end

local citys, err = red:smembers("city")
for i, value in ipairs(citys) do
	cache:set("city" .. value, value)
    ngx.log(ngx.NOTICE,"city: ",value)
end

local citys, err = red:smembers("uid")
for i, value in ipairs(citys) do
        cache:set("uid" .. value, value)
    ngx.log(ngx.NOTICE,"uid: ",value)
end

cache:set("initialize", true)
ngx.log(ngx.NOTICE, "initialize: OK!")

ok, err = red:close()
if not ok then
    ngx.log(ngx.ERR, "close redis error:", err)
end

ngx.exec("@proxy");			
			
			

2.20.5.2. 重新初始化

			
[root@netkiller ~]# cat /srv/openresty/nginx/lua/flush.lua 
local cache = ngx.shared.cache
local status, err, forcible = cache:delete("initialize")
ngx.header['Content-Type'] = 'application/json; charset=utf-8'

local json = require("cjson")
local data = {}
data["status"] = status
data["key"] = "initialize"
data["value"] = ""
local json_string = json.encode(data)
ngx.say(json_string)
ngx.log(ngx.NOTICE, "delete: ", json_string)			
			
			

2.20.5.3. 

			
[root@netkiller ~]# cat /srv/openresty/nginx/lua/grey.lua 
local cache = ngx.shared.cache
local upstream = ngx.shared.upstream
local args = ngx.req.get_uri_args()
local platform = args['platform']
local city = args['city']
local uid = args['uid']
local result = nil 
local address = "production"
local key = ngx.var.server_addr
local ups = upstream:get(key)

if ups ~= nil then
	ngx.log(ngx.NOTICE, "hit: [", ups,"] from upstream")
	return ups 
else
	ngx.log(ngx.NOTICE, "miss: [", ups,"] from upstream")
end

if platform and platform ~= "" then
	result = cache:get("platform" .. platform)
elseif city and city ~= "" then
	result = cache:get("city" .. city)
elseif uid and uid ~= "" then
	result = cache:get("uid" .. uid)
end

if result and result ~= "" then
        address = 'grey'
	ngx.log(ngx.NOTICE, "hit: ", "\""..address.."\"")
else
	ngx.log(ngx.NOTICE, "miss: ", "\""..address.."\"")
end

args.platform = nil
ngx.req.set_uri_args(args)

return address
			
			

2.20.5.4. 设置缓存

			
[root@netkiller nginx]# cat /srv/openresty/nginx/lua/set.lua 
local cache = ngx.shared.cache
local args = ngx.req.get_uri_args()
local exptime = tonumber(args['exptime'])
local key = args['key']
local value = args['value']

local host = "127.0.0.1"
local port = 6379
-- local password = "passw0rd"
local password = ""

if not exptime then
        exptime = 0
end

local status, err, forcible = cache:set(key .. value, value, exptime)

ngx.header['Content-Type'] = 'application/json; charset=utf-8'

local redis = require("resty.redis")
local red = redis:new()

red:set_timeout(5000)

local ok, err = red:connect(host, port)

if not ok then
	ngx.log(ngx.ERR, "connect to redis error : ", err)
	return
elseif password and password ~= "" then
	ok, err = red:auth(password)
	if not ok then
		ngx.log(ngx.ERR, "failed to authenticate: ", err)
		return
	end
end

if exptime == 0 then
	ok, err = red:sadd(key, value)
	if not ok then
	    ngx.log(ngx.ERR, "add set error:", err)
	end
else
    ngx.log(ngx.NOTICE, "ignore redis exptime TTL ", exptime)

end

ok, err = red:close()
if not ok then
    ngx.log(ngx.ERR, "close redis error:", err)
end

local json = require("cjson")
local data = {}
data["status"] = status
data["key"] = key
data['value'] = value
data["exptime"] = exptime
local json_string = json.encode(data)
ngx.say(json_string)
ngx.log(ngx.NOTICE, "set: ", json_string)
			
			

2.20.5.5. 删除缓存

			
[root@netkiller nginx]# cat /srv/openresty/nginx/lua/del.lua 
local cache = ngx.shared.cache
local args = ngx.req.get_uri_args()
local key = args['key']
local value = args['value']

local host = "127.0.0.1"
local port = 6379
-- local password = "passw0rd"
local password = ""

local status, err, forcible = cache:delete(key .. value)
ngx.header['Content-Type'] = 'application/json; charset=utf-8'

local redis = require("resty.redis")
local red = redis:new()

red:set_timeout(5000)

local ok, err = red:connect(host, port)

if not ok then
	ngx.log(ngx.ERR, "connect to redis error : ", err)
	return
elseif password and password ~= "" then
	ok, err = red:auth(password)
	if not ok then
		ngx.log(ngx.ERR, "failed to authenticate: ", err)
		return
	end
end

ok, err = red:srem(key, value)
if not ok then
    ngx.log(ngx.ERR, "delete set error:", err)
end


ok, err = red:close()
if not ok then
    ngx.log(ngx.ERR, "close redis error:", err)
end

local json = require("cjson")
local data = {}
data["status"] = status
data["key"] = key
data["value"] = value
local json_string = json.encode(data)
ngx.say(json_string)
ngx.log(ngx.NOTICE, "delete: ", json_string)
			
			

2.20.5.6. 获取缓存状态

			
[root@netkiller nginx]# cat /srv/openresty/nginx/lua/get.lua 
local cache = ngx.shared.cache
local args = ngx.req.get_uri_args()
local value = cache:get(args['key'] .. args['value'])

local json = require("cjson")
local data = {}
data["key"] = args['key']
if not value then
        data["status"] = false
        data['value'] = ''
else
        data["status"] = true
        data['value'] = value
end
local json_string = json.encode(data)
ngx.say(json_string)
ngx.log(ngx.NOTICE, "get: ", json_string)
			
			

2.20.5.7. Nginx 配置

			
lua_shared_dict cache 128m;	
lua_shared_dict upstream 1m;
upstream production {
	server 127.0.0.1:81;
}
upstream grey {
        server 127.0.0.1:82;
}



server {
    listen       80;
    server_name  localhost;

    #charset koi8-r;
    #access_log  logs/host.access.log  main;

	rewrite_log on;
	error_log logs/lua.log notice;
 
	lua_need_request_body on;
       
	location /grey/get {
		content_by_lua_file lua/get.lua;
	}

    location /grey/set {
		content_by_lua_file lua/set.lua;
    }

	location /grey/del {
		content_by_lua_file lua/del.lua;
	}
    location /grey/switch{
       	content_by_lua_file lua/switch.lua;
    }
	location /grey/check {
		content_by_lua_file lua/check.lua;
	}
	location /grey/debug{
		set_by_lua_file $hit lua/grey.lua;
		echo $hit;
	}
	location /grey/log {
		content_by_lua_file lua/log.lua;
	}
	location /grey/flush {
	    content_by_lua_file lua/flush.lua;
	}
	location / {
	    content_by_lua_file lua/cache.lua;
	}

	location @proxy {
		set_by_lua_file $proxy_pass_url lua/grey.lua;
        proxy_redirect     off;
        proxy_set_header   Host             $host;
        proxy_set_header   X-Real-IP        $remote_addr;
        proxy_set_header   X-Forwarded-For  $proxy_add_x_forwarded_for;
 
		proxy_pass http://$proxy_pass_url/;
	}

    #error_page  404              /404.html;

    # redirect server error pages to the static page /50x.html
    #
    error_page   500 502 503 504  /50x.html;
    location = /50x.html {
        root   html;
    }
}