| 知乎专栏 |
A/B测试 vs 灰度测试 vs 蓝绿部署 vs 金丝雀发布概念类似，手段类似，只是侧重点不同。
它们都需要两套环境，用户被主动或被动地分配到这两套环境中的一套上去。
灰度测试是指将产品/产品新功能，少量给一部分目标人群使用，通过用户的反馈结果来决定是否进一步扩大用户群，直到这个新功能覆盖所有用户，灰度测试能用最少的试错成本收集用户反馈，并及时修改产品的一些不足和缺陷，完善产品的功能，使产品的质量得到提高。
灰度测试解决方案有很多，并自由统一的标准，只要最终实现需求，达到效果即可。
这里的解决方案使用的是 Openresty
lua_shared_dict cache 128m;
lua_shared_dict upstream 1m;
upstream production {
server 192.168.0.2:80;
}
upstream grey {
server 192.168.0.3:80;
}
upstream balance {
server 192.168.0.2:80;
server 192.168.0.3:80;
}
server {
listen 80;
listen 443 ssl http2;
server_name www.netkiller.cn;
ssl_certificate /etc/nginx/cert/netkiller.cn.pem;
ssl_certificate_key /etc/nginx/cert/netkiller.cn.key;
ssl_session_cache shared:SSL:20m;
ssl_session_timeout 60m;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
error_log logs/lua.log notice;
location / {
set_by_lua_file $proxy_pass_url lua/grey.lua;
proxy_redirect off;
#proxy_set_header Host $host;
proxy_set_header Host www.netkiller.cn;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_pass http://$proxy_pass_url;
}
location /grey/get {
content_by_lua_file lua/get.lua;
}
location /grey/set {
content_by_lua_file lua/set.lua;
}
location /grey/del {
content_by_lua_file lua/del.lua;
}
location /grey/switch{
content_by_lua_file lua/switch.lua;
}
location /grey/check {
content_by_lua_file lua/check.lua;
}
location /grey/log {
content_by_lua_file lua/log.lua;
}
location /grey/debug{
set_by_lua_file $hit lua/grey.lua;
echo $hit;
}
location /nacos {
proxy_set_header Host www.netkiller.cn;
proxy_pass http://grey;
}
}
[root@netkiller ~]# cat lua/check.lua
json = require("cjson")
local cache = ngx.shared.cache
local args = ngx.req.get_uri_args()
local platform = args['platform']
local city = args['city']
local uid = args['uid']
local result = nil
local status = false
local data = {}
data["data"] = {}
if platform and platform ~= "" then
result = cache:get("platform" .. platform)
if result and result ~= "" then
status = true
end
else
if city and city ~= "" then
result = cache:get("city" .. city)
if result and result ~= "" then
status = true
end
end
if uid and uid ~= "" then
result = cache:get("uid" .. uid)
if result and result ~= "" then
status = true
end
end
end
if status then
data["code"] = 200
data["msg"] = "SUCCESS"
data["data"]["environment"] = 'grey'
else
data["code"] = 400
data["msg"] = "FAILURE"
data["data"]["environment"] = 'production'
end
json_string = json.encode(data)
ngx.log(ngx.NOTICE, "upgrade: ", "\"".. json_string .."\"")
ngx.header['Content-Type'] = 'application/json; charset=utf-8'
ngx.say(json_string)
[root@netkiller ~]# cat lua/del.lua
local cache = ngx.shared.cache
local args = ngx.req.get_uri_args()
local status, err, forcible = cache:delete(args['key'] .. args['value'])
ngx.header['Content-Type'] = 'application/json; charset=utf-8'
json = require("cjson")
data = {}
data["status"] = status
data["key"] = args['key']
data["value"] = args['value']
json_string = json.encode(data)
ngx.say(json_string)
ngx.log(ngx.NOTICE, "det: ", json_string)
[root@netkiller ~]# cat lua/get.lua
local cache = ngx.shared.cache
local args = ngx.req.get_uri_args()
local value = cache:get(args['key'] .. args['value'])
json = require("cjson")
data = {}
data["key"] = args['key']
if not value then
data["status"] = false
data['value'] = ''
else
data["status"] = true
data['value'] = value
end
json_string = json.encode(data)
ngx.header['Content-Type'] = 'application/json; charset=utf-8'
ngx.say(json_string)
ngx.log(ngx.ERR, "get: ", json_string)
[root@netkiller ~]# cat lua/grey.lua
local cache = ngx.shared.cache
local upstream = ngx.shared.upstream
local args = ngx.req.get_uri_args()
local platform = args['platform']
local city = args['city']
local uid = args['uid']
local result = nil
local address = "production"
local key = ngx.var.server_addr
local ups = upstream:get(key)
if ups ~= nil then
ngx.log(ngx.NOTICE, "hit [", ups,"] from ngx.shared.upstream")
return ups
-- else
-- ngx.log(ngx.NOTICE, "miss [", ups,"] from ngx.shared.upstream")
end
if platform and platform ~= "" then
result = cache:get("platform" .. platform)
elseif city and city ~= "" then
result = cache:get("city" .. city)
elseif uid and uid ~= "" then
result = cache:get("uid" .. uid)
end
if result and result ~= "" then
address = 'grey'
ngx.log(ngx.NOTICE, "hit: ", "\""..address.."\"")
end
args.platform = nil
ngx.req.set_uri_args(args)
return address
[root@netkiller ~]# cat lua/log.lua
local keyword = ngx.req.get_uri_args()["keyword"]
local logfile = "/usr/local/openresty/nginx/logs/lua.log"
local command = ""
if keyword == nil or keyword == "" then
command = "/usr/bin/tail -n 100 "..logfile.." 2>&1"
else
command = "/usr/bin/tail -n 1000 "..logfile.. " | grep " ..keyword.." 2>&1"
end
local file = io.popen(command)
ngx.header.content_type = "text/plain"
if file == nil then
ngx.say('open file fail! '..logfile)
else
-- ngx.say(command)
local output = file:read("*a")
ngx.say(output)
end
-- for line in file:lines() do
-- ngx.say(line)
-- end
file:close()
[root@netkiller ~]# cat lua/set.lua
local cache = ngx.shared.cache
local args = ngx.req.get_uri_args()
local exptime = tonumber(args['exptime'])
local key = args['key']
local value = args['value']
if not exptime then
exptime = 0
end
local status, err, forcible = cache:set(key .. value, value, exptime)
ngx.header['Content-Type'] = 'application/json; charset=utf-8'
json = require("cjson")
data = {}
data["status"] = status
data["key"] = key
data['value'] = value
data["exptime"] = exptime
json_string = json.encode(data)
ngx.say(json_string)
ngx.log(ngx.NOTICE, "set: ", json_string)
[root@netkiller ~]# cat lua/switch.lua
local to = ngx.req.get_uri_args()["upstream"]
if to == nil or to == "" then
ngx.say("upstream is nil")
return nil
end
-- local key = ngx.var.http_host
local key = ngx.var.server_addr
local upstream = ngx.shared.upstream
local from = upstream:get(key)
local status, err, forcible = upstream:set(key, to)
ngx.log(ngx.WARN, key, " Status is: ",status, ", Change upstream from ", from, " to ", to)
ngx.say(key, " Status is ",status, ", Change upstream from ", from, " to ", to)
这里有三个缓存，分别是: platform、city、uid 将需要灰度的条件，设置到缓存中，当匹配到key 和 value 后，做出响应，将流量分发到目标集群
https://[*.netkiller.cn]{/xxxx/xxxx/xxxx}?[platform|city|uid]=<string>
域名部分 ｜ 服务部分 ｜ 参数部分
三个参数的优先级顺序 platform|city|uid
传递参数 ?platform=<string> 或 ?city=<string> 或 ?uid=<string> 三个参数都可以同时使用，platform 优先级最高 |city 其次 |uid 最低。 http://www.netkiller.cn/grey/check?platform=111 http://www.netkiller.cn/grey/check?city=111 http://www.netkiller.cn/grey/check?uid=111 同时传递三个参数 http://www.netkiller.cn/grey/check?platform=XXXX&city=111&uid=111
调用接口回返信息 code = 200 表示流量去向灰度，code = 400 表示流程不变
{"code":200,"msg":"SUCCESS","data":{"environment":"grey"}}
{"code":400,"msg":"FAILURE","data":{"environment":"production"}}
http://www.netkiller.cn/grey/log 日志关键字中 hit 表示灰度匹配成功，流量转发到 grey 那边。 日志关键字中 miss 表示没有在灰度列表中匹配到platform/city/uid，流量回到默认的 prodution 原生产环境
默认返回 production 表示生产环境
[root@netkiller nginx]# curl http://www.netkiller.cn/grey/debug production
添加灰度列表
[root@netkiller nginx]# curl 'http://www.netkiller.cn/grey/set?key=city&value=111'
{"exptime":0,"status":true,"value":"111","key":"city"}
再次测试 返回 grey
[root@netkiller nginx]# curl 'http://www.netkiller.cn/grey/debug?city=111' grey
去掉 city 标识，默认回 production
[root@netkiller nginx]# curl http://www.netkiller.cn/grey/debug production
[root@netkiller nginx]# curl 'http://www.netkiller.cn/grey/set?key=platform&value=111'
{"status":true,"key":"platform","exptime":0,"value":"111"}
只做 30分钟灰度分发 10 秒）加入参数 exptime=10
[root@netkiller nginx]# curl 'http://www.netkiller.cn/grey/set?key=platform&value=111&exptime=10'
{"status":true,"key":"platform","exptime":10,"value":"111"}
[root@netkiller nginx]# curl 'http://www.netkiller.cn/grey/get?key=platform&value=111'
{"status":true,"key":"platform","value":"111"}
10秒钟过去之后，自动从灰度列表中删除
[root@netkiller nginx]# curl 'http://www.netkiller.cn/grey/get?key=platform&value=111'
{"status":false,"key":"platform","value":""}
查询成功，111在灰度列表中
[root@netkiller nginx]# curl 'http://www.netkiller.cn/grey/get?key=platform&value=111'
{"status":true,"key":"platform","value":"111"}
查询失败，灰度列表中未找到222
[root@netkiller nginx]# curl 'http://www.netkiller.cn/grey/get?key=platform&value=222'
{"status":false,"key":"platform","value":""}
停止某用户灰度，将正在分发的灰度用户从列表中移除
[root@netkiller nginx]# curl 'http://www.netkiller.cn/grey/get?key=platform&value=111'
{"status":true,"key":"platform","value":"111"}
[root@netkiller nginx]# curl 'http://www.netkiller.cn/grey/del?key=platform&value=111'
{"status":true,"key":"platform","value":"111"}
[root@netkiller nginx]# curl 'http://www.netkiller.cn/grey/get?key=platform&value=111'
{"status":false,"key":"platform","value":""}
[root@netkiller ~]# cat /srv/openresty/nginx/lua/cache.lua
local cache = ngx.shared.cache
local init, err = cache:get("initialize")
if init then
ngx.log(ngx.NOTICE, "redis: miss")
ngx.exec("@proxy");
else
ngx.log(ngx.NOTICE, "redis: hit")
end
local host = "127.0.0.1"
local port = 6379
-- local password = "passw0rd"
local password = ""
local redis = require("resty.redis")
local red = redis:new()
red:set_timeout(5000)
local ok, err = red:connect(host, port)
if not ok then
ngx.log(ngx.ERR, "connect to redis error : ", err)
return
elseif password and password ~= "" then
ok, err = red:auth(password)
if not ok then
ngx.log(ngx.ERR, "failed to authenticate: ", err)
return
end
end
local citys, err = red:smembers("platform")
for i, value in ipairs(citys) do
cache:set("platform" .. value, value)
ngx.log(ngx.NOTICE,"platform: ",value)
end
local citys, err = red:smembers("city")
for i, value in ipairs(citys) do
cache:set("city" .. value, value)
ngx.log(ngx.NOTICE,"city: ",value)
end
local citys, err = red:smembers("uid")
for i, value in ipairs(citys) do
cache:set("uid" .. value, value)
ngx.log(ngx.NOTICE,"uid: ",value)
end
cache:set("initialize", true)
ngx.log(ngx.NOTICE, "initialize: OK!")
ok, err = red:close()
if not ok then
ngx.log(ngx.ERR, "close redis error:", err)
end
ngx.exec("@proxy");
[root@netkiller ~]# cat /srv/openresty/nginx/lua/flush.lua
local cache = ngx.shared.cache
local status, err, forcible = cache:delete("initialize")
ngx.header['Content-Type'] = 'application/json; charset=utf-8'
local json = require("cjson")
local data = {}
data["status"] = status
data["key"] = "initialize"
data["value"] = ""
local json_string = json.encode(data)
ngx.say(json_string)
ngx.log(ngx.NOTICE, "delete: ", json_string)
[root@netkiller ~]# cat /srv/openresty/nginx/lua/grey.lua
local cache = ngx.shared.cache
local upstream = ngx.shared.upstream
local args = ngx.req.get_uri_args()
local platform = args['platform']
local city = args['city']
local uid = args['uid']
local result = nil
local address = "production"
local key = ngx.var.server_addr
local ups = upstream:get(key)
if ups ~= nil then
ngx.log(ngx.NOTICE, "hit: [", ups,"] from upstream")
return ups
else
ngx.log(ngx.NOTICE, "miss: [", ups,"] from upstream")
end
if platform and platform ~= "" then
result = cache:get("platform" .. platform)
elseif city and city ~= "" then
result = cache:get("city" .. city)
elseif uid and uid ~= "" then
result = cache:get("uid" .. uid)
end
if result and result ~= "" then
address = 'grey'
ngx.log(ngx.NOTICE, "hit: ", "\""..address.."\"")
else
ngx.log(ngx.NOTICE, "miss: ", "\""..address.."\"")
end
args.platform = nil
ngx.req.set_uri_args(args)
return address
[root@netkiller nginx]# cat /srv/openresty/nginx/lua/set.lua
local cache = ngx.shared.cache
local args = ngx.req.get_uri_args()
local exptime = tonumber(args['exptime'])
local key = args['key']
local value = args['value']
local host = "127.0.0.1"
local port = 6379
-- local password = "passw0rd"
local password = ""
if not exptime then
exptime = 0
end
local status, err, forcible = cache:set(key .. value, value, exptime)
ngx.header['Content-Type'] = 'application/json; charset=utf-8'
local redis = require("resty.redis")
local red = redis:new()
red:set_timeout(5000)
local ok, err = red:connect(host, port)
if not ok then
ngx.log(ngx.ERR, "connect to redis error : ", err)
return
elseif password and password ~= "" then
ok, err = red:auth(password)
if not ok then
ngx.log(ngx.ERR, "failed to authenticate: ", err)
return
end
end
if exptime == 0 then
ok, err = red:sadd(key, value)
if not ok then
ngx.log(ngx.ERR, "add set error:", err)
end
else
ngx.log(ngx.NOTICE, "ignore redis exptime TTL ", exptime)
end
ok, err = red:close()
if not ok then
ngx.log(ngx.ERR, "close redis error:", err)
end
local json = require("cjson")
local data = {}
data["status"] = status
data["key"] = key
data['value'] = value
data["exptime"] = exptime
local json_string = json.encode(data)
ngx.say(json_string)
ngx.log(ngx.NOTICE, "set: ", json_string)
[root@netkiller nginx]# cat /srv/openresty/nginx/lua/del.lua
local cache = ngx.shared.cache
local args = ngx.req.get_uri_args()
local key = args['key']
local value = args['value']
local host = "127.0.0.1"
local port = 6379
-- local password = "passw0rd"
local password = ""
local status, err, forcible = cache:delete(key .. value)
ngx.header['Content-Type'] = 'application/json; charset=utf-8'
local redis = require("resty.redis")
local red = redis:new()
red:set_timeout(5000)
local ok, err = red:connect(host, port)
if not ok then
ngx.log(ngx.ERR, "connect to redis error : ", err)
return
elseif password and password ~= "" then
ok, err = red:auth(password)
if not ok then
ngx.log(ngx.ERR, "failed to authenticate: ", err)
return
end
end
ok, err = red:srem(key, value)
if not ok then
ngx.log(ngx.ERR, "delete set error:", err)
end
ok, err = red:close()
if not ok then
ngx.log(ngx.ERR, "close redis error:", err)
end
local json = require("cjson")
local data = {}
data["status"] = status
data["key"] = key
data["value"] = value
local json_string = json.encode(data)
ngx.say(json_string)
ngx.log(ngx.NOTICE, "delete: ", json_string)
[root@netkiller nginx]# cat /srv/openresty/nginx/lua/get.lua
local cache = ngx.shared.cache
local args = ngx.req.get_uri_args()
local value = cache:get(args['key'] .. args['value'])
local json = require("cjson")
local data = {}
data["key"] = args['key']
if not value then
data["status"] = false
data['value'] = ''
else
data["status"] = true
data['value'] = value
end
local json_string = json.encode(data)
ngx.say(json_string)
ngx.log(ngx.NOTICE, "get: ", json_string)
lua_shared_dict cache 128m;
lua_shared_dict upstream 1m;
upstream production {
server 127.0.0.1:81;
}
upstream grey {
server 127.0.0.1:82;
}
server {
listen 80;
server_name localhost;
#charset koi8-r;
#access_log logs/host.access.log main;
rewrite_log on;
error_log logs/lua.log notice;
lua_need_request_body on;
location /grey/get {
content_by_lua_file lua/get.lua;
}
location /grey/set {
content_by_lua_file lua/set.lua;
}
location /grey/del {
content_by_lua_file lua/del.lua;
}
location /grey/switch{
content_by_lua_file lua/switch.lua;
}
location /grey/check {
content_by_lua_file lua/check.lua;
}
location /grey/debug{
set_by_lua_file $hit lua/grey.lua;
echo $hit;
}
location /grey/log {
content_by_lua_file lua/log.lua;
}
location /grey/flush {
content_by_lua_file lua/flush.lua;
}
location / {
content_by_lua_file lua/cache.lua;
}
location @proxy {
set_by_lua_file $proxy_pass_url lua/grey.lua;
proxy_redirect off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_pass http://$proxy_pass_url/;
}
#error_page 404 /404.html;
# redirect server error pages to the static page /50x.html
#
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root html;
}
}