| 知乎专栏 ｜ 多维度架构 |
目录
http://www.elasticsearch.org/
Add the Elastic Helm Chart Repo: helm repo add elastic https://helm.elastic.co Install Elasticsearch: helm install --name elasticsearch elastic/elasticsearch Install Kibana: helm install --name kibana elastic/kibana
version: '3.8'
services:
es01:
image: docker.elastic.co/elasticsearch/elasticsearch:7.9.2
container_name: es01
environment:
- node.name=es01
- cluster.name=elasticsearch-cluster
- bootstrap.memory_lock=true
- discovery.zen.ping.unicast.hosts=es01,es02,es03
- discovery.zen.minimum_master_nodes=2
- discovery.zen.ping_timeout=5s
- node.master=true
- node.data=true
- node.ingest=false
- ES_JAVA_OPTS=-Xms256m -Xmx256m
ulimits:
memlock:
soft: -1
hard: -1
volumes:
- data01:/usr/share/elasticsearch/data
ports:
- 9200:9200
- 9300:9300
networks:
- elastic
es02:
image: docker.elastic.co/elasticsearch/elasticsearch:7.9.2
container_name: es02
environment:
- node.name=es02
- cluster.name=elasticsearch-cluster
- bootstrap.memory_lock=true
- discovery.zen.ping.unicast.hosts=es01,es02,es03
- discovery.zen.minimum_master_nodes=2
- discovery.zen.ping_timeout=5s
- node.master=true
- node.data=true
- node.ingest=false
- ES_JAVA_OPTS=-Xms256m -Xmx256m
ulimits:
memlock:
soft: -1
hard: -1
volumes:
- data02:/usr/share/elasticsearch/data
networks:
- elastic
depends_on:
- es01
es03:
image: docker.elastic.co/elasticsearch/elasticsearch:7.9.2
container_name: es03
environment:
- node.name=es03
- cluster.name=elasticsearch-cluster
- bootstrap.memory_lock=true
- discovery.zen.ping.unicast.hosts=es01,es02,es03
- discovery.zen.minimum_master_nodes=2
- discovery.zen.ping_timeout=5s
- node.master=true
- node.data=true
- node.ingest=true
- ES_JAVA_OPTS=-Xms256m -Xmx256m
ulimits:
memlock:
soft: -1
hard: -1
volumes:
- data03:/usr/share/elasticsearch/data
networks:
- elastic
depends_on:
- es01
kibana:
image: docker.elastic.co/kibana/kibana:7.9.2
container_name: kibana
environment:
# SERVER_NAME: kibana.example.org
ELASTICSEARCH_HOSTS: http://es01:9200
ports:
- 5601:5601
networks:
- elastic
depends_on:
- es01
volumes:
data01:
driver: local
data02:
driver: local
data03:
driver: local
networks:
elastic:
driver: bridge
查看节点信息
neo@MacBook-Pro-Neo ~/workspace/docker % curl "http://localhost:9200/_cat/nodes?v&pretty" ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name 172.19.0.4 48 86 35 4.86 2.90 1.84 dilmrt * es03 172.19.0.3 67 86 35 4.86 2.90 1.84 dlmrt - es02 172.19.0.2 45 86 35 4.86 2.90 1.84 dlmrt - es01
apiVersion: v1
kind: ConfigMap
metadata:
name: elasticsearch-logging-config
namespace: kube-public
labels:
app: elasticsearch-logging
data:
limits.conf: |-
elasticsearch soft memlock unlimited
elasticsearch hard memlock unlimited
elasticsearch hard nofile 65536
elasticsearch soft nofile 65536
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: elasticsearch-logging
namespace: kube-public
labels:
app: elasticsearch-logging
spec:
serviceName: elasticsearch-logging
replicas: 1
selector:
matchLabels:
app: elasticsearch-logging
template:
metadata:
labels:
app: elasticsearch-logging
spec:
initContainers:
- name: elasticsearch-logging-init
image: alpine:latest
imagePullPolicy: IfNotPresent
command: ["/sbin/sysctl", "-w", "vm.max_map_count=262144"]
securityContext:
privileged: true
containers:
- name: elasticsearch-logging
image: docker.elastic.co/elasticsearch/elasticsearch:7.9.2
imagePullPolicy: IfNotPresent
resources:
limits:
cpu: 200m
# memory: "1Gi"
requests:
cpu: 200m
# memory: "1Gi"
env:
- name: "NAMESPACE"
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: node.name
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: "cluster.name"
value: "elasticsearch-cluster"
- name: "bootstrap.memory_lock"
# value: "true"
value: "false"
- name: "discovery.seed_hosts"
value: "elasticsearch-logging-0,elasticsearch-logging-1,elasticsearch-logging-2"
- name: "cluster.initial_master_nodes"
value: "elasticsearch-logging-0"
- name: "discovery.find_peers_interval"
value: "5s"
- name: "gateway.expected_nodes"
value: "2"
- name: "gateway.expected_master_nodes"
value: "1"
- name: "http.cors.enabled"
value: "true"
- name: "http.cors.allow-origin"
value: "*"
- name: "ES_JAVA_OPTS"
value: "-Xms1g -Xmx1g"
- name: RLIMIT_MEMLOCK
value: "unlimited"
ports:
- containerPort: 9200
name: restful
protocol: TCP
- containerPort: 9300
name: transport
protocol: TCP
# readinessProbe:
# httpGet:
# scheme: HTTP
# path: /_cluster/health?local=true
# port: 9200
# initialDelaySeconds: 5
# livenessProbe:
# tcpSocket:
# port: transport
# initialDelaySeconds: 20
# periodSeconds: 10
volumeMounts:
- name: elasticsearch-config
mountPath: /etc/security/limits.conf
subPath: limits.conf
- name: elasticsearch-data
mountPath: /data
volumes:
- name: elasticsearch-data
emptyDir: {}
# hostPath:
# path: /data
- name: elasticsearch-config
configMap:
name: elasticsearch-logging-config
---
apiVersion: v1
kind: Service
metadata:
name: elasticsearch-logging
namespace: kube-public
labels:
k8s-app: elasticsearch-logging
kubernetes.io/cluster-service: "true"
kubernetes.io/name: "Elasticsearch"
spec:
selector:
app: elasticsearch-logging
type: NodePort
# type: ClusterIP
# clusterIP: None
ports:
- name: restful
port: 9200
protocol: TCP
targetPort: restful
nodePort: 30092
- name: transport
port: 9300
targetPort: transport
创建 docker.py 文件
#!/usr/bin/env python3
from netkiller.docker import *
volume = Volumes('elasticsearch')
elasticsearch = Services('elasticsearch')
elasticsearch.container_name('elasticsearch').environment([
'discovery.type=single-node',
'ES_JAVA_OPTS=-Xms512m -Xmx2048m'
])
elasticsearch.image('docker.elastic.co/elasticsearch/elasticsearch:7.15.0').ports(['9200:9200','9300:9300']).volumes(['elasticsearch:/usr/share/elasticsearch/data'])
experiment = Composes('experiment')
experiment.version('3.9')
experiment.volumes(volume)
experiment.services(elasticsearch)
if __name__ == '__main__':
try:
docker = Docker()
docker.sysctl([{'vm.max_map_count':'262144'}])
docker.environment(experiment)
docker.main()
except KeyboardInterrupt:
print ("Crtl+C Pressed. Shutting down.")
[root@localhost ~]# python3 docker.py -e experiment -l
experiment :
fluentd
redis
mongo
mysql
elasticsearch
[root@localhost ~]# python3 docker.py -e experiment up elasticsearch
Starting elasticsearch ... done
[root@localhost ~]# python3 docker.py -e experiment ps elasticsearch
Name Command State Ports
-------------------------------------------------------------------------------------------------------------------------------------------
elasticsearch /bin/tini -- /usr/local/bi ... Up 0.0.0.0:9200->9200/tcp,:::9200->9200/tcp, 0.0.0.0:9300->9300/tcp,:::9300->9300/tcp
测试
[root@localhost ~]# curl -s -X GET "localhost:9200/_cat/nodes?v=true&pretty" ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name 172.21.0.2 16 99 5 1.68 0.60 0.26 cdfhilmrstw * aeedaff1ac15
停止
[root@localhost ~]# python3 docker.py -e experiment stop elasticsearch Stopping elasticsearch ... done
使用 yum 安装默认为最新版本，我们常常会遇到一个问题 elasticsearch-analysis-ik 的版本晚于 Elasticsearch。如果使用 yum 安装 Elasticsearch 可能 elasticsearch-analysis-ik 插件不支持这个版本，有些版本的 elasticsearch-analysis-ik 可以修改插件配置文件中的版本号，使其与elasticsearch版本相同，可以欺骗 elasticsearch 跳过版本不一致异常。
最佳的解决方案是去 elasticsearch-analysis-ik github 找到兼容的版本，安装我们安装 elasticsearch-analysis-ik 的版本需求来指定安装 elasticsearch
Versions IK version ES version master 5.x -> master 5.6.0 5.6.0 5.5.3 5.5.3 5.4.3 5.4.3 5.3.3 5.3.3 5.2.2 5.2.2 5.1.2 5.1.2 1.10.1 2.4.1 1.9.5 2.3.5 1.8.1 2.2.1 1.7.0 2.1.1 1.5.0 2.0.0 1.2.6 1.0.0 1.2.5 0.90.x 1.1.3 0.20.x 1.0.0 0.16.2 -> 0.19.0
最新版是 elasticsearch 5.6.1 但分词插件 elasticsearch-analysis-ik 仅能支持到 elasticsearch 版本是 5.6.0
root@netkiller /var/log % yum --showduplicates list elasticsearch | expand | tail Repository epel is listed more than once in the configuration elasticsearch.noarch 5.5.3-1 elasticsearch-5.x elasticsearch.noarch 5.6.0-1 elasticsearch-5.x elasticsearch.noarch 5.6.1-1 elasticsearch-5.x
安装 5.6.0
# yum install elasticsearch-5.6.0-1 Loaded plugins: fastestmirror, langpacks Repository epel is listed more than once in the configuration Loading mirror speeds from cached hostfile Resolving Dependencies --> Running transaction check ---> Package elasticsearch.noarch 0:5.6.0-1 will be installed --> Finished Dependency Resolution Dependencies Resolved ========================================================================================================================================================================================================== Package Arch Version Repository Size ========================================================================================================================================================================================================== Installing: elasticsearch noarch 5.6.0-1 elasticsearch-5.x 32 M Transaction Summary ========================================================================================================================================================================================================== Install 1 Package Total download size: 32 M Installed size: 36 M Is this ok [y/d/N]: y
Elasticsearch 提供了插件管理命令 elasticsearch-plugin
root@netkiller ~ % /usr/share/elasticsearch/bin/elasticsearch-plugin -h A tool for managing installed elasticsearch plugins Commands -------- list - Lists installed elasticsearch plugins install - Install a plugin remove - removes a plugin from Elasticsearch Non-option arguments: command Option Description ------ ----------- -h, --help show help -s, --silent show minimal output -v, --verbose show verbose output
安装插件
root@netkiller ~ % /usr/share/elasticsearch/bin/elasticsearch-plugin install https://github.com/medcl/elasticsearch-analysis-ik/releases/download/v5.5.1/elasticsearch-analysis-ik-5.5.1.zip -> Downloading https://github.com/medcl/elasticsearch-analysis-ik/releases/download/v5.5.1/elasticsearch-analysis-ik-5.5.1.zip [=================================================] 100% -> Installed analysis-ik
curl -XPOST http://localhost:9200/index/fulltext/_mapping -d'
{
"properties": {
"content": {
"type": "text",
"analyzer": "ik_max_word",
"search_analyzer": "ik_max_word"
}
}
}'