클러스터링 작업은 하지 않고 단독서버 구성하는 내용으로 정리함.
패키지 다운로드
centos 7 환경에 맞는 rpm파일 다운로드.
https://www.elastic.co/downloads/elasticsearch https://www.elastic.co/downloads/kibana
로컬 레파지토리 구성
centos 7 iso파일 /media로 mount 했다는 가정하에 레파지토리 구성을 진행
# df -h Filesystem Size Used Avail Use% Mounted on ... /dev/sr0 4.3G 4.3G 0 100% /media ...
서버 hosts명칭 부여
elk01로 host파일에 추가
# vi /etc/hosts ... 192.168.20.130 elk01 ...
SELinux 설정
# vi /etc/sysconfig/selinux SELINUX=disabled getenforce
NTP 설치
# yum install -y install net-tools ntp systemd-devel wget libicu*
# systemctl enable ntpd
# systemctl start ntpd
# ntpq -p
remote refid st t when poll reach delay offset jitter
==============================================================================
+send.mx.cdnetwo 125.185.190.74 2 u 271 1024 377 33.210 -0.148 9.244
+ec2-13-209-84-5 17.253.114.125 2 u 21 1024 377 53.811 6.450 7.516
*163.152.23.171 118.220.200.235 2 u 197 1024 377 36.031 2.234 11.738
OpenJDK 설치
# yum install -y install java-1.8.0-openjdk-1.8.0.242.b08-1.el7.x86_64 java-1.8.0-openjdk-devel- 1.8.0.242.b08-1.el7.x86_64 # java -version openjdk version "1.8.0_282" OpenJDK Runtime Environment (build 1.8.0_282-b08) OpenJDK 64-Bit Server VM (build 25.282-b08, mixed mode)
rpm 현황
# ls -al *.rpm -rw-r--r--. 1 root root 319584600 9월 3 2020 elasticsearch-7.9.1-x86_64.rpm -rw-r--r--. 1 root root 158616924 9월 3 2020 logstash-7.9.1.rpm
Elasticsearch 설치
# rpm -ivh elasticsearch-7.9.1-x86_64.rpm warning: elasticsearch-7.9.1-x86_64.rpm: Header V4 RSA/SHA512 Signature, key ID d88e42b4: NOKEY Preparing... ################################# [100%] Creating elasticsearch group... OK Creating elasticsearch user... OK Updating / installing... 1:elasticsearch-0:7.9.1-1 ################################# [100%] ### NOT starting on installation, please execute the following statements to configure elasticsearch service to start automatically using systemd sudo systemctl daemon-reload sudo systemctl enable elasticsearch.service ### You can start elasticsearch service by executing sudo systemctl start elasticsearch.service Created elasticsearch keystore in /etc/elasticsearch/elasticsearch.keystore
Logstash 설치
# rpm -ivh logstash-7.9.1.rpm warning: logstash-7.9.1.rpm: Header V4 RSA/SHA512 Signature, key ID d88e42b4: NOKEY Preparing... ################################# [100%] Updating / installing... 1:logstash-1:7.9.1-1 ################################# [100%] Using provided startup.options file: /etc/logstash/startup.options /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/pleaserun- 0.0.31/lib/pleaserun/platform/base.rb:112: warning: constant ::Fixnum is deprecated Successfully created system startup script for Logstash
Elasticsearch 설정
# vi /etc/elasticsearch/elasticsearch.yml cluster.name: chohi-elasticsearch node.name: elk01 network.host: _site_ bootstrap.memory_lock: true discovery.seed_hosts: ["elk01"] cluster.initial_master_nodes: ["elk01"]
Java Heap 메모리 설정
# vi /etc/elasticsearch/jvm.options -Xms4g -Xmx4g # vi /etc/sysconfig/elasticsearch ES_JAVA_OPTS="-Xms4g -Xmx4g" MAX_LOCKED_MEMORY=unlimited # vi /etc/security/limits.conf ### Mobigen Elasticsearch Paramiter elasticsearch soft memlock unlimited elasticsearch hard memlock unlimited # vi /usr/lib/systemd/system/elasticsearch.service LimitMEMLOCK=infinity
Elastics Search 서비스 활성화
# systemctl daemon-reload # systemctl restart elasticsearch.service # systemctl status elasticsearch.service # systemctl enable elasticsearch.service
Elastics Search 서비스 상태 체크
# curl elk01:9200
{
"name" : "elk01",
"cluster_name" : "chohi-elasticsearch",
"cluster_uuid" : "0m67oHWCRLuANCFgI2CqOw",
"version" : {
"number" : "7.9.1",
"build_flavor" : "default",
"build_type" : "rpm",
"build_hash" : "083627f112ba94dffc1232e8b42b73492789ef91",
"build_date" : "2020-09-01T21:22:21.964974Z",
"build_snapshot" : false,
"lucene_version" : "8.6.2",
"minimum_wire_compatibility_version" : "6.8.0",
"minimum_index_compatibility_version" : "6.0.0-beta1"
},
"tagline" : "You Know, for Search"
}
노드상태 확인
# curl -XGET http://elk01:9200/_cluster/health?pretty
{
"cluster_name" : "chohi-elasticsearch",
"status" : "yellow",
"timed_out" : false,
"number_of_nodes" : 1,
"number_of_data_nodes" : 1,
"active_primary_shards" : 1,
"active_shards" : 1,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 1,
"delayed_unassigned_shards" : 0,
"number_of_pending_tasks" : 0,
"number_of_in_flight_fetch" : 0,
"task_max_waiting_in_queue_millis" : 0,
"active_shards_percent_as_number" : 50.0
}
# curl http://elk01:9200/_cat/nodes?v
ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name
192.168.20.130 57 96 33 0.00 0.01 0.05 dilmrt * elk01
# curl http://elk01:9200/_cat/master
AJfqTXxqRS2d1rquondCNw 192.168.20.130 192.168.20.130 elk01
Logstash 구성
#logstash.conf 구성
input{
stdin {}
jdbc {
jdbc_validate_connection => true
jdbc_driver_class => "org.postgresql.Driver" #jdbc 드라이버명
jdbc_driver_library => "/usr/share/logstash/bin/postgresql-42.2.17.jar" #jdbc jar파일
jdbc_connection_string => "jdbc:postgresql://192.168.33.11:5432/DB명"
jdbc_user => "ID"
jdbc_password => "패스워드"
schedule => "50 * * * *" #cron 스케줄 매시간 50분
use_column_value => true
lowercase_column_names => false
tracking_column => "unix_ts_in_secs"
tracking_column_type => "numeric"
statement => "select * from 뷰 또는 Inline view SQL"
}
}
filter {
mutate {
copy => { "euid" => "[@metadata][_id]"}
}
}
output{
stdout {}
elasticsearch {
hosts => ["192.168.20.130:9200"] #Elastic Search 서버IP, Port
index => "search_items" #색인명
document_id => "%{[@metadata][_id]}"
}
}
#실행
nohup /usr/share/logstash/bin/logstash --path.settings /etc/logstash -f /etc/logstash/conf.d/logstash.conf > /dev/null &
#로그확인
tail -f /var/log/logstash/logstash-plain.log
#인덱스 체크
# curl -X GET 192.168.20.130:9200/_cat/indices?pretty
yellow open search_items 8Aof45UlSZOazwh3wFP7PA 1 1 19322 0 14.6mb 14.6mb
# curl -X GET 192.168.20.130:9200/search_items/_count
{"count":19322,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0}}
# curl --header "Content-Type: application/json" \
--request POST \
--data '{ "query": { "match_all": {} }}' \
192.168.20.130:9200/search_items/_delete_by_query
{"took":1612,"timed_out":false,"total":19322,"deleted":19322,"batches":20,"version_conflicts":0,"noops":0,"retries":{"bulk":0,"search":0},"throttled_millis":0,"requests_per_second":-1.0,"throttled_until_millis":0,"failures":[]}
# curl -X GET 192.168.20.130:9200/search_items/_count
{"count":0,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0}}
