Logstash

安装

官网下载安装包

https://www.elastic.co/cn/downloads/logstash

解压

tar -zxvf logstash-7.x.tar.gz

配置

conf配置

mkdir conf
vim test.conf
# test.conf
input {
	stdin {}
	jdbc {
		jdbc_connection_string => "jdbc:mysql://10.10.10.10:3306/portal?characterEncoding=UTF-8&useUnicode=true&useSSL=false&tinyInt1isBit=false&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai"
		jdbc_driver_class => "com.mysql.cj.jdbc.Driver"
		jdbc_user => "root"
		jdbc_password => "root_password"
		jdbc_paging_enabled => "true"
		jdbc_page_size => "10"
		jdbc_default_timezone => "Asia/Shanghai"
		use_column_value => "true"
		tracking_column => "project_id"
		record_last_run => "true"
		last_run_metadata_path => "/home/es/logstash/lastrun/.test_last_run"
		statement_filepath => "/home/es/logstash/sql/test.sql"
		schedule => "* * * * *"
		type => "test"
	}
}

filter {
	mutate {
		remove_field => ["@version", "@timestamp", "risktype"]
  }
}

output {
	if [type] == "test" {
		elasticsearch {
			hosts => ["10.10.6.125:9200"]
			user => "elastic"
			password => "123456"
			index => "test"
			document_id => "%{project_id}"
		}
		stdout {
			codec => json_lines
		}
	} else {
		stdout {}
	}
}

sql配置

mkdir sql
vim test.sql
# test.sql
select * from test where id > :sql_last_value

lastrun配置

mkdir lastrun

启动

nohup ./bin/logstash -f ./conf/ &