Liunx 搭建 Elk 集群配置 之 Logstash 搭建

# 解压命令
tar -zxvf logstash-7.11.0-linux-x86_64.tar.gz -C /opt

# 到 bin目录下 执行如下命令
./logstash -e 'input { stdin { } } output { stdout {} }'

# 这里我报错了 由于我的java 版本太高 报一下错误信息

Using JAVA_HOME defined java: /usr/local/jdk-15
WARNING, using JAVA_HOME while Logstash distribution comes with a bundled JDK
Unrecognized VM option 'UseConcMarkSweepGC'
Error: Could not create the Java Virtual Machine.
Error: A fatal exception has occurred. Program will exit.
# 直接修改配置文件
vim /opt/logstash-7.11.0/config/jvm.options

root@fjj001:/opt/logstash-7.11.0/bin# ./logstash -e 'input { stdin { } } output { stdout {} }'
Using JAVA_HOME defined java: /usr/local/jdk-15
WARNING, using JAVA_HOME while Logstash distribution comes with a bundled JDK
Sending Logstash logs to /opt/logstash-7.11.0/logs which is now configured via log4j2.properties
2023-02-06 15:01:46,291 main ERROR No ScriptEngine found for language JavaScript. Available languages are: ruby, jruby
2023-02-06 15:01:46,311 main ERROR No ScriptEngine found for language JavaScript. Available languages are: ruby, jruby
2023-02-06 15:01:46,485 main ERROR No ScriptEngine found for language JavaScript. Available languages are: ruby, jruby
[2023-02-06T15:01:46,527][INFO ][logstash.runner          ] Starting Logstash {"logstash.version"=>"7.11.0", "jruby.version"=>"jruby 9.2.13.0 (2.5.7) 2020-08-03 9a89c94bcc Java HotSpot(TM) 64-Bit Server VM 15+36-1562 on 15+36-1562 +indy +jit [linux-x86_64]"}
[2023-02-06T15:01:47,094][WARN ][logstash.config.source.multilocal] Ignoring the 'pipelines.yml' file because modules or command line options are specified
[2023-02-06T15:01:48,750][INFO ][org.reflections.Reflections] Reflections took 53 ms to scan 1 urls, producing 23 keys and 47 values 
[2023-02-06T15:01:49,585][INFO ][logstash.javapipeline    ][main] Starting pipeline {:pipeline_id=>"main", "pipeline.workers"=>16, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>50, "pipeline.max_inflight"=>2000, "pipeline.sources"=>["config string"], :thread=>"#<Thread:0x446cbc44 run>"}
[2023-02-06T15:01:50,925][INFO ][logstash.javapipeline    ][main] Pipeline Java execution initialization time {"seconds"=>1.33}
WARNING: An illegal reflective access operation has occurred
WARNING: Illegal reflective access by com.jrubystdinchannel.StdinChannelLibrary$Reader (file:/opt/logstash-7.11.0/vendor/bundle/jruby/2.5.0/gems/jruby-stdin-channel-0.2.0-java/lib/jruby_stdin_channel/jruby_stdin_channel.jar) to field java.io.FilterInputStream.in
WARNING: Please consider reporting this to the maintainers of com.jrubystdinchannel.StdinChannelLibrary$Reader
WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations
WARNING: All illegal access operations will be denied in a future release
[2023-02-06T15:01:50,999][INFO ][logstash.javapipeline    ][main] Pipeline started {"pipeline.id"=>"main"}
The stdin plugin is now waiting for input:
[2023-02-06T15:01:51,073][INFO ][logstash.agent           ] Pipelines running {:count=>1, :running_pipelines=>[:main], :non_running_pipelines=>[]}
[2023-02-06T15:01:51,367][INFO ][logstash.agent           ] Successfully started Logstash API endpoint {:port=>9600}
aaa
{
          "host" => "fjj001",
      "@version" => "1",
    "@timestamp" => 2023-02-06T07:01:59.843Z,
       "message" => "aaa"
}

root@fjj001:/opt/logstash-7.11.0/bin# cat test_es.conf 
input{
        stdin{}
}
output{
        elasticsearch{
        #集群的地址
                hosts=>["192.168.6.39:9200","192.168.6.39:9201","192.168.6.39:9203"]
      	#可设置索引名 如果不设置系统也会自己生成按照时间的索引名
      #          index=>"testeslogstash"
        }
        stdout{codec=>rubydebug}
}
root@fjj001:/opt/logstash-7.11.0/bin#

./logstash  -f ./test_es.conf
[2023-02-06T16:48:55,501][INFO ][logstash.agent           ] Successfully started Logstash API endpoint {:port=>9600}
[2023-02-06T16:49:53,674][WARN ][logstash.outputs.elasticsearch][main] Marking url as dead. Last error: [LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError] Elasticsearch Unreachable: [http://192.168.6.39:9200/][Manticore::SocketTimeout] Read timed out {:url=>http://192.168.6.39:9200/, :error_message=>"Elasticsearch Unreachable: [http://192.168.6.39:9200/][Manticore::SocketTimeout] Read timed out", :error_class=>"LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError"}
[2023-02-06T16:49:53,683][ERROR][logstash.outputs.elasticsearch][main] Failed to install template. {:message=>"Elasticsearch Unreachable: [http://192.168.6.39:9200/][Manticore::SocketTimeout] Read timed out", :class=>"LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError", :backtrace=>["/opt/logstash-7.11.0/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.8.1-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:319:in `perform_request_to_url'", "/opt/logstash-7.11.0/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.8.1-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:304:in `block in perform_request'", "/opt/logstash-7.11.0/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.8.1-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:399:in `with_connection'", "/opt/logstash-7.11.0/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.8.1-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:303:in `perform_request'", "/opt/logstash-7.11.0/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.8.1-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:311:in `block in Pool'", "/opt/logstash-7.11.0/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.8.1-java/lib/logstash/outputs/elasticsearch/http_client.rb:342:in `exists?'", "/opt/logstash-7.11.0/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.8.1-java/lib/logstash/outputs/elasticsearch/http_client.rb:347:in `template_exists?'", "/opt/logstash-7.11.0/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.8.1-java/lib/logstash/outputs/elasticsearch/http_client.rb:82:in `template_install'", "/opt/logstash-7.11.0/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.8.1-java/lib/logstash/outputs/elasticsearch/template_manager.rb:31:in `install'", "/opt/logstash-7.11.0/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.8.1-java/lib/logstash/outputs/elasticsearch/template_manager.rb:17:in `install_template'", "/opt/logstash-7.11.0/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.8.1-java/lib/logstash/outputs/elasticsearch.rb:424:in `install_template'", "/opt/logstash-7.11.0/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.8.1-java/lib/logstash/outputs/elasticsearch.rb:274:in `block in register'", "/opt/logstash-7.11.0/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.8.1-java/lib/logstash/plugin_mixins/elasticsearch/common.rb:137:in `block in setup_after_successful_connection'"]}
[2023-02-06T16:49:53,735][INFO ][logstash.outputs.elasticsearch][main] Creating rollover alias <logstash-{now/d}-000001>
[2023-02-06T16:49:54,570][INFO ][logstash.outputs.elasticsearch][main] Installing ILM policy {"policy"=>{"phases"=>{"hot"=>{"actions"=>{"rollover"=>{"max_size"=>"50gb", "max_age"=>"30d"}}}}}} to _ilm/policy/logstash-policy
{
      "@version" => "1",
    "@timestamp" => 2023-02-06T08:48:55.115Z,
          "host" => "fjj001",
       "message" => "dddaaa"
}
{
      "@version" => "1",
    "@timestamp" => 2023-02-06T08:48:55.085Z,
          "host" => "fjj001",
       "message" => "fjj"
}
{
      "@version" => "1",
    "@timestamp" => 2023-02-06T08:48:55.112Z,
          "host" => "fjj001",
       "message" => ""
}
{
      "@version" => "1",
    "@timestamp" => 2023-02-06T08:48:55.116Z,
          "host" => "fjj001",
       "message" => "aaa"
}

http://192.168.6.39:9201/logstash-2023.02.06-000001/_search