Merge pull request #17 from Cryptophobia/master

Cryptophobia · web-flow · commit 4381637df6c8 · 2020-11-07T20:14:57.000-05:00
Updates to fluentd from drycc/fluentd
diff --git a/.travis.yml b/.travis.yml
@@ -1,8 +1,8 @@
 sudo: required
 language: go
 services:
-- docker
-script: make test
+  - docker
+script: make docker-build test
 notifications:
   slack:
     secure: pWDCV3od8gxvzxh9DrOTvBL54XoCfWYhZZlwd2ZbyyOz6SS12Psg/ZuCT2253p4yMfF/LPlsz76mr7NgcCrMI0ReveTa/rnt3XBZtyY+1rlsQsy2oxgdAzbO587ENCQeMw2F/OWHaixMT8NDqxEqQd6xafK9Zmg6BeBjwgs7XfXKcR3WzNIuCO0ZG05+Yd0FIxmd/8Xm5tGiFEYr05+Ix6MLdF9MSCXZUPeu1EsYXhDljokLq49w63W1UMU10tm4t7VCEdaO+X9w6EJ5Ov8HDxb6L6IviUYY6+IGTZ01nwIoM6OrGQqfEAytYqgTKdehgQzQnAbLI6TW2wJ0twqEsLrlbTa4NW4j0KkazQJkN5kqcKYQvaeKJJhvJIG44Gi/u78pW3S6W7NU5DhrlE6bbxdIBHJW1vJBimkqu2oBNrO5ZoBB9MS9zflBsU5g/pQpVeHWMnWE8fcYDGa1PqAcr7q6wtdPsrVZhnHmmARN3PwZzIVVVsXbaIQG8VLC5grLGnwMf1Y1fz2nK3sVpCftvrYZT3G6CNAASo+eLOwYdZdiJ9jIS7WNLN1GtpIEvjeDt3QRqsDyH8YoAKUvY5h/v8IWPP/BaSwQbJwep4+Dj7xkpXX5/4wm4jEnVFV1p4xE0lD1AXvEMAVHtPhhggvscNhF9j6oeoPju6eTPcxG+5o=
diff --git a/LICENSE b/LICENSE
@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) Microsoft Corporation. All rights reserved.
+Copyright (c) Team Hephy. All rights reserved.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/Makefile b/Makefile
@@ -1,7 +1,7 @@
 SHORT_NAME ?= fluentd
 BUILD_TAG ?= git-$(shell git rev-parse --short HEAD)
-DEIS_REGISTRY ?= ${DEV_REGISTRY}
-IMAGE_PREFIX ?= deis
+HEPHY_REGISTRY ?= ${DEV_REGISTRY}
+IMAGE_PREFIX ?= hephy
 
 include versioning.mk
 
@@ -22,4 +22,4 @@ upgrade:
 	helm upgrade fluentd charts/fluentd --namespace deis --set org=${IMAGE_PREFIX},docker_tag=${VERSION}
 
 uninstall:
-	helm delete fluentd --purge
+	helm delete fluentd --purge
diff --git a/README.md b/README.md
@@ -1,20 +1,24 @@
 
 ## Description
-[![Build Status](https://ci.deis.io/job/fluentd/badge/icon)](https://ci.deis.io/job/fluentd)
+[![Build Status](https://api.travis-ci.org/teamhephy/fluentd.svg?branch=master)](https://travis-ci.org/github/teamhephy/fluentd)
 
 Deis (pronounced DAY-iss) is an open source PaaS that makes it easy to deploy and manage
 applications on your own servers. Deis builds on [Kubernetes](http://kubernetes.io/) to provide
 a lightweight, [Heroku-inspired](http://heroku.com) workflow.
 
-![Deis Graphic](https://getdeis.blob.core.windows.net/get-deis/deis-graphic-small.png)
-
 ## About
 This is an centos7 based image for running [fluentd](http://fluentd.org). It is built for the purpose of running on a kubernetes cluster.
 
 This work is based on the [docker-fluentd](https://github.com/fabric8io/docker-fluentd) and [docker-fluentd-kubernetes](https://github.com/fabric8io/docker-fluentd-kubernetes) images by the fabric8 team. This image is in with [deis](https://github.com/teamhephy/deis) v2 to send all log data to the [logger](https://github.com/teamhephy/logger) component.
 
 ## Configuration
 
+### Support containerd log format
+By default, fluent parses logs in docker json format. If you use containerd, you only need to set the following environment variables
+
+* `CONTAINER_TAIL_PARSER_TYPE="/^(?<time>.+) (?<stream>stdout|stderr)( (?<tags>.))? (?<log>.*)$/"`
+
+
 ### Enable more verbose logging
 By default we do not capture kubernetes system logs. However, it is possible to tell fluentd to capture those logs just by specifying a few new environment variables.
 
diff --git a/charts/fluentd/Chart.yaml b/charts/fluentd/Chart.yaml
@@ -1,7 +1,7 @@
 name: fluentd
 home: https://github.com/teamhephy/fluentd
 version: <Will be populated by the ci before publishing the chart>
-description: Log shipping mechanism for Deis Workflow.
+description: Log shipping mechanism for Hephy Workflow.
 maintainers:
-  - name: Deis Team
-    email: engineering@deis.com
+  - name: Team Hephy
+    email: team@teamhephy.com
diff --git a/charts/fluentd/templates/logger-fluentd-daemon.yaml b/charts/fluentd/templates/logger-fluentd-daemon.yaml
@@ -1,3 +1,4 @@
+{{ $nsqdNodeCount := .Values.nsqd.replicas | int }}
 apiVersion: {{ template "APIVersion" . }}
 kind: DaemonSet
 metadata:
@@ -84,6 +85,8 @@ spec:
           - name: {{ $key }}
             value: {{ $value | quote }}
           {{- end }}
+          - name: DEIS_NSQD_ADDRS
+            value: "{{range $i := until $nsqdNodeCount}}deis-nsqd-{{$i}}.deis-nsqd.{{ $.Release.Namespace }}.svc.cluster.local:{{$.Values.nsqd.tcp_port}}{{if lt (add 1 $i) $nsqdNodeCount}},{{end}}{{end}}"
         volumeMounts:
         - name: varlog
           mountPath: /var/log
diff --git a/charts/fluentd/values.yaml b/charts/fluentd/values.yaml
@@ -27,8 +27,13 @@ boot:
 # Any custom fluentd environment variables (https://github.com/teamhephy/fluentd#configuration)
 # can be specified as key-value pairs under daemon_environment.
 daemon_environment:
-  #<example-env>: <example-value>
+  CONTAINER_TAIL_PARSER_TYPE: json
+
+nsqd:
+  replicas: 1
+  tcp_port: 4150
+  http_port: 4151
 
 # Role-Based Access Control for Kubernetes >= 1.5
 global:
-  use_rbac: false
+  use_rbac: true
diff --git a/manifests/deis-logger-fluentd-daemon.yaml b/manifests/deis-logger-fluentd-daemon.yaml
@@ -1,4 +1,4 @@
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   name: deis-logger-fluentd
@@ -20,7 +20,7 @@ spec:
       serviceAccount: deis-logger-fluentd
       containers:
       - name: deis-logger-fluentd
-        image: quay.io/deisci/fluentd:canary
+        image: hephy/fluentd:canary
         imagePullPolicy: Always
         volumeMounts:
         - name: varlog
diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile
@@ -1,26 +1,29 @@
-FROM fluent/fluentd:v1.3
+FROM fluent/fluentd:v1.9-1
+
+USER root
 
 COPY fluentd /fluentd
 COPY entrypoint.sh /bin/
 RUN chmod +x /bin/entrypoint.sh
+ENV FLUENTD_CONF="/fluentd/etc/fluent.conf"
 
-RUN apk add bash findutils
-RUN apk add --update --virtual .build-deps sudo build-base ruby-dev \
-    &&  gem install bundler \
-    &&  bundle config --global silence_root_warning 1 \
-    &&  bundle install --gemfile=/fluentd/deis-output/Gemfile \
-    &&  rake --rakefile=/fluentd/deis-output/Rakefile build \
-    &&  fluent-gem install --no-document fluent-plugin-kubernetes_metadata_filter -v 2.1.4 \
-    &&  fluent-gem install --no-document fluent-plugin-elasticsearch -v 4.0.4 \
-    &&  fluent-gem install --no-document fluent-plugin-remote_syslog -v 1.0.0 \
-    &&  fluent-gem install --no-document fluent-plugin-sumologic-mattk42 -v 0.0.4 \
-    &&  fluent-gem install --no-document fluent-plugin-gelf-hs -v 1.0.7 \
-    &&  fluent-gem install --no-document influxdb -v 0.6.1 \
-    &&  fluent-gem install --no-document nsq-ruby -v 2.3.1 \
-    &&  fluent-gem install --local /fluentd/deis-output/pkg/fluent-plugin-deis_output-0.1.0.gem \
-    &&  sudo gem sources --clear-all \
-    &&  apk del .build-deps \
-    &&  rm -rf /var/cache/apk/* /root/.gem/ruby/*/cache/*.gem
+RUN apk add bash findutils \
+    && apk add --update --virtual .build-deps sudo build-base ruby-dev \
+    && gem install bundler:2.1.4 \
+    && bundle config --global silence_root_warning 1 \
+    && bundle install --gemfile=/fluentd/deis-output/Gemfile \
+    && rake --rakefile=/fluentd/deis-output/Rakefile build \
+    && fluent-gem install --no-document fluent-plugin-kubernetes_metadata_filter -v 2.5.2 \
+    && fluent-gem install --no-document fluent-plugin-elasticsearch -v 4.1.1 \
+    && fluent-gem install --no-document fluent-plugin-remote_syslog -v 1.0.0 \
+    && fluent-gem install --no-document fluent-plugin-sumologic-mattk42 -v 0.0.4 \
+    && fluent-gem install --no-document fluent-plugin-gelf-hs -v 1.0.8 \
+    && fluent-gem install --no-document influxdb -v 0.8.0 \
+    && fluent-gem install --no-document nsq-ruby -v 2.3.1 \
+    && fluent-gem install --local /fluentd/deis-output/pkg/fluent-plugin-deis_output-0.1.0.gem \
+    && sudo gem sources --clear-all \
+    && apk del .build-deps \
+    && rm -rf /var/cache/apk/* /root/.gem/ruby/*/cache/*.gem
 
 ENTRYPOINT ["/bin/entrypoint.sh"]
 
diff --git a/rootfs/entrypoint.sh b/rootfs/entrypoint.sh
@@ -1,3 +1,3 @@
-#!/bin/sh
+#!/bin/bash
 
 exec "$@"
diff --git a/rootfs/fluentd/deis-output/Gemfile.lock b/rootfs/fluentd/deis-output/Gemfile.lock
@@ -3,18 +3,18 @@ PATH
   specs:
     fluent-plugin-deis_output (0.1.0)
       fluent-mixin-config-placeholders
-      fluent-mixin-plaintextformatter
-      fluent-mixin-rewrite-tag-name
-      fluentd
-      influxdb (~> 0.3)
-      nsq-ruby
-      yajl-ruby
+       fluent-mixin-plaintextformatter
+       fluent-mixin-rewrite-tag-name
+       fluentd
+       influxdb
+       nsq-ruby
+       yajl-ruby
 
 GEM
   remote: https://rubygems.org/
   specs:
-    cool.io (1.5.3)
-    dig_rb (1.0.1)
+    concurrent-ruby (1.1.6)
+    cool.io (1.6.0)
     fluent-mixin-config-placeholders (0.4.0)
       fluentd
       uuidtools (>= 2.1.5)
@@ -23,43 +23,44 @@ GEM
       ltsv
     fluent-mixin-rewrite-tag-name (0.1.0)
       fluentd
-    fluentd (1.1.0)
+    fluentd (1.11.1)
       cool.io (>= 1.4.5, < 2.0.0)
-      dig_rb (~> 1.0.0)
       http_parser.rb (>= 0.5.1, < 0.7.0)
-      msgpack (>= 0.7.0, < 2.0.0)
+      msgpack (>= 1.3.1, < 2.0.0)
       serverengine (>= 2.0.4, < 3.0.0)
       sigdump (~> 0.2.2)
       strptime (>= 0.2.2, < 1.0.0)
-      tzinfo (~> 1.0)
+      tzinfo (>= 1.0, < 3.0)
       tzinfo-data (~> 1.0)
       yajl-ruby (~> 1.0)
     http_parser.rb (0.6.0)
-    influxdb (0.5.3)
-    ltsv (0.1.0)
-    msgpack (1.2.2)
-    nsq-ruby (2.2.0)
-    power_assert (0.3.0)
-    rake (12.3.3)
-    serverengine (2.0.6)
+    influxdb (0.8.0)
+    ltsv (0.1.2)
+    msgpack (1.3.3)
+    nsq-ruby (2.3.1)
+    power_assert (1.2.0)
+    rake (13.0.1)
+    serverengine (2.2.1)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
-    strptime (0.2.3)
-    test-unit (3.1.9)
+    strptime (0.2.4)
+    test-unit (3.3.6)
       power_assert
-    thread_safe (0.3.6)
-    tzinfo (1.2.4)
-      thread_safe (~> 0.1)
-    tzinfo-data (1.2018.3)
+    tzinfo (2.0.2)
+      concurrent-ruby (~> 1.0)
+    tzinfo-data (1.2020.1)
       tzinfo (>= 1.0.0)
-    uuidtools (2.1.5)
-    yajl-ruby (1.3.1)
+    uuidtools (2.2.0)
+    yajl-ruby (1.4.1)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (~> 1.3)
+  bundler
   fluent-plugin-deis_output!
-  rake (~> 12.3)
-  test-unit (~> 3.1.7)
+  rake
+  test-unit
+
+BUNDLED WITH
+   2.1.4
diff --git a/rootfs/fluentd/deis-output/LICENSE b/rootfs/fluentd/deis-output/LICENSE
@@ -1,13 +1,21 @@
-Copyright Engine Yard, Inc.
+The MIT License (MIT)
 
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
+Copyright (c) Team Hephy. All rights reserved.
 
-    http://www.apache.org/licenses/LICENSE-2.0
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
 
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/rootfs/fluentd/deis-output/README.md b/rootfs/fluentd/deis-output/README.md
@@ -1,7 +1,7 @@
 # Deis Fluentd plugin
 Deis (pronounced DAY-iss) Workflow is an open source Platform as a Service (PaaS) that adds a developer-friendly layer to any [Kubernetes](http://kubernetes.io) cluster, making it easy to deploy and manage applications on your own servers.
 
-For more information about the Deis Workflow, please visit the main project page at https://github.com/teamhephy/workflow.
+For more information about the Hephy Workflow, please visit the main project page at https://github.com/teamhephy/workflow.
 
 ## About
 [Fluentd](http://www.fluentd.org/) is an integral part of the [Deis Workflow](https://github.com/teamhephy/workflow) stack. It is responsible for consuming all of the log data produced by the Kuberenetes cluster. This includes the kubernetes components, Workflow components, and user deployed applications. This plugin takes that log data and will forward it to the appropriate Workflow components.
diff --git a/rootfs/fluentd/deis-output/fluent-plugin-deis_output.gemspec b/rootfs/fluentd/deis-output/fluent-plugin-deis_output.gemspec
@@ -24,12 +24,12 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "fluent-mixin-plaintextformatter"
   gem.add_runtime_dependency "fluent-mixin-config-placeholders"
   gem.add_runtime_dependency "fluent-mixin-rewrite-tag-name"
-  gem.add_runtime_dependency "influxdb", '~> 0.3'
+  gem.add_runtime_dependency "influxdb"
   gem.add_runtime_dependency "nsq-ruby"
-  gem.add_runtime_dependency 'yajl-ruby'
+  gem.add_runtime_dependency "yajl-ruby"
 
-  gem.add_development_dependency "bundler", "~> 1.3"
-  gem.add_development_dependency "rake", "~> 12.3"
-  gem.add_development_dependency "test-unit", "~> 3.1.7"
+  gem.add_development_dependency "bundler"
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "test-unit"
 
 end
diff --git a/rootfs/fluentd/deis-output/lib/fluent/mixin/deis.rb b/rootfs/fluentd/deis-output/lib/fluent/mixin/deis.rb
@@ -7,10 +7,10 @@ module Fluent
   module Mixin
     module Deis
       LOGGER_URL = "http://#{ENV['DEIS_LOGGER_SERVICE_HOST']}:#{ENV['DEIS_LOGGER_SERVICE_PORT_HTTP']}/logs"
-      INFLUX_HOST = "#{ENV['DEIS_MONITOR_INFLUXAPI_SERVICE_HOST']}"
-      INFLUX_PORT = "#{ENV['DEIS_MONITOR_INFLUXAPI_SERVICE_PORT_TRANSPORT']}"
+      INFLUX_HOST = "#{ENV['DEIS_MONITOR_INFLUX_SERVICE_HOST']}"
+      INFLUX_PORT = "#{ENV['DEIS_MONITOR_INFLUX_SERVICE_PORT_TRANSPORT']}"
       INFLUX_DATABASE = ENV['INFLUX_DATABASE'] || "kubernetes"
-      NSQ_URL = "#{ENV['DEIS_NSQD_SERVICE_HOST']}:#{ENV['DEIS_NSQD_SERVICE_PORT_TRANSPORT']}"
+      NSQ_URLs = "#{ENV['DEIS_NSQD_ADDRS']}".split(",")
 
       def kubernetes?(message)
         return message["kubernetes"] != nil
@@ -108,8 +108,8 @@ def debug?
 
       def get_nsq_producer(topic)
         begin
-          puts "Creating nsq producer (#{NSQ_URL}) for topic:#{topic}"
-          return Nsq::Producer.new(nsqd: NSQ_URL, topic: topic)
+          puts "Creating nsq producer (#{NSQ_URLs}) for topic:#{topic}"
+          return Nsq::Producer.new(nsqd: NSQ_URLs, topic: topic)
         rescue Exception => e
           puts "Error:#{e.message}"
           return nil
diff --git a/rootfs/fluentd/sbin/boot b/rootfs/fluentd/sbin/boot
@@ -1,5 +1,5 @@
 #!/bin/bash
-FLUENTD_CONF="/fluentd/etc/fluentd.conf"
+FLUENTD_CONF=${FLUENTD_CONF:-"/fluentd/etc/fluent.conf"}
 INSTALL_BUILD_TOOLS=${INSTALL_BUILD_TOOLS:-"false"}
 BUILD_TOOLS='g++ gcc make ruby-dev'
 
diff --git a/rootfs/fluentd/sbin/sources b/rootfs/fluentd/sbin/sources
@@ -1,16 +1,19 @@
 #!/bin/bash
 if [ -e "/var/log/containers" ]
 then
+
 cat << EOF >> $FLUENTD_CONF
 <source>
   @type tail
   path /var/log/containers/*.log
   pos_file /var/log/containers.log.pos
-  time_key time
-  time_format %Y-%m-%dT%H:%M:%S.%L%z
   tag kubernetes.*
-  format json
   read_from_head true
+  <parse>
+    @type "#{ENV['CONTAINER_TAIL_PARSER_TYPE'] || 'json'}"
+    time_key time
+    time_format %Y-%m-%dT%H:%M:%S.%L%z
+  </parse>
 </source>
 EOF
 

Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,3 @@`
`1`		`-#!/bin/sh`
	`1`	`+#!/bin/bash`
`2`	`2`
`3`	`3`	`exec "$@"`