ELK Integration

So I noticed there isn't really anything built in for using elasticsearch with Mattermost so I configured logstash for postgres and wanted to share for others benefit or opinions.

Only change I made was in the omnibus configuration for gitlab/mattermost is using standalone postgres on port 5432 rather then the embedded socket version then I hooked up logstash to it.

My logstash config looks like

# file: simple-out.conf
input {
    jdbc {
        # Postgres jdbc connection string to our database, mydb
        jdbc_connection_string => "jdbc:postgresql://localhost/mattermost_production"
        # The user we wish to execute our statement as
        jdbc_user => "mmuser"
        jdbc_password => "PASSWORDHERE"
        # The path to our downloaded jdbc driver
        jdbc_driver_library => "/var/lib/logstash/postgresql-9.2-1004.jdbc41.jar"
        # The name of the driver class for Postgresql
        jdbc_driver_class => "org.postgresql.Driver"
        # our query
        statement => "SELECT * FROM users" 
	# channels type O or D
	type => "mattermost_users"
    }
    jdbc {
        # Postgres jdbc connection string to our database, mydb
        jdbc_connection_string => "jdbc:postgresql://localhost/mattermost_production"
        # The user we wish to execute our statement as
        jdbc_user => "mmuser"
        jdbc_password => "PASSWORDHERE"
        # The path to our downloaded jdbc driver
        jdbc_driver_library => "/var/lib/logstash/postgresql-9.2-1004.jdbc41.jar"
        # The name of the driver class for Postgresql
        jdbc_driver_class => "org.postgresql.Driver"
        # our query
        statement => "SELECT posts.id,posts.createat,posts.updateat,posts.deleteat,posts.userid,posts.channelid,posts.rootid,posts.parentid,posts.originalid,posts.message,posts.type AS post_type,posts.props,posts.hashtags,posts.filenames,users.username,users.email,users.nickname,users.firstname,users.lastname,users.teamid,channels.name,channels.type AS channel_type,split_part(channels.name, '__', 1) AS channelname1,split_part(channels.name, '__', 2) AS channelname2 from posts LEFT JOIN users ON posts.userid=users.id LEFT JOIN channels ON posts.channelid=channels.id"
	# channels type O or D
	type => "mattermost_posts"
    }
}

filter {
	if [type] == "mattermost_posts" {
		mutate {
			rename => { "@timestamp" => "retrieved_at" }
		}
		date {
			match => ["createat", "UNIX_MS"]
		}
		if [channel_type] == "D" {
			if [channelname1] == [userid] {
				translate {
					field => "channelname2"
					destination => "to_username"
					dictionary_path => "/etc/logstash/dictionaries/id_to_username.yml"
				}
			} else {
				translate {
					field => "channelname1"
					destination => "to_username"
					dictionary_path => "/etc/logstash/dictionaries/id_to_username.yml"
				}
			}
			mutate {
				remove_field => ["channelname1","channelname2"]
			}
		}
	}
}

output {
	if [type] == "mattermost_users" {
		file {
			path => "/etc/logstash/dictionaries/id_to_username.yml"
			codec => "line" 
			message_format => "%{id}: %{username}"
		 }
	}
	if [type] == "mattermost_posts" {
		stdout { codec => rubydebug }
	}
}

And I run it using /opt/logstash/bin/logstash -f /etc/logstash/conf.d/01-simple.conf

My output looks like

{
              "id" => "gtmp8n7peffym89m5jn96jhqzw",
        "createat" => 1457308690436,
        "updateat" => 1457308690436,
        "deleteat" => 0,
          "userid" => "iy5jn6qjgjnairfmqo7o4uptrw",
       "channelid" => "ota4snk1xfbofj8pz5wexzisny",
          "rootid" => "",
        "parentid" => "",
      "originalid" => "",
         "message" => "hey",
       "post_type" => "",
           "props" => "{}",
        "hashtags" => "",
       "filenames" => "[]",
        "username" => "user1",
           "email" => "user1@domain.com",
        "nickname" => "",
       "firstname" => "Firstname",
        "lastname" => "Lastname",
          "teamid" => "6rjmkf8fhj8hpkmrqcr8fnecjy",
            "name" => "iy5jn6qjgjnairfmqo7o4uptrw__p3uoftq91bnmzmwcqxcd6yakar",
    "channel_type" => "D",
        "@version" => "1",
            "type" => "mattermost_posts",
    "retrieved_at" => "2016-03-07T00:07:59.511Z",
      "@timestamp" => "2016-03-06T23:58:10.436Z",
     "to_username" => "user2"
}
{
              "id" => "wpxik46dd7ykpxa46b7xdkmbzo",
        "createat" => 1457302752744,
        "updateat" => 1457302752744,
        "deleteat" => 0,
          "userid" => "p3uoftq91bnmzmwcqxcd6yakar",
       "channelid" => "ota4snk1xfbofj8pz5wexzisny",
          "rootid" => "",
        "parentid" => "",
      "originalid" => "",
         "message" => "test",
       "post_type" => "",
           "props" => "{}",
        "hashtags" => "",
       "filenames" => "[]",
        "username" => "user2",
           "email" => "user2@domain.com",
        "nickname" => "",
       "firstname" => "Firstname",
        "lastname" => "Lastname",
          "teamid" => "6rjmkf8fhj8hpkmrqcr8fnecjy",
            "name" => "iy5jn6qjgjnairfmqo7o4uptrw__p3uoftq91bnmzmwcqxcd6yakar",
    "channel_type" => "D",
        "@version" => "1",
            "type" => "mattermost_posts",
    "retrieved_at" => "2016-03-07T00:07:59.512Z",
      "@timestamp" => "2016-03-06T22:19:12.744Z",
     "to_username" => "user1"
}

Just thought I would share. Thanks!