Some fixes for non compliant lines, rsyslog logs and submission #4

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open

yo000 wants to merge 105 commits into youyo:master from yo000:master

Makefile

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -1,5 +1,5 @@
  
    .DEFAULT_GOAL := help

    Owner := youyo

    Owner := yo000

    Name := postfix-log-parser

    Repository := "github.com/$(Owner)/$(Name)"

    GithubToken := ${GITHUB_TOKEN}

    @@ -28,7 +28,7 @@ vendoring:
  
    ## Build

    build:

    	go get

    	goxz -os=darwin,linux -arch=amd64 -d=pkg ./$(Name)

    	goxz -os=freebsd,darwin,linux -arch=amd64 -d=pkg ./$(Name)

    ## Release

    release:

README.md

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -5,23 +5,27 @@ Parse postfix log, and output json format
  
    ## Install

    Place a `postfix-log-parser` command to your PATH and set an executable flag.  

    Download the latest release from github. https://github.com/youyo/postfix-log-parser/releases/latest

    Download the latest release from github. https://github.com/yo000/postfix-log-parser/releases/latest

    ## Usage

    Input postfix logs as os stdin.

    ``` console

    # cat /var/log/maillog | ./postfix-log-parser

    # cat /var/log/maillog | ./postfix-log-parser | jq

    {

      "time": "0000-10-10T15:59:29+09:00",

      "hostname": "mail",

      "process": "postfix/smtpd[1827]",

      "queue_id": "3D74ADB7400B",

      "client_hostname": "example.com",

      "client_ip": "127.0.0.1",

      "sasl_method": "PLAIN",

      "sasl_username": "[email protected]",

      "message_id": "[email protected]",

      "from": "[email protected]",

      "size": "1988",

      "nrcpt": "1",

      "messages": [

        {

          "time": "0000-10-10T15:59:30+09:00",

    @@ -42,6 +46,70 @@ Input postfix logs as os stdin.
  
    .

    ```

    Use -f flag to flatten json structure:

    ``` console

    # cat /var/log/maillog | ./postfix-log-parser -f | jq

    {

      "time": "0000-10-10T15:59:29+09:00",

      "hostname": "mail",

      "process": "postfix/smtpd[1827]",

      "queue_id": "3D74ADB7400B",

      "client_hostname": "example.com",

      "client_ip": "127.0.0.1",

      "sasl_method": "PLAIN",

      "sasl_username": "[email protected]",

      "message_id": "[email protected]",

      "from": "[email protected]",

      "size": "1988",

      "nrcpt": "1",

      "time_sent": "0000-10-10T15:59:30+09:00",

      "to": "[email protected]",

      "status": "sent",

      "message": "to=<[email protected]>, relay=example.to[192.168.0.20]:25, delay=1.7, delays=0.02/0/1.7/0.06, dsn=2.0.0, status=sent (250 [Sniper] OK 1539154772 snipe-queue 10549)"

    }

    {

      "time": "0000-10-10T15:59:29+09:00",

      "hostname": "mail",

      "process": "postfix/smtpd[1827]",

      "queue_id": "3D74ADB7400B",

      "client_hostname": "example.com",

      "client_ip": "127.0.0.1",

      "sasl_method": "PLAIN",

      "sasl_username": "[email protected]",

      "message_id": "[email protected]",

      "from": "[email protected]",

      "size": "1988",

      "nrcpt": "1",

      "time_sent": "0000-10-10T15:59:30+09:00",

      "to": "[email protected]",

      "status": "sent",

      "message": "to=<[email protected]>, relay=example.to[192.168.0.20]:25, delay=1.7, delays=0.02/0/1.7/0.06, dsn=2.0.0, status=sent (250 [Sniper] OK 1539154772 snipe-queue 10549)"

    }

    .

    .

    .

    ```

    Use "-o filename.json" to write output to file.

    ## Piping rsyslog to postfix-log-parser

    You can feed syslog to postfix-log-parser by using "omprog" rsyslog module, with template "RSYSLOG_FileFormat" :

    ``` console

    module(load="omprog")

    [...]

    mail.info                                                       /var/log/maillog

    & action(

        type="omprog"

        binary="/usr/local/bin/postfix-log-parser -f -o /var/log/maillog.json"

        template="RSYSLOG_FileFormat")

    & stop

    ```

    ## Library usage

    ```

    @@ -75,8 +143,12 @@ postfixlog.LogFormat{
  
      Messages:       "to=<[email protected]>, relay=mail.example-to.com[192.168.0.10]:25, delay=5.3, delays=0.26/0/0.31/4.7, dsn=2.0.0, status=sent (250 2.0.0 Ok: queued as C598F1B0002D)",

      ClientHostname: "",

      ClinetIp:       "",

      SaslMethod:     "",

      SaslUsername:   "",

      MessageId:      "",

      From:           "",

      Size:           "",

      NRcpt:          "",

      To:             "[email protected]",

      Status:         "sent",

    }

go.mod

-Original file line number
+Diff line change
@@ -1,8 +1,25 @@
-    module github.com/youyo/postfix-log-parser
+    module github.com/yo000/postfix-log-parser
+    go 1.21
+    toolchain go1.23.2
     require (
-    	github.com/spf13/cobra v0.0.3
-    	github.com/spf13/pflag v1.0.3 // indirect
+    	github.com/prometheus/client_golang v1.21.0
+    	github.com/spf13/cobra v1.9.1
+    	github.com/tabalt/pidfile v1.1.0
     )
-    go 1.13
+    require (
+    	github.com/beorn7/perks v1.0.1 // indirect
+    	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+    	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+    	github.com/klauspost/compress v1.17.11 // indirect
+    	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+    	github.com/prometheus/client_model v0.6.1 // indirect
+    	github.com/prometheus/common v0.62.0 // indirect
+    	github.com/prometheus/procfs v0.15.1 // indirect
+    	github.com/spf13/pflag v1.0.6 // indirect
+    	golang.org/x/sys v0.28.0 // indirect
+    	google.golang.org/protobuf v1.36.1 // indirect
+    )

go.sum

-Original file line number
+Diff line change
@@ -1,4 +1,43 @@
-    github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8=
-    github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
-    github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
-    github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+    github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+    github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+    github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+    github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+    github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
+    github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+    github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+    github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+    github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+    github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+    github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+    github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
+    github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
+    github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+    github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+    github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+    github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+    github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+    github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+    github.com/prometheus/client_golang v1.21.0 h1:DIsaGmiaBkSangBgMtWdNfxbMNdku5IK6iNhrEqWvdA=
+    github.com/prometheus/client_golang v1.21.0/go.mod h1:U9NM32ykUErtVBxdvD3zfi+EuFkkaBvMb09mIfe0Zgg=
+    github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
+    github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
+    github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io=
+    github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I=
+    github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
+    github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
+    github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+    github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo=
+    github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0=
+    github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
+    github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+    github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+    github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+    github.com/tabalt/pidfile v1.1.0 h1:Q7qQGZ4MoAXE+rvM5tB4/eAIrawewYewByhMiPoDE50=
+    github.com/tabalt/pidfile v1.1.0/go.mod h1:7F1QwNrjfAApsuX4Nyah3RsbHVAdY/D9qZWp0nnJ/Uw=
+    golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
+    golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+    google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk=
+    google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
+    gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+    gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+    gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

postfix-log-parser.go

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -1,19 +1,46 @@
  
    package postfixlog

    import (

    	"errors"

    	"regexp"

    	"time"

    )

    const (

    	SyslogPri                  = `(?:<\d{1,3}>)?`

    	TimeFormat                 = "Jan  2 15:04:05"

    	TimeFormatISO8601          = "2006-01-02T15:04:05.999999-07:00"

    	TimeRegexpFormat           = `([A-Za-z]{3}\s*[0-9]{1,2} [0-9]{2}:[0-9]{2}:[0-9]{2}|^\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+(?:[+-][0-2]\d:[0-5]\d|Z))`

    	HostRegexpFormat           = `([0-9A-Za-z\.]*)`

    	ProcessRegexpFormat        = `(postfix/[a-z]*\[[0-9]{1,5}\])?`

    	// Capture group 1

    	TimeRegexpFormat           = `([A-Za-z]{3}\s*[0-9]{1,2} [0-9]{2}:[0-9]{2}:[0-9]{2}|^\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d(?:\.\d+)?(?:[+-][0-2]\d:[0-5]\d|Z))`

    	// Capture group 2

    	HostRegexpFormat           = `([0-9A-Za-z\-\.]*)`

    	// Capture group 3

    	ProcessRegexpFormat        = `(postfix.*(?:\/[a-z]*)+\[[0-9]{1,7}\])?`

    	// Capture group 4

    	QueueIdRegexpFormat        = `([0-9A-Z]*)`

    	MessageDetailsRegexpFormat = `((?:client=(.+)\[(.+)\])?(?:message-id=<(.+)>)?(?:from=<(.+@.+)>)?(?:to=<(.+@.+)>.*status=([a-z]+))?.*)`

    	RegexpFormat               = TimeRegexpFormat + ` ` + HostRegexpFormat + ` ` + ProcessRegexpFormat + `: ` + QueueIdRegexpFormat + `(?:\: )?` + MessageDetailsRegexpFormat

    	// These are the main messages : Capture group 5 get all the following

    	// Capture groups 6, 7, 8, 9

    	ClientRegexpFormat         = `(?:client=(.+)\[(.+)\](?:, sasl_method=(.+), sasl_username=(.+))?)?`

    	// CG 10

    	MessageIdRegexpFormat      = `(?:message-id=<(.+)>)?`

    	// CG 11, 12, 13

    	FromRegexpFormat           = `(?:from=<(.+@.+)>(?:, size=(\d+), nrcpt=(\d+))?)?`

    	// CG 14, 15

    	ToRegexpFormat             = `(?:to=<(.+@.+)>.*status=([a-z]+))?`

    	// CG 16

    	SenderNDNRegexpFormat      = `(?:sender non-delivery notification: ([0-9A-Z]*))?`

    	// CG 17, 18, 19, 20, 21

    	MilterRegexpFormat         = `(?:(milter-.*): .* from (.+)\[(.+)\]: .*from=<(.+@.+)?> to=<(.+@.+)> .*)?`

    	// CG 22, 23, 24, 25

    	AuthentFailedRegexpFormat  = `(?:warning: (.+)\[(.+)\]: SASL (.*) authentication failed: (.*))?`

    	// CG 26, 27, 28, 29. from regex is voluntarily lazy, b/c some senders really do not put '@' in from

    	PostfixRejectRegexpFormat  = `(?:reject: RCPT from (.+)\[(.+)\]: [0-9]{3} [0-9\.]{5} [^;]*; from=<(.+)?> to=<(.+@[^>]+)>.*$)?`

    	// CG 30, 31, 32

    	PostfixRLimitExceedFormat  = `(?:warning: Message delivery request rate limit exceeded: ([0-9]+) from (.+)\[(.+)\] for service [a-zA-Z]+)?`

    	// CG 33, 34, 35, 36, 37, 38

    	FilterRcptRegexpFormat     = `(?:filter: RCPT from (.+)\[(.+)\]: <(.+@.+)?>: ([^;]+); from=<(.+@.+)?> to=<(.+@[^>]+)> proto=[^ ]+ helo=<.+>)?`

    	MessageDetailsRegexpFormat = `(` + ClientRegexpFormat + MessageIdRegexpFormat + FromRegexpFormat + ToRegexpFormat + SenderNDNRegexpFormat + MilterRegexpFormat + AuthentFailedRegexpFormat + PostfixRejectRegexpFormat + PostfixRLimitExceedFormat + FilterRcptRegexpFormat + `.*)`

    	RegexpFormat               = SyslogPri + TimeRegexpFormat + ` ` + HostRegexpFormat + ` ` + ProcessRegexpFormat + `:? ` + QueueIdRegexpFormat + `(?:\: )?` + MessageDetailsRegexpFormat

    )

    type (

    @@ -27,13 +54,18 @@ type (
  
    		Hostname       string     `json:"hostname"`

    		Process        string     `json:"process"`

    		QueueId        string     `json:"queue_id"`

    		Messages       string     `json:"messages"`

    		Message       string      `json:"message"`

    		ClientHostname string     `json:"client_hostname"`

    		ClinetIp       string     `json:"client_ip"`

    		SaslMethod     string     `json:"sasl_method"`

    		SaslUsername   string     `json:"sasl_username"`

    		MessageId      string     `json:"message_id"`

    		From           string     `json:"from"`

    		Size           string     `json:"size"`

    		NRcpt          string     `json:"nrcpt"`

    		To             string     `json:"to"`

    		Status         string     `json:"status"`

    		BounceId       string     `json:"bounce_id"`

    	}

    )

    @@ -46,6 +78,10 @@ func NewPostfixLog() *PostfixLog {
  
    func (p *PostfixLog) Parse(text []byte) (LogFormat, error) {

    	re := p.Regexp.Copy()

    	group := re.FindSubmatch(text)

    	if len(group) == 0 {

    		err := errors.New("Error: Line do not match regex")

    		return LogFormat{}, err

    	}

    	var t time.Time

    	t, err := time.ParseInLocation(TimeFormat, string(group[1]), time.Local)

    	if err != nil {

    @@ -56,17 +92,58 @@ func (p *PostfixLog) Parse(text []byte) (LogFormat, error) {
  
    	}

    	logFormat := LogFormat{

    		Time:           &t,

    		Hostname:       string(group[2]),

    		Process:        string(group[3]),

    		QueueId:        string(group[4]),

    		Messages:       string(group[5]),

    		ClientHostname: string(group[6]),

    		ClinetIp:       string(group[7]),

    		MessageId:      string(group[8]),

    		From:           string(group[9]),

    		To:             string(group[10]),

    		Status:         string(group[11]),

    		Time:         &t,

    		Hostname:     string(group[2]),

    		Process:      string(group[3]),

    		QueueId:      string(group[4]),

    		Message:      string(group[5]),

    		SaslMethod:   string(group[8]),

    		SaslUsername: string(group[9]),

    		MessageId:    string(group[10]),

    		Size:         string(group[12]),

    		NRcpt:        string(group[13]),

    		BounceId:     string(group[16]),

    	}

    	// Milter reject|hold put values far in the group

    	if len(group[17]) > 0 {

    		logFormat.Status = string(group[17])

    		logFormat.ClientHostname = string(group[18])

    		logFormat.ClinetIp = string(group[19])

    		logFormat.From = string(group[20])

    		logFormat.To = string(group[21])

    	// Authentication failure

    	} else if len(group[22]) > 0 {

    		logFormat.ClientHostname = string(group[22])

    		logFormat.ClinetIp = string(group[23])

    		logFormat.SaslMethod = string(group[24])

    		logFormat.Status = "auth-failed"

    	// postfix-reject

    	} else if len(group[26]) > 0 {

    		logFormat.ClientHostname = string(group[26])

    		logFormat.ClinetIp = string(group[27])

    		logFormat.From = string(group[28])

    		logFormat.To = string(group[29])

    		logFormat.Status = "postfix-reject"

    	// postfix rate-limit

    	} else if len(group[30]) > 0 {

    		logFormat.ClientHostname = string(group[31])

    		logFormat.ClinetIp = string(group[32])

    		logFormat.Status = "postfix-ratelimit"

    	// postfix filter RCPT

    	} else if len(group[33]) > 0 {

    		logFormat.ClientHostname = string(group[33])

    		logFormat.ClinetIp = string(group[34])

    		logFormat.From = string(group[37])

    		logFormat.To = string(group[38])

    		logFormat.Status = "postfix-filter"

    	// bounced by remote

    	} else {

    		logFormat.Status = string(group[15])

    		logFormat.ClientHostname = string(group[6])

    		logFormat.ClinetIp = string(group[7])

    		logFormat.From = string(group[11])

    		logFormat.To = string(group[14])

    	}

    	return logFormat, nil

postfix-log-parser/cmd/execute.go

This file was deleted.

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Some fixes for non compliant lines, rsyslog logs and submission #4

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!

Some fixes for non compliant lines, rsyslog logs and submission #4

Are you sure you want to change the base?

Uh oh!

Some fixes for non compliant lines, rsyslog logs and submission #4

Uh oh!

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!