diff --git a/CHANGES.txt b/CHANGES.txt index 9f348253f..44f9330e4 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -6,6 +6,8 @@ Backwards Incompatibilities Features -------- +Added Sandbox decoders for Microsoft iis and sharepoint uls logs + Bug Handling ------------ diff --git a/docs/source/_themes/mozilla b/docs/source/_themes/mozilla deleted file mode 160000 index 6c054de5f..000000000 --- a/docs/source/_themes/mozilla +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 6c054de5f69fbb3b24396ed77e64b879a8b8cbd2 diff --git a/docs/source/config/decoders/iis.rst b/docs/source/config/decoders/iis.rst new file mode 100644 index 000000000..b76cdfcef --- /dev/null +++ b/docs/source/config/decoders/iis.rst @@ -0,0 +1,13 @@ +.. _config_iis_log_decoder: + +Microsoft iis log Decoder +======================== + +.. versionadded:: 0.1 + +| Plugin Name: **SandboxDecoder** +| File Name: **lua_decoders/iis.lua** + +.. include:: /../../sandbox/lua/decoders/iis.lua + :start-after: --[[ + :end-before: --]] diff --git a/docs/source/config/decoders/index.rst b/docs/source/config/decoders/index.rst index 60e88cc9b..e52400f28 100644 --- a/docs/source/config/decoders/index.rst +++ b/docs/source/config/decoders/index.rst @@ -13,6 +13,7 @@ Available Decoder Plugins apache_access geoip graylog_extended + iis linux_cpu_stats linux_disk_stats linux_load_avg @@ -27,4 +28,5 @@ Available Decoder Plugins rsyslog sandbox scribble + sharepoint_iis stats_to_fields \ No newline at end of file diff --git a/docs/source/config/decoders/index_noref.rst b/docs/source/config/decoders/index_noref.rst index a5208e09c..a6b7f55f1 100644 --- a/docs/source/config/decoders/index_noref.rst +++ b/docs/source/config/decoders/index_noref.rst @@ -56,3 +56,9 @@ Decoders .. include:: /config/decoders/stats_to_fields.rst :start-line: 1 + +.. include:: /config/decoders/iis.rst + :start-line: 1 + +.. include:: /config/decoders/sharepoint_uls.rst + :start-line: 1 \ No newline at end of file diff --git a/docs/source/config/decoders/sharepoint_uls.rst b/docs/source/config/decoders/sharepoint_uls.rst new file mode 100644 index 000000000..bfed9d5c8 --- /dev/null +++ b/docs/source/config/decoders/sharepoint_uls.rst @@ -0,0 +1,13 @@ +.. _config_sharepoint_uls_log_decoder: + +Microsoft sharepoint uls log Decoder +======================== + +.. versionadded:: 0.1 + +| Plugin Name: **SandboxDecoder** +| File Name: **lua_decoders/sharepoint_uls.lua** + +.. include:: /../../sandbox/lua/decoders/sharepoint_uls.lua + :start-after: --[[ + :end-before: --]] diff --git a/sandbox/lua/decoders/iis.lua b/sandbox/lua/decoders/iis.lua new file mode 100644 index 000000000..e023bb6bb --- /dev/null +++ b/sandbox/lua/decoders/iis.lua @@ -0,0 +1,151 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +--[[ +Parses the iis logs based on the Microsoft iis log formats. This decoder is tested for iis verions 7 and 8. + +Config: + +- payload_keep (bool, optional, default false) + Always preserve the original log line in the message payload. + +- iis_version_7 (bool, optional, default flase) + Default configuration asssumes iis log format for version 8. + For version 7 and similar formats, set this to true + +*Example Heka Configuration* + +.. code-block:: ini + +[hekad] +share_dir = 'C:\heka-agent\heka\share\heka' +base_dir = 'C:\var\cache\hekad' + +[IISLogs] +type = "LogstreamerInput" +log_directory = 'F:\Web_Logs' +file_match = '(?P\w+)(?P\S+)u_ex(?P\d+)\.log' +differentiator = ["dir"] +priority = ["Index"] +decoder = "IISDecoder" + +[IISDecoder] +type = "SandboxDecoder" +script_type = "lua" +filename = 'lua_decoders\iis.lua' + +[IISDecoder.config] +payload_keep = true +iis_version_7 = true +tz = "UTC" + +*Example Heka Message* + +2015/08/02 00:34:43 +:Timestamp: 2014-09-22 06:32:29 +0000 UTC +:Type: iis +:Hostname: iis-host +:Pid: 0 +:Uuid: 2dd1d363-02e2-4d61-ade8-e4ed6657fcd6 +:Logger: W3SVC1368505715 +:Payload: 2014-09-22 06:32:29 101.181.48.45 GET / - 6005 - 10.181.72.190 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0) 401 0 64 46 + +:EnvVersion: +:Severity: 7 +:Fields: + | name:"substatus" type:string value:"0" + | name:"client_ip" type:string value:"101.181.72.190" + | name:"cs_method" type:string value:"GET" + | name:"cs_username" type:string value:"" + | name:"cs_user_agent" type:string value:"Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0)" + | name:"time_taken" type:double value:46 + | name:"port" type:string value:"6005" + | name:"win32_status" type:string value:"64" + | name:"status" type:double value:401 + | name:"cs_uri_stem" type:string value:"/" + | name:"host_ip" type:string value:"10.181.48.45" + | name:"cs_uri_query" type:string value:"" + +--]] + +local dt = require "date_time" +local l = require 'lpeg' +l.locale(l) + +local sp = l.space +local num = l.digit^1 / tonumber + +local function extract_quote(openp,endp) + openp = l.P(openp) + endp = endp and l.P(endp) or openp + local upto_endp = (1 - endp)^1 + return openp * l.C(upto_endp) * endp +end + +local sp = l.space + +local timestamp = l.Cg(dt.build_strftime_grammar("%Y-%m-%d %H:%M:%S") / dt.time_to_ns, "timestamp") +local host_ip = l.Cg(extract_quote(" ", " "), "host_ip") +local cs_method = l.Cg(extract_quote("", " "), "cs_method") +local cs_uri_stem = l.Cg(extract_quote("", " "), "cs_uri_stem") +local cs_uri_query = l.Cg(extract_quote("", " "), "cs_uri_query") +local port = l.Cg(extract_quote("", " "), "port") +local cs_username = l.Cg(extract_quote("", " "), "cs_username") +local client_ip = l.Cg(extract_quote("", " "), "client_ip") +local cs_user_agent = l.Cg(extract_quote("", " "), "cs_user_agent") +local cs_referer = l.Cg(extract_quote("", " "), "cs_referer") +local status = l.Cg(num, "status") +local substatus = l.Cg(extract_quote(" ", " "), "substatus") +local win32_status = l.Cg(extract_quote("", " "), "win32_status") +local time_taken = l.Cg(num, "time_taken") +local version_8 = timestamp * host_ip * cs_method * cs_uri_stem * cs_uri_query * port * cs_username * client_ip * cs_user_agent * cs_referer * status * substatus * win32_status * time_taken +local version_7 = timestamp * host_ip * cs_method * cs_uri_stem * cs_uri_query * port * cs_username * client_ip * cs_user_agent * status * substatus * win32_status * time_taken + +local grammar = l.Ct(version_8) + +local iis_version = read_config("iis_version_7") +local payload_keep = read_config("payload_keep") + +if iis_version then + grammar = l.Ct(version_7) +end + +local msg = { + Timestamp = nil, + Payload = nil, + Hostname = nil, + Fields = nil, + Type = "iis" +} + +function process_message() + + local data = read_message("Payload") + local host = read_message("Hostname") + local fields = grammar:match(data) + + if not fields then + return -1 + end + + msg.Timestamp = fields.timestamp + msg.Hostname = string.lower(host) + fields.timestamp = nil + msg.Fields = fields + + if msg.Fields.cs_username == "-" then + msg.Fields.cs_username = "" + end + + if msg.Fields.cs_uri_query == "-" then + msg.Fields.cs_uri_query = "" + end + + if payload_keep then + msg.Payload = data + end + + inject_message(msg) + return 0 +end diff --git a/sandbox/lua/decoders/sharepoint_uls.lua b/sandbox/lua/decoders/sharepoint_uls.lua new file mode 100644 index 000000000..4a97595b9 --- /dev/null +++ b/sandbox/lua/decoders/sharepoint_uls.lua @@ -0,0 +1,124 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +--[[ +Parses the Microsoft sharepoint uls logs based on the uls log format. + +Config: + +- payload_keep (bool, optional, default false) + Always preserve the original log line in the message payload. + +*Example Heka Configuration* + +.. code-block:: ini + +[hekad] +share_dir = 'C:\heka-agent\heka\\share\heka' +base_dir = 'C:\var\cache\hekad' + +[SharePointULSLogs] +type = "LogstreamerInput" +log_directory = 'F:\Trace_log' +file_match = '(?P\w+)-(?P\S+)-(?P\d{4})(?P\d{2})(?P\d{2})-(?P