diff --git a/listener/configuration_directive.go b/listener/configuration_directive.go index d69687c..b554233 100644 --- a/listener/configuration_directive.go +++ b/listener/configuration_directive.go @@ -1,8 +1,8 @@ package listener import ( - "github.com/coreruleset/seclang_parser/parser" "github.com/coreruleset/crslang/types" + "github.com/coreruleset/seclang_parser/parser" ) func (l *ExtendedSeclangParserListener) EnterEngine_config_directive_with_param(ctx *parser.Engine_config_directive_with_paramContext) { @@ -62,7 +62,7 @@ func (l *ExtendedSeclangParserListener) EnterSec_marker_directive(ctx *parser.Se } l.appendDirective = func() { l.DirectiveList.Marker = *l.configurationDirective - l.ConfigurationList.DirectiveList = append(l.ConfigurationList.DirectiveList, *l.DirectiveList) - l.DirectiveList = new(types.DirectiveList) + l.Ruleset.Groups = append(l.Ruleset.Groups, *l.DirectiveList) + l.DirectiveList = new(types.Group) } } diff --git a/listener/extended_seclang_parser_listener.go b/listener/extended_seclang_parser_listener.go index 2356a0f..e46f889 100644 --- a/listener/extended_seclang_parser_listener.go +++ b/listener/extended_seclang_parser_listener.go @@ -43,8 +43,8 @@ type ExtendedSeclangParserListener struct { varExcluded bool varCount bool parameter string - DirectiveList *types.DirectiveList - ConfigurationList types.ConfigurationList + DirectiveList *types.Group + Ruleset types.Ruleset } func doNothingFunc() {} @@ -52,7 +52,7 @@ func doNothingFunc() {} func doNothingFuncString(value string) {} func (l *ExtendedSeclangParserListener) EnterConfiguration(ctx *parser.ConfigurationContext) { - l.DirectiveList = new(types.DirectiveList) + l.DirectiveList = new(types.Group) l.setParam = doNothingFuncString l.appendDirective = doNothingFunc l.appendComment = func(value string) { @@ -63,7 +63,7 @@ func (l *ExtendedSeclangParserListener) EnterConfiguration(ctx *parser.Configura func (l *ExtendedSeclangParserListener) ExitConfiguration(ctx *parser.ConfigurationContext) { if l.DirectiveList != nil && (len(l.DirectiveList.Directives) > 0 || l.DirectiveList.Marker.Name != "") { - l.ConfigurationList.DirectiveList = append(l.ConfigurationList.DirectiveList, *l.DirectiveList) + l.Ruleset.Groups = append(l.Ruleset.Groups, *l.DirectiveList) } } diff --git a/listener_test.go b/listener_test.go index 1d2bc66..0497f35 100644 --- a/listener_test.go +++ b/listener_test.go @@ -38,7 +38,7 @@ func mustNewSetvarAction(collection types.CollectionName, operation types.VarOpe type testCase struct { name string payload string - expected types.ConfigurationList + expected types.Ruleset } var ( @@ -59,8 +59,8 @@ var ( # https://owasp.org/www-project-modsecurity-core-rule-set/ # `, - expected: types.ConfigurationList{ - DirectiveList: []types.DirectiveList{ + expected: types.Ruleset{ + Groups: []types.Group{ { Directives: []types.SeclangDirective{ types.CommentMetadata{ @@ -88,8 +88,8 @@ https://owasp.org/www-project-modsecurity-core-rule-set/ payload: ` SecRuleEngine On `, - expected: types.ConfigurationList{ - DirectiveList: []types.DirectiveList{ + expected: types.Ruleset{ + Groups: []types.Group{ { Directives: []types.SeclangDirective{ types.ConfigurationDirective{ @@ -139,8 +139,8 @@ SecAction \ setvar:'tx.outbound_anomaly_score_pl3=0',\ setvar:'tx.outbound_anomaly_score_pl4=0',\ setvar:'tx.anomaly_score=0'"`, - expected: types.ConfigurationList{ - DirectiveList: []types.DirectiveList{ + expected: types.Ruleset{ + Groups: []types.Group{ { Directives: []types.SeclangDirective{ &types.SecAction{ @@ -239,8 +239,8 @@ SecRule REQUEST_LINE "@rx (?i)^(?:get /[^#\?]*(?:\?[^\s\v#]*)?(?:#[^\s\v]*)?|(?: ver:'OWASP_CRS/4.0.1-dev',\ severity:'WARNING',\ setvar:'tx.inbound_anomaly_score_pl1=+%{tx.warning_anomaly_score}'"`, - expected: types.ConfigurationList{ - DirectiveList: []types.DirectiveList{ + expected: types.Ruleset{ + Groups: []types.Group{ { Directives: []types.SeclangDirective{ &types.SecRule{ @@ -319,8 +319,8 @@ SecRule ARGS_GET:fbclid "@rx [a-zA-Z0-9_-]{61,61}" \ tag:'OWASP_CRS',\ ctl:ruleRemoveTargetById=942440;ARGS:fbclid,\ ver:'OWASP_CRS/4.0.1-dev'"`, - expected: types.ConfigurationList{ - DirectiveList: []types.DirectiveList{ + expected: types.Ruleset{ + Groups: []types.Group{ { Directives: []types.SeclangDirective{ &types.SecRule{ @@ -376,8 +376,8 @@ SecRule ARGS_GET:fbclid|!ARGS_GET|ARGS_GET:fbclid|!ARGS_GET:fbclid|ARGS_GET:test "id:942441,\ phase:2,\ pass"`, - expected: types.ConfigurationList{ - DirectiveList: []types.DirectiveList{ + expected: types.Ruleset{ + Groups: []types.Group{ { Directives: []types.SeclangDirective{ &types.SecRule{ @@ -415,8 +415,8 @@ SecRule ARGS:/^id_/|!ARGS:id_1 "@rx test" \ "id:942441,\ phase:2,\ pass"`, - expected: types.ConfigurationList{ - DirectiveList: []types.DirectiveList{ + expected: types.Ruleset{ + Groups: []types.Group{ { Directives: []types.SeclangDirective{ &types.SecRule{ @@ -472,8 +472,8 @@ SecRule REQUEST_LINE "@streq GET /" \ "t:none,\ ctl:ruleRemoveByTag=OWASP_CRS,\ ctl:auditEngine=Off"`, - expected: types.ConfigurationList{ - DirectiveList: []types.DirectiveList{ + expected: types.Ruleset{ + Groups: []types.Group{ { Directives: []types.SeclangDirective{ &types.SecRule{ @@ -550,8 +550,8 @@ SecRuleRemoveByTag "attack-sqli" SecRuleRemoveByMsg FAIL `, - expected: types.ConfigurationList{ - DirectiveList: []types.DirectiveList{ + expected: types.Ruleset{ + Groups: []types.Group{ { Directives: []types.SeclangDirective{ types.RemoveRuleDirective{ @@ -590,8 +590,8 @@ SecRule REQBODY_PROCESSOR "!@rx (?:URLENCODED|MULTIPART|XML|JSON)" \ msg:'Enabling body inspection',\ ctl:forceRequestBodyVariable=On,\ ver:'OWASP_CRS/4.0.0-rc1'"`, - expected: types.ConfigurationList{ - DirectiveList: []types.DirectiveList{ + expected: types.Ruleset{ + Groups: []types.Group{ { Directives: []types.SeclangDirective{ &types.SecRule{ @@ -637,8 +637,8 @@ SecCollectionTimeout 600 SecMarker "END-TEST" SecComponentSignature "OWASP_CRS/4.0.1-dev"`, - expected: types.ConfigurationList{ - DirectiveList: []types.DirectiveList{ + expected: types.Ruleset{ + Groups: []types.Group{ { Directives: []types.SeclangDirective{ types.ConfigurationDirective{ @@ -674,7 +674,7 @@ SecComponentSignature "OWASP_CRS/4.0.1-dev"`, func TestLoadSecLang(t *testing.T) { for _, test := range listenerTestCases { t.Run(test.name, func(t *testing.T) { - got := types.ConfigurationList{} + got := types.Ruleset{} input := antlr.NewInputStream(test.payload) lexer := parser.NewSecLangLexer(input) stream := antlr.NewCommonTokenStream(lexer, 0) @@ -682,7 +682,7 @@ func TestLoadSecLang(t *testing.T) { start := p.Configuration() var seclangListener listener.ExtendedSeclangParserListener antlr.ParseTreeWalkerDefault.Walk(&seclangListener, start) - got = seclangListener.ConfigurationList + got = seclangListener.Ruleset require.Equalf(t, test.expected, got, test.name) }) diff --git a/main.go b/main.go index 629a1b7..dc34e79 100644 --- a/main.go +++ b/main.go @@ -16,7 +16,9 @@ var ( ) func main() { - toSeclang := flag.Bool("s", false, "Transalates the specified CRSLang file to Seclang files, instead of the default Seclang to CRSLang.") + toSeclang := flag.Bool("s", false, "Translates the specified CRSLang file to Seclang files, instead of the default Seclang to CRSLang.") + // Experimental flag + dirMode := flag.Bool("d", false, "Directory mode. In Seclang -> CRSLang (default), split the output into multiple YAML files. In CRSLang -> Seclang (-s), treat the input path as a directory containing ruleset.yaml/group.yaml instead of a single YAML file.") output := flag.String("o", "", "Output file name used in translation from Seclang to CRSLang. Output folder used in translation from CRSLang to Seclang.") flag.Usage = func() { @@ -47,25 +49,46 @@ Flags: } configList = *translator.ToCRSLang(configList) + if !*dirMode { + if *output == "" { + *output = "crslang" + } - if *output == "" { - *output = "crslang" + err = translator.PrintYAML(configList, *output+".yaml") + if err != nil { + log.Fatal(err.Error()) + } + } else { + if *output == "" { + *output = "crslang" + } + err := translator.WriteRuleSeparately(configList, *output) + if err != nil { + log.Fatal(err.Error()) + } } - err = translator.PrintYAML(configList, *output+".yaml") - if err != nil { - log.Fatal(err.Error()) - } } else { - if filepath.Ext(pathArg) != ".yaml" { - log.Fatal("Only .yaml files are allowed") - } + if !*dirMode { + if filepath.Ext(pathArg) != ".yaml" { + log.Fatal("Only .yaml files are allowed") + } - configList := types.LoadDirectivesWithConditionsFromFile(pathArg) - - err := translator.PrintSeclang(configList, *output) - if err != nil { - log.Fatal(err.Error()) + configList := types.LoadDirectivesWithConditionsFromFile(pathArg) + err := translator.PrintSeclang(configList, *output) + if err != nil { + log.Fatal(err.Error()) + } + } else { + /* Load rule from dir */ + configList, err := translator.LoadRulesFromDirectory(pathArg) + if err != nil { + log.Fatal(err.Error()) + } + err = translator.PrintSeclang(configList, *output) + if err != nil { + log.Fatal(err.Error()) + } } } } diff --git a/translator/crslang.go b/translator/crslang.go index 2b286a9..efe4aa5 100644 --- a/translator/crslang.go +++ b/translator/crslang.go @@ -1,11 +1,195 @@ package translator -import "github.com/coreruleset/crslang/types" +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "strconv" + + "github.com/coreruleset/crslang/types" + "go.yaml.in/yaml/v4" +) // ToCRSLang process previously loaded seclang directives to CRSLang schema directives -func ToCRSLang(configList types.ConfigurationList) *types.ConfigurationList { +func ToCRSLang(configList types.Ruleset) *types.Ruleset { configListWithConditions := types.ToDirectiveWithConditions(configList) configListWithConditions.ExtractDefaultValues() + + for i := range configListWithConditions.Groups { + configListWithConditions.Groups[i].ExtractDefaultValues() + } + return configListWithConditions } + +// WriteRuleSeparately writes each rule in a separate file, and creates a group.yaml file for each group and a ruleset.yaml file for the whole ruleset. The output is organized in the following structure: +// output/ +// ├── ruleset.yaml +// ├── group1/ +// │ ├── group.yaml +// │ └── rules/ +// │ ├── 1.yaml +// │ ├── 2.yaml +// │ └── ... +func WriteRuleSeparately(ruleset types.Ruleset, output string) error { + output = filepath.Clean(output) + if err := os.MkdirAll(output, 0755); err != nil { + return err + } + + groups := []string{} + + // EXPERIMENTAL: output each group and rule in separate files + for _, group := range ruleset.Groups { + groups = append(groups, group.Id) + + groupFolder := filepath.Join(output, group.Id) + ruleFolder := filepath.Join(groupFolder, "rules") + err := os.MkdirAll(ruleFolder, 0755) + if err != nil { + return err + } + + ruleIds := []int{} + comments := []string{} + configs := []types.ConfigurationDirective{} + for _, directive := range group.Directives { + if directive.GetKind() == types.RuleKind { + rule, ok := directive.(*types.RuleWithCondition) + if !ok { + return fmt.Errorf("error casting directive of type %T in group %s to *types.RuleWithCondition", directive, group.Id) + } + + fileName := filepath.Join(ruleFolder, strconv.Itoa(rule.Metadata.Id)+".yaml") + err := PrintYAML(directive, fileName) + if err != nil { + return err + } + ruleIds = append(ruleIds, rule.Metadata.Id) + + } else if directive.GetKind() == types.CommentKind { + comment, ok := directive.(types.CommentDirective) + if !ok { + return fmt.Errorf("error casting directive of type %T in group %s to types.CommentDirective", directive, group.Id) + } + comments = append(comments, comment.Metadata.Comment) + } else if directive.GetKind() == types.ConfigurationKind { + config, ok := directive.(types.ConfigurationDirective) + if !ok { + return fmt.Errorf("error casting directive of type %T in group %s to types.ConfigurationDirective", directive, group.Id) + } + configs = append(configs, config) + } + } + newGroup := types.Group{ + Id: group.Id, + Tags: group.Tags, + Comments: comments, + Rules: ruleIds, + Configurations: configs, + Marker: group.Marker, + } + err = PrintYAML(newGroup, filepath.Join(groupFolder, "group.yaml")) + if err != nil { + return err + } + } + + newRuleset := types.Ruleset{ + Global: ruleset.Global, + GroupsIds: groups, + } + err := PrintYAML(newRuleset, filepath.Join(output, "ruleset.yaml")) + if err != nil { + return err + } + return nil +} + +// LoadRulesFromDirectory loads a ruleset from a directory containing a ruleset.yaml file and group subdirectories with group.yaml and rule yaml files. The structure of the directory should be as follows: +// dir/ +// ├── ruleset.yaml +// ├── group1/ +// │ ├── group.yaml +// │ └── rules/ +// │ ├── 1.yaml +// │ ├── 2.yaml +// │ └── ... +func LoadRulesFromDirectory(dir string) (types.Ruleset, error) { + info, err := os.Stat(dir) + + if err != nil { + return types.Ruleset{}, err + } else if !info.IsDir() { + return types.Ruleset{}, fmt.Errorf("path is not a directory: %s", dir) + } + dir = filepath.Clean(dir) + + rFile, err := os.ReadFile(filepath.Join(dir, "ruleset.yaml")) + + if err != nil { + return types.Ruleset{}, err + } + + ruleset := types.Ruleset{} + err = yaml.Unmarshal([]byte(rFile), &ruleset) + + if err != nil { + return types.Ruleset{}, err + } + + for _, groupId := range ruleset.GroupsIds { + if err := verifyId(groupId); err != nil { + return types.Ruleset{}, err + } + groupFile, err := os.ReadFile(filepath.Join(dir, groupId, "group.yaml")) + if err != nil { + return types.Ruleset{}, err + } + group := types.Group{} + err = yaml.Unmarshal([]byte(groupFile), &group) + if err != nil { + return types.Ruleset{}, err + } + for _, comment := range group.Comments { + group.Directives = append(group.Directives, types.CommentDirective{ + Kind: types.CommentKind, + Metadata: types.CommentMetadata{ + Comment: comment, + }, + }) + } + group.Comments = nil + for _, config := range group.Configurations { + group.Directives = append(group.Directives, config) + } + group.Configurations = nil + for _, ruleId := range group.Rules { + ruleFile, err := os.ReadFile(filepath.Join(dir, groupId, "rules", strconv.Itoa(ruleId)+".yaml")) + if err != nil { + return types.Ruleset{}, err + } + rule := types.RuleWithCondition{} + err = yaml.Unmarshal([]byte(ruleFile), &rule) + if err != nil { + return types.Ruleset{}, err + } + group.Directives = append(group.Directives, &rule) + } + group.Rules = nil + ruleset.Groups = append(ruleset.Groups, group) + } + ruleset.GroupsIds = nil + return ruleset, nil +} + +var validID = regexp.MustCompile(`^[A-Za-z0-9_-]+$`) + +func verifyId(id string) error { + if !validID.MatchString(id) { + return fmt.Errorf("invalid id: %s. Ids can only contain letters, numbers, underscores and hyphens", id) + } + return nil +} diff --git a/translator/seclang.go b/translator/seclang.go index 3e6aa3e..8c5da9d 100644 --- a/translator/seclang.go +++ b/translator/seclang.go @@ -14,7 +14,7 @@ import ( // assignDirectiveIDs assigns a base id (and an indexed suffix when there are // multiple directive lists) to each entry produced by a single parse run. -func assignDirectiveIDs(directives []types.DirectiveList, id string) { +func assignDirectiveIDs(directives []types.Group, id string) { for i := range directives { directives[i].Id = id if len(directives) > 1 { @@ -23,9 +23,9 @@ func assignDirectiveIDs(directives []types.DirectiveList, id string) { } } -// LoadSeclangFromString loads seclang directives from a string and returns a ConfigurationList. +// LoadSeclangFromString loads seclang directives from a string and returns a Ruleset. // The id parameter is used to name the resulting directive list. -func LoadSeclangFromString(content string, id string) (types.ConfigurationList, error) { +func LoadSeclangFromString(content string, id string) (types.Ruleset, error) { input := antlr.NewInputStream(content) lexer := parser.NewSecLangLexer(input) stream := antlr.NewCommonTokenStream(lexer, 0) @@ -33,14 +33,14 @@ func LoadSeclangFromString(content string, id string) (types.ConfigurationList, start := p.Configuration() var seclangListener listener.ExtendedSeclangParserListener antlr.ParseTreeWalkerDefault.Walk(&seclangListener, start) - assignDirectiveIDs(seclangListener.ConfigurationList.DirectiveList, id) - return seclangListener.ConfigurationList, nil + assignDirectiveIDs(seclangListener.Ruleset.Groups, id) + return seclangListener.Ruleset, nil } -// LoadSeclang loads seclang directives from an input file or folder and returns a ConfigurationList +// LoadSeclang loads seclang directives from an input file or folder and returns a Ruleset // if a folder is specified it loads all .conf files in the folder -func LoadSeclang(input string) (types.ConfigurationList, error) { - resultConfigs := []types.DirectiveList{} +func LoadSeclang(input string) (types.Ruleset, error) { + resultConfigs := []types.Group{} filepath.Walk(input, func(path string, info os.FileInfo, err error) error { if err != nil { return err @@ -57,17 +57,17 @@ func LoadSeclang(input string) (types.ConfigurationList, error) { var seclangListener listener.ExtendedSeclangParserListener antlr.ParseTreeWalkerDefault.Walk(&seclangListener, start) id := strings.TrimSuffix(filepath.Base(info.Name()), filepath.Ext(info.Name())) - assignDirectiveIDs(seclangListener.ConfigurationList.DirectiveList, id) - resultConfigs = append(resultConfigs, seclangListener.ConfigurationList.DirectiveList...) + assignDirectiveIDs(seclangListener.Ruleset.Groups, id) + resultConfigs = append(resultConfigs, seclangListener.Ruleset.Groups...) } return nil }) - configList := types.ConfigurationList{DirectiveList: resultConfigs} + configList := types.Ruleset{Groups: resultConfigs} return configList, nil } // PrintSeclang writes seclang directives to files specified in directive list ids. -func PrintSeclang(configList types.ConfigurationList, dir string) error { +func PrintSeclang(configList types.Ruleset, dir string) error { dir = filepath.Clean(dir) if err := os.MkdirAll(dir, 0755); err != nil { return err @@ -75,7 +75,7 @@ func PrintSeclang(configList types.ConfigurationList, dir string) error { unfDirs := types.FromCRSLangToUnformattedDirectives(configList) - for _, group := range unfDirs.DirectiveList { + for _, group := range unfDirs.Groups { seclangDirectives := group.ToSeclang() groupId := group.Id + ".conf" if strings.HasSuffix(group.Id, ".conf") { diff --git a/translator/translator_test.go b/translator/translator_test.go index 2afca2b..2842b01 100644 --- a/translator/translator_test.go +++ b/translator/translator_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/coreruleset/crslang/types" + "github.com/stretchr/testify/require" "go.yaml.in/yaml/v4" ) @@ -47,7 +48,7 @@ func TestLoadSeclangFromString(t *testing.T) { t.Fatalf("Error loading seclang from string: %v", err) } - if len(configList.DirectiveList) == 0 { + if len(configList.Groups) == 0 { t.Fatal("Expected at least one directive list, got none") } @@ -80,3 +81,333 @@ func TestFromCRSLangToSeclang(t *testing.T) { } } + +func TestWriteAndLoadRuleSeparately(t *testing.T) { + testCases := []struct { + name string + input types.Ruleset + expected types.Ruleset + }{ + { + name: "Simple ruleset with comment, config and rule", + input: types.Ruleset{ + Global: types.DefaultConfigs{ + Version: "4.0.0", + Tags: []string{"OWASP_CRS"}, + }, + Groups: []types.Group{ + { + Id: "test-group-1", + Tags: []string{"tag1", "tag2"}, + Directives: []types.SeclangDirective{ + types.CommentDirective{ + Kind: types.CommentKind, + Metadata: types.CommentMetadata{ + Comment: "Test comment", + }, + }, + types.ConfigurationDirective{ + Kind: types.ConfigurationKind, + Name: types.SecRuleEngine, + Parameter: "On", + }, + &types.RuleWithCondition{ + Kind: types.RuleKind, + Metadata: types.SecRuleMetadata{ + OnlyPhaseMetadata: types.OnlyPhaseMetadata{ + Phase: "1", + }, + Id: 12345, + Msg: "Test rule", + }, + Conditions: []types.Condition{ + { + Variables: []types.Variable{ + { + Name: types.REQUEST_URI, + }, + }, + Operator: types.Operator{ + Name: types.Rx, + Value: "/test", + }, + }, + }, + }, + }, + }, + }, + }, + expected: types.Ruleset{ + Global: types.DefaultConfigs{ + Version: "4.0.0", + Tags: []string{"OWASP_CRS"}, + }, + Groups: []types.Group{ + { + Id: "test-group-1", + Tags: []string{"tag1", "tag2"}, + Directives: []types.SeclangDirective{ + types.CommentDirective{ + Kind: types.CommentKind, + Metadata: types.CommentMetadata{ + Comment: "Test comment", + }, + }, + types.ConfigurationDirective{ + Kind: types.ConfigurationKind, + Name: types.SecRuleEngine, + Parameter: "On", + }, + &types.RuleWithCondition{ + Kind: types.RuleKind, + Metadata: types.SecRuleMetadata{ + OnlyPhaseMetadata: types.OnlyPhaseMetadata{ + Phase: "1", + }, + Id: 12345, + Msg: "Test rule", + }, + Conditions: []types.Condition{ + { + Variables: []types.Variable{ + { + Name: types.REQUEST_URI, + }, + }, + Operator: types.Operator{ + Name: types.Rx, + Value: "/test", + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "Multiple groups with different directive types", + input: types.Ruleset{ + Global: types.DefaultConfigs{ + Version: "4.0.0", + Tags: []string{"OWASP_CRS"}, + }, + Groups: []types.Group{ + { + Id: "group-a", + Tags: []string{"security", "testing"}, + Directives: []types.SeclangDirective{ + types.CommentDirective{ + Kind: types.CommentKind, + Metadata: types.CommentMetadata{ + Comment: "This is a test group", + }, + }, + &types.RuleWithCondition{ + Kind: types.RuleKind, + Metadata: types.SecRuleMetadata{ + OnlyPhaseMetadata: types.OnlyPhaseMetadata{ + Phase: "1", + }, + Id: 100, + Msg: "First rule", + }, + Conditions: []types.Condition{ + { + Variables: []types.Variable{ + { + Name: types.REQUEST_URI, + }, + }, + Operator: types.Operator{ + Name: types.Rx, + Value: "/admin", + }, + }, + }, + }, + &types.RuleWithCondition{ + Kind: types.RuleKind, + Metadata: types.SecRuleMetadata{ + OnlyPhaseMetadata: types.OnlyPhaseMetadata{ + Phase: "2", + }, + Id: 200, + Msg: "Second rule", + }, + Conditions: []types.Condition{ + { + Collections: []types.Collection{ + { + Name: types.REQUEST_HEADERS, + }, + }, + Operator: types.Operator{ + Name: types.Contains, + Value: "test", + }, + }, + }, + }, + }, + }, + { + Id: "group-b", + Directives: []types.SeclangDirective{ + types.ConfigurationDirective{ + Kind: types.ConfigurationKind, + Name: types.SecRuleEngine, + Parameter: "DetectionOnly", + }, + &types.RuleWithCondition{ + Kind: types.RuleKind, + Metadata: types.SecRuleMetadata{ + OnlyPhaseMetadata: types.OnlyPhaseMetadata{ + Phase: "3", + }, + Id: 300, + Msg: "Third rule", + }, + Conditions: []types.Condition{ + { + Variables: []types.Variable{ + { + Name: types.RESPONSE_BODY, + }, + }, + Operator: types.Operator{ + Name: types.Rx, + Value: "sensitive", + }, + }, + }, + }, + }, + }, + }, + }, + expected: types.Ruleset{ + Global: types.DefaultConfigs{ + Version: "4.0.0", + Tags: []string{"OWASP_CRS"}, + }, + Groups: []types.Group{ + { + Id: "group-a", + Tags: []string{"security", "testing"}, + Directives: []types.SeclangDirective{ + types.CommentDirective{ + Kind: types.CommentKind, + Metadata: types.CommentMetadata{ + Comment: "This is a test group", + }, + }, + &types.RuleWithCondition{ + Kind: types.RuleKind, + Metadata: types.SecRuleMetadata{ + OnlyPhaseMetadata: types.OnlyPhaseMetadata{ + Phase: "1", + }, + Id: 100, + Msg: "First rule", + }, + Conditions: []types.Condition{ + { + Variables: []types.Variable{ + { + Name: types.REQUEST_URI, + }, + }, + Operator: types.Operator{ + Name: types.Rx, + Value: "/admin", + }, + }, + }, + }, + &types.RuleWithCondition{ + Kind: types.RuleKind, + Metadata: types.SecRuleMetadata{ + OnlyPhaseMetadata: types.OnlyPhaseMetadata{ + Phase: "2", + }, + Id: 200, + Msg: "Second rule", + }, + Conditions: []types.Condition{ + { + Collections: []types.Collection{ + { + Name: types.REQUEST_HEADERS, + }, + }, + Operator: types.Operator{ + Name: types.Contains, + Value: "test", + }, + }, + }, + }, + }, + }, + { + Id: "group-b", + Directives: []types.SeclangDirective{ + types.ConfigurationDirective{ + Kind: types.ConfigurationKind, + Name: types.SecRuleEngine, + Parameter: "DetectionOnly", + }, + &types.RuleWithCondition{ + Kind: types.RuleKind, + Metadata: types.SecRuleMetadata{ + OnlyPhaseMetadata: types.OnlyPhaseMetadata{ + Phase: "3", + }, + Id: 300, + Msg: "Third rule", + }, + Conditions: []types.Condition{ + { + Variables: []types.Variable{ + { + Name: types.RESPONSE_BODY, + }, + }, + Operator: types.Operator{ + Name: types.Rx, + Value: "sensitive", + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Create temporary directory for output + tmpDir := t.TempDir() + + // Write ruleset separately + err := WriteRuleSeparately(tc.input, tmpDir) + if err != nil { + t.Fatalf("WriteRuleSeparately failed: %v", err) + } + + // Load from directory + loadedRuleset, err := LoadRulesFromDirectory(tmpDir) + if err != nil { + t.Fatalf("LoadRulesFromDirectory failed: %v", err) + } + + require.Equal(t, tc.expected, loadedRuleset, tc.name) + }) + } +} diff --git a/types/condition_directives.go b/types/condition_directives.go index caa7e8f..067629f 100644 --- a/types/condition_directives.go +++ b/types/condition_directives.go @@ -37,10 +37,10 @@ func (s *RuleWithCondition) GetKind() Kind { return s.Kind } -func ToDirectiveWithConditions(configList ConfigurationList) *ConfigurationList { - result := new(ConfigurationList) - for _, config := range configList.DirectiveList { - configWrapper := new(DirectiveList) +func ToDirectiveWithConditions(configList Ruleset) *Ruleset { + result := new(Ruleset) + for _, config := range configList.Groups { + configWrapper := new(Group) configWrapper.Id = config.Id configWrapper.Marker = config.Marker for _, directive := range config.Directives { @@ -62,7 +62,7 @@ func ToDirectiveWithConditions(configList ConfigurationList) *ConfigurationList } configWrapper.Directives = append(configWrapper.Directives, directiveWrapper) } - result.DirectiveList = append(result.DirectiveList, *configWrapper) + result.Groups = append(result.Groups, *configWrapper) } return result } @@ -131,13 +131,14 @@ func RuleToCondition(directive ChainableDirective) *RuleWithCondition { // configurationYamlLoader is a auxiliary struct to load the whole yaml file type configurationYamlLoader struct { - Global DefaultConfigs `yaml:"global,omitempty"` - DirectiveList []yamlLoaderConditionRules `yaml:"directivelist,omitempty"` + Global DefaultConfigs `yaml:"global,omitempty"` + Groups []yamlLoaderConditionRules `yaml:"groups,omitempty"` } // yamlLoaderConditionRules is a auxiliary struct to load and iterate over the yaml file type yamlLoaderConditionRules struct { Id string `yaml:"id"` + Tags []string `yaml:"tags,omitempty"` Directives []yaml.Node `yaml:"directives,omitempty"` Marker ConfigurationDirective `yaml:"marker,omitempty"` } @@ -263,7 +264,7 @@ func (s *SeclangActions) UnmarshalYAML(value *yaml.Node) error { } // LoadDirectivesWithConditionsFromFile loads condition format directives from a yaml file -func LoadDirectivesWithConditionsFromFile(filename string) ConfigurationList { +func LoadDirectivesWithConditionsFromFile(filename string) Ruleset { yamlFile, err := os.ReadFile(filename) if err != nil { panic(err) @@ -273,15 +274,15 @@ func LoadDirectivesWithConditionsFromFile(filename string) ConfigurationList { } // LoadDirectivesWithConditions loads condition format directives from a yaml file -func LoadDirectivesWithConditions(yamlFile []byte) ConfigurationList { +func LoadDirectivesWithConditions(yamlFile []byte) Ruleset { var config configurationYamlLoader err := yaml.Unmarshal(yamlFile, &config) - configs := config.DirectiveList + groups := config.Groups if err != nil { panic(err) } - var resultConfigs []DirectiveList - for _, config := range configs { + var resultConfigs []Group + for _, config := range groups { var directives []SeclangDirective for _, yamlDirective := range config.Directives { directive := loadConditionDirective(yamlDirective) @@ -291,9 +292,9 @@ func LoadDirectivesWithConditions(yamlFile []byte) ConfigurationList { directives = append(directives, directive) } } - resultConfigs = append(resultConfigs, DirectiveList{Id: config.Id, Directives: directives, Marker: config.Marker}) + resultConfigs = append(resultConfigs, Group{Id: config.Id, Tags: config.Tags, Directives: directives, Marker: config.Marker}) } - return ConfigurationList{Global: config.Global, DirectiveList: resultConfigs} + return Ruleset{Global: config.Global, Groups: resultConfigs} } // loadConditionDirective loads the different kind of directives @@ -394,11 +395,12 @@ func loadRuleWithConditions(yamlDirective yaml.Node) *RuleWithCondition { return &directive } -func FromCRSLangToUnformattedDirectives(configListWrapped ConfigurationList) *ConfigurationList { - result := new(ConfigurationList) - for _, config := range configListWrapped.DirectiveList { - configList := new(DirectiveList) +func FromCRSLangToUnformattedDirectives(configListWrapped Ruleset) *Ruleset { + result := new(Ruleset) + for _, config := range configListWrapped.Groups { + configList := new(Group) configList.Id = config.Id + configList.Tags = config.Tags configList.Marker = config.Marker for _, directiveWrapped := range config.Directives { var directive SeclangDirective @@ -415,6 +417,13 @@ func FromCRSLangToUnformattedDirectives(configListWrapped ConfigurationList) *Co for _, tag := range configListWrapped.Global.Tags { chainableDir.GetMetadata().AddTag(tag) } + // Ignore paranoia level check rules when adding group tags + lastDigits := *&directiveWrapped.(*RuleWithCondition).Metadata.Id % 1000 + if !(lastDigits < 20) { + for _, tag := range config.Tags { + chainableDir.GetMetadata().AddTag(tag) + } + } directive = chainableDir case ConfigurationDirective: directive = ConfigurationDirective{ @@ -425,7 +434,7 @@ func FromCRSLangToUnformattedDirectives(configListWrapped ConfigurationList) *Co } configList.Directives = append(configList.Directives, directive) } - result.DirectiveList = append(result.DirectiveList, *configList) + result.Groups = append(result.Groups, *configList) } return result } diff --git a/types/configuration.go b/types/configuration.go index bf54fb9..76c0179 100644 --- a/types/configuration.go +++ b/types/configuration.go @@ -9,18 +9,23 @@ type DefaultConfigs struct { Tags []string `yaml:"tags,omitempty"` } -type ConfigurationList struct { - Global DefaultConfigs `yaml:"global,omitempty"` - DirectiveList []DirectiveList `yaml:"directivelist,omitempty"` +type Ruleset struct { + Global DefaultConfigs `yaml:"global,omitempty"` + GroupsIds []string `yaml:"rule_groups,omitempty"` + Groups []Group `yaml:"groups,omitempty"` } -type DirectiveList struct { - Id string `yaml:"id"` - Directives []SeclangDirective `yaml:"directives,omitempty"` - Marker ConfigurationDirective `yaml:"marker,omitempty"` +type Group struct { + Id string `yaml:"id"` + Tags []string `yaml:"tags,omitempty"` + Comments []string `yaml:"comments,omitempty"` + Configurations []ConfigurationDirective `yaml:"configurations,omitempty"` + Directives []SeclangDirective `yaml:"directives,omitempty"` + Rules []int `yaml:"rules,omitempty"` + Marker ConfigurationDirective `yaml:"marker,omitempty"` } -func (d DirectiveList) ToSeclang() string { +func (d Group) ToSeclang() string { result := "" for _, directive := range d.Directives { result += directive.ToSeclang() + "\n" @@ -31,9 +36,9 @@ func (d DirectiveList) ToSeclang() string { return result } -func ToSeclang(configList ConfigurationList) string { +func ToSeclang(configList Ruleset) string { result := "" - for _, config := range configList.DirectiveList { + for _, config := range configList.Groups { for _, directive := range config.Directives { result += directive.ToSeclang() + "\n" } @@ -45,17 +50,17 @@ func ToSeclang(configList ConfigurationList) string { } // ExtractDefaultValues extracts default values for version and tags from the rules in the configuration list -func (c *ConfigurationList) ExtractDefaultValues() { +func (c *Ruleset) ExtractDefaultValues() { directiveFound := false version := "" tags := []string{} rules := []*RuleWithCondition{} - for i := range c.DirectiveList { - for j := range c.DirectiveList[i].Directives { + for i := range c.Groups { + for j := range c.Groups[i].Directives { // Only consider Rule directives - if c.DirectiveList[i].Directives[j].GetKind() == RuleKind { - rule := c.DirectiveList[i].Directives[j].(*RuleWithCondition) + if c.Groups[i].Directives[j].GetKind() == RuleKind { + rule := c.Groups[i].Directives[j].(*RuleWithCondition) rules = append(rules, rule) if !directiveFound { directiveFound = true @@ -93,3 +98,50 @@ func (c *ConfigurationList) ExtractDefaultValues() { c.Global.Version = version c.Global.Tags = tags } + +// TODO: merge this method with the one in Ruleset +// ExtractDefaultValues extracts default values for version and tags from the rules in the configuration list +func (g *Group) ExtractDefaultValues() { + directiveFound := false + tags := []string{} + rules := []*RuleWithCondition{} + + for j := range g.Directives { + // Only consider Rule directives + if g.Directives[j].GetKind() == RuleKind { + // Ignore paranoia level check rules + lastDigits := g.Directives[j].(*RuleWithCondition).Metadata.Id % 1000 + if lastDigits > 20 { + rule := g.Directives[j].(*RuleWithCondition) + rules = append(rules, rule) + auxTags := []string{} + if !directiveFound { + directiveFound = true + auxTags = append(auxTags, rule.Metadata.Tags...) + } else { + for _, tag := range tags { + if slices.Contains(rule.Metadata.Tags, tag) { + auxTags = append(auxTags, tag) + } + } + } + tags = auxTags + // If tags are empty after found a rule it means there is no common value + // so we can stop the search + if len(tags) == 0 { + return + } + } + } + } + + // Clear tags in rules since they are now in the global section + for _, rule := range rules { + rule.Metadata.Tags = slices.DeleteFunc(rule.Metadata.Tags, func(s string) bool { + return slices.Contains(tags, s) + }) + } + + g.Tags = append([]string{}, tags...) + +}