From 2c75c46754b1a27c628837d1537104e36205b862 Mon Sep 17 00:00:00 2001 From: Sean Cunningham Date: Fri, 6 Mar 2026 10:58:39 -0500 Subject: [PATCH] Overhaul compiler - Remove parser phase and instead parse directly to AST - Refactor AST and data structures for simplicity and better error handling - Add more tests and test data - Add strict compiler mode - Add semantic versioning to the compiler engine and enforce version constraints in the rules --- .github/workflows/build.yml | 9 +- go.mod | 27 +- go.sum | 108 +- pkg/ast/ast.go | 586 +++----- pkg/ast/ast_log.go | 267 ---- pkg/ast/ast_machine.go | 120 -- pkg/ast/ast_metrics.go | 85 -- pkg/ast/ast_script.go | 101 -- pkg/ast/ast_test.go | 403 ++---- pkg/ast/const.go | 110 ++ pkg/ast/draw.go | 356 +++++ pkg/ast/draw_test.go | 361 +++++ pkg/ast/errors.go | 99 ++ pkg/ast/errors_test.go | 85 ++ pkg/ast/opts.go | 93 ++ pkg/ast/opts_test.go | 145 ++ pkg/ast/parse.go | 144 ++ pkg/ast/parse_cre.go | 198 +++ pkg/ast/parse_cre_test.go | 278 ++++ pkg/ast/parse_event.go | 58 + pkg/ast/parse_event_test.go | 148 ++ pkg/ast/parse_extract.go | 128 ++ pkg/ast/parse_extract_test.go | 277 ++++ pkg/ast/parse_help.go | 213 +++ pkg/ast/parse_help_test.go | 396 ++++++ pkg/ast/parse_meta.go | 161 +++ pkg/ast/parse_meta_test.go | 297 ++++ pkg/ast/parse_node.go | 319 +++++ pkg/ast/parse_node_test.go | 141 ++ pkg/ast/parse_prom.go | 80 ++ pkg/ast/parse_prom_test.go | 165 +++ pkg/ast/parse_rules.go | 262 ++++ pkg/ast/parse_rules_test.go | 599 ++++++++ pkg/ast/parse_script.go | 149 ++ pkg/ast/parse_script_test.go | 286 ++++ pkg/ast/parse_terms.go | 352 +++++ pkg/ast/parse_terms_test.go | 696 +++++++++ pkg/ast/parse_test.go | 570 ++++++++ pkg/ast/proto.go | 141 ++ pkg/ast/proto_test.go | 194 +++ pkg/ast/state.go | 71 + pkg/ast/walk.go | 71 + pkg/ast/walk_test.go | 154 ++ pkg/compiler/callbacks.go | 90 +- pkg/compiler/compile_test.go | 40 + pkg/compiler/compiler.go | 202 +-- pkg/compiler/compiler_log.go | 136 +- pkg/compiler/defaults.go | 55 + pkg/compiler/error.go | 12 + pkg/compiler/plugin.go | 40 - pkg/datasrc/parse.go | 2 +- pkg/parser/parse.go | 214 --- pkg/parser/parse_test.go | 404 ------ pkg/parser/tree.go | 1253 ----------------- pkg/schema/schema.go | 23 - pkg/testdata/rules.go | 1047 +++++++------- .../00-rules-document-example.yaml | 12 +- .../01-set-single-example.yaml | 1 + .../02-set-multiple-example-bad-window.yaml | 1 + .../02-set-multiple-example-good-window.yaml | 1 + .../03-set-negative-example.yaml | 1 + .../success_examples/04-set-1x1-example.yaml | 1 + .../08-sequence-example-bad-window.yaml | 1 + .../08-sequence-example-good-window.yaml | 1 + .../09-sequence-negate-example.yaml | 1 + .../success_examples/13-string-example.yaml | 1 + .../success_examples/14-string-example.yaml | 1 + .../success_examples/15-regex-example.yaml | 1 + .../success_examples/16-regex-example.yaml | 1 + .../success_examples/17-jq-example.yaml | 1 + .../success_examples/18-jq-example.yaml | 1 + .../19-bad-literal-block-example.yaml | 1 + .../20-bad-regex-example.yaml | 1 + .../success_examples/21-negative-example.yaml | 1 + .../24-multiple-negatives.yaml | 1 + .../25-negate-options-1x1.yaml | 1 + .../success_examples/26-negate-window.yaml | 1 + .../27-negate-window-shorter.yaml | 1 + .../success_examples/27-negate-window.yaml | 1 + .../28-negate-anchor-shorter.yaml | 1 + .../success_examples/28-negate-anchor.yaml | 1 + .../29-negate-slide-anchor-1-window.yaml | 1 + .../29-negate-slide-anchor-1.yaml | 1 + .../success_examples/29-negate-slide.yaml | 1 + .../success_examples/30-negate-absolute.yaml | 1 + pkg/testdata/success_examples/41-nested.yaml | 88 +- pkg/version/version.go | 63 + pkg/version/version_test.go | 92 ++ 88 files changed, 9106 insertions(+), 4198 deletions(-) delete mode 100644 pkg/ast/ast_log.go delete mode 100644 pkg/ast/ast_machine.go delete mode 100644 pkg/ast/ast_metrics.go delete mode 100644 pkg/ast/ast_script.go create mode 100644 pkg/ast/const.go create mode 100644 pkg/ast/draw.go create mode 100644 pkg/ast/draw_test.go create mode 100644 pkg/ast/errors.go create mode 100644 pkg/ast/errors_test.go create mode 100644 pkg/ast/opts.go create mode 100644 pkg/ast/opts_test.go create mode 100644 pkg/ast/parse.go create mode 100644 pkg/ast/parse_cre.go create mode 100644 pkg/ast/parse_cre_test.go create mode 100644 pkg/ast/parse_event.go create mode 100644 pkg/ast/parse_event_test.go create mode 100644 pkg/ast/parse_extract.go create mode 100644 pkg/ast/parse_extract_test.go create mode 100644 pkg/ast/parse_help.go create mode 100644 pkg/ast/parse_help_test.go create mode 100644 pkg/ast/parse_meta.go create mode 100644 pkg/ast/parse_meta_test.go create mode 100644 pkg/ast/parse_node.go create mode 100644 pkg/ast/parse_node_test.go create mode 100644 pkg/ast/parse_prom.go create mode 100644 pkg/ast/parse_prom_test.go create mode 100644 pkg/ast/parse_rules.go create mode 100644 pkg/ast/parse_rules_test.go create mode 100644 pkg/ast/parse_script.go create mode 100644 pkg/ast/parse_script_test.go create mode 100644 pkg/ast/parse_terms.go create mode 100644 pkg/ast/parse_terms_test.go create mode 100644 pkg/ast/parse_test.go create mode 100644 pkg/ast/proto.go create mode 100644 pkg/ast/proto_test.go create mode 100644 pkg/ast/state.go create mode 100644 pkg/ast/walk.go create mode 100644 pkg/ast/walk_test.go create mode 100644 pkg/compiler/compile_test.go create mode 100644 pkg/compiler/defaults.go create mode 100644 pkg/compiler/error.go delete mode 100644 pkg/compiler/plugin.go delete mode 100644 pkg/parser/parse.go delete mode 100644 pkg/parser/parse_test.go delete mode 100644 pkg/parser/tree.go delete mode 100644 pkg/schema/schema.go create mode 100644 pkg/version/version.go create mode 100644 pkg/version/version_test.go diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7734665..e760085 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go-version: [1.24.2] + go-version: [1.26.3] os: [ubuntu-24.04] steps: - name: Checkout Repository @@ -34,9 +34,4 @@ jobs: - name: Run prequel-compiler tests run: | - pushd pkg/parser - go test - popd - pushd pkg/ast - go test - popd + go test -v ./... diff --git a/go.mod b/go.mod index 44c00be..3eead53 100644 --- a/go.mod +++ b/go.mod @@ -1,21 +1,26 @@ module github.com/prequel-dev/prequel-compiler -go 1.25 +go 1.26 require ( - github.com/btcsuite/btcutil v1.0.2 - github.com/prequel-dev/prequel-logmatch v0.0.20 - github.com/rs/zerolog v1.34.0 - gopkg.in/yaml.v3 v3.0.1 + github.com/Masterminds/semver/v3 v3.5.0 + github.com/goccy/go-yaml v1.19.2 + github.com/jedib0t/go-pretty/v6 v6.7.10 + github.com/prequel-dev/prequel-logmatch v0.0.25 + github.com/rs/zerolog v1.35.1 + github.com/thediveo/go-asciitree v1.0.2 ) require ( - github.com/goccy/go-yaml v1.19.2 // indirect - github.com/itchyny/gojq v0.12.18 // indirect - github.com/itchyny/timefmt-go v0.1.7 // indirect + github.com/clipperhouse/uax29/v2 v2.7.0 // indirect + github.com/itchyny/gojq v0.12.19 // indirect + github.com/itchyny/timefmt-go v0.1.8 // indirect github.com/mattn/go-colorable v0.1.14 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-isatty v0.0.22 // indirect + github.com/mattn/go-runewidth v0.0.23 // indirect github.com/philhofer/fwd v1.2.0 // indirect - github.com/tinylib/msgp v1.6.3 // indirect - golang.org/x/sys v0.40.0 // indirect + github.com/tinylib/msgp v1.6.4 // indirect + golang.org/x/net v0.52.0 // indirect + golang.org/x/sys v0.44.0 // indirect + golang.org/x/text v0.36.0 // indirect ) diff --git a/go.sum b/go.sum index 6d792af..b513c43 100644 --- a/go.sum +++ b/go.sum @@ -1,68 +1,56 @@ -github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= -github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= -github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= -github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= -github.com/btcsuite/btcutil v1.0.2 h1:9iZ1Terx9fMIOtq1VrwdqfsATL9MC2l8ZrUY6YZ2uts= -github.com/btcsuite/btcutil v1.0.2/go.mod h1:j9HUFwoQRsZL3V4n+qG+CUnEGHOarIxfC3Le2Yhbcts= -github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= -github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= -github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= -github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= -github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= -github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE= +github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= +github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/itchyny/gojq v0.12.18 h1:gFGHyt/MLbG9n6dqnvlliiya2TaMMh6FFaR2b1H6Drc= -github.com/itchyny/gojq v0.12.18/go.mod h1:4hPoZ/3lN9fDL1D+aK7DY1f39XZpY9+1Xpjz8atrEkg= -github.com/itchyny/timefmt-go v0.1.7 h1:xyftit9Tbw+Dc/huSSPJaEmX1TVL8lw5vxjJLK4GMMA= -github.com/itchyny/timefmt-go v0.1.7/go.mod h1:5E46Q+zj7vbTgWY8o5YkMeYb4I6GeWLFnetPy5oBrAI= -github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= -github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 h1:5iH8iuqE5apketRbSFBy+X1V0o+l+8NF1avt4HWl7cA= +github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/itchyny/gojq v0.12.19 h1:ttXA0XCLEMoaLOz5lSeFOZ6u6Q3QxmG46vfgI4O0DEs= +github.com/itchyny/gojq v0.12.19/go.mod h1:5galtVPDywX8SPSOrqjGxkBeDhSxEW1gSxoy7tn1iZY= +github.com/itchyny/timefmt-go v0.1.8 h1:1YEo1JvfXeAHKdjelbYr/uCuhkybaHCeTkH8Bo791OI= +github.com/itchyny/timefmt-go v0.1.8/go.mod h1:5E46Q+zj7vbTgWY8o5YkMeYb4I6GeWLFnetPy5oBrAI= +github.com/jedib0t/go-pretty/v6 v6.7.10 h1:B/2qW2Bkv2L6n14PP8o1kx75kWzHOQ3YTluWzg9icac= +github.com/jedib0t/go-pretty/v6 v6.7.10/go.mod h1:YwC5CE4fJ1HFUDeivSV1r//AmANFHyqczZk+U6BDALU= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= -github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4= +github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= +github.com/mattn/go-runewidth v0.0.23 h1:7ykA0T0jkPpzSvMS5i9uoNn2Xy3R383f9HDx3RybWcw= +github.com/mattn/go-runewidth v0.0.23/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/onsi/ginkgo/v2 v2.20.2 h1:7NVCeyIWROIAheY21RLS+3j2bb52W0W82tkberYytp4= +github.com/onsi/ginkgo/v2 v2.20.2/go.mod h1:K9gyxPIlb+aIvnZ8bd9Ak+YP18w3APlR+5coaZoE2ag= +github.com/onsi/gomega v1.34.2 h1:pNCwDkzrsv7MS9kpaQvVb1aVLahQXyJ/Tv5oAZMI3i8= +github.com/onsi/gomega v1.34.2/go.mod h1:v1xfxRgk0KIsG+QOdm7p8UosrOzPYRo60fd3B/1Dukc= github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/prequel-dev/prequel-logmatch v0.0.20 h1:PNhc+1sBZVlaUvDHpPfdxi3+dPsYkUhhzy5LbDkJjmY= -github.com/prequel-dev/prequel-logmatch v0.0.20/go.mod h1:Vw1nuvH++C6139OXTm8+U2/IJubLT+GCalHe33m7wB4= -github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= -github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= -github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= -github.com/tinylib/msgp v1.6.3 h1:bCSxiTz386UTgyT1i0MSCvdbWjVW+8sG3PjkGsZQt4s= -github.com/tinylib/msgp v1.6.3/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= -golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= -golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prequel-dev/prequel-logmatch v0.0.25 h1:q1nhZ0Mnt3/iBG9rKZCPhAKTvVMeet11j0c67l0+yFU= +github.com/prequel-dev/prequel-logmatch v0.0.25/go.mod h1:TwkRa7euFLgoemo8vHVaBgx/CXgkaghZYUTehlkvMiI= +github.com/rs/zerolog v1.35.1 h1:m7xQeoiLIiV0BCEY4Hs+j2NG4Gp2o2KPKmhnnLiazKI= +github.com/rs/zerolog v1.35.1/go.mod h1:EjML9kdfa/RMA7h/6z6pYmq1ykOuA8/mjWaEvGI+jcw= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/thediveo/go-asciitree v1.0.2 h1:V/w9cd0FdPqDx7f/CJEIoQ8oeXPoPUoBpFsWG4p7Sho= +github.com/thediveo/go-asciitree v1.0.2/go.mod h1:2OACJb6C/MZf+Jp/TktVMeEZuf+QIO0Kh2Lp9rWHAv4= +github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ= +github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= +golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= +golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= +golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= +golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= +golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= +golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s= +golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/ast/ast.go b/pkg/ast/ast.go index 846139a..f37495c 100644 --- a/pkg/ast/ast.go +++ b/pkg/ast/ast.go @@ -1,486 +1,232 @@ package ast import ( - "errors" "fmt" - "io" - "os" - "strconv" - "strings" "time" - "github.com/prequel-dev/prequel-compiler/pkg/parser" - "github.com/prequel-dev/prequel-compiler/pkg/schema" "github.com/prequel-dev/prequel-logmatch/pkg/match" - "github.com/rs/zerolog/log" ) +const AstVersion = "v1" + const ( - AstVersion = 1 + SeverityCritical = 0 + SeverityHigh = 1 + SeverityMedium = 2 + SeverityLow = 3 + SeverityInfo = 4 ) -var ( - ErrInvalidEventType = errors.New("invalid event type") - ErrInvalidNodeType = errors.New("invalid node type") - ErrRootNodeWithoutEventSrc = errors.New("root node has no event source") - ErrInvalidWindow = errors.New("invalid window") - ErrMissingOrigin = errors.New("missing origin event") - ErrMultipleOrigin = errors.New("multiple origin events") - ErrInvalidAnchor = errors.New("invalid negate anchor") - ErrNoTermIdx = errors.New("no term idx") +const ( + KindPrequel = "prequel" + KindCustom = "custom" ) -type AstT struct { - Nodes []*AstNodeT `json:"nodes"` +type AstRuleT struct { + Root AstNode + Cre *AstCreT + Metadata AstMetadataT } -type AstNodeAddressT struct { - Version string `json:"version"` // Version of the address format - Name string `json:"name"` // Name of the node. Currently using type - RuleHash string `json:"rule_hash"` // unique semantic identifier for the rule - Depth uint32 `json:"depth"` // Depth of the node in the rule tree - NodeId uint32 `json:"node_id"` // globally unique identifier for the match in the rule tree - TermIdx *uint32 `json:"term_idx"` // Index of term/condition into parent's conditions. Used for assertion to assign term idx into parent machines +type AstNode interface { + Type() AstNodeType + Scope() AstScopeT + Address() AstNodeAddressT + Parent() *AstNodeAddressT } -type AstNodeT struct { - Metadata AstMetadataT `json:"metadata"` // Metadata for the node - Children []*AstNodeT `json:"children"` // Children of the node - Object any `json:"object"` // Object for the node (e.g. log matcher, state machine, descriptor, etc.) +type AstNodeAddressT struct { + Type AstNodeType // Type of node + RuleId string // RuleId is the unique identifier for the rule + RuleHash string // unique semantic identifier for the rule + Rank uint32 // Index of term/condition into parent's conditions. Used for assertion to assign term idx into parent machines + Depth uint32 // Depth of the node in the rule tree + NodeId uint32 // Globally unique identifier for the match in the rule tree +} + +func (a AstNodeAddressT) String() string { + return fmt.Sprintf("%s.%s.%s.d%d.n%d.t%d", + AstVersion, + a.Type, + a.RuleHash, + a.Depth, + a.NodeId, + a.Rank, + ) } type AstMetadataT struct { - Type schema.NodeTypeT `json:"type"` // Type of the node - Address *AstNodeAddressT `json:"address"` // Address of this node in the rule tree. Must be globally unique in the tree - ParentAddress *AstNodeAddressT `json:"parent_address"` // Address of the parent node - NegateOpts *AstNegateOptsT `json:"negate_opts"` // Optional egate options for the node - RuleId string `json:"rule_id"` // Consistent identifier for the rule that remains consistent through rule logic changes - Scope string `json:"scope"` // Scope can be an individual node, a cluster, or a set of clusters - NegIdx int `json:"neg_idx"` // Index into children where negative conditions begin. Equals -1 if no children or no negative conditions + Name string + Id string + Hash string + Kind string + Gen uint32 } -// NegateOptsT contains optional negate settings for the matcher object -type AstNegateOptsT struct { - Window time.Duration `json:"window"` - Slide time.Duration `json:"slide"` - Anchor uint32 `json:"anchor"` - Absolute bool `json:"absolute"` +type AstTermT struct { + Term AstNode + NegateOpts *AstNegateOptsT } -type AstExtractT struct { - Name string `json:"name"` - JqValue string `json:"jq_value,omitempty"` - RegexValue string `json:"regex_value,omitempty"` +type AstInnerNodeT struct { + baseAst + Window time.Duration + Correlations []string + Terms []AstTermT + Negate []AstTermT } type AstFieldT struct { - Field string `json:"field"` - TermValue match.TermT `json:"term_value"` - NegateOpts *AstNegateOptsT `json:"negate_opts"` - Extracts []AstExtractT `json:"extracts"` -} - -type AstEventT struct { - Origin bool `json:"origin"` - Source string `json:"source"` -} - -type builderT struct { - CurrentNodeId uint32 - CurrentDepth uint32 - OriginCnt int + Count uint64 + Field string + TermValue match.TermT + NegateOpts *AstNegateOptsT + Extracts []AstExtractT } -func NewBuilder() *builderT { - return &builderT{ - CurrentNodeId: uint32(0), - CurrentDepth: uint32(0), - OriginCnt: 0, - } -} - -func (b *builderT) descendTree(fn func() error) error { - b.CurrentDepth++ - defer func() { b.CurrentDepth-- }() - return fn() -} - -func Build(data []byte) (*AstT, error) { - - parseTree, err := parser.Parse(data) - if err != nil { - log.Error().Any("err", err).Msg("Parser failed") - return nil, err - } - - return BuildTree(parseTree) +type AstMatchLeafT struct { + baseAst + Window time.Duration + Correlations []string + Terms []AstFieldT + Negate []AstFieldT + Event AstEventT } -// Build AST from the given parser node in pre-order DFS traversal -func BuildTree(tree *parser.TreeT) (*AstT, error) { - var ( - ast = &AstT{ - Nodes: make([]*AstNodeT, 0), - } - ) - - for _, parserNode := range tree.Nodes { - - var ( - rb = NewBuilder() - err error - termIdx = uint32(0) - rule *AstNodeT - ) - - // Recursively build tree - if rule, err = rb.buildTree(parserNode, nil, &termIdx); err != nil { - return nil, err - } - - switch { - case rb.OriginCnt == 0: - return nil, parserNode.WrapError(ErrMissingOrigin) - case rb.OriginCnt > 1: - return nil, parserNode.WrapError(ErrMultipleOrigin) - } - - ast.Nodes = append(ast.Nodes, rule) - } +type AstNodeType int - return ast, nil -} - -func (b *builderT) buildTree(parserNode *parser.NodeT, parentMachineAddress *AstNodeAddressT, termIdx *uint32) (*AstNodeT, error) { - - var ( - machineAddress = b.newAstNodeAddress(parserNode.Metadata.RuleHash, parserNode.Metadata.Type.String(), termIdx) - ) - - children, err := b.buildChildrenNodes(parserNode, machineAddress, termIdx) - if err != nil { - return nil, err - } - - // Build state machine after recursively building children - machineMatchNode, err := b.buildStateMachine(parserNode, parentMachineAddress, machineAddress, children) - if err != nil { - return nil, err - } - - machineMatchNode.Children = append(machineMatchNode.Children, children...) - - return machineMatchNode, nil -} - -func (b *builderT) buildChildrenNodes(parserNode *parser.NodeT, machineAddress *AstNodeAddressT, termIdx *uint32) (children []*AstNodeT, err error) { - - leaf, err := b.buildLeafChild(parserNode, machineAddress, termIdx) +const ( + AstNodeTypeSet AstNodeType = iota + AstNodeTypeSeq + AstNodeTypeLogSet + AstNodeTypeLogSeq + AstNodeTypePromQL + AstNodeTypeScript +) - switch { - case err != nil: - return nil, err - case leaf != nil: - return []*AstNodeT{leaf}, nil - case parserNode.IsScriptNode(): - children, err = b.buildScriptChildren(parserNode, machineAddress) +func (t AstNodeType) String() string { + switch t { + case AstNodeTypeSet: + return nodeTypeSet + case AstNodeTypeSeq: + return nodeTypeSeq + case AstNodeTypeLogSet: + return nodeTypeLogSet + case AstNodeTypeLogSeq: + return nodeTypeLogSeq + case AstNodeTypePromQL: + return nodeTypePromQL + case AstNodeTypeScript: + return nodeTypeScript default: - children, err = b.buildMachineChildren(parserNode, machineAddress) + return nodeTypeUnknown } - - return children, err } -func (b *builderT) buildLeafChild(parserNode *parser.NodeT, machineAddress *AstNodeAddressT, termIdx *uint32) (leaf *AstNodeT, err error) { +type AstScopeT int - switch { - case parserNode.IsMatcherNode(): - leaf, err = b.buildMatcherChild(parserNode, machineAddress, termIdx) - case parserNode.IsPromNode(): - leaf, err = b.buildPromQLChild(parserNode, machineAddress, termIdx) - } - return -} +const ( + AstScopeNode AstScopeT = iota + AstScopeCluster + AstScopeOrganization + AstScopeGlobal +) -func (b *builderT) newAstNodeAddress(ruleHash, name string, termIdx *uint32) *AstNodeAddressT { - var address = &AstNodeAddressT{ - Version: "v" + strconv.FormatInt(int64(AstVersion), 10), - Name: name, - RuleHash: ruleHash, - Depth: b.CurrentDepth, - NodeId: b.CurrentNodeId, - TermIdx: termIdx, +func (s AstScopeT) String() string { + switch s { + case AstScopeNode: + return scopeTypeNode + case AstScopeCluster: + return scopeTypeCluster + case AstScopeOrganization: + return scopeTypeOrganization + case AstScopeGlobal: + return scopeTypeGlobal + default: + return scopeTypeUnknown } - - b.CurrentNodeId++ - - return address } -func newAstNode(parserNode *parser.NodeT, typ schema.NodeTypeT, scope string, parentAddress, address *AstNodeAddressT) *AstNodeT { - return &AstNodeT{ - Metadata: AstMetadataT{ - RuleId: parserNode.Metadata.RuleId, - Address: address, - ParentAddress: parentAddress, - NegIdx: parserNode.NegIdx, - Type: typ, - Scope: scope, - }, - } +type AstCreT struct { + Id string + Severity uint + Title string + Category string + Tags []string + Author string + Description string + Impact string + ImpactScore uint + Cause string + Mitigation string + MitigationScore uint + References []string + Reports uint + Applications []AstAppT } -func (b *builderT) buildMatcherChild(parserNode *parser.NodeT, machineAddress *AstNodeAddressT, termIdx *uint32) (*AstNodeT, error) { - - var ( - matchNode *AstNodeT - err error - ) - - if parserNode.Metadata.Event == nil { - return nil, parserNode.WrapError(ErrRootNodeWithoutEventSrc) - } - - if parserNode.Metadata.Event.Source == "" { - log.Error(). - Any("address", machineAddress). - Msg("Event missing source") - return nil, parserNode.WrapError(ErrInvalidEventType) - } - - // This appears to be a legacy hack to support rules that don't specify origin but have event sources. - // We should consider removing this and requiring explicit origin specification in the rules. - if b.CurrentDepth == 0 && !parserNode.Metadata.Event.Origin { - parserNode.Metadata.Event.Origin = true - } - - err = b.descendTree(func() error { - if matchNode, err = b.buildMatcherNodes(parserNode, machineAddress, termIdx); err != nil { - return err - } - return nil - }) - if err != nil { - return nil, err - } - - return matchNode, nil +type AstEventT struct { + Source string + Origin bool } -func (b *builderT) buildMatcherNodes(parserNode *parser.NodeT, machineAddress *AstNodeAddressT, termIdx *uint32) (*AstNodeT, error) { - - // Validation - switch parserNode.Metadata.Type { - case schema.NodeTypeLogSeq: - case schema.NodeTypeLogSet: - case schema.NodeTypePromQL: - return b.buildPromQLNode(parserNode, machineAddress, termIdx) - default: - return nil, parserNode.WrapError(ErrInvalidNodeType) - } - - // We currently only support building log matchers in this package - return b.buildLogMatcherNode(parserNode, machineAddress, termIdx) +type AstAppT struct { + Name string + ProcessName string + ProcessPath string + ContainerName string + ImageUrl string + RepoUrl string + Version string } -func (b *builderT) buildMachineChildren(parserNode *parser.NodeT, machineAddress *AstNodeAddressT) ([]*AstNodeT, error) { - - var ( - children = make([]*AstNodeT, 0) - ) - - for i, child := range parserNode.Children { - var ( - negateOpts *parser.NegateOptsT - termIdx = uint32(i) - parserChildNode *parser.NodeT - matchNode *AstNodeT - ok bool - err error - ) - - if parserChildNode, ok = child.(*parser.NodeT); !ok { - return nil, parserNode.WrapError(ErrInvalidNodeType) - } - - if parserChildNode.Metadata.NegateOpts != nil { - negateOpts = parserChildNode.Metadata.NegateOpts - - if negateOpts.Anchor > uint32(len(parserNode.Children)) { - log.Error(). - Msg("Negate anchor is greater than the number of children") - return nil, parserNode.WrapError(ErrInvalidAnchor) - } - } - - // Process nested state machine - if parserChildNode.Metadata.Event == nil { - err = b.descendTree(func() error { - if matchNode, err = b.buildTree(parserChildNode, machineAddress, &termIdx); err != nil { - return err - } - addNegateOpts(matchNode, negateOpts) - children = append(children, matchNode) - return nil - }) - if err != nil { - return nil, err - } - continue - } - - // If the child has an event/data source, then it is not a state machine. Build it via buildMatcherNodes - - if parserChildNode.Metadata.Event.Source == "" { - log.Error(). - Any("address", machineAddress). - Msg("Event missing source") - return nil, parserChildNode.WrapError(ErrInvalidEventType) - } - - err = b.descendTree(func() error { - if matchNode, err = b.buildMatcherNodes(parserChildNode, machineAddress, &termIdx); err != nil { - return err - } - addNegateOpts(matchNode, negateOpts) - children = append(children, matchNode) - return nil - }) - if err != nil { - return nil, err - } - - } - - return children, nil -} - -func addNegateOpts(assert *AstNodeT, negateOpts *parser.NegateOptsT) { - if negateOpts == nil { - return - } - - assert.Metadata.NegateOpts = &AstNegateOptsT{ - Window: negateOpts.Window, - Slide: negateOpts.Slide, - Anchor: negateOpts.Anchor, - Absolute: negateOpts.Absolute, - } +type AstExtractT struct { + Name string + JqValue string + RegexValue string } -func (b *builderT) buildStateMachine(parserNode *parser.NodeT, parentMachineAddress, machineAddress *AstNodeAddressT, children []*AstNodeT) (*AstNodeT, error) { - - switch parserNode.Metadata.Type { - case schema.NodeTypeSeq, schema.NodeTypeLogSeq: - if parserNode.Metadata.Window == 0 { - log.Error(). - Any("address", machineAddress). - Msg("Window is required for sequences") - return nil, parserNode.WrapError(ErrInvalidWindow) - } - case schema.NodeTypeSet, schema.NodeTypeLogSet, schema.NodeTypePromQL: - case schema.NodeTypeScript: - return b.buildScriptNode(parserNode, parentMachineAddress, machineAddress) - - default: - log.Error(). - Any("address", machineAddress). - Str("type", parserNode.Metadata.Type.String()). - Msg("Invalid node type") - return nil, parserNode.WrapError(ErrInvalidNodeType) - } - - return b.buildMachineNode(parserNode, parentMachineAddress, machineAddress, children) +type AstPromT struct { + baseAst + Expr string + For time.Duration + Interval time.Duration + Event *AstEventT } -func (a *AstNodeAddressT) String() string { - - var ( - addressStr string - ) - - addressStr = fmt.Sprintf("%s.%s.%s.d%d.n%d", - a.Version, - a.Name, - a.RuleHash, - a.Depth, - a.NodeId, - ) - - if a.TermIdx != nil { - addressStr += fmt.Sprintf(".t%d", *a.TermIdx) - } - - return addressStr +type AstScriptT struct { + baseAst + Code string + Language string + Timeout time.Duration + Input AstNode } -func (a *AstNodeAddressT) GetTermIdx() (uint32, error) { - if a.TermIdx == nil { - return 0, ErrNoTermIdx - } - return *a.TermIdx, nil +type AstNegateOptsT struct { + Window time.Duration + Slide time.Duration + Anchor uint32 + Absolute bool } -func (a *AstNodeAddressT) GetDepth() uint32 { - return a.Depth +type baseAst struct { + scope AstScopeT + address AstNodeAddressT + parent *AstNodeAddressT } -func (a *AstNodeAddressT) GetRuleHash() string { - return a.RuleHash +func (b baseAst) Address() AstNodeAddressT { + return b.address } -func (a *AstNodeAddressT) GetNodeId() uint32 { - return a.NodeId +func (b baseAst) Type() AstNodeType { + return b.address.Type } -func traverseTree(node *AstNodeT, wr io.Writer, depth int) error { - - var ( - obj string - parent = "nil" - err error - ) - - if node.Metadata.ParentAddress != nil { - parent = node.Metadata.ParentAddress.String() - } - - obj = fmt.Sprintf("addr=%s parent=%s scope=%s", - node.Metadata.Address.String(), - parent, - node.Metadata.Scope, - ) - - indent := strings.Repeat(" ", depth) - - if _, err = fmt.Fprintf(wr, "depth_%d: %s%s\n", depth, indent, obj); err != nil { - return err - } - - for _, c := range node.Children { - if err = traverseTree(c, wr, depth+1); err != nil { - return err - } - } - - return nil +func (b baseAst) Scope() AstScopeT { + return b.scope } -func DrawTree(tree *AstT, path string) error { - var ( - f *os.File - err error - ) - - if f, err = os.Create(path); err != nil { - return err - } - - for _, node := range tree.Nodes { - if err = traverseTree(node, f, 0); err != nil { - return err - } - } - - return nil +func (b baseAst) Parent() *AstNodeAddressT { + return b.parent } diff --git a/pkg/ast/ast_log.go b/pkg/ast/ast_log.go deleted file mode 100644 index f050775..0000000 --- a/pkg/ast/ast_log.go +++ /dev/null @@ -1,267 +0,0 @@ -package ast - -import ( - "errors" - "time" - - "github.com/prequel-dev/prequel-compiler/pkg/parser" - "github.com/prequel-dev/prequel-compiler/pkg/schema" - "github.com/prequel-dev/prequel-logmatch/pkg/match" - "github.com/rs/zerolog/log" -) - -var ( - ErrSeqPosConditions = errors.New("sequences require two or more positive conditions") - ErrMissingScalar = errors.New("missing string, jq, or regex condition") - ErrExtractTerm = errors.New("invalid extract (must have name and one of jq or regex)") - ErrNegateCount = errors.New("negate fields cannot have count > 1") - ErrExtractNegate = errors.New("negate fields cannot have extracts") -) - -type AstLogMatcherT struct { - Event AstEventT - Match []AstFieldT - Negate []AstFieldT - Correlations []string - Window time.Duration -} - -func validateLogSeq(n *parser.NodeT, matches int) error { - - if matches <= 1 { - log.Error(). - Any("node", n). - Msg("Sequences require two or more positive conditions") - return n.WrapError(ErrSeqPosConditions) - } - - if n.Metadata.Window == 0 { - log.Error(). - Any("node", n). - Msg("Sequence requires a window") - return n.WrapError(ErrInvalidWindow) - } - - return nil -} - -func validateLogSet(n *parser.NodeT, matches int) error { - - // Only one positive condition with a window is not allowed - if matches == 1 && n.Metadata.Window != 0 { - log.Error(). - Any("node", n). - Msg("Windows require two or more positive conditions") - return n.WrapError(ErrInvalidWindow) - } - - // More than one positive condition with no window is not allowed - if matches > 1 && n.Metadata.Window == 0 { - log.Error(). - Any("node", n). - Msg("Window requires two or more positive conditions") - return n.WrapError(ErrInvalidWindow) - } - - return nil -} - -func (b *builderT) buildLogMatcherNode(parserNode *parser.NodeT, machineAddress *AstNodeAddressT, termIdx *uint32) (*AstNodeT, error) { - - var ( - matchFields = make([]AstFieldT, 0) - negateFields = make([]AstFieldT, 0) - zlog = log.With().Any("address", machineAddress).Logger() - err error - ) - - for _, child := range parserNode.Children { - var ( - match *parser.MatcherT - term AstFieldT - ok bool - ) - - // Children are expected to be scalar matcher values - if match, ok = child.(*parser.MatcherT); !ok { - zlog.Error().Msg("Expected scalar value") - return nil, parserNode.WrapError(ErrMissingScalar) - } - - // Count match fields and remember values - for _, field := range match.Match.Fields { - for range max(field.Count, 1) { - if term, err = newMatchTerm(field); err != nil { - zlog.Error().Err(err).Msg("Invalid match field term") - return nil, parserNode.WrapError(err) - } - matchFields = append(matchFields, term) - } - } - - // Count negate fields and remember values - for _, field := range match.Negate.Fields { - if field.Count > 1 { - err = ErrNegateCount - zlog.Error().Err(err).Int("count", field.Count).Msg("Negate field with count > 1") - return nil, parserNode.WrapError(err) - - } - if term, err = newNegateTerm(field, uint32(len(match.Negate.Fields))); err != nil { - zlog.Error().Err(err).Msg("Invalid negate field term") - return nil, parserNode.WrapError(err) - } - negateFields = append(negateFields, term) - - } - } - - switch parserNode.Metadata.Type { - case schema.NodeTypeLogSet: - if err = validateLogSet(parserNode, len(matchFields)); err != nil { - return nil, err - } - case schema.NodeTypeLogSeq: - if err = validateLogSeq(parserNode, len(matchFields)); err != nil { - return nil, err - } - default: - log.Error(). - Any("type", parserNode.Metadata.Type.String()). - Msg("Invalid node type") - return nil, parserNode.WrapError(ErrInvalidNodeType) - } - - return b.doBuildLogMatcherNode(parserNode, machineAddress, termIdx, matchFields, negateFields) -} - -func (b *builderT) doBuildLogMatcherNode(parserNode *parser.NodeT, machineAddress *AstNodeAddressT, termIdx *uint32, matchFields []AstFieldT, negateFields []AstFieldT) (*AstNodeT, error) { - var ( - address = b.newAstNodeAddress(parserNode.Metadata.RuleHash, parserNode.Metadata.Type.String(), termIdx) - matchNode = newAstNode(parserNode, parserNode.Metadata.Type, schema.ScopeNode, machineAddress, address) - ) - - matchNode.Object = &AstLogMatcherT{ - Event: AstEventT{ - Origin: parserNode.Metadata.Event.Origin, - Source: parserNode.Metadata.Event.Source, - }, - Match: matchFields, - Negate: negateFields, - Window: parserNode.Metadata.Window, - Correlations: parserNode.Metadata.Correlations, - } - - if parserNode.Metadata.Event.Origin { - b.OriginCnt++ - } - - return matchNode, nil -} - -func newMatchTerm(field parser.FieldT) (AstFieldT, error) { - var ( - t AstFieldT - count = 0 - ) - - t = AstFieldT{ - Field: field.Field, - } - - if len(field.Extract) > 0 { - extracts, err := extractTerms(field.Extract) - if err != nil { - return AstFieldT{}, err - } - - t.Extracts = extracts - } - - if field.StrValue != "" { - t.TermValue = match.TermT{ - Type: match.TermRaw, - Value: field.StrValue, - } - count++ - } - if field.JqValue != "" { - t.TermValue = match.TermT{ - Type: match.TermJqJson, - Value: field.JqValue, - } - count++ - } - if field.RegexValue != "" { - t.TermValue = match.TermT{ - Type: match.TermRegex, - Value: field.RegexValue, - } - count++ - } - - if count > 1 { - log.Error().Msg("Only one of str, json, or regex value can be set") - return AstFieldT{}, ErrInvalidNodeType - } - - return t, nil - -} - -func newNegateTerm(field parser.FieldT, anchors uint32) (AstFieldT, error) { - - var ( - t AstFieldT - err error - ) - - if len(field.Extract) > 0 { - log.Error().Msg("Negate terms cannot have extracts") - return AstFieldT{}, ErrExtractNegate - } - - if t, err = newMatchTerm(field); err != nil { - return AstFieldT{}, err - } - - if field.NegateOpts != nil { - - if field.NegateOpts.Anchor > anchors { - return AstFieldT{}, ErrInvalidAnchor - } - - t.NegateOpts = &AstNegateOptsT{ - Window: field.NegateOpts.Window, - Slide: field.NegateOpts.Slide, - Anchor: field.NegateOpts.Anchor, - Absolute: field.NegateOpts.Absolute, - } - } - - return t, nil -} - -func extractTerms(terms []parser.ExtractT) ([]AstExtractT, error) { - var extracts []AstExtractT - for _, term := range terms { - var ( - cnt int - e = AstExtractT{Name: term.Name} - ) - - if term.RegexValue != "" { - cnt++ - e.RegexValue = term.RegexValue - } - if term.JqValue != "" { - cnt++ - e.JqValue = term.JqValue - } - if cnt != 1 { - return nil, ErrExtractTerm - } - extracts = append(extracts, e) - } - return extracts, nil -} diff --git a/pkg/ast/ast_machine.go b/pkg/ast/ast_machine.go deleted file mode 100644 index e3a7a72..0000000 --- a/pkg/ast/ast_machine.go +++ /dev/null @@ -1,120 +0,0 @@ -package ast - -import ( - "time" - - "github.com/prequel-dev/prequel-compiler/pkg/parser" - "github.com/prequel-dev/prequel-compiler/pkg/schema" - "github.com/rs/zerolog/log" -) - -type AstSeqMatcherT struct { - Order []*AstMetadataT - Negate []*AstMetadataT - Correlations []string - Window time.Duration -} - -type AstSetMatcherT struct { - Match []*AstMetadataT - Negate []*AstMetadataT - Correlations []string - Window time.Duration -} - -func (b *builderT) buildMachineNode(parserNode *parser.NodeT, parentMachineAddress, machineAddress *AstNodeAddressT, children []*AstNodeT) (*AstNodeT, error) { - var ( - matchNode = newAstNode(parserNode, parserNode.Metadata.Type, schema.ScopeCluster, parentMachineAddress, machineAddress) - ) - - switch parserNode.Metadata.Type { - case schema.NodeTypeSeq, schema.NodeTypeLogSeq: - matchNode.Metadata.Type = schema.NodeTypeSeq - if seqMatcher, err := buildSeqMatcher(parserNode, children); err != nil { - return nil, err - } else { - matchNode.Object = seqMatcher - } - case schema.NodeTypeSet, schema.NodeTypeLogSet: - matchNode.Metadata.Type = schema.NodeTypeSet - if setMatcher, err := buildSetMatcher(parserNode, children); err != nil { - return nil, err - } else { - matchNode.Object = setMatcher - } - case schema.NodeTypePromQL: - matchNode.Metadata.Type = schema.NodeTypePromQL - if promMatcher, err := b.buildPromQLNode(parserNode, machineAddress, nil); err != nil { - return nil, err - } else { - matchNode.Object = promMatcher - } - default: - log.Error(). - Str("type", parserNode.Metadata.Type.String()). - Msg("Invalid node type") - return nil, ErrInvalidNodeType - } - - return matchNode, nil -} - -// Iterate over children. Create descs and add them to the rule along with correlations -func buildSeqMatcher(n *parser.NodeT, children []*AstNodeT) (*AstSeqMatcherT, error) { - var ( - sm = &AstSeqMatcherT{ - Correlations: make([]string, 0), - Window: n.Metadata.Window, - } - ) - - if n.Metadata.Correlations != nil { - sm.Correlations = n.Metadata.Correlations - } - - sm.Order, sm.Negate = buildTermDescriptors(n, children) - - return sm, nil -} - -// Iterate over children. Create descs and add them to the rule along with correlations -func buildSetMatcher(n *parser.NodeT, children []*AstNodeT) (*AstSetMatcherT, error) { - - var ( - sm = &AstSetMatcherT{ - Correlations: make([]string, 0), - Window: n.Metadata.Window, - } - ) - - if n.Metadata.Correlations != nil { - sm.Correlations = n.Metadata.Correlations - } - - sm.Match, sm.Negate = buildTermDescriptors(n, children) - - return sm, nil -} - -func buildTermDescriptors(parserNode *parser.NodeT, children []*AstNodeT) ([]*AstMetadataT, []*AstMetadataT) { - var ( - match = make([]*AstMetadataT, 0) - negate = make([]*AstMetadataT, 0) - descPos int - ) - - for _, child := range children { - if parserNode.NegIdx > 0 { - if descPos < parserNode.NegIdx { - match = append(match, &child.Metadata) - } else { - negate = append(negate, &child.Metadata) - } - } else { - match = append(match, &child.Metadata) - } - descPos++ - } - - return match, negate -} diff --git a/pkg/ast/ast_metrics.go b/pkg/ast/ast_metrics.go deleted file mode 100644 index d28b2ae..0000000 --- a/pkg/ast/ast_metrics.go +++ /dev/null @@ -1,85 +0,0 @@ -package ast - -import ( - "time" - - "github.com/prequel-dev/prequel-compiler/pkg/parser" - "github.com/prequel-dev/prequel-compiler/pkg/schema" - "github.com/rs/zerolog/log" -) - -type AstPromQL struct { - Expr string - For time.Duration - Interval time.Duration - Event *AstEventT -} - -func (b *builderT) buildPromQLChild(parserNode *parser.NodeT, machineAddress *AstNodeAddressT, termIdx *uint32) (*AstNodeT, error) { - var child *AstNodeT - - err := b.descendTree(func() error { - node, err := b.buildPromQLNode(parserNode, machineAddress, termIdx) - if err != nil { - return err - } - child = node - return nil - }) - - return child, err - -} - -func (b *builderT) buildPromQLNode(parserNode *parser.NodeT, machineAddress *AstNodeAddressT, termIdx *uint32) (*AstNodeT, error) { - - // Expects one child of type ParsePromQL - - if len(parserNode.Children) != 1 { - log.Error().Int("child_count", len(parserNode.Children)).Msg("PromQL node must have exactly one child") - return nil, parserNode.WrapError(ErrInvalidNodeType) - } - - promNode, ok := parserNode.Children[0].(*parser.PromQLT) - - if !ok { - log.Error().Any("promql", parserNode.Children[0]).Msg("Failed to build PromQL node") - return nil, parserNode.WrapError(ErrMissingScalar) - } - - if promNode.Expr == "" { - log.Error().Msg("PromQL Expr string is empty") - return nil, parserNode.WrapError(ErrMissingScalar) - } - - pn := &AstPromQL{ - Expr: promNode.Expr, - } - - if parserNode.Metadata.Event != nil { - pn.Event = &AstEventT{ - Source: parserNode.Metadata.Event.Source, - Origin: parserNode.Metadata.Event.Origin, - } - if parserNode.Metadata.Event.Origin { - b.OriginCnt++ - } - } - - if promNode.Interval != nil { - pn.Interval = *promNode.Interval - } - - if promNode.For != nil { - pn.For = *promNode.For - } - - var ( - address = b.newAstNodeAddress(parserNode.Metadata.RuleHash, parserNode.Metadata.Type.String(), termIdx) - node = newAstNode(parserNode, parserNode.Metadata.Type, schema.ScopeCluster, machineAddress, address) - ) - - node.Object = pn - return node, nil - -} diff --git a/pkg/ast/ast_script.go b/pkg/ast/ast_script.go deleted file mode 100644 index 5bc6cfc..0000000 --- a/pkg/ast/ast_script.go +++ /dev/null @@ -1,101 +0,0 @@ -package ast - -import ( - "time" - - "github.com/prequel-dev/prequel-compiler/pkg/parser" - "github.com/prequel-dev/prequel-compiler/pkg/schema" - "github.com/rs/zerolog/log" -) - -type AstScriptT struct { - Code string - Language string - Timeout time.Duration -} - -// Build the child Ast nodes for the script. -// -// Script nodes are internal nodes with one input node. -// The parser node for a script contains a ScriptT struct as its first child, followed by one input node. -// Build the the child nodes for the script node by building each of the parser node's children; -// the first child is skipped since it is the script definition, and the remaining child is built as the input to the script node. - -func (b *builderT) buildScriptChildren(parserNode *parser.NodeT, machineAddress *AstNodeAddressT) ([]*AstNodeT, error) { - - if len(parserNode.Children) != 2 { - log.Error().Int("child_count", len(parserNode.Children)).Msg("Script node must have two children") - return nil, parserNode.WrapError(ErrInvalidNodeType) - } - - termIdx := uint32(1) - - child := parserNode.Children[1] - parserChildNode, ok := child.(*parser.NodeT) - if !ok { - log.Error().Any("child", child).Msg("Failed to build Script child node") - return nil, parserNode.WrapError(ErrInvalidNodeType) - } - - leaf, err := b.buildLeafChild(parserChildNode, machineAddress, &termIdx) - - var childList []*AstNodeT - - switch { - case err != nil: - // fallthrough - case leaf != nil: - childList = []*AstNodeT{leaf} - default: - err = b.descendTree(func() error { - node, err := b.buildTree(parserChildNode, machineAddress, &termIdx) - if err != nil { - return err - } - childList = []*AstNodeT{node} - return nil - }) - } - - return childList, err -} - -// Validate script definitions and build the script node. - -func (b *builderT) buildScriptNode(parserNode *parser.NodeT, parentMachineAddress, machineAddress *AstNodeAddressT) (*AstNodeT, error) { - - // Expects exactly two children, the first should be parser.ScriptT, the following is the script input node. - - if len(parserNode.Children) != 2 { - log.Error().Int("child_count", len(parserNode.Children)).Msg("Script node must have exactly two children") - return nil, parserNode.WrapError(ErrInvalidNodeType) - } - - scriptNode, ok := parserNode.Children[0].(*parser.ScriptT) - - if !ok { - log.Error().Any("script", parserNode.Children[0]).Msg("Failed to build Script node") - return nil, parserNode.WrapError(ErrMissingScalar) - } - - if scriptNode.Code == "" { - log.Error().Msg("Script code string is empty") - return nil, parserNode.WrapError(ErrMissingScalar) - } - - pn := &AstScriptT{ - Code: scriptNode.Code, - Language: scriptNode.Language, - } - - if scriptNode.Timeout != nil { - pn.Timeout = *scriptNode.Timeout - } - - var ( - node = newAstNode(parserNode, parserNode.Metadata.Type, schema.ScopeCluster, parentMachineAddress, machineAddress) - ) - - node.Object = pn - return node, nil -} diff --git a/pkg/ast/ast_test.go b/pkg/ast/ast_test.go index c08ee7b..e827b3e 100644 --- a/pkg/ast/ast_test.go +++ b/pkg/ast/ast_test.go @@ -1,325 +1,164 @@ package ast import ( - "errors" - "fmt" - "os" - "path/filepath" "reflect" "testing" + "time" - "github.com/prequel-dev/prequel-compiler/pkg/parser" - "github.com/prequel-dev/prequel-compiler/pkg/pqerr" - "github.com/prequel-dev/prequel-compiler/pkg/testdata" - "github.com/rs/zerolog/log" + "github.com/prequel-dev/prequel-logmatch/pkg/match" ) -// traverses the tree and collects node types in DFS pre-order (root, then children) -func gatherNodeTypes(node *AstNodeT, out *[]string) { - - if node == nil { - return +func TestAstNodeAddressT_String(t *testing.T) { + addr := AstNodeAddressT{ + Type: AstNodeTypeSeq, + RuleId: "rule1", + RuleHash: "hash1", + Rank: 2, + Depth: 3, + NodeId: 4, } - - *out = append(*out, node.Metadata.Type.String()) - for _, child := range node.Children { - gatherNodeTypes(child, out) + got := addr.String() + want := "v1.machine_seq.hash1.d3.n4.t2" + if got != want { + t.Errorf("AstNodeAddressT.String() = %q, want %q", got, want) } } -func gatherNodeAddresses(node *AstNodeT, out *[]string) { - if node == nil { - return - } - - *out = append(*out, node.Metadata.Address.String()) - for _, child := range node.Children { - gatherNodeAddresses(child, out) +func TestAstNodeType_String(t *testing.T) { + tests := []struct { + typ AstNodeType + want string + }{ + {AstNodeTypeSet, nodeTypeSet}, + {AstNodeTypeSeq, nodeTypeSeq}, + {AstNodeTypeLogSet, nodeTypeLogSet}, + {AstNodeTypeLogSeq, nodeTypeLogSeq}, + {AstNodeTypePromQL, nodeTypePromQL}, + {AstNodeTypeScript, nodeTypeScript}, + {AstNodeType(99), nodeTypeUnknown}, + } + for _, tt := range tests { + got := tt.typ.String() + if got != tt.want { + t.Errorf("AstNodeType(%d).String() = %q, want %q", tt.typ, got, tt.want) + } } } -func TestAstSuccess(t *testing.T) { - - var tests = map[string]struct { - rule string - expectedNodeTypes []string +func TestAstScopeT_String(t *testing.T) { + tests := []struct { + scope AstScopeT + want string }{ - "Success_Simple1": { - rule: testdata.TestSuccessSimpleRule1, - expectedNodeTypes: []string{"machine_seq", "log_seq"}, - }, - "Success_Complex2": { - rule: testdata.TestSuccessComplexRule2, - expectedNodeTypes: []string{"machine_seq", "log_seq", "log_set", "machine_seq", "log_seq", "log_set", "log_set"}, - }, - "Success_Complex3": { - rule: testdata.TestSuccessComplexRule3, - expectedNodeTypes: []string{"machine_seq", "log_seq", "log_set"}, - }, - "Success_Complex4": { - rule: testdata.TestSuccessComplexRule4, - expectedNodeTypes: []string{"machine_seq", "log_seq", "machine_seq", "log_seq", "log_set", "log_set", "machine_seq", "log_seq", "log_set", "log_set", "log_set"}, - }, - "Success_NegateOptions1": { - rule: testdata.TestSuccessNegateOptions1, - expectedNodeTypes: []string{"machine_seq", "log_seq"}, - }, - "Success_NegateOptions2": { - rule: testdata.TestSuccessNegateOptions2, - expectedNodeTypes: []string{"machine_seq", "log_seq", "log_set", "log_set"}, - }, - "Success_Extract1": { - rule: testdata.TestSuccessSimpleExtraction, - expectedNodeTypes: []string{"machine_seq", "log_seq"}, - }, - "Success_PromQLMetric": { - rule: testdata.TestSuccessSimplePromQL, - expectedNodeTypes: []string{"machine_set", "promql", "log_set"}, - }, - "Success_ChildScript": { - rule: testdata.TestSuccessChildScript, - expectedNodeTypes: []string{"machine_seq", "script", "log_seq", "log_set"}, - }, - "Success_ChildScriptMultipleInputs": { - rule: testdata.TestSuccessChildScriptMultipleInputs, - expectedNodeTypes: []string{"machine_set", "script", "machine_seq", "log_seq", "log_set"}, - }, - "Success_ChildScriptPromQLInput": { - rule: testdata.TestSuccessChildScriptPromQLInput, - expectedNodeTypes: []string{"machine_set", "script", "promql"}, - }, - } - - for name, test := range tests { - t.Run(name, func(t *testing.T) { - - var dupeAddresses = make(map[string]struct{}) - - ast, err := Build([]byte(test.rule)) - if err != nil { - t.Fatalf("Error parsing rule: %v", err) - } - - if err = DrawTree(ast, fmt.Sprintf("rule_%s.dot", name)); err != nil { - t.Fatalf("Error drawing tree: %v", err) - } - - if len(ast.Nodes) == 0 { - t.Fatalf("No nodes found in AST") - } - - if err = validateTree(ast.Nodes[0]); err != nil { - t.Fatalf("Error validating tree: %v", err) - } - - var actualNodes []string - gatherNodeTypes(ast.Nodes[0], &actualNodes) - - var actualAddresses []string - gatherNodeAddresses(ast.Nodes[0], &actualAddresses) - - for _, address := range actualAddresses { - if _, ok := dupeAddresses[address]; ok { - t.Errorf("Duplicate address found: %s", address) - } - dupeAddresses[address] = struct{}{} - } - - if ast.Nodes[0].Metadata.ParentAddress != nil { - t.Errorf("Root node has parent address: %s", ast.Nodes[0].Metadata.ParentAddress.String()) - } - - if !reflect.DeepEqual(actualNodes, test.expectedNodeTypes) { - t.Errorf("gathered types = %v, want %v", actualNodes, test.expectedNodeTypes) - } - }) + {AstScopeNode, scopeTypeNode}, + {AstScopeCluster, scopeTypeCluster}, + {AstScopeOrganization, scopeTypeOrganization}, + {AstScopeGlobal, scopeTypeGlobal}, + {AstScopeT(99), scopeTypeUnknown}, + } + for _, tt := range tests { + got := tt.scope.String() + if got != tt.want { + t.Errorf("AstScopeT(%d).String() = %q, want %q", tt.scope, got, tt.want) + } } } -func TestAstFail(t *testing.T) { - - var tests = map[string]struct { - rule string - err error - line int - col int - }{ - "Fail_MissingPositiveCondition": { - rule: testdata.TestFailMissingPositiveCondition, - err: parser.ErrMissingMatch, - line: 33, - col: 7, - }, - "Fail_BadNegativeCondition1": { - rule: testdata.TestFailNegativeCondition1, - err: parser.ErrMissingMatch, - line: 34, - col: 7, - }, - "Fail_BadNegativeCondition2": { - rule: testdata.TestFailNegativeCondition2, - err: parser.ErrMissingMatch, - line: 33, - col: 7, - }, - "Fail_BadNegativeCondition3": { - rule: testdata.TestFailNegateOptions3, - err: parser.ErrMissingMatch, - line: 41, - col: 7, - }, - "Fail_BadNegativeCondition4": { - rule: testdata.TestFailNegateOptions4, - err: parser.ErrMissingMatch, - line: 42, - col: 7, - }, - "Fail_TermsSemanticError1": { - rule: testdata.TestFailTermsSemanticError1, - err: ErrSeqPosConditions, - line: 36, - col: 15, - }, - "Fail_TermsSemanticError2": { - rule: testdata.TestFailTermsSemanticError2, - err: ErrRootNodeWithoutEventSrc, - line: 11, - col: 9, - }, - "Fail_TermsSemanticError3": { - rule: testdata.TestFailTermsSemanticError3, - err: ErrMissingOrigin, - line: 11, - col: 9, - }, - "Fail_TermsSemanticError4": { - rule: testdata.TestFailTermsSemanticError4, - err: ErrInvalidEventType, - line: 14, - col: 11, - }, - "Fail_TermsSemanticError5": { - rule: testdata.TestFailTermsSemanticError5, - err: ErrInvalidAnchor, - line: 11, - col: 9, - }, - "Fail_MultipleOrigin": { - rule: testdata.TestFailMultipleOrigin, - err: ErrMultipleOrigin, - line: 11, - col: 17, - }, +func TestBaseAst_Methods(t *testing.T) { + addr := AstNodeAddressT{ + Type: AstNodeTypeLogSeq, + RuleId: "rule2", + RuleHash: "hash2", + Rank: 1, + Depth: 2, + NodeId: 3, } - - for name, test := range tests { - t.Run(name, func(t *testing.T) { - _, err := Build([]byte(test.rule)) - if err == nil { - t.Fatalf("Expected error building ast for rule") - } - - if !errors.Is(err, test.err) { - log.Info().Type("err_type", err).Msg("error") - t.Errorf("Expected error %v, got %v", test.err, err) - } - - if pos, ok := pqerr.PosOf(err); ok { - if pos.Line != test.line { - t.Errorf("Expected error position line=%d, got line=%d", test.line, pos.Line) - } - if pos.Col != test.col { - t.Errorf("Expected error position col=%d, got col=%d", test.col, pos.Col) - } - } else { - t.Errorf("Expected wrapped pqerr error %v, got %v", test.err, err) - } - }) + parent := &AstNodeAddressT{Type: AstNodeTypeSet} + b := baseAst{ + scope: AstScopeCluster, + address: addr, + parent: parent, } -} - -func TestSuccessExamples(t *testing.T) { - - rules, err := filepath.Glob(filepath.Join("../testdata", "success_examples", "*.yaml")) - if err != nil { - t.Fatalf("Error finding CRE test files: %v", err) + if !reflect.DeepEqual(b.Address(), addr) { + t.Errorf("baseAst.Address() = %+v, want %+v", b.Address(), addr) } - - for _, rule := range rules { - - // Read the test file - testData, err := os.ReadFile(rule) - if err != nil { - t.Fatalf("Error reading test file %s: %v", rule, err) - } - - _, err = Build(testData) - if err != nil { - t.Fatalf("Error building rule %s: %v", rule, err) - } + if b.Type() != AstNodeTypeLogSeq { + t.Errorf("baseAst.Type() = %v, want %v", b.Type(), AstNodeTypeLogSeq) + } + if b.Scope() != AstScopeCluster { + t.Errorf("baseAst.Scope() = %v, want %v", b.Scope(), AstScopeCluster) + } + if b.Parent() != parent { + t.Errorf("baseAst.Parent() = %+v, want %+v", b.Parent(), parent) } } -func TestFailureExamples(t *testing.T) { - - rules, err := filepath.Glob(filepath.Join("../testdata", "failure_examples", "*.yaml")) - if err != nil { - t.Fatalf("Error finding CRE test files: %v", err) +func TestAstFieldT_ZeroValue(t *testing.T) { + var f AstFieldT + if f.Count != 0 || f.Field != "" || f.TermValue != (match.TermT{}) || f.NegateOpts != nil || f.Extracts != nil { + t.Errorf("AstFieldT zero value not as expected: %+v", f) } +} - for _, rule := range rules { - - // Read the test file - testData, err := os.ReadFile(rule) - if err != nil { - t.Fatalf("Error reading test file %s: %v", rule, err) - } - - _, err = Build(testData) - if err == nil { - t.Fatalf("Expected error building rule %s", rule) - } +func TestAstEventT_ZeroValue(t *testing.T) { + var e AstEventT + if e.Source != "" || e.Origin != false { + t.Errorf("AstEventT zero value not as expected: %+v", e) } } -// Validate the following invariants on the tree: -// 1. No duplicate addresses -// 2. Root node has no parent address -// 3. Node ids are unique -// 4. Depth is consistent with distance from root - -func validateTree(node *AstNodeT) error { - if node == nil { - return fmt.Errorf("Root node is nil") +func TestAstAppT_ZeroValue(t *testing.T) { + var a AstAppT + if a.Name != "" || a.ProcessName != "" || a.ProcessPath != "" || a.ContainerName != "" || + a.ImageUrl != "" || a.RepoUrl != "" || a.Version != "" { + t.Errorf("AstAppT zero value not as expected: %+v", a) } +} - if node.Metadata.ParentAddress != nil { - return fmt.Errorf("Root node has parent address: %s", node.Metadata.ParentAddress.String()) +func TestAstExtractT_ZeroValue(t *testing.T) { + var e AstExtractT + if e.Name != "" || e.JqValue != "" || e.RegexValue != "" { + t.Errorf("AstExtractT zero value not as expected: %+v", e) } - - return _validateTree(node, 0, make(map[uint32]struct{})) // start at depth 0 for root } -func _validateTree(node *AstNodeT, depth uint32, ids map[uint32]struct{}) error { - - if node == nil { - return nil +func TestAstNegateOptsT_ZeroValue(t *testing.T) { + var n AstNegateOptsT + if n.Window != 0 || n.Slide != 0 || n.Anchor != 0 || n.Absolute != false { + t.Errorf("AstNegateOptsT zero value not as expected: %+v", n) } +} - if node.Metadata.Address.Depth != depth { - return fmt.Errorf("Node %s has depth %d, expected %d", node.Metadata.Address.String(), node.Metadata.Address.Depth, depth) +func TestAstMetadataT_ZeroValue(t *testing.T) { + var m AstMetadataT + if m.Name != "" || m.Id != "" || m.Hash != "" || m.Kind != "" || m.Gen != 0 { + t.Errorf("AstMetadataT zero value not as expected: %+v", m) } +} - if _, exists := ids[node.Metadata.Address.NodeId]; exists { - return fmt.Errorf("Duplicate node ID %d found", node.Metadata.Address.NodeId) +func TestAstCreT_ZeroValue(t *testing.T) { + var c AstCreT + if c.Id != "" || c.Severity != 0 || c.Title != "" || c.Category != "" || c.Tags != nil || + c.Author != "" || c.Description != "" || c.Impact != "" || c.ImpactScore != 0 || + c.Cause != "" || c.Mitigation != "" || c.MitigationScore != 0 || c.References != nil || + c.Reports != 0 || c.Applications != nil { + t.Errorf("AstCreT zero value not as expected: %+v", c) } - ids[node.Metadata.Address.NodeId] = struct{}{} +} - for _, child := range node.Children { - if err := _validateTree(child, depth+1, ids); err != nil { - return err - } +func TestAstMatchLeafT_Fields(t *testing.T) { + leaf := AstMatchLeafT{ + Window: 5 * time.Second, + Correlations: []string{"foo", "bar"}, + Terms: []AstFieldT{{Field: "f"}}, + Negate: []AstFieldT{{Field: "n"}}, + Event: AstEventT{Source: "syslog", Origin: true}, + } + if leaf.Window != 5*time.Second || + !reflect.DeepEqual(leaf.Correlations, []string{"foo", "bar"}) || + len(leaf.Terms) != 1 || leaf.Terms[0].Field != "f" || + len(leaf.Negate) != 1 || leaf.Negate[0].Field != "n" || + leaf.Event.Source != "syslog" || !leaf.Event.Origin { + t.Errorf("AstMatchLeafT fields not as expected: %+v", leaf) } - - return nil } diff --git a/pkg/ast/const.go b/pkg/ast/const.go new file mode 100644 index 0000000..8cf5617 --- /dev/null +++ b/pkg/ast/const.go @@ -0,0 +1,110 @@ +package ast + +const ( + // Defaults + defaultMaxGen = 128 + defaultMaxRank = 64 + defaultMaxDepth = 16 + + // Node type strings + nodeTypeSet = "machine_set" + nodeTypeSeq = "machine_seq" + nodeTypeLogSet = "log_set" + nodeTypeLogSeq = "log_seq" + nodeTypePromQL = "promql" + nodeTypeScript = "script" + nodeTypeUnknown = "unknown" + + // Scope type strings + scopeTypeNode = "node" + scopeTypeCluster = "cluster" + scopeTypeOrganization = "organization" + scopeTypeGlobal = "global" + scopeTypeUnknown = "unknown" + + // Root keys + kwRules = "rules" + kwMetadata = "metadata" + kwCre = "cre" + kwRule = "rule" + kwCompiler = "compiler" + + // Nodes + kwSequence = "sequence" + kwSet = "set" + kwEvent = "event" + kwMatch = "match" + kwOrder = "order" + kwNegate = "negate" + kwCorrelations = "correlations" + + // Metadata + kwName = "name" + kwId = "id" + kwHash = "hash" + kwGen = "gen" + kwKind = "kind" + + // CRE + kwCreId = "id" + kwSeverity = "severity" + kwTitle = "title" + kwCategory = "category" + kwTags = "tags" + kwAuthor = "author" + kwDescription = "description" + kwImpact = "impact" + kwImpactScore = "impactScore" + kwCause = "cause" + kwMitigation = "mitigation" + kwMitigationScore = "mitigationScore" + kwReferences = "references" + kwReports = "reports" + kwApplications = "applications" + + // CRE Applications + kwAppName = "name" + kwAppProcessName = "processName" + kwAppProcessPath = "processPath" + kwAppContainer = "containerName" + kwAppImage = "imageUrl" + kwAppRepo = "repoUrl" + kwAppVersion = "version" + + // Event + kwSource = "source" + kwOrigin = "origin" + + // Extract + kwExtractName = "name" + kwExtractJq = "jq" + kwExtractRegex = "regex" + + // Term + kwField = "field" + kwValue = "value" + kwJq = "jq" + kwRegex = "regex" + kwCount = "count" + kwExtract = "extract" + kwPromQL = "promql" + kwScript = "script" + + // Negate opts + kwWindow = "window" + kwSlide = "slide" + kwAnchor = "anchor" + kwAbsolute = "absolute" + + // PromQL + kwPromExpr = "expr" + kwPromInterval = "interval" + kwPromFor = "for" + kwPromEvent = "event" + + // Script + kwScriptCode = "code" + kwScriptLang = "language" + kwScriptTimeout = "timeout" + kwScriptInput = "input" +) diff --git a/pkg/ast/draw.go b/pkg/ast/draw.go new file mode 100644 index 0000000..fcd62fd --- /dev/null +++ b/pkg/ast/draw.go @@ -0,0 +1,356 @@ +package ast + +import ( + "errors" + "fmt" + "strings" + + "github.com/jedib0t/go-pretty/v6/text" + "github.com/rs/zerolog/log" + "github.com/thediveo/go-asciitree" +) + +type DrawOpt func(*drawOpts) + +type drawOpts struct { + colorize bool + parent bool +} + +func (o drawOpts) styleAddr(addr string) string { + if o.colorize { + return text.FgHiYellow.Sprint(addr) + } + return addr +} + +func WithColor() DrawOpt { + return func(opts *drawOpts) { + opts.colorize = true + } +} + +func WithParent() DrawOpt { + return func(opts *drawOpts) { + opts.parent = true + } +} + +func parseDrawOpts(opts []DrawOpt) drawOpts { + var o drawOpts + for _, opt := range opts { + opt(&o) + } + return o +} + +func Draw(r AstRuleT, opts ...DrawOpt) string { + o := parseDrawOpts(opts) + + type nodeT struct { + depth uint32 + Label string `asciitree:"label"` + Props []string `asciitree:"properties"` + Children []*nodeT `asciitree:"children"` + } + + root := &nodeT{ + Label: fmt.Sprintf("Rule: %s", o.styleId(r.Metadata.Id)), + Props: []string{ + fmt.Sprintf("hash=%s", o.styleId(r.Metadata.Hash)), + fmt.Sprintf("gen=%d", r.Metadata.Gen), + }, + } + + if r.Cre != nil { + root.Props = append(root.Props, + fmt.Sprintf("cre_id=%s", r.Cre.Id), + ) + } + + fmtLabel := func(n AstNode, nopts *AstNegateOptsT) string { + return fmt.Sprintf( + "%s %s %s", + renderScopeLabel(n.Scope(), nopts, o), + o.styleAddr(n.Address().String()), + renderNegateOpts(nopts, o), + ) + } + + var stack = []*nodeT{} + + findParent := func(depth uint32) int { + for i := len(stack) - 1; i >= 0; i-- { + if stack[i].depth == depth { + return i + } + } + return -1 + } + + walker := func(n AstNode, nopts *AstNegateOptsT) error { + + var ( + curDepth uint32 + addr = n.Address() + ) + + if len(stack) > 0 { + curDepth = stack[len(stack)-1].depth + } + + switch { + case len(stack) == 0: + // first node, add to root + child := &nodeT{ + Label: fmtLabel(n, nopts), + Props: extractProps(n, o), + } + root.Children = append(root.Children, child) + stack = append(stack, child) + + case addr.Depth == curDepth: + // sibling node, add to current parent + parIdx := findParent(addr.Depth - 1) + if parIdx < 0 { + return errors.New("parent not found for node") + } + parent := stack[parIdx] + child := &nodeT{ + depth: addr.Depth, + Label: fmtLabel(n, nopts), + Props: extractProps(n, o), + } + parent.Children = append(parent.Children, child) + stack = append(stack, child) + case addr.Depth > curDepth: + // child node, add to stack + parent := stack[len(stack)-1] + child := &nodeT{ + depth: addr.Depth, + Label: fmtLabel(n, nopts), + Props: extractProps(n, o), + } + parent.Children = append(parent.Children, child) + stack = append(stack, child) + + case addr.Depth < curDepth: + // moving back up the tree, pop stack until we find the correct parent + parIdx := findParent(addr.Depth - 1) + if parIdx < 0 { + return errors.New("parent not found for node") + } + parent := stack[parIdx] + child := &nodeT{ + depth: addr.Depth, + Label: fmtLabel(n, nopts), + Props: extractProps(n, o), + } + parent.Children = append(parent.Children, child) + stack = stack[:parIdx+1] + stack = append(stack, child) + } + + return nil + } + + if err := r.Walk(walker); err != nil { + log.Error().Err(err).Msg("Error walking AST for drawing") + } + return asciitree.RenderFancy(root) +} + +func extractProps(n AstNode, o drawOpts) []string { + var props []string + + if o.parent { + if addr := n.Parent(); addr != nil { + props = append(props, + fmt.Sprintf("parent=%s", addr.String()), + ) + } + } + + switch node := n.(type) { + case *AstInnerNodeT: + props = append(props, fmt.Sprintf("type=%s", node.Type().String())) + if node.Window != 0 { + props = append(props, fmt.Sprintf("window=%s", node.Window.String())) + } + if len(node.Correlations) > 0 { + props = append(props, fmt.Sprintf("correlations=%v", node.Correlations)) + } + + case *AstMatchLeafT: + props = append(props, "type=line_match") + if node.Window != 0 { + props = append(props, fmt.Sprintf("window=%s", node.Window.String())) + } + if len(node.Correlations) > 0 { + props = append(props, fmt.Sprintf("correlations=%v", node.Correlations)) + } + + props = append(props, fmt.Sprintf("event_src=%s", o.styleEventSrc(node.Event.Source))) + if node.Event.Origin { + props = append(props, "origin=true") + } + + for i, term := range node.Terms { + cnt := term.Count + if cnt == 0 { + cnt = 1 + } + + opts := fmt.Sprintf("%d,%s,%d", i, term.TermValue.Type.String(), cnt) + if term.Field != "" { + opts = fmt.Sprintf("%s,field=%s", opts, term.Field) + } + + props = append(props, + fmt.Sprintf("term [%s]=%s", opts, o.styleTermValue(term.TermValue.Value)), + ) + + } + for i, term := range node.Negate { + + cnt := term.Count + if cnt == 0 { + cnt = 1 + } + + opts := fmt.Sprintf("%d,%s,%d", i, term.TermValue.Type.String(), cnt) + if term.Field != "" { + opts = fmt.Sprintf("%s,field=%s", opts, term.Field) + } + + negateProps := renderNegateOpts(term.NegateOpts, o) + + props = append(props, + fmt.Sprintf( + "negate [%s]%s=%s", + opts, + negateProps, + o.styleTermValue(term.TermValue.Value)), + ) + } + case *AstPromT: + props = append(props, "type=promql") + props = append(props, fmt.Sprintf("event_src=%s", o.styleEventSrc(node.Event.Source))) + if node.Event.Origin { + props = append(props, "origin=true") + } + props = append(props, fmt.Sprintf("expr=%s", o.styleTermValue(node.Expr))) + if node.Interval != 0 { + props = append(props, fmt.Sprintf("interval=%v", node.Interval)) + } + if node.For != 0 { + props = append(props, fmt.Sprintf("for=%v", node.For)) + } + + case *AstScriptT: + props = append(props, "type=script") + // Print first few lines of code + codePreview := node.Code + if len(codePreview) > 100 { + codePreview = codePreview[:100] + "..." + } + props = append(props, fmt.Sprintf("code=%s", o.styleTermValue(codePreview))) + + if node.Language != "" { + props = append(props, fmt.Sprintf("language=%s", node.Language)) + } + if node.Timeout != 0 { + props = append(props, fmt.Sprintf("timeout=%s", node.Timeout.String())) + } + default: + props = append(props, "type=unknown") + } + + return props +} + +func renderScopeLabel(scope AstScopeT, nopts *AstNegateOptsT, o drawOpts) string { + + var c string + switch scope { + case AstScopeGlobal: + c = "G" + case AstScopeOrganization: + c = "O" + case AstScopeCluster: + c = "C" + case AstScopeNode: + c = "N" + default: + c = "?" + } + + if nopts != nil { + return o.styleNegate(fmt.Sprintf("[¬ %s]", c)) + } + + return o.styleScope(fmt.Sprintf("[%s]", c)) +} + +func renderNegateOpts(nopts *AstNegateOptsT, o drawOpts) string { + if nopts == nil { + return "" + } + + var s []string + if nopts.Window != 0 { + s = append(s, fmt.Sprintf("window=%s", nopts.Window.String())) + } + if nopts.Slide != 0 { + s = append(s, fmt.Sprintf("slide=%s", nopts.Slide.String())) + } + if nopts.Anchor != 0 { + s = append(s, fmt.Sprintf("anchor=%d", nopts.Anchor)) + } + if nopts.Absolute { + s = append(s, "absolute=true") + } + + if len(s) == 0 { + return "" + } + + negateProps := fmt.Sprintf("[%s]", strings.Join(s, ",")) + + return o.styleNegate(negateProps) +} + +func (o drawOpts) styleScope(addr string) string { + if o.colorize { + return text.FgHiGreen.Sprint(addr) + } + return addr +} + +func (o drawOpts) styleTermValue(val string) string { + if o.colorize { + return text.FgCyan.Sprint(val) + } + return val +} + +func (o drawOpts) styleEventSrc(val string) string { + if o.colorize { + return text.FgHiMagenta.Sprint(val) + } + return val +} + +func (o drawOpts) styleId(val string) string { + if o.colorize { + return text.FgHiGreen.Sprint(val) + } + return val +} + +func (o drawOpts) styleNegate(val string) string { + if o.colorize { + return text.FgRed.Sprint(val) + } + return val +} diff --git a/pkg/ast/draw_test.go b/pkg/ast/draw_test.go new file mode 100644 index 0000000..1eec946 --- /dev/null +++ b/pkg/ast/draw_test.go @@ -0,0 +1,361 @@ +package ast + +import ( + "strings" + "testing" + "time" + + "github.com/prequel-dev/prequel-compiler/pkg/testdata" + "github.com/prequel-dev/prequel-logmatch/pkg/match" +) + +// Dummy AstNode for unknown type +type drawDummyNode struct{} + +func (d drawDummyNode) Type() AstNodeType { return AstNodeType(99) } +func (d drawDummyNode) Scope() AstScopeT { return AstScopeT(99) } +func (d drawDummyNode) Address() AstNodeAddressT { return AstNodeAddressT{} } +func (d drawDummyNode) Parent() *AstNodeAddressT { return nil } + +func TestDrawOpts_WithColorAndParent(t *testing.T) { + opts := parseDrawOpts([]DrawOpt{WithColor(), WithParent()}) + if !opts.colorize { + t.Errorf("WithColor() did not set colorize") + } + if !opts.parent { + t.Errorf("WithParent() did not set parent") + } +} + +func TestDrawOpts_Defaults(t *testing.T) { + opts := parseDrawOpts(nil) + if opts.colorize { + t.Errorf("Default colorize should be false") + } + if opts.parent { + t.Errorf("Default parent should be false") + } +} + +func TestDrawOpts_StyleMethods(t *testing.T) { + o := drawOpts{colorize: false} + if got := o.styleAddr("foo"); got != "foo" { + t.Errorf("styleAddr() = %q, want %q", got, "foo") + } + if got := o.styleScope("bar"); got != "bar" { + t.Errorf("styleScope() = %q, want %q", got, "bar") + } + if got := o.styleTermValue("baz"); got != "baz" { + t.Errorf("styleTermValue() = %q, want %q", got, "baz") + } + if got := o.styleEventSrc("src"); got != "src" { + t.Errorf("styleEventSrc() = %q, want %q", got, "src") + } + if got := o.styleId("id"); got != "id" { + t.Errorf("styleId() = %q, want %q", got, "id") + } + if got := o.styleNegate("neg"); got != "neg" { + t.Errorf("styleNegate() = %q, want %q", got, "neg") + } +} + +func TestRenderScopeLabel(t *testing.T) { + o := drawOpts{} + tests := []struct { + scope AstScopeT + want string + }{ + {AstScopeGlobal, "[G]"}, + {AstScopeOrganization, "[O]"}, + {AstScopeCluster, "[C]"}, + {AstScopeNode, "[N]"}, + {AstScopeT(99), "[?]"}, + } + for _, tt := range tests { + got := renderScopeLabel(tt.scope, nil, o) + if got != tt.want { + t.Errorf("renderScopeLabel(%v) = %q, want %q", tt.scope, got, tt.want) + } + } +} + +func TestRenderNegateOpts(t *testing.T) { + o := drawOpts{} + // nil + if got := renderNegateOpts(nil, o); got != "" { + t.Errorf("renderNegateOpts(nil) = %q, want empty", got) + } + // all fields + n := &AstNegateOptsT{ + Window: 2 * time.Second, + Slide: 1 * time.Second, + Anchor: 3, + Absolute: true, + } + got := renderNegateOpts(n, o) + if !strings.Contains(got, "window=2s") || + !strings.Contains(got, "slide=1s") || + !strings.Contains(got, "anchor=3") || + !strings.Contains(got, "absolute=true") { + t.Errorf("renderNegateOpts() = %q, missing expected fields", got) + } + // only one field + n = &AstNegateOptsT{Window: 1 * time.Second} + got = renderNegateOpts(n, o) + if !strings.Contains(got, "window=1s") { + t.Errorf("renderNegateOpts() = %q, want window=1s", got) + } +} + +func TestExtractProps_UnknownType(t *testing.T) { + props := extractProps(drawDummyNode{}, drawOpts{}) + if len(props) == 0 || props[0] != "type=unknown" { + t.Errorf("extractProps(dummyNode) = %v, want [type=unknown]", props) + } +} + +func TestExtractProps_AstInnerNodeT(t *testing.T) { + node := &AstInnerNodeT{ + Window: 5 * time.Second, + Correlations: []string{"foo", "bar"}, + } + props := extractProps(node, drawOpts{}) + if !containsAny(props, []string{"type=machine_set", "type=machine_seq", "type=log_set", "type=log_seq"}) { + t.Errorf("extractProps(AstInnerNodeT) missing type: %v", props) + } + if !contains(props, "window=5s") { + t.Errorf("extractProps(AstInnerNodeT) missing window: %v", props) + } + if !containsAny(props, []string{"correlations=[foo bar]", "correlations=[foo,bar]"}) { + t.Errorf("extractProps(AstInnerNodeT) missing correlations: %v", props) + } +} + +func TestExtractProps_AstMatchLeafT(t *testing.T) { + node := &AstMatchLeafT{ + Window: 3 * time.Second, + Correlations: []string{"x"}, + Event: AstEventT{Source: "syslog", Origin: true}, + Terms: []AstFieldT{ + {Field: "f", TermValue: match.TermT{Type: match.TermRaw, Value: "v"}, Count: 2}, + }, + Negate: []AstFieldT{ + {Field: "nf", TermValue: match.TermT{Type: match.TermRegex, Value: "nv"}, Count: 1, NegateOpts: &AstNegateOptsT{Window: 1 * time.Second}}, + }, + } + props := extractProps(node, drawOpts{}) + if !contains(props, "type=line_match") { + t.Errorf("extractProps(AstMatchLeafT) missing type=line_match: %v", props) + } + if !contains(props, "window=3s") { + t.Errorf("extractProps(AstMatchLeafT) missing window: %v", props) + } + if !containsAny(props, []string{"correlations=[x]", "correlations=[x]"}) { + t.Errorf("extractProps(AstMatchLeafT) missing correlations: %v", props) + } + if !contains(props, "event_src=syslog") { + t.Errorf("extractProps(AstMatchLeafT) missing event_src=syslog: %v", props) + } + if !contains(props, "origin=true") { + t.Errorf("extractProps(AstMatchLeafT) missing origin=true: %v", props) + } + if !containsAny(props, []string{"term [0,raw,2,field=f]=v", "term [0,raw,2,field=f]=v"}) { + t.Errorf("extractProps(AstMatchLeafT) missing term: %v", props) + } + if !containsAny(props, []string{"negate [0,regex,1,field=nf][window=1s]=nv", "negate [0,regex,1,field=nf][window=1s]=nv"}) { + t.Errorf("extractProps(AstMatchLeafT) missing negate: %v", props) + } +} + +func TestExtractProps_AstPromT(t *testing.T) { + node := &AstPromT{ + Expr: "up", + For: 10 * time.Second, + Interval: 5 * time.Second, + Event: &AstEventT{Source: "prom", Origin: false}, + } + props := extractProps(node, drawOpts{}) + if !contains(props, "type=promql") { + t.Errorf("extractProps(AstPromT) missing type=promql: %v", props) + } + if !contains(props, "event_src=prom") { + t.Errorf("extractProps(AstPromT) missing event_src=prom: %v", props) + } + if !contains(props, "expr=up") { + t.Errorf("extractProps(AstPromT) missing expr=up: %v", props) + } + if !contains(props, "interval=5s") { + t.Errorf("extractProps(AstPromT) missing interval=5s: %v", props) + } + if !contains(props, "for=10s") { + t.Errorf("extractProps(AstPromT) missing for=10s: %v", props) + } +} + +func TestExtractProps_AstScriptT(t *testing.T) { + node := &AstScriptT{ + Code: "print(1)", + Language: "python", + Timeout: 2 * time.Second, + } + props := extractProps(node, drawOpts{}) + if !contains(props, "type=script") { + t.Errorf("extractProps(AstScriptT) missing type=script: %v", props) + } + if !contains(props, "code=print(1)") { + t.Errorf("extractProps(AstScriptT) missing code: %v", props) + } + if !contains(props, "language=python") { + t.Errorf("extractProps(AstScriptT) missing language=python: %v", props) + } + if !contains(props, "timeout=2s") { + t.Errorf("extractProps(AstScriptT) missing timeout=2s: %v", props) + } +} + +func contains(slice []string, s string) bool { + for _, v := range slice { + if v == s { + return true + } + } + return false +} + +func containsAny(slice []string, options []string) bool { + for _, v := range slice { + for _, o := range options { + if v == o { + return true + } + } + } + return false +} + +const ( + testOutput_TestSuccessComplexRule4 = `Rule: J7uRQTGpGMyL1iFpssnBeS +│ • hash=2KdXQZDAfRbYcH9FBDteBS +│ • gen=0 +│ • cre_id=TestSuccessComplexRule4 +└─ [C] v1.machine_seq.2KdXQZDAfRbYcH9FBDteBS.d0.n0.t0 + │ • type=machine_seq + │ • window=30s + │ • correlations=[hostname] + ├─ [N] v1.log_seq.2KdXQZDAfRbYcH9FBDteBS.d1.n1.t0 + │ • type=line_match + │ • window=10s + │ • event_src=rabbitmq + │ • origin=true + │ • term [0,raw,10]=Discarding message + │ • term [1,raw,1]=Mnesia overloaded + │ • negate [0,raw,1]=SIGTERM + ├─ [C] v1.machine_seq.2KdXQZDAfRbYcH9FBDteBS.d1.n2.t1 + │ │ • type=machine_seq + │ │ • window=5s + │ │ • correlations=[container_id] + │ ├─ [N] v1.log_seq.2KdXQZDAfRbYcH9FBDteBS.d2.n3.t0 + │ │ • type=line_match + │ │ • window=1s + │ │ • event_src=nginx + │ │ • term [0,raw,1]=error message + │ │ • term [1,raw,1]=shutdown + │ ├─ [N] v1.log_set.2KdXQZDAfRbYcH9FBDteBS.d2.n4.t1 + │ │ • type=line_match + │ │ • event_src=nginx + │ │ • term [0,raw,1]=90% + │ └─ [N] v1.log_set.2KdXQZDAfRbYcH9FBDteBS.d2.n5.t2 + │ • type=line_match + │ • event_src=k8s + │ • term [0,raw,1,field=reason]=Killing + ├─ [C] v1.machine_seq.2KdXQZDAfRbYcH9FBDteBS.d1.n6.t2 + │ │ • type=machine_seq + │ │ • window=5s + │ │ • correlations=[container_id] + │ ├─ [N] v1.log_seq.2KdXQZDAfRbYcH9FBDteBS.d2.n7.t0 + │ │ • type=line_match + │ │ • window=1s + │ │ • event_src=nginx + │ │ • term [0,raw,1]=error message + │ │ • term [1,raw,1]=shutdown + │ ├─ [N] v1.log_set.2KdXQZDAfRbYcH9FBDteBS.d2.n8.t1 + │ │ • type=line_match + │ │ • event_src=nginx + │ │ • term [0,raw,1]=90% + │ └─ [N] v1.log_set.2KdXQZDAfRbYcH9FBDteBS.d2.n9.t2 + │ • type=line_match + │ • event_src=k8s + │ • term [0,raw,1,field=reason]=Killing + └─ [¬ N] v1.log_set.2KdXQZDAfRbYcH9FBDteBS.d1.n10.t3 + • type=line_match + • event_src=k8s + • term [0,raw,1,field=reason]=NodeShutdown +` + testOutput_TestSuccessSimpleRule1 = `Rule: J7uRQTGpGMyL1iFpssnBeS +│ • hash=rdJLgqYgkEp8jg8Qks1qiq +│ • gen=1 +│ • cre_id=TestSuccessSimpleRule1 +└─ [C] v1.machine_set.rdJLgqYgkEp8jg8Qks1qiq.d0.n0.t0 + │ • type=machine_set + └─ [N] v1.log_set.rdJLgqYgkEp8jg8Qks1qiq.d1.n1.t0 + • type=line_match + • window=10s + • event_src=kafka + • origin=true + • term [0,raw,3]=io.vertx.core.VertxException: Thread blocked +` + testOutput_TestSuccessSimpleRule1WithColor = "Rule: \x1b[92mJ7uRQTGpGMyL1iFpssnBeS\x1b[0m\n│ • hash=\x1b[92mrdJLgqYgkEp8jg8Qks1qiq\x1b[0m\n│ • gen=1\n│ • cre_id=TestSuccessSimpleRule1\n└─ \x1b[92m[C]\x1b[0m \x1b[93mv1.machine_set.rdJLgqYgkEp8jg8Qks1qiq.d0.n0.t0\x1b[0m \n │ • type=machine_set\n └─ \x1b[92m[N]\x1b[0m \x1b[93mv1.log_set.rdJLgqYgkEp8jg8Qks1qiq.d1.n1.t0\x1b[0m \n • type=line_match\n • window=10s\n • event_src=\x1b[95mkafka\x1b[0m\n • origin=true\n • term [0,raw,3]=\x1b[36mio.vertx.core.VertxException: Thread blocked\x1b[0m\n" +) + +// NOTE: This test is somewhat brittle since it relies on the exact formatting of the output, +// but it serves as a regression test to ensure that the Draw function continues to walk the +// node tree and include all relevant properties in the output. If the formatting changes, +// this test will need to be updated accordingly. +func TestDraw_WalksNodeTree_Brittle(t *testing.T) { + + tests := []struct { + name string + input string + output string + color bool + }{ + { + name: "TestSuccessComplexRule4", + input: testdata.TestSuccessComplexRule4, + output: testOutput_TestSuccessComplexRule4, + }, + { + name: "TestSuccessSimpleRule1", + input: testdata.TestSuccessSimpleRule1, + output: testOutput_TestSuccessSimpleRule1, + }, + { + name: "TestSuccessSimpleRule1 with color", + input: testdata.TestSuccessSimpleRule1, + output: testOutput_TestSuccessSimpleRule1WithColor, + color: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + + astRules, err := ParseRules([]byte(tt.input)) + if err != nil { + t.Fatalf("ParseRules() error = %v", err) + } + + opts := []DrawOpt{} + if tt.color { + opts = append(opts, WithColor()) + } + + out := Draw(astRules[0], opts...) + + if out != tt.output { + t.Errorf("Draw() output = '%s', want '%s'", out, tt.output) + } + }) + } + +} diff --git a/pkg/ast/errors.go b/pkg/ast/errors.go new file mode 100644 index 0000000..05d0274 --- /dev/null +++ b/pkg/ast/errors.go @@ -0,0 +1,99 @@ +package ast + +import ( + "errors" + "fmt" + + "github.com/goccy/go-yaml" + "github.com/goccy/go-yaml/token" +) + +var ( + ErrBadAnchor = errors.New("anchor value is out of range") + ErrBadExtractName = errors.New("extract name is not valid") + ErrBadField = errors.New("field is not valid") + ErrBadGen = errors.New("gen is not valid") + ErrBadHash = errors.New("hash is not valid") + ErrBadIdentifier = errors.New("identifier is not valid") + ErrBadJq = errors.New("invalid jq expression") + ErrBadKind = errors.New("kind is not valid") + ErrBadPromQL = errors.New("invalid PromQL query") + ErrBadRegex = errors.New("invalid regex pattern") + ErrBadScriptCode = errors.New("script code is not valid") + ErrBadScriptLang = errors.New("script language is not valid") + ErrBadSeverity = errors.New("severity is not valid") + ErrDupeExtractName = errors.New("duplicate extract name") + ErrMaxDepthExceeded = errors.New("maximum depth exceeded") + ErrMaxRankExceeded = errors.New("maximum rank exceeded") + ErrMissingEvent = errors.New("event is required when using field terms") + ErrMissingKey = errors.New("missing key") + ErrMissingOrigin = errors.New("missing origin") + ErrMissingScriptInput = errors.New("script input is required") + ErrMissingSource = errors.New("source is required in event") + ErrMissingTerm = errors.New("at least one term is required") + ErrMissingWindow = errors.New("window is required when using multiple terms") + ErrMissingVersion = errors.New("compiler version is required in strict mode") + ErrMultipleOrigin = errors.New("multiple origin events are not allowed") + ErrNegateCount = errors.New("negate fields cannot have count > 1") + ErrOverflow = errors.New("value overflow") + ErrShortSequence = errors.New("sequence must have at least 2 terms") + ErrTermRedefined = errors.New("term previously defined") + ErrTermTypeConflict = errors.New("term type conflicts with previous term type") + ErrUnexpectedKey = errors.New("unexpected key") + ErrUnexpectedType = errors.New("unexpected type") + ErrUndefinedAnchor = errors.New("undefined anchor") + ErrUnknownNodeType = errors.New("unknown node type") + ErrWindowNegative = errors.New("window duration cannot be negative") + ErrZeroCount = errors.New("count value must be a positive integer") +) + +type ErrRule struct { + Meta AstMetadataT + Err error +} + +func (e ErrRule) Error() string { + return fmt.Sprintf("rule error id=%s hash=%s: %v", e.Meta.Id, e.Meta.Hash, e.Err) +} + +func (e ErrRule) Unwrap() error { + return e.Err +} + +type ParseError struct { + token *token.Token + err error +} + +func (e ParseError) Offset() int { + if e.token == nil { + return 0 + } + return e.token.Position.Offset +} + +func (e ParseError) Line() int { + if e.token == nil { + return 0 + } + return e.token.Position.Line +} + +func (e ParseError) Column() int { + if e.token == nil { + return 0 + } + return e.token.Position.Column +} + +func (e ParseError) Unwrap() error { + return e.err +} + +func (e ParseError) Error() string { + return e.err.Error() +} + +func (e ParseError) Format(colored, inclSource bool) string { + return yaml.FormatErrorWithToken(e.err.Error(), e.token, colored, inclSource) +} diff --git a/pkg/ast/errors_test.go b/pkg/ast/errors_test.go new file mode 100644 index 0000000..3733649 --- /dev/null +++ b/pkg/ast/errors_test.go @@ -0,0 +1,85 @@ +package ast + +import ( + "errors" + "testing" + + "github.com/goccy/go-yaml/token" +) + +func TestErrRule_ErrorAndUnwrap(t *testing.T) { + meta := AstMetadataT{Id: "id1", Hash: "h1"} + origErr := errors.New("something bad") + e := ErrRule{Meta: meta, Err: origErr} + + msg := e.Error() + if want := "rule error id=id1 hash=h1: something bad"; msg != want { + t.Errorf("ErrRule.Error() = %q, want %q", msg, want) + } + + if !errors.Is(e, origErr) { + t.Errorf("ErrRule.Unwrap() did not return the original error") + } +} + +func TestParseError_OffsetLineColumn(t *testing.T) { + tok := &token.Token{ + Position: &token.Position{ + Offset: 42, + Line: 7, + Column: 13, + }, + } + e := ParseError{token: tok, err: errors.New("parse fail")} + + if got := e.Offset(); got != 42 { + t.Errorf("ParseError.Offset() = %d, want 42", got) + } + if got := e.Line(); got != 7 { + t.Errorf("ParseError.Line() = %d, want 7", got) + } + if got := e.Column(); got != 13 { + t.Errorf("ParseError.Column() = %d, want 13", got) + } +} + +func TestParseError_OffsetLineColumn_NilToken(t *testing.T) { + e := ParseError{token: nil, err: errors.New("parse fail")} + if got := e.Offset(); got != 0 { + t.Errorf("ParseError.Offset() with nil token = %d, want 0", got) + } + if got := e.Line(); got != 0 { + t.Errorf("ParseError.Line() with nil token = %d, want 0", got) + } + if got := e.Column(); got != 0 { + t.Errorf("ParseError.Column() with nil token = %d, want 0", got) + } +} + +func TestParseError_ErrorAndUnwrap(t *testing.T) { + origErr := errors.New("parse fail") + e := ParseError{err: origErr} + if got := e.Error(); got != "parse fail" { + t.Errorf("ParseError.Error() = %q, want %q", got, "parse fail") + } + if !errors.Is(e, origErr) { + t.Errorf("ParseError.Unwrap() did not return the original error") + } +} + +// Note: Format() just calls yaml.FormatErrorWithToken, which is a passthrough. +// You can check that it returns a non-empty string. +func TestParseError_Format(t *testing.T) { + tok := &token.Token{ + Position: &token.Position{ + Offset: 1, + Line: 2, + Column: 3, + }, + } + e := ParseError{token: tok, err: errors.New("parse fail")} + out := e.Format(false, false) + if out == "" { + t.Errorf("ParseError.Format() returned empty string") + } +} diff --git a/pkg/ast/opts.go b/pkg/ast/opts.go new file mode 100644 index 0000000..2295126 --- /dev/null +++ b/pkg/ast/opts.go @@ -0,0 +1,93 @@ +package ast + +type ( + ParseOpt func(*optT) + ValidatorFunc func(string) error +) + +type optT struct { + maxGen uint32 + maxRank uint32 + maxDepth uint32 + strict bool + jqValidator ValidatorFunc + luaValidator ValidatorFunc + promQLValidator ValidatorFunc +} + +// WithStrict sets the strict mode for parsing. +// In strict mode, the parser will return an error if it encounters any unexpected keys in the YAML input. In non-strict mode, the parser will ignore unexpected keys and continue parsing. +// When disabled, the parser will ignore any non operational keys in the YAML input +// This is particularly true in the metadata sections where additional keys do +// not have operational impact on the rules engine. + +func WithStrict(strict bool) ParseOpt { + return func(opts *optT) { + opts.strict = strict + } +} + +func WithJQValidator(validator ValidatorFunc) ParseOpt { + return func(opts *optT) { + opts.jqValidator = selectValidator(validator) + } +} + +func WithLuaValidator(validator ValidatorFunc) ParseOpt { + return func(opts *optT) { + opts.luaValidator = selectValidator(validator) + } +} + +func WithPromQLValidator(validator ValidatorFunc) ParseOpt { + return func(opts *optT) { + opts.promQLValidator = selectValidator(validator) + } +} + +func WithMaxGen(maxGen uint32) ParseOpt { + return func(opts *optT) { + opts.maxGen = maxGen + } +} + +// WithMaxRank sets the maximum allowed rank for terms in the YAML input. +// This is a safeguard against excessively large numbers of terms that could lead to performance issues during parsing. +func WithMaxRank(maxRank uint32) ParseOpt { + return func(opts *optT) { + opts.maxRank = maxRank + } +} + +// WithMaxDepth sets the maximum allowed depth for rule definitions in the YAML input. +// This is a safeguard against excessively nested structures that could lead to stack overflows or performance issues during parsing. +func WithMaxDepth(maxDepth uint32) ParseOpt { + return func(opts *optT) { + opts.maxDepth = maxDepth + } +} + +func selectValidator(validator ValidatorFunc) ValidatorFunc { + if validator == nil { + return stubValidator + } + return validator +} + +var stubValidator = func(string) error { return nil } + +func parseOpts(opts ...ParseOpt) optT { + opt := optT{ + maxGen: defaultMaxGen, + maxRank: defaultMaxRank, + maxDepth: defaultMaxDepth, + strict: false, + luaValidator: stubValidator, + promQLValidator: stubValidator, + jqValidator: stubValidator, + } + for _, f := range opts { + f(&opt) + } + return opt +} diff --git a/pkg/ast/opts_test.go b/pkg/ast/opts_test.go new file mode 100644 index 0000000..51fc181 --- /dev/null +++ b/pkg/ast/opts_test.go @@ -0,0 +1,145 @@ +package ast + +import ( + "errors" + "testing" +) + +func TestWithStrict(t *testing.T) { + opt := parseOpts(WithStrict(true)) + if !opt.strict { + t.Errorf("WithStrict(true) did not set strict to true") + } + opt = parseOpts(WithStrict(false)) + if opt.strict { + t.Errorf("WithStrict(false) did not set strict to false") + } +} + +func TestWithMaxGen(t *testing.T) { + opt := parseOpts(WithMaxGen(123)) + if opt.maxGen != 123 { + t.Errorf("WithMaxGen(123) did not set maxGen to 123, got %d", opt.maxGen) + } +} + +func TestWithMaxRank(t *testing.T) { + opt := parseOpts(WithMaxRank(456)) + if opt.maxRank != 456 { + t.Errorf("WithMaxRank(456) did not set maxRank to 456, got %d", opt.maxRank) + } +} + +func TestWithMaxDepth(t *testing.T) { + opt := parseOpts(WithMaxDepth(789)) + if opt.maxDepth != 789 { + t.Errorf("WithMaxDepth(789) did not set maxDepth to 789, got %d", opt.maxDepth) + } +} + +func TestWithJQValidator(t *testing.T) { + called := false + validator := func(s string) error { + called = true + if s == "fail" { + return errors.New("fail") + } + return nil + } + opt := parseOpts(WithJQValidator(validator)) + if err := opt.jqValidator("ok"); err != nil { + t.Errorf("jqValidator returned error for 'ok': %v", err) + } + if !called { + t.Errorf("jqValidator was not called") + } + called = false + err := opt.jqValidator("fail") + if err == nil || err.Error() != "fail" { + t.Errorf("jqValidator did not return expected error for 'fail'") + } +} + +func TestWithLuaValidator(t *testing.T) { + called := false + validator := func(s string) error { + called = true + if s == "bad" { + return errors.New("bad") + } + return nil + } + opt := parseOpts(WithLuaValidator(validator)) + if err := opt.luaValidator("ok"); err != nil { + t.Errorf("luaValidator returned error for 'ok': %v", err) + } + if !called { + t.Errorf("luaValidator was not called") + } + called = false + err := opt.luaValidator("bad") + if err == nil || err.Error() != "bad" { + t.Errorf("luaValidator did not return expected error for 'bad'") + } +} + +func TestWithPromQLValidator(t *testing.T) { + called := false + validator := func(s string) error { + called = true + if s == "nope" { + return errors.New("nope") + } + return nil + } + opt := parseOpts(WithPromQLValidator(validator)) + if err := opt.promQLValidator("ok"); err != nil { + t.Errorf("promQLValidator returned error for 'ok': %v", err) + } + if !called { + t.Errorf("promQLValidator was not called") + } + called = false + err := opt.promQLValidator("nope") + if err == nil || err.Error() != "nope" { + t.Errorf("promQLValidator did not return expected error for 'nope'") + } +} + +func TestSelectValidator(t *testing.T) { + // nil returns stubValidator + v := selectValidator(nil) + if v == nil { + t.Errorf("selectValidator(nil) should not return nil") + } + if err := v("anything"); err != nil { + t.Errorf("stubValidator should always return nil") + } + // non-nil returns the same + called := false + myValidator := func(string) error { called = true; return nil } + v = selectValidator(myValidator) + v("test") + if !called { + t.Errorf("selectValidator did not return the provided validator") + } +} + +func TestParseOpts_Defaults(t *testing.T) { + opt := parseOpts() + if opt.maxGen != defaultMaxGen { + t.Errorf("default maxGen = %d, want %d", opt.maxGen, defaultMaxGen) + } + if opt.maxRank != defaultMaxRank { + t.Errorf("default maxRank = %d, want %d", opt.maxRank, defaultMaxRank) + } + if opt.maxDepth != defaultMaxDepth { + t.Errorf("default maxDepth = %d, want %d", opt.maxDepth, defaultMaxDepth) + } + if opt.strict { + t.Errorf("default strict should be false") + } + if opt.jqValidator == nil || opt.luaValidator == nil || opt.promQLValidator == nil { + t.Errorf("default validators should not be nil") + } +} diff --git a/pkg/ast/parse.go b/pkg/ast/parse.go new file mode 100644 index 0000000..875edd7 --- /dev/null +++ b/pkg/ast/parse.go @@ -0,0 +1,144 @@ +package ast + +import ( + "errors" + "fmt" + + "github.com/goccy/go-yaml" + "github.com/goccy/go-yaml/ast" + "github.com/goccy/go-yaml/parser" +) + +type parserT struct { + maxGen uint32 + maxRank uint32 + maxDepth uint32 + strict bool + root ast.Node + validateJQ ValidatorFunc + validateLua ValidatorFunc + validatePromQL ValidatorFunc +} + +func ParseRules(yamlInput []byte, opts ...ParseOpt) ([]AstRuleT, error) { + o := parseOpts(opts...) + + p := parserT{ + maxGen: o.maxGen, + maxRank: o.maxRank, + maxDepth: o.maxDepth, + strict: o.strict, + validateJQ: o.jqValidator, + validateLua: o.luaValidator, + validatePromQL: o.promQLValidator, + } + + return p.parse(yamlInput) +} + +func (p *parserT) parse(yamlInput []byte) ([]AstRuleT, error) { + + // First parse the YAML into a yaml AST ignoring comments. + doc, err := parser.ParseBytes(yamlInput, 0) + if err != nil { + return nil, err + } + + // A yaml file can contain multiple documents; + // iterate across the documents and parse each one separately as a rule document. + + var ( + rules []AstRuleT + errList []error + ) + + for _, d := range doc.Docs { + nRules, err := p.parseDocument(d) + + switch err { + case nil: + rules = append(rules, nRules...) + default: + errList = append(errList, err) + } + } + + return rules, errors.Join(errList...) +} + +// A single YAML document should container a map with a single key "rules" that maps to a list of rules. + +func (p *parserT) parseDocument(doc *ast.DocumentNode) ([]AstRuleT, error) { + + // Set the root of the document for error reporting purposes. + p.root = doc.Body + defer func() { + p.root = nil + }() + + mapping, err := p.nodeToMapping(doc.Body) + if err != nil { + return nil, err + } + + var ( + hasRules bool + rules []AstRuleT + ) + + for _, v := range mapping.Values { + + key, err := p.nodeToString(v.Key) + if err != nil { + return nil, err + } + + switch key { + + case kwRules: + hasRules = true + if rules, err = p.parseRulesNode(v.Value); err != nil { + return nil, err + } + + default: + err := fmt.Errorf("unexpected key '%s' in document body", key) + return nil, p.wrapError(v.Key, err) + } + } + + if !hasRules { + err := fmt.Errorf("%w: %s", ErrMissingKey, kwRules) + return nil, p.wrapError(mapping, err) + } + + return rules, nil +} + +func (p *parserT) rewriteError(err error) error { + + if yErr, ok := err.(yaml.Error); ok { + return ParseError{ + token: yErr.GetToken(), + err: err, + } + } + + return err +} + +func (p *parserT) wrapErrorParent(node ast.Node, err error) error { + parent := ast.Parent(p.root, node) + return p.wrapError(parent, err) +} + +func (p *parserT) wrapError(node ast.Node, err error) error { + if node == nil { + return p.rewriteError(err) + } + + return ParseError{ + token: node.GetToken(), + err: err, + } +} diff --git a/pkg/ast/parse_cre.go b/pkg/ast/parse_cre.go new file mode 100644 index 0000000..ab874ae --- /dev/null +++ b/pkg/ast/parse_cre.go @@ -0,0 +1,198 @@ +package ast + +import ( + "fmt" + "regexp" + + "github.com/goccy/go-yaml/ast" +) + +var validCreIdRegex = regexp.MustCompile(`^[A-Za-z0-9-]{4,}$`) + +func (p *parserT) parseCreNode(node ast.Node) (*AstCreT, error) { + + mapping, err := p.nodeToMapping(node) + if err != nil { + return nil, err + } + + var ( + cre AstCreT + ) + + for _, v := range mapping.Values { + + key, err := p.nodeToString(v.Key) + if err != nil { + return nil, err + } + + switch key { + case kwCreId: + cre.Id, err = p.parseCreId(v.Value) + + case kwSeverity: + cre.Severity, err = p.parseSeverityNode(v.Value) + + case kwTitle: + cre.Title, err = p.nodeToString(v.Value) + + case kwCategory: + cre.Category, err = p.nodeToString(v.Value) + + case kwTags: + cre.Tags, err = p.nodeToStrs(v.Value) + + case kwAuthor: + cre.Author, err = p.nodeToString(v.Value) + + case kwDescription: + cre.Description, err = p.nodeToString(v.Value) + + case kwImpact: + cre.Impact, err = p.nodeToString(v.Value) + + case kwImpactScore: + cre.ImpactScore, err = p.nodeToUint(v.Value) + + case kwCause: + cre.Cause, err = p.nodeToString(v.Value) + + case kwMitigation: + cre.Mitigation, err = p.nodeToString(v.Value) + + case kwMitigationScore: + cre.MitigationScore, err = p.nodeToUint(v.Value) + + case kwReferences: + cre.References, err = p.nodeToStrs(v.Value) + + case kwReports: + cre.Reports, err = p.nodeToUint(v.Value) + + case kwApplications: + cre.Applications, err = p.parseApplicationsNode(v.Value) + + default: + if p.strict { + err = p.wrapError(v.Key, ErrUnexpectedKey) + } + } + + if err != nil { + return nil, err + } + } + + if cre.Id == "" { + err := fmt.Errorf("%w: %s", ErrMissingKey, kwCreId) + return nil, p.wrapErrorParent(mapping, err) + } + + return &cre, nil +} + +func (p *parserT) parseCreId(v ast.Node) (string, error) { + s, err := p.nodeToString(v) + if err != nil { + return "", err + } + + if !validCreIdRegex.MatchString(s) { + err := fmt.Errorf("%w: id value must be at least 4 characters and contain only letters, numbers, or hyphens", ErrBadIdentifier) + return "", p.wrapError(v, err) + } + return s, nil +} + +func (p *parserT) parseSeverityNode(v ast.Node) (uint, error) { + severity, err := p.nodeToUint(v) + if err != nil { + return 0, err + } + + switch severity { + case SeverityCritical, SeverityHigh, SeverityMedium, SeverityLow, SeverityInfo: + + default: + + if p.strict { + err := fmt.Errorf("%w: severity value must be between 0 and 4", ErrBadSeverity) + return 0, p.wrapError(v, err) + } + } + return severity, nil +} + +func (p *parserT) parseApplicationsNode(v ast.Node) ([]AstAppT, error) { + apps := []AstAppT{} + + seq, err := p.nodeToSequence(v) + if err != nil { + return nil, err + } + + for _, appNode := range seq.Values { + app, err := p.parseApplicationNode(appNode) + if err != nil { + return nil, err + } + apps = append(apps, *app) + } + + return apps, nil +} + +func (p *parserT) parseApplicationNode(node ast.Node) (*AstAppT, error) { + + mapping, err := p.nodeToMapping(node) + if err != nil { + return nil, err + } + + var ( + app AstAppT + ) + + for _, v := range mapping.Values { + + key, err := p.nodeToString(v.Key) + if err != nil { + return nil, err + } + + switch key { + case kwAppName: + app.Name, err = p.nodeToString(v.Value) + + case kwAppProcessName: + app.ProcessName, err = p.nodeToString(v.Value) + + case kwAppProcessPath: + app.ProcessPath, err = p.nodeToString(v.Value) + + case kwAppContainer: + app.ContainerName, err = p.nodeToString(v.Value) + + case kwAppImage: + app.ImageUrl, err = p.nodeToString(v.Value) + + case kwAppRepo: + app.RepoUrl, err = p.nodeToString(v.Value) + + case kwAppVersion: + app.Version, err = p.nodeToString(v.Value) + + default: + if p.strict { + err = p.wrapError(v, fmt.Errorf("%w: %s", ErrUnexpectedKey, key)) + } + } + + if err != nil { + return nil, err + } + } + + return &app, nil +} diff --git a/pkg/ast/parse_cre_test.go b/pkg/ast/parse_cre_test.go new file mode 100644 index 0000000..3c41bbe --- /dev/null +++ b/pkg/ast/parse_cre_test.go @@ -0,0 +1,278 @@ +package ast + +import ( + "reflect" + "testing" + + "github.com/goccy/go-yaml/ast" + "github.com/goccy/go-yaml/parser" +) + +func mustParseYAMLNode(t *testing.T, src string) ast.Node { + t.Helper() + + doc, err := parser.ParseBytes([]byte(src), 0) + if err != nil { + t.Fatalf("failed to parse yaml: %v", err) + } + + if len(doc.Docs) != 1 { + t.Fatalf("expected exactly one document, got %d", len(doc.Docs)) + } + + return doc.Docs[0].Body +} + +func TestParseCreNode_TableDriven(t *testing.T) { + tests := []struct { + name string + yaml string + strict bool + wantErr error + wantPos int + wants AstCreT + }{ + { + name: "valid CRE minimal", + yaml: ` +id: CRE-1234 +title: Example CRE +`, + strict: true, + wants: AstCreT{ + Id: "CRE-1234", + Title: "Example CRE", + }, + }, + { + name: "valid CRE with everything", + yaml: ` +id: PREQUEL-2024-0006 +severity: 2 +title: Kafka Topic Operator Thread Blocked +category: message-queue-problem +author: Prequel +description: | + There is a known issue in the Strimzi Kafka. +cause: | + The Kafka Topic Operator is using a single thread to process events. +impact: | + No Kafka topics will be created or updated. +tags: +- known-problem +- kafka +- strimzi +mitigation: | + - Add additional CPU resources and restart the Kafka Topic Operator + - Use the Zookeeper store instead of the Kafka Streams store for the Strimzi Kafka Topic Operator +mitigationScore: 2 +impactScore: 8 +reports: 11 +references: +- https://github.com/strimzi/strimzi-kafka-operator/issues/6046 +applications: +- name: "kafka" +`, + strict: true, + wants: AstCreT{ + Id: "PREQUEL-2024-0006", + Title: "Kafka Topic Operator Thread Blocked", + Severity: 2, + Category: "message-queue-problem", + Author: "Prequel", + Description: "There is a known issue in the Strimzi Kafka.\n", + Cause: "The Kafka Topic Operator is using a single thread to process events.\n", + Impact: "No Kafka topics will be created or updated.\n", + Tags: []string{"known-problem", "kafka", "strimzi"}, + Mitigation: "- Add additional CPU resources and restart the Kafka Topic Operator\n- Use the Zookeeper store instead of the Kafka Streams store for the Strimzi Kafka Topic Operator\n", + MitigationScore: 2, + ImpactScore: 8, + Reports: 11, + References: []string{"https://github.com/strimzi/strimzi-kafka-operator/issues/6046"}, + Applications: []AstAppT{{Name: "kafka"}}, + }, + }, + { + name: "cre id is wrong type", + yaml: `id: 112333`, + strict: true, + wantErr: ErrUnexpectedType, + wantPos: 5, // Seems to be one based. + }, + { + name: "invalid id (too short)", + yaml: ` +id: ab +title: Bad CRE +`, + strict: true, + wantErr: ErrBadIdentifier, + wantPos: 6, // Pos of 'ab', one based. + }, + { + name: "unexpected key in strict mode", + yaml: ` +id: CRE-9999 +title: Strict CRE +unexpected: value +`, + strict: true, + wantErr: ErrUnexpectedKey, + wantPos: 33, // Pos of 'unexpected', one based. + }, + { + name: "unexpected key in non-strict mode", + yaml: ` +id: CRE-9999 +title: NonStrict CRE +unexpected: value +`, + strict: false, + wants: AstCreT{ + Id: "CRE-9999", + Title: "NonStrict CRE", + }, + }, + { + name: "valid CRE with applications", + yaml: ` +id: CRE-8888 +title: App CRE +applications: + - name: app1 + version: v1 + processName: nginx + processPath: /usr/sbin/nginx + containerName: nginx-container + imageUrl: nginx:latest + repoUrl: github.com/nginx/nginx + - name: app2 + version: v2 +`, + strict: true, + wants: AstCreT{ + Id: "CRE-8888", + Title: "App CRE", + Applications: []AstAppT{ + { + Name: "app1", + Version: "v1", + ProcessName: "nginx", + ProcessPath: "/usr/sbin/nginx", + ContainerName: "nginx-container", + ImageUrl: "nginx:latest", + RepoUrl: "github.com/nginx/nginx", + }, + {Name: "app2", Version: "v2"}, + }, + }, + }, + { + name: "strict app with extra key", + yaml: ` +applications: + - name: app1 + unexpected: value +`, + strict: true, + wantErr: ErrUnexpectedKey, + wantPos: 45, // Pos of ':' in 'unexpected:' + }, + { + name: "strict app with extra key non-strict", + yaml: ` +id: CRE-7777 +applications: + - name: app1 + unexpected: value +`, + strict: false, + wants: AstCreT{ + Id: "CRE-7777", + Applications: []AstAppT{ + {Name: "app1"}, + }, + }, + }, + { + name: "bad app type", + yaml: ` +applications: badtype +`, + wantErr: ErrUnexpectedType, + wantPos: 16, // Pos of 'b' in 'badtype' + }, + { + name: "bad app value type", + yaml: ` +applications: + - notamapping +`, + wantErr: ErrUnexpectedType, + wantPos: 20, // Pos of 'n' in 'notamapping' + }, + { + name: "bad app key", + yaml: ` +applications: + - 11: badkey +`, + strict: true, + wantErr: ErrUnexpectedType, + wantPos: 20, // Pos of '11' in '11: badkey' + }, + { + name: "valid severity value", + yaml: ` +id: CRE-5555 +severity: 3 +`, + strict: true, + wants: AstCreT{ + Id: "CRE-5555", + Severity: SeverityLow, + }, + }, + { + name: "negative severity value", + yaml: `severity: -1`, + strict: true, + wantErr: ErrUnexpectedType, + wantPos: 11, // Pos of '-' in '-1' + }, + { + name: "invalid severity value", + yaml: `severity: 11`, + strict: true, + wantErr: ErrBadSeverity, + }, + { + name: "invalid node type", + yaml: `shrubbery`, + wantErr: ErrUnexpectedType, + wantPos: 1, // Pos of 's' in 'shrubbery' + }, + { + name: "invalid mapping key type", + yaml: `11: invalid key type`, + wantErr: ErrUnexpectedType, + wantPos: 1, // Pos of '11' in '11: invalid key type' + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + node := mustParseYAMLNode(t, tt.yaml) + p := &parserT{strict: tt.strict} + cre, err := p.parseCreNode(node) + + if !checkParserError(t, err, tt.wantErr, tt.wantPos) { + return + } + + if !reflect.DeepEqual(*cre, tt.wants) { + t.Errorf("expected CRE %+v, got %+v", tt.wants, *cre) + } + }) + } +} diff --git a/pkg/ast/parse_event.go b/pkg/ast/parse_event.go new file mode 100644 index 0000000..e479857 --- /dev/null +++ b/pkg/ast/parse_event.go @@ -0,0 +1,58 @@ +package ast + +import ( + "github.com/goccy/go-yaml/ast" +) + +func (p *parserT) parseEventNode(state ruleState, node ast.Node) (*AstEventT, error) { + + mapping, err := p.nodeToMapping(node) + if err != nil { + return nil, err + } + + var ( + event AstEventT + ) + + for _, v := range mapping.Values { + key, err := p.nodeToString(v.Key) + if err != nil { + return nil, err + } + + switch key { + case kwOrigin: + event.Origin, err = p.parseOrigin(state, v.Value) + case kwSource: + event.Source, err = p.nodeToString(v.Value) + default: + if p.strict { + err = p.wrapError(v.Key, ErrUnexpectedKey) + } + } + + if err != nil { + return nil, err + } + } + + if event.Source == "" { + return nil, p.wrapErrorParent(node, ErrMissingSource) + } + + return &event, nil +} + +func (p *parserT) parseOrigin(state ruleState, node ast.Node) (bool, error) { + b, err := p.nodeToBool(node) + if err != nil { + return false, err + } + + if b && (state.incOrigin() > 1) { + return false, p.wrapError(node, ErrMultipleOrigin) + } + + return b, nil +} diff --git a/pkg/ast/parse_event_test.go b/pkg/ast/parse_event_test.go new file mode 100644 index 0000000..9a6b6b1 --- /dev/null +++ b/pkg/ast/parse_event_test.go @@ -0,0 +1,148 @@ +package ast + +import ( + "reflect" + "testing" + + "github.com/goccy/go-yaml/ast" +) + +func TestParseEventNode(t *testing.T) { + tests := []struct { + name string + yamlInput string + strict bool + want *AstEventT + wantErr error + wantPos int + }{ + { + name: "valid event with both fields", + yamlInput: ` +event: + source: "syslog" + origin: true +`, + strict: true, + want: &AstEventT{ + Source: "syslog", + Origin: true, + }, + }, + { + name: "valid event with only source", + yamlInput: ` +event: + source: "auditd" +`, + strict: true, + want: &AstEventT{ + Source: "auditd", + Origin: false, + }, + }, + { + name: "invalid event missing source", + yamlInput: ` +event: + origin: true +`, + strict: true, + wantErr: ErrMissingSource, + wantPos: 7, + }, + { + name: "invalid type for source", + yamlInput: ` +event: + source: 123 + origin: true +`, + strict: true, + want: nil, + wantErr: ErrUnexpectedType, + wantPos: 19, // Pos of '1' in '123' + }, + { + name: "invalid type for origin", + yamlInput: ` +event: + source: "syslog" + origin: "yes" +`, + strict: true, + want: nil, + wantErr: ErrUnexpectedType, + wantPos: 38, // Pos of '"' in '"yes"' + }, + { + name: "unexpected key in strict mode", + yamlInput: ` +event: + source: "syslog" + origin: true + extra: "field" +`, + strict: true, + want: nil, + wantErr: ErrUnexpectedKey, + wantPos: 45, // Pos of ':' in 'extra:' + }, + { + name: "unexpected key in non-strict mode", + yamlInput: ` +event: + source: "syslog" + origin: true + extra: "field" +`, + strict: false, + want: &AstEventT{ + Source: "syslog", + Origin: true, + }, + }, + { + name: "unexpected mapping type for event", + yamlInput: ` +event: wrongtype +`, + strict: false, + wantErr: ErrUnexpectedType, + wantPos: 9, // Pos of 'w' in 'wrongtype' + }, + { + name: "unexpected key type", + yamlInput: ` +event: + 123: "syslog" +`, + strict: false, + wantErr: ErrUnexpectedType, + wantPos: 11, // Pos of '1' in '123' + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + node := mustParseYAMLNode(t, tt.yamlInput) + // Should be mapping Node with single key "event" + mapping, ok := node.(*ast.MappingNode) + if !ok || len(mapping.Values) != 1 { + t.Fatalf("expected a mapping node with one key, got %T with %d keys", node, len(mapping.Values)) + } + v := mapping.Values[0] + p := &parserT{strict: tt.strict, root: node} + state := newRuleState(&AstMetadataT{Id: "test", Hash: "hash"}) + got, err := p.parseEventNode(state, v.Value) + + if !checkParserError(t, err, tt.wantErr, tt.wantPos) { + return + } + + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("parseEventNode() = %+v, want %+v", got, tt.want) + } + }) + } +} diff --git a/pkg/ast/parse_extract.go b/pkg/ast/parse_extract.go new file mode 100644 index 0000000..2b79465 --- /dev/null +++ b/pkg/ast/parse_extract.go @@ -0,0 +1,128 @@ +package ast + +import ( + "fmt" + "regexp" + + "github.com/goccy/go-yaml/ast" +) + +var validateExtractName = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_]*$`) + +func (p *parserT) parseExtracts(node ast.Node) ([]AstExtractT, error) { + + seq, err := p.nodeToSequence(node) + if err != nil { + return nil, err + } + + var ( + extracts []AstExtractT + dupes = make(map[string]struct{}, len(seq.Values)) // Track extract names to detect duplicates. + ) + + for _, v := range seq.Values { + + extract, err := p.parseExtractNode(v, dupes) + if err != nil { + return nil, err + } + + extracts = append(extracts, *extract) + } + + if len(extracts) == 0 && p.strict { + err := fmt.Errorf("%w: 'extract' must contain at least one extract definition", ErrMissingKey) + return nil, p.wrapError(node, err) + } + + return extracts, nil +} + +func (p *parserT) parseExtractNode(node ast.Node, dupeMap map[string]struct{}) (*AstExtractT, error) { + + mapping, err := p.nodeToMapping(node) + if err != nil { + return nil, err + } + + var ( + hasValue bool + extract AstExtractT + ) + + checkConflict := func(v ast.Node, key string) error { + if hasValue { + err := fmt.Errorf("%w: multiple value keys in extract: %s", ErrUnexpectedKey, key) + return p.wrapError(v, err) + } + hasValue = true + return nil + } + + for _, v := range mapping.Values { + + key, err := p.nodeToString(v.Key) + if err != nil { + return nil, err + } + + switch key { + case kwExtractName: + extract.Name, err = p.parseExtractName(v.Value) + + if err == nil { + if _, exists := dupeMap[extract.Name]; exists { + err = fmt.Errorf("%w: '%s' is duplicated", ErrDupeExtractName, extract.Name) + return nil, p.wrapError(v.Value, err) + } + dupeMap[extract.Name] = struct{}{} + } + + case kwExtractJq: + if err := checkConflict(v, key); err != nil { + return nil, err + } + extract.JqValue, err = p.nodeToJq(v.Value) + + case kwExtractRegex: + if err := checkConflict(v, key); err != nil { + return nil, err + } + var exp *regexp.Regexp + if exp, err = p.nodeToRegex(v.Value); err == nil { + extract.RegexValue = exp.String() + } + + default: + err = p.wrapError(v.Key, ErrUnexpectedKey) + } + + if err != nil { + return nil, err + } + } + + if extract.Name == "" { + err := fmt.Errorf("%w: missing required '%s' key in extract definition", ErrMissingKey, kwExtractName) + return nil, p.wrapErrorParent(node, err) + } + + return &extract, nil +} + +func (p *parserT) parseExtractName(v ast.Node) (string, error) { + + s, err := p.nodeToString(v) + if err != nil { + return "", err + } + + // Ignore strict here; a valid extract name is required for correct operation. + if !validateExtractName.MatchString(s) { + err := fmt.Errorf("%w: extract name must start with a letter and contain only letters, numbers, or underscores", ErrBadExtractName) + return "", p.wrapError(v, err) + } + return s, nil + +} diff --git a/pkg/ast/parse_extract_test.go b/pkg/ast/parse_extract_test.go new file mode 100644 index 0000000..ea855e2 --- /dev/null +++ b/pkg/ast/parse_extract_test.go @@ -0,0 +1,277 @@ +package ast + +import ( + "fmt" + "reflect" + "testing" + + "github.com/goccy/go-yaml/ast" +) + +func TestParseExtracts(t *testing.T) { + tests := []struct { + name string + yamlInput string + strict bool + want []AstExtractT + wantErr error + wantPos int + failJQ bool + }{ + { + name: "valid extract with name and jq", + yamlInput: ` +extract: + - name: "example" + jq: ".field" +`, + strict: true, + want: []AstExtractT{{ + Name: "example", + JqValue: ".field", + }, + }, + }, + { + name: "valid extract with name and jq in unusual order", + yamlInput: ` +extract: + - jq: ".field" + name: "example" +`, + strict: true, + want: []AstExtractT{{ + Name: "example", + JqValue: ".field", + }, + }, + }, + { + name: "multiple valid extractions", + yamlInput: ` +extract: + - name: "example" + regex: ".*" +`, + strict: true, + want: []AstExtractT{{ + Name: "example", + RegexValue: ".*", + }, + }, + }, + { + name: "multiple valid extractions with different types", + yamlInput: ` +extract: + - name: "example" + regex: ".*" + - name: "another_example" + jq: ".field" + - name: "yet_another_example" + regex: "xxx.*xxx" +`, + strict: true, + want: []AstExtractT{ + { + Name: "example", + RegexValue: ".*", + }, + { + Name: "another_example", + JqValue: ".field", + }, + { + Name: "yet_another_example", + RegexValue: "xxx.*xxx", + }, + }, + }, + { + name: "invalid name", + yamlInput: ` +extract: + - name: "???" + jq: ".field" +`, + wantErr: ErrBadExtractName, + wantPos: 21, // Position of first '?' in '???' + + }, + { + name: "empty name", + yamlInput: ` +extract: + - name: "" + jq: ".field" +`, + wantErr: ErrBadExtractName, + wantPos: 21, // Position of empty string in 'name: ""' + + }, + { + name: "invalid name type", + yamlInput: ` +extract: + - name: 123 + jq: ".field" +`, + wantErr: ErrUnexpectedType, + wantPos: 21, // Position of first '1' in '123' + + }, + { + name: "missing name", + yamlInput: ` +extract: + - jq: ".field" +`, + wantErr: ErrMissingKey, + wantPos: 13, + }, + { + name: "duplicate extract names", + yamlInput: ` +extract: + - name: "example" + jq: ".field" + - name: "example" + regex: ".*" +`, + wantErr: ErrDupeExtractName, + wantPos: 58, + }, + { + name: "bad regex value", + yamlInput: ` +extract: + - name: "example" + regex: "[abc" +`, + wantErr: ErrBadRegex, + wantPos: 42, + }, + { + name: "bad jq value", + yamlInput: ` +extract: + - name: "example" + jq: "(.foo" +`, + wantErr: ErrBadJq, + wantPos: 39, + failJQ: true, + }, + { + name: "bad jq value but no validator", + yamlInput: ` +extract: + - name: "example" + jq: "(.foo" +`, + failJQ: false, + }, + { + name: "conflicting values", + yamlInput: ` +extract: + - name: "example" + jq: "(.foo" + regex: ".*" +`, + wantErr: ErrUnexpectedKey, + wantPos: 56, + }, + { + name: "conflicting values reversed", + yamlInput: ` +extract: + - name: "example" + regex: ".*" + jq: "(.foo" +`, + wantErr: ErrUnexpectedKey, + wantPos: 53, + }, + { + name: "no extracts defined [strict mode]", + yamlInput: ` +extract: [] +`, + strict: true, + wantErr: ErrMissingKey, + wantPos: 11, // Position of '[' in 'extract: []' + }, + { + name: "no extracts defined [non strict mode]", + yamlInput: ` +extract: [] +`, + strict: false, + }, + { + name: "bad extracts type", + yamlInput: ` +extract: "not a sequence" +`, + wantErr: ErrUnexpectedType, + wantPos: 11, // Position of '"' in 'not a sequence' + }, + { + name: "bad extract type", + yamlInput: ` +extract: +- not a mapping +`, + wantErr: ErrUnexpectedType, + wantPos: 13, // Position of 'n' in '- not a mapping' + }, + { + name: "bad extract key type", + yamlInput: ` +extract: +- 123: "value" +`, + wantErr: ErrUnexpectedType, + wantPos: 13, // Position of '1' in '- 123: "value"' + }, + { + name: "unexpected key in extract", + yamlInput: ` +extract: + - name: "example" + jq: ".field" + shrubbery: "value" +`, + wantErr: ErrUnexpectedKey, + wantPos: 52, // Position of 'shrubbery' in 'shrubbery: "value"' + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + node := mustParseYAMLNode(t, tt.yamlInput) + // Should be mapping Node with single key "extract" + mapping, ok := node.(*ast.MappingNode) + if !ok || len(mapping.Values) != 1 { + t.Fatalf("expected a mapping node with one key, got %T with %d keys", node, len(mapping.Values)) + } + v := mapping.Values[0] + p := &parserT{strict: tt.strict, root: node, validateJQ: stubValidator} + if tt.failJQ { + p.validateJQ = func(v string) error { + return fmt.Errorf("invalid jq expression: %s", v) + } + } + got, err := p.parseExtracts(v.Value) + + if !checkParserError(t, err, tt.wantErr, tt.wantPos) { + return + } + + if tt.want != nil && !reflect.DeepEqual(got, tt.want) { + t.Errorf("parseExtracts() = %+v, want %+v", got, tt.want) + } + }) + } +} diff --git a/pkg/ast/parse_help.go b/pkg/ast/parse_help.go new file mode 100644 index 0000000..8ff6a32 --- /dev/null +++ b/pkg/ast/parse_help.go @@ -0,0 +1,213 @@ +package ast + +import ( + "errors" + "fmt" + "regexp" + "time" + + "github.com/goccy/go-yaml/ast" +) + +func (p *parserT) nodeToMapping(node ast.Node) (*ast.MappingNode, error) { + if node == nil { + return nil, fmt.Errorf("%w: expected yaml mapping, got null", ErrUnexpectedType) + } + mapping, ok := node.(*ast.MappingNode) + if !ok { + err := fmt.Errorf("%w: expected yaml mapping, got %s", ErrUnexpectedType, node.Type()) + return nil, p.wrapError(node, err) + } + return mapping, nil +} + +func (p *parserT) nodeToSequence(node ast.Node) (*ast.SequenceNode, error) { + if node == nil { + return nil, fmt.Errorf("%w: expected yaml sequence, got null", ErrUnexpectedType) + } + seq, ok := node.(*ast.SequenceNode) + if !ok { + err := fmt.Errorf("%w: expected yaml sequence, got %s", ErrUnexpectedType, node.Type()) + return nil, p.wrapError(node, err) + } + return seq, nil +} + +func (p *parserT) nodeToString(node ast.Node) (string, error) { + if node == nil { + return "", fmt.Errorf("%w: expected yaml string, got null", ErrUnexpectedType) + } + + var s string + + switch v := node.(type) { + case *ast.StringNode: + s = v.Value + case *ast.LiteralNode: + if v.Value == nil { + err := fmt.Errorf("%w: literal node value is null", ErrUnexpectedType) + return "", p.wrapError(node, err) + } + s = v.Value.Value + default: + err := fmt.Errorf("%w: expected yaml string, got %s", ErrUnexpectedType, node.Type()) + return "", p.wrapError(node, err) + } + return s, nil +} + +func (p *parserT) nodeToInt64(node ast.Node) (int64, error) { + if node == nil { + return 0, fmt.Errorf("%w: expected yaml integer, got null", ErrUnexpectedType) + } + v, ok := node.(*ast.IntegerNode) + if !ok { + err := fmt.Errorf("%w: %s", ErrUnexpectedType, node.Type()) + return 0, p.wrapError(node, err) + } + + ival, ok := v.Value.(int64) + if !ok { + err := fmt.Errorf("%w: integer value out of range: %v", ErrUnexpectedType, v.Value) + return 0, p.wrapError(node, err) + } + + return ival, nil +} + +func (p *parserT) nodeToUint64(node ast.Node) (uint64, error) { + if node == nil { + return 0, fmt.Errorf("%w: expected yaml integer, got null", ErrUnexpectedType) + } + v, ok := node.(*ast.IntegerNode) + if !ok { + err := fmt.Errorf("%w: %s", ErrUnexpectedType, node.Type()) + return 0, p.wrapError(node, err) + } + + ival, ok := v.Value.(uint64) + if !ok { + err := fmt.Errorf("%w: integer value out of range: %v", ErrUnexpectedType, v.Value) + return 0, p.wrapError(node, err) + } + + return ival, nil +} + +func (p *parserT) nodeToUint(v ast.Node) (uint, error) { + n, err := p.nodeToUint64(v) + if err != nil { + return 0, err + } + // if n > math.MaxUint { + // // This is a theoretical limit since uint is typically either 32 or 64 bits depending on the platform, + // // but we enforce it to prevent potential overflow issues when converting from uint64 to uint. + // err := fmt.Errorf("%w: value must be a positive integer", ErrOverflow) + // return 0, p.wrapError(v, err) + // } + return uint(n), nil +} + +func (p *parserT) nodeToBool(node ast.Node) (bool, error) { + if node == nil { + return false, fmt.Errorf("%w: expected yaml boolean, got null", ErrUnexpectedType) + } + v, ok := node.(*ast.BoolNode) + if !ok { + err := fmt.Errorf("%w: %s", ErrUnexpectedType, node.Type()) + return false, p.wrapError(node, err) + } + return v.Value, nil +} + +func (p *parserT) nodeToStrs(node ast.Node) ([]string, error) { + seq, err := p.nodeToSequence(node) + if err != nil { + return nil, err + } + + var strs []string + for _, v := range seq.Values { + ss, err := p.nodeToString(v) + if err != nil { + return nil, err + } + strs = append(strs, ss) + } + + return strs, nil +} + +func (p *parserT) nodeToRegex(node ast.Node) (*regexp.Regexp, error) { + v, err := p.nodeToString(node) + if err != nil { + return nil, p.wrapError(node, err) + } + if v == "" { + err := fmt.Errorf("%w: regex pattern cannot be empty", ErrBadRegex) + return nil, p.wrapError(node, err) + } + exp, err := regexp.Compile(v) + if err != nil { + err = errors.Join(ErrBadRegex, err) + return nil, p.wrapError(node, err) + } + return exp, nil +} + +func (p *parserT) nodeToJq(node ast.Node) (string, error) { + v, err := p.nodeToString(node) + if err != nil { + return "", p.wrapError(node, err) + } + if v == "" { + err := fmt.Errorf("%w: jq expression cannot be empty", ErrBadJq) + return "", p.wrapError(node, err) + } + if err := p.validateJQ(v); err != nil { + err := errors.Join(ErrBadJq, err) + return "", p.wrapError(node, err) + } + + return v, nil +} + +func (p *parserT) nodeToDuration(node ast.Node) (time.Duration, error) { + v, err := p.nodeToString(node) + if err != nil { + return 0, p.wrapError(node, err) + } + + w, err := time.ParseDuration(v) + if err != nil { + err := fmt.Errorf("%w: invalid duration format: %v", ErrUnexpectedType, err) + return 0, p.wrapError(node, err) + } + + return w, nil +} + +func (p *parserT) nodeToDurationPositive(node ast.Node) (time.Duration, error) { + dur, err := p.nodeToDuration(node) + if err != nil { + return 0, err + } + if dur <= 0 { + err := fmt.Errorf("%w: duration must be positive", ErrUnexpectedType) + return 0, p.wrapError(node, err) + } + return dur, nil +} + +func findKey(node ast.Node, key string) ast.Node { + mapping, ok := node.(*ast.MappingNode) + if !ok { + return nil + } + for _, v := range mapping.Values { + if k, ok := v.Key.(*ast.StringNode); ok && k.Value == key { + return v.Key + } + } + return nil +} diff --git a/pkg/ast/parse_help_test.go b/pkg/ast/parse_help_test.go new file mode 100644 index 0000000..c969744 --- /dev/null +++ b/pkg/ast/parse_help_test.go @@ -0,0 +1,396 @@ +package ast + +import ( + "errors" + "reflect" + "regexp" + "testing" + "time" + + "github.com/goccy/go-yaml/ast" +) + +func TestNodeToMapping(t *testing.T) { + p := &parserT{} + tests := []struct { + name string + node ast.Node + wantErr bool + }{ + {"nil node", nil, true}, + {"not mapping", &ast.StringNode{Value: "foo"}, true}, + {"mapping", &ast.MappingNode{}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := p.nodeToMapping(tt.node) + if (err != nil) != tt.wantErr { + t.Errorf("nodeToMapping() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestNodeToSequence(t *testing.T) { + p := &parserT{} + tests := []struct { + name string + node ast.Node + wantErr bool + }{ + {"nil node", nil, true}, + {"not sequence", &ast.StringNode{Value: "foo"}, true}, + {"sequence", &ast.SequenceNode{}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := p.nodeToSequence(tt.node) + if (err != nil) != tt.wantErr { + t.Errorf("nodeToSequence() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestNodeToString(t *testing.T) { + p := &parserT{} + tests := []struct { + name string + node ast.Node + want string + wantErr bool + }{ + {"nil node", nil, "", true}, + {"string node", &ast.StringNode{Value: "foo"}, "foo", false}, + {"literal node", &ast.LiteralNode{Value: &ast.StringNode{Value: "bar"}}, "bar", false}, + {"literal node nil value", &ast.LiteralNode{Value: nil}, "", true}, + {"wrong type", &ast.IntegerNode{Value: int64(1)}, "", true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := p.nodeToString(tt.node) + if (err != nil) != tt.wantErr { + t.Errorf("nodeToString() error = %v, wantErr %v", err, tt.wantErr) + } + if got != tt.want { + t.Errorf("nodeToString() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestNodeToInt64(t *testing.T) { + p := &parserT{} + tests := []struct { + name string + node ast.Node + want int64 + wantErr bool + }{ + {"nil node", nil, 0, true}, + {"not integer", &ast.StringNode{Value: "foo"}, 0, true}, + {"int64 value", &ast.IntegerNode{Value: int64(42)}, 42, false}, + {"wrong value type", &ast.IntegerNode{Value: "notint"}, 0, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := p.nodeToInt64(tt.node) + if (err != nil) != tt.wantErr { + t.Errorf("nodeToInt64() error = %v, wantErr %v", err, tt.wantErr) + } + if got != tt.want { + t.Errorf("nodeToInt64() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestNodeToUint64(t *testing.T) { + p := &parserT{} + tests := []struct { + name string + node ast.Node + want uint64 + wantErr bool + }{ + {"nil node", nil, 0, true}, + {"not integer", &ast.StringNode{Value: "foo"}, 0, true}, + {"uint64 value", &ast.IntegerNode{Value: uint64(42)}, 42, false}, + {"wrong value type", &ast.IntegerNode{Value: "notuint"}, 0, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := p.nodeToUint64(tt.node) + if (err != nil) != tt.wantErr { + t.Errorf("nodeToUint64() error = %v, wantErr %v", err, tt.wantErr) + } + if got != tt.want { + t.Errorf("nodeToUint64() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestNodeToUint(t *testing.T) { + p := &parserT{} + tests := []struct { + name string + node ast.Node + want uint + wantErr bool + }{ + {"ok", &ast.IntegerNode{Value: uint64(42)}, 42, false}, + {"bad type", &ast.StringNode{Value: "foo"}, 0, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := p.nodeToUint(tt.node) + if (err != nil) != tt.wantErr { + t.Errorf("nodeToUint() error = %v, wantErr %v", err, tt.wantErr) + } + if got != tt.want { + t.Errorf("nodeToUint() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestNodeToBool(t *testing.T) { + p := &parserT{} + tests := []struct { + name string + node ast.Node + want bool + wantErr bool + }{ + {"nil node", nil, false, true}, + {"not bool", &ast.StringNode{Value: "foo"}, false, true}, + {"bool true", &ast.BoolNode{Value: true}, true, false}, + {"bool false", &ast.BoolNode{Value: false}, false, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := p.nodeToBool(tt.node) + if (err != nil) != tt.wantErr { + t.Errorf("nodeToBool() error = %v, wantErr %v", err, tt.wantErr) + } + if got != tt.want { + t.Errorf("nodeToBool() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestNodeToStrs(t *testing.T) { + p := &parserT{} + tests := []struct { + name string + node ast.Node + want []string + wantErr bool + }{ + { + "ok", + &ast.SequenceNode{Values: []ast.Node{ + &ast.StringNode{Value: "a"}, + &ast.StringNode{Value: "b"}, + }}, + []string{"a", "b"}, + false, + }, + { + "not sequence", + &ast.StringNode{Value: "foo"}, + nil, + true, + }, + { + "element not string", + &ast.SequenceNode{Values: []ast.Node{ + &ast.IntegerNode{Value: int64(1)}, + }}, + nil, + true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := p.nodeToStrs(tt.node) + if (err != nil) != tt.wantErr { + t.Errorf("nodeToStrs() error = %v, wantErr %v", err, tt.wantErr) + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("nodeToStrs() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestNodeToRegex(t *testing.T) { + p := &parserT{} + tests := []struct { + name string + node ast.Node + want *regexp.Regexp + wantErr bool + }{ + { + "ok", + &ast.StringNode{Value: "^foo$"}, + regexp.MustCompile("^foo$"), + false, + }, + { + "bad regex", + &ast.StringNode{Value: "["}, + nil, + true, + }, + { + "not string", + &ast.IntegerNode{Value: int64(1)}, + nil, + true, + }, + { + "empty regex", + &ast.StringNode{Value: ""}, + nil, + true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := p.nodeToRegex(tt.node) + if (err != nil) != tt.wantErr { + t.Errorf("nodeToRegex() error = %v, wantErr %v", err, tt.wantErr) + } + if tt.want != nil && got != nil && got.String() != tt.want.String() { + t.Errorf("nodeToRegex() = %v, want %v", got, tt.want) + } + if (tt.want == nil) != (got == nil) { + t.Errorf("nodeToRegex() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestNodeToJq(t *testing.T) { + p := &parserT{ + validateJQ: func(s string) error { + if s == "bad" { + return errors.New("bad jq") + } + return nil + }, + } + tests := []struct { + name string + node ast.Node + want string + wantErr bool + }{ + {"ok", &ast.StringNode{Value: ".foo"}, ".foo", false}, + {"bad jq", &ast.StringNode{Value: "bad"}, "", true}, + {"not string", &ast.IntegerNode{Value: int64(1)}, "", true}, + {"empty string", &ast.StringNode{Value: ""}, "", true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := p.nodeToJq(tt.node) + if (err != nil) != tt.wantErr { + t.Errorf("nodeToJq() error = %v, wantErr %v", err, tt.wantErr) + } + if got != tt.want { + t.Errorf("nodeToJq() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestNodeToDuration(t *testing.T) { + p := &parserT{} + tests := []struct { + name string + node ast.Node + want time.Duration + wantErr bool + }{ + {"ok", &ast.StringNode{Value: "1s"}, time.Second, false}, + {"bad duration", &ast.StringNode{Value: "notdur"}, 0, true}, + {"not string", &ast.IntegerNode{Value: int64(1)}, 0, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := p.nodeToDuration(tt.node) + if (err != nil) != tt.wantErr { + t.Errorf("nodeToDuration() error = %v, wantErr %v", err, tt.wantErr) + } + if got != tt.want { + t.Errorf("nodeToDuration() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestNodeToDurationPositive(t *testing.T) { + p := &parserT{} + tests := []struct { + name string + node ast.Node + want time.Duration + wantErr bool + }{ + {"ok", &ast.StringNode{Value: "1s"}, time.Second, false}, + {"zero", &ast.StringNode{Value: "0s"}, 0, true}, + {"negative", &ast.StringNode{Value: "-1s"}, 0, true}, + {"bad duration", &ast.StringNode{Value: "notdur"}, 0, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := p.nodeToDurationPositive(tt.node) + if (err != nil) != tt.wantErr { + t.Errorf("nodeToDurationPositive() error = %v, wantErr %v", err, tt.wantErr) + } + if got != tt.want { + t.Errorf("nodeToDurationPositive() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestFindKey(t *testing.T) { + node := &ast.MappingNode{ + Values: []*ast.MappingValueNode{ + { + Key: &ast.StringNode{Value: "foo"}, + Value: &ast.StringNode{Value: "bar"}, + }, + { + Key: &ast.StringNode{Value: "baz"}, + Value: &ast.StringNode{Value: "qux"}, + }, + }, + } + tests := []struct { + name string + key string + want ast.Node + }{ + {"found", "foo", &ast.StringNode{Value: "foo"}}, + {"not found", "nope", nil}, + {"not mapping", "foo", nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var n ast.Node = node + if tt.name == "not mapping" { + n = &ast.StringNode{Value: "foo"} + } + got := findKey(n, tt.key) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("findKey() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/ast/parse_meta.go b/pkg/ast/parse_meta.go new file mode 100644 index 0000000..374c8d5 --- /dev/null +++ b/pkg/ast/parse_meta.go @@ -0,0 +1,161 @@ +package ast + +import ( + "fmt" + "regexp" + + "github.com/goccy/go-yaml/ast" +) + +var validBase58Regex = regexp.MustCompile(`^[1-9A-Za-z]{12,}$`) + +func (p *parserT) parseMetadataNode(node ast.Node) (*AstMetadataT, error) { + + mapping, err := p.nodeToMapping(node) + if err != nil { + return nil, err + } + + var ( + meta AstMetadataT + ) + + for _, v := range mapping.Values { + + key, err := p.nodeToString(v.Key) + if err != nil { + return nil, err + } + + switch key { + + case kwName: + meta.Name, err = p.nodeToString(v.Value) + + case kwId: + meta.Id, err = p.parseIdNode(v.Value) + + case kwHash: + meta.Hash, err = p.parseHash(v.Value) + + case kwGen: + meta.Gen, err = p.parseGen(v.Value) + + case kwKind: + meta.Kind, err = p.parseKind(v.Value) + + default: + if p.strict { + err = p.wrapError(v.Key, ErrUnexpectedKey) + } + } + + if err != nil { + return nil, err + } + } + + if meta.Id == "" { + err := fmt.Errorf("%w: %s", ErrMissingKey, kwId) + return nil, p.wrapErrorParent(mapping, err) + } + + if meta.Hash == "" { + err := fmt.Errorf("%w: %s", ErrMissingKey, kwHash) + return nil, p.wrapErrorParent(mapping, err) + } + + return &meta, nil +} + +func (p *parserT) parseMetadataNodeMinimal(node ast.Node) (*AstMetadataT, error) { + + mapping, err := p.nodeToMapping(node) + if err != nil { + return nil, err + } + + var meta AstMetadataT + + for _, v := range mapping.Values { + + key, err := p.nodeToString(v.Key) + if err != nil { + continue + } + + switch key { + + case kwHash: + meta.Hash, _ = p.parseHash(v.Value) + + case kwId: + meta.Id, _ = p.parseIdNode(v.Value) + } + } + + // Only an error if both id and hash are missing, since the name is not required for diagnostics. + if meta.Id == "" && meta.Hash == "" { + err := fmt.Errorf("%w: %s and %s", ErrMissingKey, kwId, kwHash) + return nil, p.wrapErrorParent(mapping, err) + } + + return &meta, nil +} + +func (p *parserT) parseIdNode(v ast.Node) (string, error) { + s, err := p.nodeToString(v) + if err != nil { + return "", err + } + + // Expect string to be a randomized 16 byte wide base58 encoded string. + // Ignore strict here; a valid hash is required for correct operation. + if !validBase58Regex.MatchString(s) { + return "", p.wrapError(v, ErrBadIdentifier) + } + return s, nil +} + +func (p *parserT) parseHash(v ast.Node) (string, error) { + s, err := p.nodeToString(v) + if err != nil { + return "", err + } + // Expect base58 encoded sha256 hash. + // Ignore strict here; a valid hash is required for correct operation. + if !validBase58Regex.MatchString(s) { + return "", p.wrapError(v, ErrBadHash) + } + return s, nil +} + +func (p *parserT) parseGen(v ast.Node) (uint32, error) { + gen, err := p.nodeToUint64(v) + if err != nil { + return 0, err + } + // Sanity check generation number; should be a positive integer, and not unreasonably high. + if gen > uint64(p.maxGen) { + err := fmt.Errorf("%w: generation value must be a positive integer less or equal to %d", ErrBadGen, p.maxGen) + return 0, p.wrapError(v, err) + } + + return uint32(gen), nil +} + +func (p *parserT) parseKind(v ast.Node) (string, error) { + s, err := p.nodeToString(v) + if err != nil { + return "", err + } + switch s { + case KindPrequel, KindCustom: + default: + if p.strict { + err := fmt.Errorf("%w: kind value must be either 'prequel' or 'custom'", ErrBadKind) + return "", p.wrapError(v, err) + } + } + return s, nil +} diff --git a/pkg/ast/parse_meta_test.go b/pkg/ast/parse_meta_test.go new file mode 100644 index 0000000..da1b3fd --- /dev/null +++ b/pkg/ast/parse_meta_test.go @@ -0,0 +1,297 @@ +package ast + +import ( + "reflect" + "testing" + + "github.com/goccy/go-yaml/ast" +) + +func TestParseMetaNode(t *testing.T) { + tests := []struct { + name string + yamlInput string + strict bool + want *AstMetadataT + wantErr error + wantPos int + }{ + { + name: "valid meta with all fields", + yamlInput: ` +metadata: + name: TestRule + id: eeJwJiWQa9TyH3qTYYSZM9 + hash: 9GJSdx4smGJeJCdiw6tiK5 + gen: 10 + kind: prequel +`, + strict: true, + want: &AstMetadataT{ + Name: "TestRule", + Id: "eeJwJiWQa9TyH3qTYYSZM9", + Hash: "9GJSdx4smGJeJCdiw6tiK5", + Gen: 10, + Kind: KindPrequel, + }, + }, + { + name: "missing id", + yamlInput: ` +metadata: + hash: 9GJSdx4smGJeJCdiw6tiK5 +`, + strict: true, + wantErr: ErrMissingKey, + wantPos: 10, // Position of the metadata mapping node + }, + { + name: "missing hash", + yamlInput: ` +metadata: + id: eeJwJiWQa9TyH3qTYYSZM9 +`, + strict: true, + wantErr: ErrMissingKey, + wantPos: 10, // Position of the metadata mapping node + }, + { + name: "only required id,hash", + yamlInput: ` +metadata: + id: eeJwJiWQa9TyH3qTYYSZM9 + hash: 9GJSdx4smGJeJCdiw6tiK5 +`, + strict: true, + want: &AstMetadataT{ + Id: "eeJwJiWQa9TyH3qTYYSZM9", + Hash: "9GJSdx4smGJeJCdiw6tiK5", + }, + }, + { + name: "unexpected key in strict mode", + yamlInput: ` +metadata: + id: eeJwJiWQa9TyH3qTYYSZM9 + hash: 9GJSdx4smGJeJCdiw6tiK5 + extra: value +`, + strict: true, + wantErr: ErrUnexpectedKey, + wantPos: 74, // Position of the "extra" key node + }, + { + name: "bad node type", + yamlInput: ` +metadata: not_a_mapping +`, + strict: true, + wantErr: ErrUnexpectedType, + wantPos: 12, // Position of the "not_a_mapping" scalar node + }, + { + name: "bad key type", + yamlInput: ` +metadata: + 999: not_a_string_key +`, + strict: true, + wantErr: ErrUnexpectedType, + wantPos: 14, // Position of the "999" key node (integer instead of string) + }, + { + name: "bad value type for id", + yamlInput: ` +metadata: + id: 12345 + hash: 9GJSdx4smGJeJCdiw6tiK5 +`, + strict: true, + wantErr: ErrUnexpectedType, + wantPos: 18, // Position of the "id" value node (integer instead of string) + }, + { + name: "bad id value", + yamlInput: ` +metadata: + id: $$$ + hash: 9GJSdx4smGJeJCdiw6tiK5 +`, + strict: true, + wantErr: ErrBadIdentifier, + wantPos: 18, // Position of "$$$" id value node + }, + { + name: "bad value type for hash", + yamlInput: ` +metadata: + id: eeJwJiWQa9TyH3qTYYSZM9 + hash: 999 +`, + strict: true, + wantErr: ErrUnexpectedType, + wantPos: 49, // Position of the "999" value node (integer instead of string) + }, + { + name: "bad hash value", + yamlInput: ` +metadata: + id: eeJwJiWQa9TyH3qTYYSZM9 + hash: $$$ +`, + strict: true, + wantErr: ErrBadHash, + wantPos: 49, // Position of the "$$$" value node (string instead of valid hash) + }, + { + name: "bad gen type", + yamlInput: ` +metadata: + id: eeJwJiWQa9TyH3qTYYSZM9 + hash: 9GJSdx4smGJeJCdiw6tiK5 + gen: "not_a_number" +`, + strict: true, + wantErr: ErrUnexpectedType, + wantPos: 79, // Position of the "not_a_number" value node (string instead of valid number) + }, + { + name: "negative gen value", + yamlInput: ` +metadata: + id: eeJwJiWQa9TyH3qTYYSZM9 + hash: 9GJSdx4smGJeJCdiw6tiK5 + gen: -1 +`, + strict: true, + wantErr: ErrUnexpectedType, + wantPos: 79, // Position of the "not_a_number" value node (string instead of valid number) + }, + { + name: "out of range gen value", + yamlInput: ` +metadata: + id: eeJwJiWQa9TyH3qTYYSZM9 + hash: 9GJSdx4smGJeJCdiw6tiK5 + gen: 11 # We set maxGen to 10 in the test. +`, + strict: true, + wantErr: ErrBadGen, + wantPos: 79, // Position of the "11" value node (integer instead of valid range) + }, + { + name: "bad kind type", + yamlInput: ` +metadata: + id: eeJwJiWQa9TyH3qTYYSZM9 + hash: 9GJSdx4smGJeJCdiw6tiK5 + kind: 123 +`, + strict: true, + wantErr: ErrUnexpectedType, + wantPos: 80, // Position of the "123" value node (integer instead of valid kind) + }, + { + name: "bad kind value strict mode", + yamlInput: ` +metadata: + id: eeJwJiWQa9TyH3qTYYSZM9 + hash: 9GJSdx4smGJeJCdiw6tiK5 + kind: unknown_kind +`, + strict: true, + wantErr: ErrBadKind, + wantPos: 80, // Position of the "unknown_kind" value node (string instead of valid kind) + }, + { + name: "bad kind value tolerant mode", + yamlInput: ` +metadata: + id: eeJwJiWQa9TyH3qTYYSZM9 + hash: 9GJSdx4smGJeJCdiw6tiK5 + kind: unknown_kind +`, + strict: false, + want: &AstMetadataT{ + Id: "eeJwJiWQa9TyH3qTYYSZM9", + Hash: "9GJSdx4smGJeJCdiw6tiK5", + Kind: "unknown_kind", // Should accept unknown kind in tolerant mode + }, + }, + { + name: "no hash or id on minimal parse", + yamlInput: ` +metadata: + kind: unknown_kind +`, + wantErr: ErrMissingKey, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + node := mustParseYAMLNode(t, tt.yamlInput) + // Should be mapping Node with single key kwMetadata + mapping, ok := node.(*ast.MappingNode) + if !ok || len(mapping.Values) != 1 { + t.Fatalf("expected a mapping node with one key, got %T with %d keys", node, len(mapping.Values)) + } + v := mapping.Values[0] + p := &parserT{strict: tt.strict, root: node, maxGen: 10} + + got, err := p.parseMetadataNode(v.Value) + + if !checkParserError(t, err, tt.wantErr, tt.wantPos) { + return + } + + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("parseMetaNode() = %+v, want %+v", got, tt.want) + } + }) + } +} + +func TestParseMetaNodeMinimal(t *testing.T) { + tests := []struct { + name string + yamlInput string + strict bool + want *AstMetadataT + wantErr error + wantPos int + }{ + + { + name: "no hash or id on minimal parse", + yamlInput: ` +metadata: + kind: unknown_kind +`, + wantErr: ErrMissingKey, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + node := mustParseYAMLNode(t, tt.yamlInput) + // Should be mapping Node with single key kwMetadata + mapping, ok := node.(*ast.MappingNode) + if !ok || len(mapping.Values) != 1 { + t.Fatalf("expected a mapping node with one key, got %T with %d keys", node, len(mapping.Values)) + } + v := mapping.Values[0] + p := &parserT{strict: tt.strict, root: node, maxGen: 10} + + got, err := p.parseMetadataNodeMinimal(v.Value) + + if !checkParserError(t, err, tt.wantErr, tt.wantPos) { + return + } + + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("parseMetaNode() = %+v, want %+v", got, tt.want) + } + }) + } +} diff --git a/pkg/ast/parse_node.go b/pkg/ast/parse_node.go new file mode 100644 index 0000000..a5c0504 --- /dev/null +++ b/pkg/ast/parse_node.go @@ -0,0 +1,319 @@ +package ast + +import ( + "fmt" + "time" + + "github.com/goccy/go-yaml/ast" +) + +// parseNode is a utility function to parse either a set or sequence node based on the provided type. +// This is used to handle the common logic for parsing both sets and sequences, +// since they share the same structure and keys, with the main difference being the +// node type and validation rules. + +func (p *parserT) parseNode(state ruleState, ty AstNodeType, node ast.Node) (AstNode, error) { + + mapping, err := p.nodeToMapping(node) + if err != nil { + return nil, err + } + + // Push the child node to the state and reset the rank. + // The parent rank should not affect the rank of a child node. + child := state.pushNode(ty).setRank(0) + + // Sanity check on child depth; this should be after pushing the child node + // since that is when the depth is incremented. + // Note: maxDepth is one based, whereas addr.Depth is zero based, so we check if Depth >= maxDepth. + if child.addr.Depth >= p.maxDepth { + err := fmt.Errorf("%w: %d", ErrMaxDepthExceeded, p.maxDepth) + return nil, p.wrapErrorParent(node, err) + } + + // Parse the node into a protoNode, which is an intermediate representation + // that captures the relevant information from the YAML node in a structured way. + proto, err := p._parseNode(child, ty, mapping) + if err != nil { + return nil, err + } + + // Construct the appropriate AST node (SetNode, SequenceNode, etc) based on the protoNode. + return p.constructNode(state, child, mapping, proto) +} + +// _parseNode is the internal implementation of parseNode which does the actual parsing of the node into a protoNode. +// The node is recursively parsed, generating a protoNode. +// The protoNode is used as an intermediate representation to facilitate validation and transformation. +// +// Expected layout of the node is a mapping with the following optional keys: +// Window string `yaml:"window,omitempty"` +// Correlations []string `yaml:"correlations,omitempty"` +// Event *ParseEventT `yaml:"event,omitempty"` +// Match []ParseTermT `yaml:"match,omitempty"` +// Order []ParseTermT `yaml:"order,omitempty"` +// Negate []ParseTermT `yaml:"negate,omitempty"` + +func (p *parserT) _parseNode(state ruleState, ty AstNodeType, node *ast.MappingNode) (*protoNode, error) { + + var ( + negNode ast.Node + proto = protoNode{ty: ty, window: -1} // Default to -1 to indicate no window specified; a window of 0 is valid and means "match events that occur at the same time".} + ) + + for _, v := range node.Values { + + key, err := p.nodeToString(v.Key) + if err != nil { + return nil, err + } + + switch key { + + case kwEvent: + if proto.event, err = p.parseEventNode(state, v.Value); err != nil { + return nil, err + } + + case kwWindow: + if proto.window, err = p.parseWindow(v.Value); err != nil { + return nil, err + } + + case kwMatch: + if proto.ty != AstNodeTypeSet { + err := fmt.Errorf("%w: '%s' key is not allowed in this context", ErrUnexpectedKey, key) + return nil, p.wrapError(v.Key, err) + } + if proto.terms, err = p.parseTerms(state, v.Value, 0); err != nil { + return nil, err + } + + case kwOrder: + if proto.ty != AstNodeTypeSeq { + err := fmt.Errorf("%w: '%s' key is not allowed in this context", ErrUnexpectedKey, key) + return nil, p.wrapError(v.Key, err) + } + if proto.terms, err = p.parseTerms(state, v.Value, 0); err != nil { + return nil, err + } + + case kwNegate: + // Defer parsing the negate node until the end in case it occurs before the match/order node. + // This is necessary to get the addressing consistent; ie. negative terms successively addressed + // after positive terms. + negNode = v.Value + + case kwCorrelations: + if proto.correlations, err = p.nodeToStrs(v.Value); err != nil { + return nil, err + } + + default: + err := fmt.Errorf("%w: %s", ErrUnexpectedKey, key) + return nil, p.wrapError(v.Key, err) + } + } + + // Sanity checks + switch { + case len(proto.terms) == 0: + // At least one term has to exist. + // No terms means there is nothing to match/order, which is not valid. + // Negate only terms are not allowed. + return nil, p.wrapErrorParent(node, ErrMissingTerm) + case ty == AstNodeTypeSeq && len(proto.terms) == 1: + // A sequence with only one term is not allowed. + return nil, p.wrapError(findKey(node, kwOrder), ErrShortSequence) + case proto.window >= 0: + // Fallthrough; window is defined and valid; zero is allowable. + case len(proto.terms) > 1 || proto.terms[0].count() > 1: + // A window is required if there are multiple terms to time bound the match. + return nil, p.wrapErrorParent(node, ErrMissingWindow) + default: + // Windows was not defined on a single term which is allowed. + // Reset window to 0 for consistency. + proto.window = 0 + } + + // Process a negate node if it exists. + // This has been deferred until now to ensure that the match/order terms have been parsed + // and the rank/offset can be correctly assigned to the negate terms. + if negNode != nil { + // Fix up the rank on the state to include the already parsed match/order terms, + // so that negate terms are ranked after them. + var ( + err error + negateOffset = len(proto.terms) + negateState = state.setRank(uint32(negateOffset)) + ) + if proto.negate, err = p.parseTerms(negateState, negNode, negateOffset); err != nil { + return nil, err + } + } + + // Possibly assign origin if not already set. + p.maybeFixupEventOrigin(proto, state) + + return &proto, nil +} + +// Simple rules may not have the origin explicit set in the event. +// This is deprecated behavior; new rules should explicitly specify an origin. +// If not strict and no origin specified, determine if there is exactly +// one leaf term in the rule, and if so, assign origin to that term. + +func (p *parserT) maybeFixupEventOrigin(proto protoNode, state ruleState) { + + switch { + case state.getOrigin() > 0: + // If origin is already set, do not attempt to infer or assign it. + case p.strict: + // In strict mode, origin must be explicitly specified; do not attempt to infer or assign it. + case state.addr != nil && state.addr.Depth > 0: + // If this is not the root node, do not attempt to infer or assign origin; it must be set at the root level if applicable. + case proto.terms[0].field == nil: + // Either all fields or no fields; if the first term is not a field, do not attempt to infer or assign origin. + case proto.event == nil: + // If there is no event, do not attempt to infer or assign origin; origin only applies if there is an event. + default: + // Force origin to be true and increment the origin count in the state to reflect this assignment. + proto.event.Origin = true + state.incOrigin() + } +} + +// Window is expected to be a duration string, which we parse into a time.Duration. +// A negative duration is not valid, since a window cannot be negative. + +func (p *parserT) parseWindow(node ast.Node) (time.Duration, error) { + window, err := p.nodeToDuration(node) + if err != nil { + return 0, err + } + if window < 0 { + return 0, p.wrapError(node, ErrWindowNegative) + } + return window, nil +} + +// Interpret the prototype and construct the appropriate AST node (SetNode, SequenceNode, etc). + +func (p *parserT) constructNode(parent, child ruleState, mapping *ast.MappingNode, proto *protoNode) (AstNode, error) { + + // If there are negate terms, they must either all be field terms or all child node terms, + // and they must match the type of the match/order terms. + allFields := proto.terms[0].field != nil + + if len(proto.negate) > 0 { + if negateAllFields := proto.negate[0].field != nil; allFields != negateAllFields { + err := fmt.Errorf("%w: positive terms and negate terms must both be either field nodes or child nodes", ErrTermTypeConflict) + return nil, p.wrapError(p.findNegateTerm(mapping), err) + } + } + + // Confirm that the event key is set if required, and not set otherwise. + switch { + case allFields && proto.event == nil: + return nil, p.wrapErrorParent(mapping, ErrMissingEvent) + + case !allFields && proto.event != nil: + err := fmt.Errorf("%w: an event is not allowed when using child node terms", ErrUnexpectedKey) + return nil, p.wrapError(findKey(mapping, kwEvent), err) + } + + // TODO: Validate anchors in negate terms; should be in range of [1, len(terms)) + + var node AstNode + + switch { + case !allFields: + node = p.constructInnerNode(parent, child, proto) + default: + node = p.constructLeafNode(parent, child, proto) + } + + return node, nil +} + +func (p *parserT) constructLeafNode(parent, child ruleState, proto *protoNode) AstNode { + + // A non root leaf node is constructed as a normal leaf node with the appropriate parent and address. + if child.addr.Depth > 0 { + return p._constructLeafNode(parent, child, proto) + } + + // Leaf nodes are not allowed at the root level; + // they must be contained within an Cluster scoped inner node. + // This allows the engine to evaluate the match at the cluster level where the publish logic is executing. + + // Construct a placeholder inner node to hold the leaf terms, with the appropriate parent and address. + // This will assume the original child's address, and the leaf node will be addressed as a child of this placeholder node. + root := &AstInnerNodeT{ + baseAst: baseAst{address: *child.addr, parent: nil, scope: AstScopeCluster}, + } + + grandChild := child.pushNode(AstNodeTypeSet) // Only sets support 1 term. + + leaf := p._constructLeafNode(child, grandChild, proto) + + term := AstTermT{ + Term: leaf, + } + + root.Terms = []AstTermT{term} + + return root +} + +func (p *parserT) _constructLeafNode(parent, child ruleState, proto *protoNode) *AstMatchLeafT { + + // Translate node type from proto to the appropriate match type; ie. Set -> MatchSet, Seq -> MatchSeq. + translatedType := proto.ty + switch proto.ty { + case AstNodeTypeSet: + translatedType = AstNodeTypeLogSet + case AstNodeTypeSeq: + translatedType = AstNodeTypeLogSeq + } + + child.addr.Type = translatedType + + return &AstMatchLeafT{ + baseAst: baseAst{address: *child.addr, parent: parent.addr, scope: AstScopeNode}, + Window: proto.window, + Correlations: proto.correlations, + Terms: protoTermsToAstFields(proto.terms), + Negate: protoTermsToAstFields(proto.negate), + Event: *proto.event, + } +} + +func (p *parserT) constructInnerNode(parent, child ruleState, proto *protoNode) AstNode { + + return &AstInnerNodeT{ + baseAst: baseAst{address: *child.addr, parent: parent.addr, scope: AstScopeCluster}, + Window: proto.window, + Correlations: proto.correlations, + Terms: protoTermsToAstTerms(proto.terms), + Negate: protoTermsToAstTerms(proto.negate), + } +} + +// Given mapping node a level up, find the first negate term node for accurate error reporting on negate term issues. +func (p *parserT) findNegateTerm(mapping *ast.MappingNode) (outNode ast.Node) { + // Default to the mapping node for error reporting if + // we cannot find the negate terms for some reason; + // this should not happen since the negate terms + // should have already been parsed successfully + // at this point, but we defensively code against it just in case. + outNode = mapping + + for _, v := range mapping.Values { + if k, ok := v.Key.(*ast.StringNode); ok && k.Value == kwNegate { + outNode = v.Value + break + } + } + return +} diff --git a/pkg/ast/parse_node_test.go b/pkg/ast/parse_node_test.go new file mode 100644 index 0000000..c12e1da --- /dev/null +++ b/pkg/ast/parse_node_test.go @@ -0,0 +1,141 @@ +package ast + +import ( + "testing" + + "github.com/goccy/go-yaml/ast" +) + +func TestParseNode(t *testing.T) { + tests := []struct { + name string + yamlInput string + ty AstNodeType + strict bool + wantErr error + wantPos int + }{ + { + name: "bad mapping type", + ty: AstNodeTypeSet, + yamlInput: ` +set: not a mapping set +`, + wantErr: ErrUnexpectedType, + wantPos: 7, // Position of the 'set' key + }, + { + name: "missing window on count with count > 1", + ty: AstNodeTypeSet, + yamlInput: ` +set: + event: + source: kafka + origin: true + match: + - value: "serenity" + count: 2 +`, + wantErr: ErrMissingWindow, + wantPos: 5, // Position of the 'set' key + }, + { + name: "bad mapping key", + ty: AstNodeTypeSet, + yamlInput: ` +set: + 123: not a string +`, + wantErr: ErrUnexpectedType, + wantPos: 9, // Position of the '123' key + }, + { + name: "bad correlations", + ty: AstNodeTypeSet, + yamlInput: ` +set: + correlations: not a sequence +`, + wantErr: ErrUnexpectedType, + wantPos: 23, + }, + { + name: "negative window", + ty: AstNodeTypeSet, + yamlInput: ` +set: + window: -5s +`, + wantErr: ErrWindowNegative, + wantPos: 17, + }, + { + name: "fail mixed positive and negative field types", + ty: AstNodeTypeSet, + yamlInput: ` +set: + window: 5s + event: + source: kafka + origin: true + match: + - value: "serenity" + - "now" + negate: + - set: + window: 10s + event: + source: kafka + match: + - "nope" +`, + wantErr: ErrTermTypeConflict, + wantPos: 123, // Position of the 'nope' key in the negate term, which is the first negate term and should be highlighted for the error + }, + { + name: "disallow event on inner nodes", + ty: AstNodeTypeSet, + yamlInput: ` +set: + window: 5s + event: + source: kafka + match: + - set: + event: + source: kafka + origin: true + match: + - "serenity" +`, + wantErr: ErrUnexpectedKey, + wantPos: 22, // Position of the 'event' key in the inner set node, which should be highlighted for the error + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + node := mustParseYAMLNode(t, tt.yamlInput) + // Should be mapping Node with single key kwMetadata + mapping, ok := node.(*ast.MappingNode) + if !ok || len(mapping.Values) != 1 { + t.Fatalf("expected a mapping node with one key, got %T with %d keys", node, len(mapping.Values)) + } + v := mapping.Values[0] + p := &parserT{ + strict: tt.strict, + root: node, + maxGen: 10, + maxRank: 10, + maxDepth: 10, + } + state := newRuleState( + &AstMetadataT{Id: "test", Hash: "hash"}, + ) + + _, err := p.parseNode(state, tt.ty, v.Value) + + checkParserError(t, err, tt.wantErr, tt.wantPos) + }) + } +} diff --git a/pkg/ast/parse_prom.go b/pkg/ast/parse_prom.go new file mode 100644 index 0000000..947d638 --- /dev/null +++ b/pkg/ast/parse_prom.go @@ -0,0 +1,80 @@ +package ast + +import ( + "errors" + "fmt" + + "github.com/goccy/go-yaml/ast" +) + +func (p *parserT) parsePromQLNode(state ruleState, node ast.Node) (*AstPromT, error) { + + mapping, err := p.nodeToMapping(node) + if err != nil { + return nil, err + } + + child := state.pushNode(AstNodeTypePromQL) + + var ( + prom = AstPromT{ + baseAst: baseAst{ + scope: AstScopeCluster, + address: *child.addr, + parent: state.addr, + }, + } + ) + + for _, v := range mapping.Values { + + key, err := p.nodeToString(v.Key) + if err != nil { + return nil, err + } + + switch key { + + case kwPromExpr: + prom.Expr, err = p.parsePromExpr(v.Value) + + case kwPromInterval: + prom.Interval, err = p.nodeToDurationPositive(v.Value) + + case kwPromFor: + prom.For, err = p.nodeToDurationPositive(v.Value) + + case kwPromEvent: + prom.Event, err = p.parseEventNode(child, v.Value) + + default: + err = p.wrapError(v.Key, ErrUnexpectedKey) + } + + if err != nil { + return nil, err + } + } + + return &prom, nil +} + +func (p *parserT) parsePromExpr(node ast.Node) (string, error) { + + s, err := p.nodeToString(node) + if err != nil { + return "", err + } + + if s == "" { + err := fmt.Errorf("%w: PromQL expression cannot be empty", ErrBadPromQL) + return "", p.wrapError(node, err) + } + + if err := p.validatePromQL(s); err != nil { + err := errors.Join(ErrBadPromQL, err) + return "", p.wrapError(node, err) + } + + return s, nil +} diff --git a/pkg/ast/parse_prom_test.go b/pkg/ast/parse_prom_test.go new file mode 100644 index 0000000..b3d67c0 --- /dev/null +++ b/pkg/ast/parse_prom_test.go @@ -0,0 +1,165 @@ +package ast + +import ( + "reflect" + "testing" + "time" + + "github.com/goccy/go-yaml/ast" +) + +func TestParsePromNode(t *testing.T) { + tests := []struct { + name string + yamlInput string + strict bool + failQL bool + want *AstPromT + wantErr error + wantPos int + }{ + { + name: "valid promql query", + yamlInput: ` +promql: + expr: "sum(rate(http_request_duration_seconds_count[5m])) by (job)" + interval: 1m + for: 5m + event: + source: stubSource + origin: true +`, + want: &AstPromT{ + baseAst: baseAst{ + scope: AstScopeCluster, + address: AstNodeAddressT{ + Type: AstNodeTypePromQL, + RuleId: stubRuleId, + RuleHash: stubRuleHash, + Rank: 0, + Depth: 0, + NodeId: 0, + }, + }, + Expr: "sum(rate(http_request_duration_seconds_count[5m])) by (job)", + Interval: time.Minute, + For: time.Minute * 5, + Event: &AstEventT{ + Source: stubSource, + Origin: true, + }, + }, + }, + { + name: "bad promql", + yamlInput: ` +promql: + expr: "bad query" + interval: 1m + for: 5m + event: + source: stubSource + origin: true +`, + failQL: true, + wantErr: ErrBadPromQL, + wantPos: 18, // position of the "expr" value in the YAML input + }, + { + name: "bad expr node type", + yamlInput: ` +promql: + expr: 12345 +`, + wantErr: ErrUnexpectedType, + wantPos: 18, // position of the "12345" value in the YAML input + }, + { + name: "bad mapping key", + yamlInput: ` +promql: + 123: "nope" +`, + strict: false, + wantErr: ErrUnexpectedType, + wantPos: 12, // position of the "123" key in the YAML input + }, + { + name: "unexpected key", + yamlInput: ` +promql: + shrubbery: "nope" +`, + wantErr: ErrUnexpectedKey, + wantPos: 12, // position of the "shrubbery" key in the YAML input + }, + { + name: "bad mapping type", + yamlInput: ` +promql: "nope" +`, + wantErr: ErrUnexpectedType, + wantPos: 10, // position of the "nope" key in the YAML input + }, + { + name: "negative interval", + yamlInput: ` +promql: + interval: -1m +`, + wantErr: ErrUnexpectedType, + wantPos: 22, // position of the "-1m" value in the YAML input + }, + { + name: "zero interval", + yamlInput: ` +promql: + interval: 0m +`, + wantErr: ErrUnexpectedType, + wantPos: 22, // position of the "0m" value in the YAML input + }, + { + name: "empty promql expression", + yamlInput: ` +promql: + expr: "" +`, + wantErr: ErrBadPromQL, + wantPos: 18, // position of the "expr" value in the YAML input + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + node := mustParseYAMLNode(t, tt.yamlInput) + // Should be mapping Node with single key "promql" + mapping, ok := node.(*ast.MappingNode) + if !ok || len(mapping.Values) != 1 { + t.Fatalf("expected a mapping node with one key, got %T with %d keys", node, len(mapping.Values)) + } + validatePromQL := stubValidator + if tt.failQL { + validatePromQL = func(query string) error { + return ErrBadPromQL + } + } + v := mapping.Values[0] + p := &parserT{ + strict: tt.strict, + root: node, + validatePromQL: validatePromQL, + } + state := newRuleState(&AstMetadataT{Id: stubRuleId, Hash: stubRuleHash}) + got, err := p.parsePromQLNode(state, v.Value) + + if !checkParserError(t, err, tt.wantErr, tt.wantPos) { + return + } + + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("parsePromNode() = %+v, want %+v", got, tt.want) + } + }) + } +} diff --git a/pkg/ast/parse_rules.go b/pkg/ast/parse_rules.go new file mode 100644 index 0000000..4f36a77 --- /dev/null +++ b/pkg/ast/parse_rules.go @@ -0,0 +1,262 @@ +package ast + +import ( + "errors" + "fmt" + + "github.com/goccy/go-yaml/ast" + "github.com/prequel-dev/prequel-compiler/pkg/version" +) + +func (p *parserT) parseRulesNode(node ast.Node) ([]AstRuleT, error) { + + seq, err := p.nodeToSequence(node) + if err != nil { + return nil, err + } + + var ( + rules []AstRuleT + errList []error + ) + + for _, ruleNode := range seq.Values { + + rule, err := p.parseRuleNode(ruleNode) + + switch { + case err != nil: + errList = append(errList, err) + default: + rules = append(rules, *rule) + } + } + + return rules, errors.Join(errList...) +} + +// Expects layout of a single mapping node with keys 'metadata', 'cre', and 'rule' +// Optionally there is a version filter; with a required version with an optional "<" prefix. + +func (p *parserT) parseRuleNode(node ast.Node) (*AstRuleT, error) { + + mapping, err := p.nodeToMapping(node) + if err != nil { + return nil, err + } + + // Check compiler version filter if present + if err := p.maybeCheckCompiler(mapping); err != nil { + return nil, err + } + + var ( + meta *AstMetadataT + cre *AstCreT + ruleDom ast.Node + ) + + maybeMeta := func(err error) error { + if meta == nil { + return err + } + return ErrRule{ + Meta: *meta, + Err: err, + } + } + + for _, v := range mapping.Values { + + key, err := p.nodeToString(v.Key) + if err != nil { + return nil, maybeMeta(err) + } + + switch key { + + case kwMetadata: + meta, err = p.parseMetadataNode(v.Value) + + case kwCre: + cre, err = p.parseCreNode(v.Value) + + case kwRule: + ruleDom = v.Value + + case kwCompiler: + // TODO: Handle compiler filter + + default: + err = p.wrapError(v.Key, ErrUnexpectedKey) + } + + if err != nil { + return nil, maybeMeta(err) + } + } + + switch { + case meta == nil: + err := fmt.Errorf("%w: %s", ErrMissingKey, kwMetadata) + return nil, p.wrapErrorParent(mapping, err) + case ruleDom == nil: + kerr := fmt.Errorf("%w: %s", ErrMissingKey, kwRule) + return nil, maybeMeta(p.wrapErrorParent(mapping, kerr)) + } + + // Parse the root + var ( + rootAst AstNode + ruleState = newRuleState(meta) + ) + + if rootAst, err = p.parseRootNode(ruleState, ruleDom); err != nil { + return nil, maybeMeta(err) + } + + if ruleState.getOrigin() < 1 { + return nil, maybeMeta(p.wrapErrorParent(ruleDom, ErrMissingOrigin)) + } + + rule := &AstRuleT{ + Cre: cre, + Metadata: *meta, + Root: rootAst, + } + + return rule, nil +} + +// At the root, expecting: +// type ParseRuleDataT struct { +// Sequence *ParseSequenceT `yaml:"sequence,omitempty"` +// Set *ParseSetT `yaml:"set,omitempty"` +// } + +func (p *parserT) parseRootNode(state ruleState, node ast.Node) (AstNode, error) { + + mapping, err := p.nodeToMapping(node) + if err != nil { + return nil, err + } + + var ( + rootNode AstNode + ) + + alreadySet := func(v ast.Node) error { + if rootNode != nil { + err := fmt.Errorf("%w: one of '%s' or '%s' expected in rule root", ErrUnexpectedKey, kwSequence, kwSet) + return p.wrapError(v, err) + } + return nil + } + + for _, v := range mapping.Values { + + key, err := p.nodeToString(v.Key) + if err != nil { + return nil, err + } + switch key { + + case kwSequence: + if err := alreadySet(v.Key); err != nil { + return nil, err + } + if rootNode, err = p.parseNode(state, AstNodeTypeSeq, v.Value); err != nil { + return nil, err + } + + case kwSet: + if err := alreadySet(v.Key); err != nil { + return nil, err + } + if rootNode, err = p.parseNode(state, AstNodeTypeSet, v.Value); err != nil { + return nil, err + } + + default: + err := fmt.Errorf("%w: only '%s' or '%s' expected in rule root, not '%s'", ErrUnexpectedKey, kwSequence, kwSet, key) + return nil, p.wrapError(v.Key, err) + + } + } + + if rootNode == nil { + err := fmt.Errorf("%w: expected rule root to contain either '%s' or '%s' key", ErrMissingKey, kwSequence, kwSet) + return nil, p.wrapError(mapping, err) + } + + return rootNode, nil +} + +// Checks that the compiler version, if specified in the rule, +// matches the version of the compiler. + +func (p *parserT) allowCompilerVersion(node ast.Node) error { + + versionExp, err := p.nodeToString(node) + + switch { + case err != nil: + return err + case versionExp != "": + if err := version.AllowVersion(versionExp); err != nil { + return p.wrapError(node, err) + } + case p.strict: + err := fmt.Errorf("%w: compiler version expression must be specified in strict mode", ErrMissingVersion) + return p.wrapError(node, err) + default: + // allow missing version in non-strict mode + } + + return nil +} + +func (p *parserT) maybeCheckCompiler(mapping *ast.MappingNode) error { + + var ( + metaNode ast.Node + compilerNode ast.Node + ) + + // Scan ahead for compiler node and metadata node for error context if version check fails + for _, v := range mapping.Values { + key, err := p.nodeToString(v.Key) + if err != nil { + continue + } + switch key { + case kwCompiler: + compilerNode = v.Value + case kwMetadata: + metaNode = v.Value + } + } + + if compilerNode == nil { + return nil + } + + err := p.allowCompilerVersion(compilerNode) + if err == nil { + return nil + } + + // Attempt to parse metadata for error context, + meta, merr := p.parseMetadataNode(metaNode) + + // If metadata parsing fails, attempt a minimal metadata parse + if merr != nil { + if meta, merr = p.parseMetadataNodeMinimal(metaNode); merr != nil { + // If even the minimal parse fails, return the + // original version error without metadata context + return err + } + } + + return ErrRule{Meta: *meta, Err: err} +} diff --git a/pkg/ast/parse_rules_test.go b/pkg/ast/parse_rules_test.go new file mode 100644 index 0000000..557e484 --- /dev/null +++ b/pkg/ast/parse_rules_test.go @@ -0,0 +1,599 @@ +package ast + +import ( + "fmt" + "testing" + + "github.com/goccy/go-yaml/ast" + "github.com/prequel-dev/prequel-compiler/pkg/version" +) + +type wantErrT struct { + err error + pos int + hash string + rule string +} + +func TestParseRulesNode(t *testing.T) { + tests := []struct { + name string + yamlInput string + strict bool + want [][]string + wantErr []wantErrT + }{ + { + name: "empty rules sequence", + yamlInput: ` +rules: [] + `, + }, + { + name: "bad compiler type", + yamlInput: ` +rules: + - compiler: 999 + `, + wantErr: []wantErrT{ + {err: ErrUnexpectedType, pos: 23}, + }, + }, + { + name: "empty compiler strict", + yamlInput: ` +rules: + - compiler: "" + `, + strict: true, + wantErr: []wantErrT{ + {err: ErrMissingVersion, pos: 23}, + }, + }, + { + name: "empty compiler non-strict", + yamlInput: ` +rules: + - cre: + id: TestSuccessSimpleRule1 + severity: 1 + metadata: + id: J7uRQTGpGMyL1iFpssnBeS + hash: rdJLgqYgkEp8jg8Qks1qiq + rule: + set: + event: + source: kafka + origin: true + match: + - value: "io.vertx.core.VertxException: Thread blocked" + compiler: "" + `, + strict: false, + want: [][]string{ + { + "v1.machine_set.rdJLgqYgkEp8jg8Qks1qiq.d0.n0.t0", + "v1.log_set.rdJLgqYgkEp8jg8Qks1qiq.d1.n1.t0", + }, + }, + }, + { + name: "compiler version specified but invalid", + yamlInput: ` +rules: + - compiler: "whoops i did it again" + `, + strict: true, + wantErr: []wantErrT{ + {err: version.ErrBadConstraint, pos: 23}, + }, + }, + { + name: "compiler version specified but invalid", + yamlInput: fmt.Sprintf(` +rules: + - compiler: "%s" + metadata: + id: "K7uRQTGpGMyL1iFpssnBeS" + hash: "sdJLgqYgkEp8jg8Qks1qiq" + 999: ignored bad key in metadata + unknown_key: also ignored in metadata + `, fmt.Sprintf("<%s", version.SemVer())), + strict: true, + wantErr: []wantErrT{ + { + err: version.ErrVersionNotAllowed{Version: version.SemVer(), Expression: fmt.Sprintf("<%s", version.SemVer())}, + pos: 23, + hash: "sdJLgqYgkEp8jg8Qks1qiq", + rule: "K7uRQTGpGMyL1iFpssnBeS", + }, + }, + }, + { + name: "mismatch compiler version with minimal metadata", + yamlInput: fmt.Sprintf(` +rules: + - compiler: "%s" + `, fmt.Sprintf("<%s", version.SemVer())), + strict: true, + wantErr: []wantErrT{ + {err: version.ErrVersionNotAllowed{Version: version.SemVer(), Expression: fmt.Sprintf("<%s", version.SemVer())}, pos: 23}, + }, + }, + { + name: "compiler version specified exact", + yamlInput: fmt.Sprintf(` +rules: + - compiler: "%s" + `, fmt.Sprintf("=%s", version.SemVer())), + strict: true, + wantErr: []wantErrT{ + {err: ErrMissingKey, pos: 11}, // fall through to missing metadata + }, + }, + { + name: "compiler version specified greater or equal too", + yamlInput: fmt.Sprintf(` +rules: + - compiler: "%s" + `, fmt.Sprintf(">= %s", version.SemVer())), + strict: true, + wantErr: []wantErrT{ + {err: ErrMissingKey, pos: 11}, // fall through to missing metadata + }, + }, + { + name: "bad rules type", + yamlInput: ` +rules: {} + `, + wantErr: []wantErrT{ + {err: ErrUnexpectedType, pos: 9}, // Position of the rules key node + }, + }, + { + name: "bad rule type", + yamlInput: ` +rules: + - not a mapping + `, + wantErr: []wantErrT{ + {err: ErrUnexpectedType, pos: 13}, // Position of the first rule node + }, + }, + { + name: "bad key type in rule mapping", + yamlInput: ` +rules: + - 123: not a string key + `, + wantErr: []wantErrT{ + {err: ErrUnexpectedType, pos: 13}, // Position of the first rule node + }, + }, + { + name: "unexpected key in rule mapping", + yamlInput: ` +rules: + - unexpected_key: value + `, + wantErr: []wantErrT{ + {err: ErrUnexpectedKey, pos: 13}, // Position of the unexpected key node + }, + }, + { + name: "missing meta", + yamlInput: ` +rules: + - cre: + id: TestSuccessSimpleRule1 + severity: 1 + rule: + set: + event: + source: kafka + origin: true + match: + - value: "io.vertx.core.VertxException: Thread blocked" + `, + wantErr: []wantErrT{ + {err: ErrMissingKey, pos: 11}, // Position of the unexpected key node + }, + }, + { + name: "missing root", + yamlInput: ` +rules: + - cre: + id: TestSuccessSimpleRule1 + severity: 1 + metadata: + id: "J7uRQTGpGMyL1iFpssnBeS" + hash: "rdJLgqYgkEp8jg8Qks1qiq" + `, + wantErr: []wantErrT{ + {err: ErrMissingKey, + pos: 11, + hash: "rdJLgqYgkEp8jg8Qks1qiq", + rule: "J7uRQTGpGMyL1iFpssnBeS", + }, + }, + }, + { + name: "first rule, ok second rule bad type", + yamlInput: ` +rules: + - cre: + id: TestSuccessSimpleRule1 + severity: 1 + metadata: + id: J7uRQTGpGMyL1iFpssnBeS + hash: rdJLgqYgkEp8jg8Qks1qiq + rule: + set: + event: + source: kafka + origin: true + match: + - value: "io.vertx.core.VertxException: Thread blocked" + - not a mapping + `, + wantErr: []wantErrT{ + {err: ErrUnexpectedType, pos: 319}, // Position of the second rule node + }, + want: [][]string{ + { + "v1.machine_set.rdJLgqYgkEp8jg8Qks1qiq.d0.n0.t0", + "v1.log_set.rdJLgqYgkEp8jg8Qks1qiq.d1.n1.t0", + }, + }, + }, + { + name: "first rule bad type, second rule ok", + yamlInput: ` +rules: + - not a mapping + - cre: + id: TestSuccessSimpleRule1 + severity: 1 + metadata: + id: J7uRQTGpGMyL1iFpssnBeS + hash: rdJLgqYgkEp8jg8Qks1qiq + rule: + set: + event: + source: kafka + origin: true + match: + - value: "io.vertx.core.VertxException: Thread blocked" +`, + wantErr: []wantErrT{ + {err: ErrUnexpectedType, pos: 13}, // Position of the first rule node + }, + want: [][]string{ + { + "v1.machine_set.rdJLgqYgkEp8jg8Qks1qiq.d0.n0.t0", + "v1.log_set.rdJLgqYgkEp8jg8Qks1qiq.d1.n1.t0", + }, + }, + }, + { + name: "two ok followed by a illegal metadata in strict mode", + yamlInput: ` +rules: + - cre: + id: TestSuccessSimpleRule1 + severity: 1 + metadata: + id: J7uRQTGpGMyL1iFpssnBe1 + hash: rdJLgqYgkEp8jg8Qks1qi3 + rule: + set: + event: + source: kafka + origin: true + match: + - value: "io.vertx.core.VertxException: Thread blocked" + - cre: + id: TestSuccessSimpleRule1 + severity: 1 + metadata: + id: J7uRQTGpGMyL1iFpssnBe2 + hash: rdJLgqYgkEp8jg8Qks1qis + rule: + set: + event: + source: kafka + origin: true + match: + - value: "io.vertx.core.VertxException: Thread blocked" + - cre: + id: TestSuccessSimpleRule1 + severity: 1 + metadata: + id: J7uRQTGpGMyL1iFpssnBe3 + hash: rdJLgqYgkEp8jg8Qks1qir + nope: not allowed key in strict mode + rule: + set: + event: + source: kafka + origin: true + match: + - value: "io.vertx.core.VertxException: Thread blocked" +`, + wantErr: []wantErrT{ + {err: ErrUnexpectedKey, pos: 769}, // Position of bad metadata key + }, + want: [][]string{ + { + "v1.machine_set.rdJLgqYgkEp8jg8Qks1qi3.d0.n0.t0", + "v1.log_set.rdJLgqYgkEp8jg8Qks1qi3.d1.n1.t0", + }, + { + "v1.machine_set.rdJLgqYgkEp8jg8Qks1qis.d0.n0.t0", + "v1.log_set.rdJLgqYgkEp8jg8Qks1qis.d1.n1.t0", + }, + }, + strict: true, + }, + { + name: "single origin", + yamlInput: ` +rules: + - cre: + id: TestSuccessSimpleRule1 + severity: 1 + metadata: + id: J7uRQTGpGMyL1iFpssnBeS + hash: rdJLgqYgkEp8jg8Qks1qiq + rule: + set: + event: + source: kafka + origin: true + match: + - value: "io.vertx.core.VertxException: Thread blocked" +`, + want: [][]string{ + { + "v1.machine_set.rdJLgqYgkEp8jg8Qks1qiq.d0.n0.t0", + "v1.log_set.rdJLgqYgkEp8jg8Qks1qiq.d1.n1.t0", + }, + }, + }, + { + name: "no origin", + yamlInput: ` +rules: + - cre: + id: TestSuccessSimpleRule1 + severity: 1 + metadata: + id: "J7uRQTGpGMyL1iFpssnBeS" + hash: "rdJLgqYgkEp8jg8Qks1qiq" + rule: + set: + event: + source: kafka + match: + - value: "io.vertx.core.VertxException: Thread blocked" +`, + wantErr: []wantErrT{ + {err: ErrMissingOrigin, pos: 163}, + }, + strict: true, + }, + { + name: "multiple origin", + yamlInput: ` +rules: + - cre: + id: TestSuccessSimpleRule1 + severity: 1 + metadata: + id: "J7uRQTGpGMyL1iFpssnBeS" + hash: "rdJLgqYgkEp8jg8Qks1qiq" + rule: + set: + match: + - set: + event: + source: kafka + origin: true + match: + - value: "io.vertx.core.VertxException: Thread blocked" + - set: + event: + source: shrubbery + origin: true + match: + - value: "io.vertx.core.VertxException: Thread blocked" +`, + wantErr: []wantErrT{ + {err: ErrMultipleOrigin, pos: 457}, // Position of the second origin node`` + }, + }, + { + name: "conflicting set and seq nodes", + yamlInput: ` +rules: + - cre: + id: TestSuccessSimpleRule1 + severity: 1 + metadata: + id: "J7uRQTGpGMyL1iFpssnBeS" + hash: "rdJLgqYgkEp8jg8Qks1qiq" + rule: + set: + window: 10s + event: + source: kafka + origin: true + match: + - "shrubbery" + sequence: + window: 10s + event: + source: kafka + origin: true + order: + - "shrubbery" + - "trees" + `, + wantErr: []wantErrT{ + {err: ErrUnexpectedKey, pos: 301}, // Position of the conflicting key node + }, + }, + { + name: "conflicting set and seq nodes", + yamlInput: ` +rules: + - cre: + id: TestSuccessSimpleRule1 + severity: 1 + metadata: + id: "J7uRQTGpGMyL1iFpssnBeS" + hash: "rdJLgqYgkEp8jg8Qks1qiq" + rule: + sequence: + window: 10s + event: + source: kafka + origin: true + order: + - "shrubbery" + - "trees" + set: + window: 10s + event: + source: kafka + origin: true + match: + - "shrubbery" + `, + wantErr: []wantErrT{ + {err: ErrUnexpectedKey, pos: 324}, // Position of the conflicting key node + }, + }, + { + name: "missing set and seq nodes", + yamlInput: ` +rules: + - cre: + id: TestSuccessSimpleRule1 + severity: 1 + metadata: + id: "J7uRQTGpGMyL1iFpssnBeS" + hash: "rdJLgqYgkEp8jg8Qks1qiq" + rule: {} + `, + wantErr: []wantErrT{ + {err: ErrMissingKey, pos: 165}, + }, + }, + { + name: "bad type in rule node", + yamlInput: ` +rules: + - rule: not a mapping + metadata: + id: "J7uRQTGpGMyL1iFpssnBeS" + hash: "rdJLgqYgkEp8jg8Qks1qiq" + `, + wantErr: []wantErrT{ + {err: ErrUnexpectedType, pos: 19}, // Position of the first rule node + }, + }, + { + name: "bad key type in rule node", + yamlInput: ` +rules: + - rule: + 123: not a string key + metadata: + id: "J7uRQTGpGMyL1iFpssnBeS" + hash: "rdJLgqYgkEp8jg8Qks1qiq" + `, + wantErr: []wantErrT{ + {err: ErrUnexpectedType, pos: 25}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + node := mustParseYAMLNode(t, tt.yamlInput) + // Should be mapping Node with single key kwMetadata + mapping, ok := node.(*ast.MappingNode) + if !ok || len(mapping.Values) != 1 { + t.Fatalf("expected a mapping node with one key, got %T with %d keys", node, len(mapping.Values)) + } + v := mapping.Values[0] + p := &parserT{ + strict: tt.strict, + root: node, + maxGen: 10, + maxDepth: 11, + maxRank: 11, + } + + got, err := p.parseRulesNode(v.Value) + + checkParserErrors(t, err, tt.wantErr) + + if len(got) != len(tt.want) { + t.Fatalf("expected %d rules, got %d", len(tt.want), len(got)) + } + + for i, tree := range tt.want { + compareTree(t, got[i], tree) + } + + }) + } +} + +func checkParserErrors(t *testing.T, err error, wantErrs []wantErrT) { + t.Helper() + + switch { + case len(wantErrs) == 0 && err == nil: + return // No errors expected and got none, all good. + case len(wantErrs) == 0 && err != nil: + t.Fatalf("unexpected error: %v", err) + case len(wantErrs) > 0 && err == nil: + t.Fatalf("expected errors but got none") + + } + + type unwrapper interface { + Unwrap() []error + } + + if uw, ok := err.(unwrapper); ok { + errs := uw.Unwrap() + if len(errs) != len(wantErrs) { + t.Fatalf("expected %d errors, got %d", len(wantErrs), len(errs)) + } + for i, want := range wantErrs { + checkParserError(t, errs[i], want.err, want.pos) + if want.hash != "" || want.rule != "" { + pe, ok := errs[i].(ErrRule) + if !ok { + t.Fatalf("expected error to be *ErrRule, got %T", errs[i]) + } + if want.hash != "" && pe.Meta.Hash != want.hash { + t.Errorf("error %d: expected hash %q, got %q", i, want.hash, pe.Meta.Hash) + } + if want.rule != "" && pe.Meta.Id != want.rule { + t.Errorf("error %d: expected rule %q, got %q", i, want.rule, pe.Meta.Id) + } + } + } + } else if len(wantErrs) == 1 { + checkParserError(t, err, wantErrs[0].err, wantErrs[0].pos) + } else { + t.Fatalf("expected multiple errors but got a single error: %v", err) + } +} diff --git a/pkg/ast/parse_script.go b/pkg/ast/parse_script.go new file mode 100644 index 0000000..1499929 --- /dev/null +++ b/pkg/ast/parse_script.go @@ -0,0 +1,149 @@ +package ast + +import ( + "fmt" + + "github.com/goccy/go-yaml/ast" +) + +// type ParseScriptT struct { +// Code string `yaml:"code"` +// Language string `yaml:"language,omitempty"` // Assumes 'lua' if empty +// Timeout string `yaml:"timeout,omitempty"` // Uses default if empty; expects duration string +// Input *ParseTermT `yaml:"input"` // Required input +// } + +const ( + scriptLua = "lua" +) + +func (p *parserT) parseScriptNode(state ruleState, node ast.Node) (*AstScriptT, error) { + + mapping, err := p.nodeToMapping(node) + if err != nil { + return nil, err + } + + child := state.pushNode(AstNodeTypeScript) + + var ( + script = AstScriptT{ + baseAst: baseAst{ + scope: AstScopeCluster, + address: *child.addr, + parent: state.addr, + }, + } + ) + + for _, v := range mapping.Values { + + key, err := p.nodeToString(v.Key) + if err != nil { + return nil, err + } + + switch key { + + case kwScriptCode: + script.Code, err = p.parseScriptCode(v.Value) + + case kwScriptLang: + script.Language, err = p.parseScriptLang(v.Value) + + case kwScriptTimeout: + script.Timeout, err = p.nodeToDuration(v.Value) + + case kwScriptInput: + script.Input, err = p.parseScriptInput(child, v.Value) + + default: + err = p.wrapError(v.Key, ErrUnexpectedKey) + } + + if err != nil { + return nil, err + } + } + + if script.Input == nil { + return nil, p.wrapErrorParent(mapping, ErrMissingScriptInput) + } + + return &script, nil +} + +func (p *parserT) parseScriptInput(state ruleState, node ast.Node) (AstNode, error) { + + mapping, err := p.nodeToMapping(node) + if err != nil { + return nil, err + } + + var inputNode AstNode + + for i, v := range mapping.Values { + if i > 0 { + err := fmt.Errorf("%w: script input mapping must have exactly one key", ErrUnexpectedKey) + return nil, p.wrapError(v.Key, err) + } + + key, ok := v.Key.(*ast.StringNode) + if !ok { + err := fmt.Errorf("%w: script input mapping keys must be strings", ErrUnexpectedType) + return nil, p.wrapError(v.Key, err) + } + + if inputNode, err = p.parseTermChild(state, key, v.Value, nil); err != nil { + return nil, err + } + } + + if inputNode == nil { + err := fmt.Errorf("%w: script input mapping must have exactly one key", ErrMissingScriptInput) + return nil, p.wrapError(node, err) + } + + return inputNode, nil +} + +func (p *parserT) parseScriptLang(node ast.Node) (string, error) { + + s, err := p.nodeToString(node) + if err != nil { + return "", err + } + + switch s { + case scriptLua: + // Fall through + + case "": + if p.strict { + err = fmt.Errorf("%w: script language cannot be empty", ErrBadScriptLang) + return "", p.wrapError(node, err) + } + + default: + err = fmt.Errorf("%w: unsupported script language: %s", ErrBadScriptLang, s) + return "", p.wrapError(node, err) + } + + return s, nil + +} + +func (p *parserT) parseScriptCode(node ast.Node) (string, error) { + + s, err := p.nodeToString(node) + if err != nil { + return "", err + } + + // Only Lua supported for now, so validate as Lua code + if err := p.validateLua(s); err != nil { + return "", p.wrapError(node, fmt.Errorf("%w: invalid Lua code: %w", ErrBadScriptCode, err)) + } + + return s, nil +} diff --git a/pkg/ast/parse_script_test.go b/pkg/ast/parse_script_test.go new file mode 100644 index 0000000..6587a49 --- /dev/null +++ b/pkg/ast/parse_script_test.go @@ -0,0 +1,286 @@ +package ast + +import ( + "reflect" + "testing" + "time" + + "github.com/goccy/go-yaml/ast" + "github.com/prequel-dev/prequel-logmatch/pkg/match" +) + +func TestParseScriptNode(t *testing.T) { + tests := []struct { + name string + yamlInput string + strict bool + failLua bool + want *AstScriptT + wantErr error + wantPos int + }{ + { + name: "valid script with all fields", + yamlInput: ` +script: + code: "print(\"Hello, world!\")" + language: "lua" + timeout: 5m + input: + set: + event: + source: "stubSource" + origin: true + match: + - "pod" +`, + want: &AstScriptT{ + baseAst: baseAst{ + scope: AstScopeCluster, + address: AstNodeAddressT{ + Type: AstNodeTypeScript, + RuleId: stubRuleId, + RuleHash: stubRuleHash, + }, + }, + Code: "print(\"Hello, world!\")", + Language: scriptLua, + Timeout: 5 * time.Minute, + Input: stubLuaInput([]string{"pod"}), + }, + }, + { + name: "bad mapping", + yamlInput: ` +script: "not a mapping" +`, + wantErr: ErrUnexpectedType, + wantPos: 10, // Position of the "not a mapping" string node, which is where the type error is detected. + }, + { + name: "bad key type", + yamlInput: ` +script: + 123: "some value" +`, + wantErr: ErrUnexpectedType, + wantPos: 12, // Position of the "123" integer node, which is where the type error is detected. + }, + { + name: "bad script code type", + yamlInput: ` +script: + code: 123 +`, + wantErr: ErrUnexpectedType, + wantPos: 18, // Position of the "123" integer node, which is where the type error for script code is detected. + }, + { + name: "bad script code", + yamlInput: ` +script: + code: "bad lua code" +`, + failLua: true, + wantErr: ErrBadScriptCode, + wantPos: 18, // Position of the "bad lua code" string node, which is where the type error for script code is detected. + }, + { + name: "bad script code no validator", + yamlInput: ` +script: + code: "bad lua code" +`, + failLua: false, + wantErr: ErrMissingScriptInput, // falls through to this error since code validation succeeds and input is required. + wantPos: 8, // Position of the "script" key node, which is where the missing input error is detected since code validation fails and input is required. + }, + { + name: "language wrong type", + yamlInput: ` +script: + language: 123 +`, + wantErr: ErrUnexpectedType, + wantPos: 22, // Position of the "123" integer node, which is where the type error for script language is detected. + }, + { + name: "empty language string", + yamlInput: ` +script: + code: "print(\"Hello, world!\")" + language: "" +`, + strict: true, + wantErr: ErrBadScriptLang, + wantPos: 57, // Position of the empty string node for language, which is where the bad script language error is detected. + }, + { + name: "empty language string not strict", + yamlInput: ` +script: + code: "print(\"Hello, world!\")" + language: "" +`, + strict: false, + wantErr: ErrMissingScriptInput, // falls through to this error since language validation succeeds and input is required. + wantPos: 8, // Position of the "script" key node, which is where the missing input error is detected since code validation fails and input is required. + }, + { + name: "unknown language", + yamlInput: ` +script: + code: "print(\"Hello, world!\")" + language: "unknown" +`, + wantErr: ErrBadScriptLang, + wantPos: 57, // Position of the "unknown" string node for language, which is where the bad script language error is detected. + }, + { + name: "input bad type", + yamlInput: ` +script: + input: 123 +`, + wantErr: ErrUnexpectedType, + wantPos: 19, // Position of the "123" integer node, which is where the type error for script input is detected. + }, + { + name: "input bad length", + yamlInput: ` +script: + input: + set: + event: + source: "stubSource" + origin: true + match: + - "pod" + two: banana +`, + wantErr: ErrUnexpectedKey, + wantPos: 122, // Position of the "two" key node, which is where the unexpected key error for script input is detected since there should only be one term definition. + }, + { + name: "input bad type in key node", + yamlInput: ` +script: + input: + 123: banana +`, + wantErr: ErrUnexpectedType, + wantPos: 23, // Position of the "123" integer node, which is where the type error for script input is detected. + }, + { + name: "empty input", + yamlInput: ` +script: + input: {} +`, + wantErr: ErrMissingScriptInput, + wantPos: 19, // Position of the empty mapping node for input, which is where the missing script input error is detected. + }, + { + name: "input malformed", + yamlInput: ` +script: + input: + setx: + event: + source: "stubSource" + origin: true + match: + - "pod" + two: banana +`, + wantErr: ErrUnexpectedKey, + wantPos: 23, // Position of the "setx" key node, which is where the unexpected key error for script input is detected since "setx" is not a valid input type. + }, + { + name: "unexpected key", + yamlInput: ` +script: + shrubbery: "nope" +`, + wantErr: ErrUnexpectedKey, + wantPos: 12, // Position of the "shrubbery" key node, which is where the unexpected key error for script input is detected. + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + node := mustParseYAMLNode(t, tt.yamlInput) + // Should be mapping Node with single key "script" + mapping, ok := node.(*ast.MappingNode) + if !ok || len(mapping.Values) != 1 { + t.Fatalf("expected a mapping node with one key, got %T with %d keys", node, len(mapping.Values)) + } + + validateLua := stubValidator + if tt.failLua { + validateLua = func(code string) error { + return ErrBadScriptCode + } + } + + v := mapping.Values[0] + p := &parserT{ + strict: tt.strict, + root: node, + maxDepth: 11, + maxRank: 11, + validateLua: validateLua, + } + state := newRuleState(&AstMetadataT{Id: stubRuleId, Hash: stubRuleHash}) + got, err := p.parseScriptNode(state, v.Value) + + if !checkParserError(t, err, tt.wantErr, tt.wantPos) { + return + } + + if tt.want != nil && !reflect.DeepEqual(got, tt.want) { + t.Errorf("parseScriptNode() = %+v, want %+v", got, tt.want) + } + }) + } +} + +func stubLuaInput(rawMatches []string) AstNode { + + var fields []AstFieldT + for _, raw := range rawMatches { + fields = append(fields, AstFieldT{ + Count: 1, + TermValue: match.TermT{ + Type: match.TermRaw, + Value: raw, + }, + }) + } + + return &AstMatchLeafT{ + baseAst: baseAst{ + scope: AstScopeNode, + address: AstNodeAddressT{ + Type: AstNodeTypeLogSet, + RuleId: stubRuleId, + RuleHash: stubRuleHash, + Depth: 1, + NodeId: 1, + }, + parent: &AstNodeAddressT{ + Type: AstNodeTypeScript, + RuleId: stubRuleId, + RuleHash: stubRuleHash, + Depth: 0, + NodeId: 0, + }, + }, + Window: 0, + Terms: fields, + Event: AstEventT{ + Source: stubSource, + Origin: true, + }, + } +} diff --git a/pkg/ast/parse_terms.go b/pkg/ast/parse_terms.go new file mode 100644 index 0000000..5cbc35d --- /dev/null +++ b/pkg/ast/parse_terms.go @@ -0,0 +1,352 @@ +package ast + +import ( + "fmt" + "math" + "regexp" + + "github.com/goccy/go-yaml/ast" +) + +// parseTerms is responsible for parsing a sequence of terms. +// +// Terms can be either line match terms (field nodes) or set/sequence/promql/script terms (child nodes). +// A non-zero negateOffset indicates that these terms are being parsed in the context of a negate clause. +// Terms must be all field nodes or all child nodes; mixing is not allowed. + +func (p *parserT) parseTerms(state ruleState, v ast.Node, negateOffset int) ([]*protoTerm, error) { + + // Expects the terms to be defined as a sequence node. If it's not a sequence, this will return an error. + seq, err := p.nodeToSequence(v) + if err != nil { + return nil, err + } + + var ( + allFields bool + terms []*protoTerm + ) + + for i, termNode := range seq.Values { + + // Sanity check on rank; this should be inside the loop since rank is incremented for each term. + // Note: maxRank is one based, whereas rank is zero based, so we check if rank+1 exceeds maxRank. + if state.rank >= p.maxRank { + err := fmt.Errorf("%w: %d", ErrMaxRankExceeded, p.maxRank) + return nil, p.wrapError(termNode, err) + } + + // Parse the term, which can be either a leaf node or an inner node. + // We determine this based on the first term, and then enforce that all subsequent terms are of the same type. + term, err := p.parseTerm(state, termNode, negateOffset) + + switch { + case err != nil: + return nil, err + + case i == 0: + // First term; determine if this is a field term or a child node term, + // and set the allFields flag accordingly. + allFields = term.field != nil + + case allFields && term.field == nil: + // This term is a child node, but previous terms were field nodes; this is not allowed. + err := fmt.Errorf("%w: all terms must be field nodes", ErrTermTypeConflict) + return nil, p.wrapError(termNode, err) + + case !allFields && term.field != nil: + // This term is a field node, but previous terms were child nodes; this is not allowed. + err := fmt.Errorf("%w: all terms must be child nodes", ErrTermTypeConflict) + return nil, p.wrapError(termNode, err) + + default: + // Term type is consistent with previous terms; continue. + } + + // Term is valid; add to list + terms = append(terms, term) + + // Increment the rank for the next term. + state = state.incRank() + } + + if p.strict && len(terms) == 0 { + return nil, p.wrapError(v, ErrMissingTerm) + } + + return terms, nil +} + +// parseTerm is responsible for parsing a single term, which can be either a simple string (field term) or a mapping (field term or child node term). +func (p *parserT) parseTerm(state ruleState, node ast.Node, negateOffset int) (*protoTerm, error) { + + // A term which can be either a simple string or a mapping. + // If the term is a mapping type, parse it as such. Otherwise, treat it as a simple string term. + if mapping, ok := node.(*ast.MappingNode); ok { + return p.parseTermAsMap(state, mapping, negateOffset) + } + + s, err := p.nodeToString(node) + if err != nil { + return nil, err + } + + return &protoTerm{ + field: &protoField{ + StrValue: s, + }, + }, nil +} + +// parseTermAsMap parses a term that is represented as a YAML mapping. +// This can represent either a field term (line match) or a child node term (set/sequence/promql/script). +// +// A non-zero negateOffset indicates that these terms are being parsed in the context of a negate clause, +// and the offset is used to validate any anchors. +// +// Layout is as follows: +// // AstFieldT fields +// Field string `yaml:"field,omitempty"` +// StrValue string `yaml:"value,omitempty"` +// JqValue string `yaml:"jq,omitempty"` +// RegexValue string `yaml:"regex,omitempty"` +// Count int `yaml:"count,omitempty"` +// Extract []ParseExtractT `yaml:"extract,omitempty"` +// +// // Child terms +// Set *ParseSetT `yaml:"set,omitempty"` +// Sequence *ParseSequenceT `yaml:"sequence,omitempty"` +// PromQL *ParsePromQL `yaml:"promql,omitempty"` +// Script *ParseScriptT `yaml:"script,omitempty"` +// +// // Applies to AstField and Child terms +// NegateOpts *ParseNegateOptsT `yaml:",inline,omitempty"` +// + +func (p *parserT) parseTermAsMap(state ruleState, mapping *ast.MappingNode, negateOffset int) (*protoTerm, error) { + + type childPair struct { + key *ast.StringNode + val ast.Node + } + + var ( + err error + child AstNode + cPair childPair + field *protoField + nOpts *AstNegateOptsT + ) + + for _, v := range mapping.Values { + + // Keep the key node around to pass to helper functions; necessary for error wrapping with context. + key, ok := v.Key.(*ast.StringNode) + if !ok { + err := fmt.Errorf("%w: %s", ErrUnexpectedType, v.Key.Type()) + return nil, p.wrapError(v.Key, err) + } + + switch key.Value { + + case kwSet, kwSequence, kwPromQL, kwScript: + // Only one child or field allowed; if either is already defined, this is an error. + if cPair.key != nil || field != nil { + return nil, p.wrapError(v.Key, ErrTermRedefined) + } + + // Defer parsing the child node until the end, + // after we've checked for field terms and negate options. + // Child node parsing requires negate options to be parsed. + cPair = childPair{key: key, val: v.Value} + + case kwField, kwValue, kwJq, kwRegex, kwCount, kwExtract: + // Only one child or field allowed; if child is already defined, this is an error. + if cPair.key != nil { + return nil, p.wrapError(v.Key, ErrTermRedefined) + } + if field == nil { + field = &protoField{Count: 1} + } + if err := p.parseTermField(key, v.Value, field, negateOffset > 0); err != nil { + return nil, err + } + + case kwWindow, kwSlide, kwAnchor, kwAbsolute: + // Negate options are only allowed if there is a non-zero negate offset, + // which indicates that we are parsing terms in the context of a negate clause. + if negateOffset == 0 { + err := fmt.Errorf("%w: negate options not allowed on positive term", ErrUnexpectedKey) + return nil, p.wrapError(v.Key, err) + } + if nOpts == nil { + nOpts = &AstNegateOptsT{} + } + if err := p.parseNegateOpts(key.Value, v.Value, nOpts, negateOffset); err != nil { + return nil, err + } + + default: + return nil, p.wrapError(v.Key, ErrUnexpectedKey) + } + } + + if cPair.key != nil { + if child, err = p.parseTermChild(state, cPair.key, cPair.val, nOpts); err != nil { + return nil, err + } + } + + if field != nil { + if err := field.validate(); err != nil { + return nil, p.wrapErrorParent(mapping, err) + } + } + + return &protoTerm{ + negateOpts: nOpts, + child: child, + field: field, + }, nil +} + +func (p *parserT) parseTermChild(state ruleState, key *ast.StringNode, val ast.Node, nOpts *AstNegateOptsT) (AstNode, error) { + + switch key.Value { + + case kwSet: + return p.parseNode(state, AstNodeTypeSet, val) + + case kwSequence: + return p.parseNode(state, AstNodeTypeSeq, val) + + case kwPromQL: + if nOpts != nil { + err := fmt.Errorf("%w: negate options cannot be used with %s terms", ErrUnexpectedKey, key.Value) + return nil, p.wrapError(key, err) + } + return p.parsePromQLNode(state, val) + + case kwScript: + if nOpts != nil { + err := fmt.Errorf("%w: negate options cannot be used with %s terms", ErrUnexpectedKey, key.Value) + return nil, p.wrapError(key, err) + } + return p.parseScriptNode(state, val) + + default: + // Should not happen; parseTermAsMap should only call this on expected keys. + return nil, p.wrapError(key, ErrUnexpectedKey) + } +} + +func (p *parserT) parseTermField(key *ast.StringNode, v ast.Node, match *protoField, allowNegate bool) error { + + var ( + err error + hasValue bool + ) + + if match.RegexValue != "" || match.JqValue != "" || match.StrValue != "" { + hasValue = true + } + + mkMatchKeyError := func() error { + kerr := fmt.Errorf("%w: '%s' key cannot be used together with other line match keys", ErrUnexpectedKey, key.Value) + return p.wrapError(key, kerr) + } + + switch key.Value { + case kwField: + match.Field, err = p.nodeToString(v) + + case kwCount: + match.Count, err = p.nodeToUint64(v) + switch { + case err != nil: + // fall through + case match.Count == 0: + err = p.wrapError(key, ErrZeroCount) + case allowNegate && match.Count > 1: + err = p.wrapError(key, ErrNegateCount) + } + + case kwExtract: + match.Extract, err = p.parseExtracts(v) + + case kwJq: + if hasValue { + err = mkMatchKeyError() + } else { + match.JqValue, err = p.nodeToJq(v) + } + + case kwRegex: + if hasValue { + err = mkMatchKeyError() + } else { + var exp *regexp.Regexp + if exp, err = p.nodeToRegex(v); err == nil { + match.RegexValue = exp.String() + } + } + + case kwValue: + if hasValue { + err = mkMatchKeyError() + } else { + match.StrValue, err = p.nodeToString(v) + if err == nil && match.StrValue == "" { + err = p.wrapError(v, fmt.Errorf("%w: value cannot be empty", ErrBadField)) + } + } + + default: + // Should not happen; parseTermAsMap should only call this on expected keys. + err = p.wrapError(key, ErrUnexpectedKey) + } + + return err +} + +func (p *parserT) parseNegateOpts(key string, node ast.Node, opts *AstNegateOptsT, negateOffset int) error { + var err error + + switch key { + case kwWindow: + opts.Window, err = p.nodeToDurationPositive(node) + + case kwSlide: + opts.Slide, err = p.nodeToDuration(node) + + case kwAnchor: + var anchorInt uint64 + anchorInt, err = p.nodeToUint64(node) + switch { + case err != nil: + // fall through + case anchorInt >= uint64(negateOffset): + nerr := fmt.Errorf("%w: anchor value must be in range of [0, %d)", ErrBadAnchor, negateOffset) + err = p.wrapError(node, nerr) + case anchorInt > math.MaxUint8: + // Anchor is later cast to a uint8 (see match.ResetT.Anchor) + nerr := fmt.Errorf("%w: anchor value must be less than or equal to %d", ErrBadAnchor, math.MaxUint8) + err = p.wrapError(node, nerr) + default: + opts.Anchor = uint32(anchorInt) + } + + case kwAbsolute: + opts.Absolute, err = p.nodeToBool(node) + + default: + // Should not happen; parseTermAsMap should only call this on expected keys. + err = fmt.Errorf("%w: %s", ErrUnexpectedKey, key) + } + + if err != nil { + ferr := fmt.Errorf("failed to parse negate option '%s': %w", key, err) + return p.wrapError(node, ferr) + } + return nil +} diff --git a/pkg/ast/parse_terms_test.go b/pkg/ast/parse_terms_test.go new file mode 100644 index 0000000..942c706 --- /dev/null +++ b/pkg/ast/parse_terms_test.go @@ -0,0 +1,696 @@ +package ast + +import ( + "errors" + "reflect" + "testing" + "time" + + "github.com/goccy/go-yaml/ast" + "github.com/prequel-dev/prequel-logmatch/pkg/match" +) + +const ( + stubRuleId = "test" + stubRuleHash = "hash" + stubSource = "stubSource" +) + +func TestParseTerms(t *testing.T) { + tests := []struct { + name string + yamlInput string + negateOff int + strict bool + want []*protoTerm + wantErr error + wantPos int + }{ + { + name: "valid term with one field", + yamlInput: ` +terms: + - "simple string match" +`, + want: []*protoTerm{ + { + field: &protoField{ + StrValue: "simple string match", + }, + }, + }, + }, + { + name: "valid term with two fields", + yamlInput: ` +terms: + - "simple string match 1" + - "simple string match 2" +`, + want: []*protoTerm{ + { + field: &protoField{ + StrValue: "simple string match 1", + }, + }, + { + field: &protoField{ + StrValue: "simple string match 2", + }, + }, + }, + }, + { + name: "valid term with one child", + yamlInput: ` +terms: + - set: + event: + source: stubSource + origin: true + match: + - "child node match" +`, + want: []*protoTerm{ + { + child: genStubChild([]string{"child node match"}), + }, + }, + }, + { + name: "valid term with two children", + strict: true, + yamlInput: ` +terms: + - set: + event: + source: stubSource + origin: true + match: + - "child node match 1" + - set: + event: + source: stubSource + match: + - "child node match 2" +`, + want: genStubChildren([]string{"child node match 1", "child node match 2"}), + }, + { + name: "invalid term with mixed field and child nodes", + yamlInput: ` +terms: + - "simple string match" + - set: + event: + source: stubSource + match: + - "child node match 2" +`, + wantErr: ErrTermTypeConflict, + wantPos: 42, // Position of the "set" key in the second term, which is where the type inconsistency is detected. + }, + { + name: "invalid term with mixed field and child nodes reversed", + yamlInput: ` +terms: + - set: + event: + source: stubSource + match: + - "child node match 1" + - "simple string match" +`, + wantErr: ErrTermTypeConflict, + wantPos: 104, // Position of the second term node (the "simple string match" string), which is where the type inconsistency is detected. + }, + { + name: "test bad terms type", + yamlInput: ` +terms: "not a sequence" +`, + wantErr: ErrUnexpectedType, + wantPos: 9, // Position of the "not a sequence" value node (string instead of sequence) + }, + { + name: "empty terms strict mode", + yamlInput: ` +terms: [] +`, + wantErr: ErrMissingTerm, + strict: true, + wantPos: 9, // Position of the empty sequence node, which is where the missing term error is detected in strict mode. + }, + { + name: "empty terms tolerant mode", + yamlInput: ` +terms: [] +`, + wantErr: nil, + strict: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + node := mustParseYAMLNode(t, tt.yamlInput) + // Should be mapping Node with single key "terms" + mapping, ok := node.(*ast.MappingNode) + if !ok || len(mapping.Values) != 1 { + t.Fatalf("expected a mapping node with one key, got %T with %d keys", node, len(mapping.Values)) + } + v := mapping.Values[0] + p := &parserT{strict: tt.strict, root: node, maxRank: 10, maxDepth: 5} + state := newRuleState(&AstMetadataT{Id: stubRuleId, Hash: stubRuleHash}) + got, err := p.parseTerms(state, v.Value, tt.negateOff) + + if !checkParserError(t, err, tt.wantErr, tt.wantPos) { + return + } + + if tt.want != nil && !reflect.DeepEqual(got, tt.want) { + t.Errorf("parseTerms() = %+v, want %+v", got, tt.want) + } + }) + } +} + +func TestParseTerm(t *testing.T) { + tests := []struct { + name string + yamlInput string + negateOff int + failJQ bool + strict bool + want *protoTerm + wantErr error + wantPos int + }{ + { + name: "simple string term", + yamlInput: ` +term: "simple string match" +`, + want: &protoTerm{ + field: &protoField{ + StrValue: "simple string match", + }, + }, + }, + { + name: "bad term type", + yamlInput: ` +term: 11 +`, + wantErr: ErrUnexpectedType, + wantPos: 8, // Position of the "11" value node (integer instead of string or mapping) + }, + { + name: "bad key type", + yamlInput: ` +term: + 123: "value" +`, + wantErr: ErrUnexpectedType, + wantPos: 10, // Position of the "123" key node (integer instead of string) + }, + { + name: "simple field term with value key and field", + yamlInput: ` +term: + field: "field1" + value: "simple string match with value key" +`, + want: &protoTerm{ + field: &protoField{ + Count: 1, + Field: "field1", + StrValue: "simple string match with value key", + }, + }, + }, + { + name: "simple field term with jq key", + yamlInput: ` +term: + jq: ".name" +`, + want: &protoTerm{ + field: &protoField{ + Count: 1, + JqValue: ".name", + }, + }, + }, + { + name: "simple field term with bad jq key", + yamlInput: ` +term: + jq: "bad jq" +`, + want: &protoTerm{ + field: &protoField{ + Count: 1, + JqValue: ".name", + }, + }, + failJQ: true, + wantErr: ErrBadJq, + wantPos: 14, // Position of the "bad jq" string node, which is where the invalid jq error is detected. + }, + { + name: "simple field term with regex key", + yamlInput: ` +term: + regex: ".*" +`, + want: &protoTerm{ + field: &protoField{ + Count: 1, + RegexValue: ".*", + }, + }, + }, + { + name: "simple field term with bad regex key", + yamlInput: ` +term: + regex: "(abc" +`, + wantErr: ErrBadRegex, + wantPos: 17, // Position of the "(abc" string node, which is where the invalid regex error is detected. + }, + { + name: "simple field term with empty regex key", + yamlInput: ` +term: + regex: "" +`, + wantErr: ErrBadRegex, + wantPos: 17, // Position of the empty string node, which is where the invalid regex error is detected. + }, + { + name: "no term defined", + yamlInput: ` +term: + field: "field1" +`, + wantErr: ErrBadField, + wantPos: 6, // Positiong of the term mapping node, which is where the missing term definition is detected. + }, + { + name: "empty string term", + yamlInput: ` +term: + value: "" +`, + wantErr: ErrBadField, + wantPos: 17, // Position of the empty string node, which is where the empty value error is detected. + }, + { + name: "regex that matches empty string", + yamlInput: ` +term: + regex: "^$" +`, + want: &protoTerm{ + field: &protoField{ + Count: 1, + RegexValue: "^$", + }, + }, + }, + { + name: "field already defined", + yamlInput: ` +term: + value: "field defined first" + promql: + expr: "rate(http_requests_total[5m])" +`, + wantErr: ErrTermRedefined, + wantPos: 41, // Position of the "promql" key node, which is where the term redefinition error is detected (second term definition in the same term). + }, + { + name: "promql already defined", + yamlInput: ` +term: + promql: + expr: "rate(http_requests_total[5m])" + value: "field defined first" +`, + wantErr: ErrTermRedefined, + wantPos: 62, // Position of the "value" key node, which is where the term redefinition error is detected (second term definition in the same term). + }, + { + name: "negate key not allowed on non-negate term", + yamlInput: ` +term: + value: "simple string match" + anchor: 0 +`, + wantErr: ErrUnexpectedKey, + wantPos: 41, // Position of the "value" key node, which is where the term redefinition error is detected (second term definition in the same term). + }, + { + name: "negate key not allowed on promql terms", + yamlInput: ` +term: + promql: + expr: "rate(http_requests_total[5m])" + anchor: 0 +`, + wantErr: ErrUnexpectedKey, + wantPos: 62, // Position of the "value" key node, which is where the term redefinition error is detected (second term definition in the same term). + }, + { + name: "negate key not allowed on promql terms reversed", + yamlInput: ` +term: + anchor: 0 + promql: + expr: "rate(http_requests_total[5m])" +`, + negateOff: 1, + wantErr: ErrUnexpectedKey, + wantPos: 22, // Position of the "value" key node, which is where the term redefinition error is detected (second term definition in the same term). + }, + { + name: "negate key not allowed on script terms", + yamlInput: ` +term: + script: + code: "console.log('hello world')" + input: + set: + event: + source: stubSource + origin: true + match: + - "child node match" + anchor: 0 +`, + wantErr: ErrUnexpectedKey, + wantPos: 192, // Position of the "anchor" key node, which is where the unexpected key error is detected in the context of a script term. + }, + { + name: "negate key not allowed on script terms reversed", + yamlInput: ` +term: + anchor: 0 + script: + code: "console.log('hello world')" + input: + set: + event: + source: stubSource + origin: true + match: + - "child node match" +`, + negateOff: 1, + wantErr: ErrUnexpectedKey, + wantPos: 22, // Position of the "script" key node, which is where the unexpected key error is detected in the context of a script term. + }, + { + name: "jq and regex keys not allowed on the same term", + yamlInput: ` +term: + jq: ".name" + regex: ".*" +`, + wantErr: ErrUnexpectedKey, + wantPos: 24, // Position of the "regex" key node, which is where the term redefinition error is detected (second term definition in the same term). + }, + { + name: "jq and regex keys not allowed on the same term reversed", + yamlInput: ` +term: + regex: ".*" + jq: ".name" +`, + wantErr: ErrUnexpectedKey, + wantPos: 24, // Position of the "jq" key node, which is where the term redefinition error is detected (second term definition in the same term). + }, + { + name: "value and regex keys not allowed on the same term", + yamlInput: ` +term: + regex: ".*" + value: "some value" +`, + wantErr: ErrUnexpectedKey, + wantPos: 24, // Position of the "value" key node, which is where the term redefinition error is detected (second term definition in the same term). + }, + { + name: "value and regex keys not allowed on the same term reversed", + yamlInput: ` +term: + value: "some value" + regex: ".*" +`, + wantErr: ErrUnexpectedKey, + wantPos: 32, // Position of the "regex" key node, which is where the term redefinition error is detected (second term definition in the same term). + }, + { + name: "simple count", + yamlInput: ` +term: + value: "some value" + count: 11 +`, + want: &protoTerm{ + field: &protoField{ + Count: 11, + StrValue: "some value", + }, + }, + }, + { + name: "negative count", + yamlInput: ` +term: + value: "some value" + count: -1 +`, + wantErr: ErrUnexpectedType, + wantPos: 39, // Position of the "-1" integer node, which is where the invalid count error is detected (negative count value). + }, + { + name: "zero count", + yamlInput: ` +term: + value: "some value" + count: 0 +`, + wantErr: ErrZeroCount, + wantPos: 32, // Position of the "count" key node, which is where the zero count error is detected. + }, + { + name: "non-one count on negative term", + yamlInput: ` +term: + value: "some value" + count: 2 +`, + negateOff: 1, + wantErr: ErrNegateCount, + wantPos: 32, // Position of the "count" key node, which is where the non-one count error is detected on a negative term. + }, + { + name: "valid negate opts", + yamlInput: ` +term: + value: "some value" + anchor: 0 + window: 1m + slide: 30s + absolute: true +`, + negateOff: 1, + want: &protoTerm{ + field: &protoField{ + Count: 1, + StrValue: "some value", + }, + negateOpts: &AstNegateOptsT{ + Anchor: 0, + Window: time.Minute, + Slide: time.Second * 30, + Absolute: true, + }, + }, + }, + { + name: "negate opts negative anchor", + yamlInput: ` +term: + value: "some value" + anchor: -1 +`, + negateOff: 1, + wantErr: ErrUnexpectedType, + wantPos: 40, // Position of the "anchor" key node, which is where the bad anchor error is detected. + }, + { + name: "negate opts bad anchor", + yamlInput: ` +term: + value: "some value" + anchor: 1 +`, + negateOff: 1, + wantErr: ErrBadAnchor, + wantPos: 40, // Position of the "anchor" key node, which is where the bad anchor error is detected. + }, + { + name: "negate opts overflow anchor", + yamlInput: ` +term: + value: "some value" + anchor: 256 +`, + negateOff: 257, + wantErr: ErrBadAnchor, + wantPos: 40, // Position of the "anchor" key node, which is where the bad anchor error is detected. + }, + { + name: "disallow negate on script term", + yamlInput: ` +term: + promql: + expr: "rate(http_requests_total[5m])" + anchor: 0 +`, + negateOff: 1, + wantErr: ErrUnexpectedKey, + wantPos: 10, // Position of the "anchor" key node, which is where the unexpected key error is detected in the context of a promql term. + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + node := mustParseYAMLNode(t, tt.yamlInput) + + // Should be mapping Node with single key "term" + mapping, ok := node.(*ast.MappingNode) + if !ok || len(mapping.Values) != 1 { + t.Fatalf("expected a mapping node with one key, got %T with %d keys", node, len(mapping.Values)) + } + v := mapping.Values[0] + + validateJQ := stubValidator + if tt.failJQ { + validateJQ = func(s string) error { + return errors.New("invalid jq") + } + } + + p := &parserT{ + strict: tt.strict, + root: node, + maxRank: 10, + maxDepth: 5, + validateJQ: validateJQ, + validatePromQL: stubValidator, + validateLua: stubValidator, + } + + state := newRuleState(&AstMetadataT{Id: stubRuleId, Hash: stubRuleHash}) + got, err := p.parseTerm(state, v.Value, tt.negateOff) + + if !checkParserError(t, err, tt.wantErr, tt.wantPos) { + return + } + + if tt.want != nil && !reflect.DeepEqual(got, tt.want) { + t.Errorf("parseTerm() = %+v, want %+v", got, tt.want) + } + }) + } +} + +func TestTermsCoverageHack(t *testing.T) { + // This test exists solely to increase coverage of the default cases in the switch statements of parseTerm and parseNegateOpts, + // which should be unreachable if parseTermAsMap is correctly implemented, but we include them for completeness. + key := &ast.StringNode{Value: "unexpected"} + p := &parserT{} + _, err := p.parseTermChild(ruleState{}, key, nil, nil) + + if !errors.Is(err, ErrUnexpectedKey) { + t.Errorf("expected ErrUnexpectedKey, got %v", err) + } + + err = p.parseNegateOpts("unexpected", nil, nil, 0) + if !errors.Is(err, ErrUnexpectedKey) { + t.Errorf("expected ErrUnexpectedKey, got %v", err) + } + + err = p.parseTermField(key, nil, &protoField{}, false) + if !errors.Is(err, ErrUnexpectedKey) { + t.Errorf("expected ErrUnexpectedKey, got %v", err) + } +} + +func genStubChildren(rawMatches []string) []*protoTerm { + children := make([]*protoTerm, 0, len(rawMatches)) + for i, v := range rawMatches { + u := uint32(i) + stub := _genStubChild(2*u, u, i == 0, []string{v}) + children = append(children, &protoTerm{child: stub}) + } + return children +} + +func genStubChild(rawMatches []string) *AstInnerNodeT { + return _genStubChild(0, 0, true, rawMatches) +} + +// _genStubChild is a helper function to generate a stub child node for testing purposes, +// given a list of raw match strings. +func _genStubChild(nodeId, rank uint32, origin bool, rawMatches []string) *AstInnerNodeT { + + parentAddr := &AstNodeAddressT{ + Type: AstNodeTypeSet, + RuleId: stubRuleId, + RuleHash: stubRuleHash, + NodeId: nodeId, + Rank: rank, + } + + leaf := &AstMatchLeafT{ + baseAst: baseAst{ + scope: AstScopeNode, + address: AstNodeAddressT{ + Type: AstNodeTypeLogSet, + RuleId: stubRuleId, + RuleHash: stubRuleHash, + Depth: 1, + NodeId: nodeId + 1, + }, + parent: parentAddr, + }, + Window: 0, + Terms: []AstFieldT{}, + Event: AstEventT{ + Source: stubSource, + Origin: origin, + }, + } + for _, raw := range rawMatches { + leaf.Terms = append(leaf.Terms, AstFieldT{ + Count: 1, + TermValue: match.TermT{ + Type: match.TermRaw, + Value: raw, + }, + }) + } + + return &AstInnerNodeT{ + baseAst: baseAst{ + scope: AstScopeCluster, + address: *parentAddr, + }, + Terms: []AstTermT{{Term: leaf}}, + } +} diff --git a/pkg/ast/parse_test.go b/pkg/ast/parse_test.go new file mode 100644 index 0000000..7917133 --- /dev/null +++ b/pkg/ast/parse_test.go @@ -0,0 +1,570 @@ +package ast + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/prequel-dev/prequel-compiler/pkg/testdata" +) + +func TestAstSuccess(t *testing.T) { + + tests := []struct { + name string + yaml string + wantOrder []string + }{ + { + name: "Success_Simple1", + yaml: testdata.TestSuccessSimpleRule1, + wantOrder: []string{ + "v1.machine_set.rdJLgqYgkEp8jg8Qks1qiq.d0.n0.t0", + "v1.log_set.rdJLgqYgkEp8jg8Qks1qiq.d1.n1.t0", + }, + }, + { + name: "Success_Complex2", + yaml: testdata.TestSuccessComplexRule2, + wantOrder: []string{ + "v1.machine_seq.rdJLgqYgkEp8jg8Qks1qiq.d0.n0.t0", + "v1.log_seq.rdJLgqYgkEp8jg8Qks1qiq.d1.n1.t0", + "v1.log_set.rdJLgqYgkEp8jg8Qks1qiq.d1.n2.t1", + "v1.machine_seq.rdJLgqYgkEp8jg8Qks1qiq.d1.n3.t2", + "v1.log_seq.rdJLgqYgkEp8jg8Qks1qiq.d2.n4.t0", + "v1.log_set.rdJLgqYgkEp8jg8Qks1qiq.d2.n5.t1", + "v1.log_set.rdJLgqYgkEp8jg8Qks1qiq.d2.n6.t2", + }, + }, + { + name: "Success_Complex3", + yaml: testdata.TestSuccessComplexRule3, + wantOrder: []string{ + "v1.machine_seq.rdJLgqYgkEp8jg8Qks1qiq.d0.n0.t0", + "v1.log_seq.rdJLgqYgkEp8jg8Qks1qiq.d1.n1.t0", + "v1.log_set.rdJLgqYgkEp8jg8Qks1qiq.d1.n2.t1", + }, + }, + { + name: "Success_Complex4", + yaml: testdata.TestSuccessComplexRule4, + wantOrder: []string{ + "v1.machine_seq.2KdXQZDAfRbYcH9FBDteBS.d0.n0.t0", + "v1.log_seq.2KdXQZDAfRbYcH9FBDteBS.d1.n1.t0", + "v1.machine_seq.2KdXQZDAfRbYcH9FBDteBS.d1.n2.t1", + "v1.log_seq.2KdXQZDAfRbYcH9FBDteBS.d2.n3.t0", + "v1.log_set.2KdXQZDAfRbYcH9FBDteBS.d2.n4.t1", + "v1.log_set.2KdXQZDAfRbYcH9FBDteBS.d2.n5.t2", + "v1.machine_seq.2KdXQZDAfRbYcH9FBDteBS.d1.n6.t2", + "v1.log_seq.2KdXQZDAfRbYcH9FBDteBS.d2.n7.t0", + "v1.log_set.2KdXQZDAfRbYcH9FBDteBS.d2.n8.t1", + "v1.log_set.2KdXQZDAfRbYcH9FBDteBS.d2.n9.t2", + "v1.log_set.2KdXQZDAfRbYcH9FBDteBS.d1.n10.t3", + }, + }, + { + name: "Success_NegateOptions1", + yaml: testdata.TestSuccessNegateOptions1, + wantOrder: []string{ + "v1.machine_set.rdJLgqYgkEp8jg8Qks1qiq.d0.n0.t0", + "v1.log_set.rdJLgqYgkEp8jg8Qks1qiq.d1.n1.t0", + }, + }, + { + name: "Success_NegateOptions2", + yaml: testdata.TestSuccessNegateOptions2, + wantOrder: []string{ + "v1.machine_seq.rdJLgqYgkEp8jg8Qks1qiq.d0.n0.t0", + "v1.log_seq.rdJLgqYgkEp8jg8Qks1qiq.d1.n1.t0", + "v1.log_set.rdJLgqYgkEp8jg8Qks1qiq.d1.n2.t1", + "v1.log_set.rdJLgqYgkEp8jg8Qks1qiq.d1.n3.t2", + }, + }, + { + name: "Success_Extract1", + yaml: testdata.TestSuccessSimpleExtraction, + wantOrder: []string{ + "v1.machine_seq.rdJLgqYgkEp8jg8Qks1qiq.d0.n0.t0", + "v1.log_seq.rdJLgqYgkEp8jg8Qks1qiq.d1.n1.t0", + }, + }, + { + name: "Success_PromQL", + yaml: testdata.TestSuccessSimplePromQL, + wantOrder: []string{ + "v1.machine_set.rdJLgqYgkEp8jg8Qks1qiq.d0.n0.t0", + "v1.promql.rdJLgqYgkEp8jg8Qks1qiq.d1.n1.t0", + "v1.log_set.rdJLgqYgkEp8jg8Qks1qiq.d1.n2.t1", + }, + }, + { + name: "Success_ChildScript", + yaml: testdata.TestSuccessChildScript, + wantOrder: []string{ + "v1.machine_seq.rdJLgqYgkEp8jg8Qks1qiq.d0.n0.t0", + "v1.script.rdJLgqYgkEp8jg8Qks1qiq.d1.n1.t0", + "v1.log_seq.rdJLgqYgkEp8jg8Qks1qiq.d2.n2.t0", + "v1.log_set.rdJLgqYgkEp8jg8Qks1qiq.d1.n3.t1", + }, + }, + { + name: "Success_ChildScriptMultipleInputs", + yaml: testdata.TestSuccessChildScriptMultipleInputs, + wantOrder: []string{ + "v1.machine_set.rdJLgqYgkEp8jg8Qks1qiq.d0.n0.t0", + "v1.script.rdJLgqYgkEp8jg8Qks1qiq.d1.n1.t0", + "v1.machine_seq.rdJLgqYgkEp8jg8Qks1qiq.d2.n2.t0", + "v1.log_seq.rdJLgqYgkEp8jg8Qks1qiq.d3.n3.t0", + "v1.log_set.rdJLgqYgkEp8jg8Qks1qiq.d3.n4.t1", + }, + }, + { + name: "Success_ChildScriptPromQLInput", + yaml: testdata.TestSuccessChildScriptPromQLInput, + wantOrder: []string{ + "v1.machine_set.rdJLgqYgkEp8jg8Qks1qiq.d0.n0.t0", + "v1.script.rdJLgqYgkEp8jg8Qks1qiq.d1.n1.t0", + "v1.promql.rdJLgqYgkEp8jg8Qks1qiq.d2.n2.t0", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + + rules, err := ParseRules([]byte(tt.yaml), WithStrict(true)) + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if len(rules) != 1 { + t.Fatalf("expected 1 rule, got %d", len(rules)) + } + + compareTree(t, rules[0], tt.wantOrder) + }) + } +} + +func TestAstFail(t *testing.T) { + tests := []struct { + name string + yaml string + strict bool + maxRank uint32 + maxDepth uint32 + wantErr error + wantPos int + }{ + { + name: "Fail_MissingPositiveCondition", + yaml: testdata.TestFailMissingPositiveCondition, + wantErr: ErrMissingTerm, + wantPos: 607, + }, + { + name: "Fail_NegativeCondition1", + yaml: testdata.TestFailNegativeCondition1, + wantErr: ErrMissingTerm, + wantPos: 635, + }, + { + name: "Fail_NegativeCondition2", + yaml: testdata.TestFailNegativeCondition2, + wantErr: ErrMissingTerm, + wantPos: 635, + }, + { + name: "Fail_NegativeCondition2_Strict", + yaml: testdata.TestFailNegativeCondition2, + wantErr: ErrUnexpectedKey, // 'imageUrl' + strict: true, + wantPos: 420, + }, + { + name: "Fail_NegativeCondition3", + yaml: testdata.TestFailNegateOptions3, + wantErr: ErrMissingTerm, + wantPos: 749, + }, + { + name: "Fail_NegativeCondition4", + yaml: testdata.TestFailNegateOptions4, + wantErr: ErrMissingTerm, + wantPos: 765, + }, + { + name: "Fail_TermsSyntaxError1", + yaml: testdata.TestFailTermsSyntaxError1, + wantErr: ErrUnexpectedKey, + wantPos: 664, + }, + { + name: "Fail_TermsSyntaxError2", + yaml: testdata.TestFailTermsSyntaxError2, + wantErr: ErrUnexpectedType, + wantPos: 672, + }, + { + name: "Fail_TermsSemanticError1", + yaml: testdata.TestFailTermsSemanticError1, + wantErr: ErrShortSequence, + wantPos: 697, + }, + { + name: "Fail_TermsSemanticError2", + yaml: testdata.TestFailTermsSemanticError2, + wantErr: ErrMissingEvent, + wantPos: 199, + }, + { + name: "Fail_TermsSemanticError3", + yaml: testdata.TestFailTermsSemanticError3, + wantErr: ErrMissingOrigin, + wantPos: 183, + }, + { + name: "Fail_TermsSemanticError4", + yaml: testdata.TestFailTermsSemanticError4, + wantErr: ErrUnexpectedType, + wantPos: 314, + }, + { + name: "Fail_TermsSemanticError5", + yaml: testdata.TestFailTermsSemanticError5, + wantErr: ErrBadAnchor, + wantPos: 411, + }, + { + name: "Fail_TermsSemanticError6", + yaml: testdata.TestFailTermsSemanticError6, + strict: true, + wantErr: ErrMissingOrigin, + wantPos: 183, + }, + { + name: "Fail_TermsSemanticError6_NonStrict", + yaml: testdata.TestFailTermsSemanticError6, + strict: false, // Normally fails in strict mode due to origin requirement, but should fail with missing origin error in non-strict mode. + }, + { + name: "Fail_MultipleOrigin", + yaml: testdata.TestFailMultipleOrigin, + wantErr: ErrMultipleOrigin, + wantPos: 502, + }, + { + name: "Fail_Typo", + yaml: testdata.TestFailTypo, + wantErr: ErrUnexpectedKey, + wantPos: 290, + }, + { + name: "Fail_MissingOrder", + yaml: testdata.TestFailMissingOrder, + wantErr: ErrUnexpectedKey, + wantPos: 279, + }, + { + name: "Fail_MissingMatch", + yaml: testdata.TestFailMissingMatch, + wantErr: ErrUnexpectedKey, + wantPos: 274, + }, + { + name: "Fail_InvalidWindow", + yaml: testdata.TestFailInvalidWindow, + wantErr: ErrUnexpectedType, + wantPos: 224, + }, + { + name: "Fail_UnsupportedRule", + yaml: testdata.TestFailUnsupportedRule, + wantErr: ErrUnexpectedKey, + wantPos: 203, + }, + { + name: "Fail_MissingCreId", + yaml: testdata.TestFailMissingCreRule, + wantErr: ErrMissingKey, + wantPos: 36, + }, + { + name: "Fail_MissingRuleId", + yaml: testdata.TestFailMissingRuleIdRule, + wantErr: ErrMissingKey, + wantPos: 100, + }, + { + name: "Fail_MissingRuleHash", + yaml: testdata.TestFailMissingRuleHashRule, + wantErr: ErrMissingKey, + wantPos: 102, + }, + { + name: "Fail_BadRuleId", + yaml: testdata.TestFailBadRuleIdRule, + wantErr: ErrBadIdentifier, + wantPos: 108, + }, + { + name: "Fail_BadCreId", + yaml: testdata.TestFailBadCreIdRule, + wantErr: ErrBadIdentifier, + wantPos: 48, + }, + { + name: "Fail_BadRuleHash", + yaml: testdata.TestFailBadRuleHashRule, + wantErr: ErrBadHash, + wantPos: 147, + }, + { + name: "Fail_ScriptRoot", + yaml: testdata.TestFailScriptRoot, + wantErr: ErrUnexpectedKey, + wantPos: 162, + }, + { + name: "Fail_ScriptNoInput", + yaml: testdata.TestFailScriptNoInput, + wantErr: ErrMissingScriptInput, + wantPos: 203, + }, + { + name: "Fail_MissingWindow", + yaml: testdata.TestFailMissingWindow, + wantErr: ErrMissingWindow, + wantPos: 151, + }, + { + name: "Fail_MaxDepthExceeded", + yaml: testdata.TestFailMaxDepthExceeded, + maxDepth: 2, + wantErr: ErrMaxDepthExceeded, + wantPos: 234, // Position of the node that exceeds the max depth + }, + { + name: "Fail_MaxRankExceeded", + yaml: testdata.TestFailMaxRankExceeded, + maxRank: 3, + wantErr: ErrMaxRankExceeded, + wantPos: 322, // Position of the node that exceeds the max rank + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + + opts := []ParseOpt{WithStrict(tt.strict)} + if tt.maxDepth > 0 { + opts = append(opts, WithMaxDepth(tt.maxDepth)) + } + if tt.maxRank > 0 { + opts = append(opts, WithMaxRank(tt.maxRank)) + } + + rules, err := ParseRules([]byte(tt.yaml), opts...) + + ok := checkParserError(t, err, tt.wantErr, tt.wantPos) + + switch { + case ok && rules == nil: + t.Errorf("expected rules to be returned, got nil") + case !ok && rules != nil: + t.Errorf("expected no rules to be returned, got %v", rules) + } + }) + } +} + +// Validate the following invariants on the tree: +// 1. No duplicate addresses +// 2. Root node has no parent address +// 3. Node ids are unique +// 4. Depth is consistent with distance from root + +func extractOrder(t *testing.T, rule AstRuleT) []string { + t.Helper() + + var ( + order []string + dupeIds = make(map[uint32]struct{}) + dupeAddr = make(map[string]struct{}) + ) + + err := rule.Walk(func(node AstNode, _ *AstNegateOptsT) error { + var ( + addr = node.Address() + addrStr = addr.String() + ) + + order = append(order, addrStr) + + if _, exists := dupeIds[addr.NodeId]; exists { + t.Fatalf("Duplicate node ID found: %d", addr.NodeId) + } + dupeIds[addr.NodeId] = struct{}{} + + if _, exists := dupeAddr[addrStr]; exists { + t.Fatalf("Duplicate address found: %s", addrStr) + } + dupeAddr[addrStr] = struct{}{} + + // Depth should be one larger than parent's depth; + // or can rewinde if going back up the tree + if parentAddr := node.Parent(); parentAddr == nil { + if addr.Depth != 0 { + t.Fatalf("Root node has non-zero depth: %d", addr.Depth) + } + } else if addr.Depth != parentAddr.Depth+1 { + t.Fatalf("Node depth inconsistent with parent's depth: %d (parent: %d)", addr.Depth, parentAddr.Depth) + } + + return nil + }) + + if err != nil { + t.Fatalf("Walk failed: %v", err) + } + + return order +} + +func TestSuccessExamples(t *testing.T) { + + rules, err := filepath.Glob(filepath.Join("../testdata", "success_examples", "*.yaml")) + if err != nil { + t.Fatalf("Error finding CRE test files: %v", err) + } + + for _, rule := range rules { + + t.Run(filepath.Base(rule), func(t *testing.T) { + testData, err := os.ReadFile(rule) + if err != nil { + t.Fatalf("Error reading test file %s: %v", rule, err) + } + + _, err = ParseRules(testData, WithStrict(true)) + if err != nil { + t.Fatalf("Error building rule %s: %v", rule, err) + } + }) + } +} + +// Test that multiple documents in a single file are handled correctly. In this case, we just want to make sure that the second document is ignored and does not cause an error. +func TestMultipleDocuments(t *testing.T) { + + docs := []string{ + testdata.TestSuccessSimpleRule1, + testdata.TestSuccessSimpleExtraction, + testdata.TestSuccessSimplePromQL, + } + + input := strings.Join(docs, "\n---\n") + + rules, err := ParseRules([]byte(input), WithStrict(true)) + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if len(rules) != 3 { + t.Fatalf("expected 3 rules, got %d", len(rules)) + } + +} + +func TestFailureExamples(t *testing.T) { + + rules, err := filepath.Glob(filepath.Join("../testdata", "failure_examples", "*.yaml")) + if err != nil { + t.Fatalf("Error finding CRE test files: %v", err) + } + + for _, rule := range rules { + + t.Run(filepath.Base(rule), func(t *testing.T) { + + testData, err := os.ReadFile(rule) + if err != nil { + t.Fatalf("Error reading test file %s: %v", rule, err) + } + + _, err = ParseRules(testData, WithStrict(true)) + + if err == nil { + t.Fatalf("expected failure, got nil") + } + + }) + } +} + +// Return true if in a non error state +func checkParserError(t *testing.T, err error, wantErr error, wantPos int) bool { + t.Helper() + + var ( + perr ParseError + hasPerr = errors.As(err, &perr) + ) + + if !errors.Is(err, wantErr) { + t.Errorf("expected error '%v', got '%v'", wantErr, err) + if hasPerr { + t.Log(perr.Format(false, true)) + } + } + + if wantErr == nil { + return true + } + + switch { + case wantPos <= 0: + case !hasPerr: + t.Errorf("expected a parser error, got '%v'", err) + case perr.Offset() != wantPos: + t.Errorf("expected error at position %d, got %d", wantPos, perr.Offset()) + t.Log(perr.Format(false, true)) + } + + return false +} + +func compareTree(t *testing.T, rule AstRuleT, wantOrder []string) { + t.Helper() + order := extractOrder(t, rule) + + minLen := min(len(order), len(wantOrder)) + + // Compare shared range. + for i := range minLen { + if order[i] != wantOrder[i] { + fmt.Println(Draw(rule, WithColor())) + t.Fatalf( + "first mismatch at position %d: expected '%s' != '%s'", + i, + wantOrder[i], + order[i], + ) + } + } + + // Handle length mismatch. + if len(order) != len(wantOrder) { + t.Fatalf( + "expected order sz %v, got %v", + len(wantOrder), + len(order), + ) + } + +} diff --git a/pkg/ast/proto.go b/pkg/ast/proto.go new file mode 100644 index 0000000..0e0a1d8 --- /dev/null +++ b/pkg/ast/proto.go @@ -0,0 +1,141 @@ +package ast + +import ( + "fmt" + "time" + + "github.com/prequel-dev/prequel-logmatch/pkg/match" +) + +type protoNode struct { + ty AstNodeType + window time.Duration + correlations []string + event *AstEventT + terms []*protoTerm + negate []*protoTerm +} + +// A protoTerm represents either a field term or a child node term in the proto representation of the rule. +type protoTerm struct { + field *protoField + child AstNode + negateOpts *AstNegateOptsT +} + +func (t protoTerm) count() uint64 { + if t.field != nil { + return t.field.Count + } + return 1 +} + +type protoField struct { + Field string + StrValue string + JqValue string + RegexValue string + Count uint64 + Extract []AstExtractT +} + +func protoTermsToAstFields(terms []*protoTerm) []AstFieldT { + var fields []AstFieldT + for _, term := range terms { + if term.field != nil { + // Convert protoField to AstField, applying negate options if present. + // Must is ok as the fields have already been validated during parsing. + fields = append(fields, term.field.MustField(term.negateOpts)) + } + } + return fields +} + +func protoTermsToAstTerms(terms []*protoTerm) []AstTermT { + var termsList []AstTermT + for _, term := range terms { + if term.child != nil { + termsList = append(termsList, AstTermT{ + Term: term.child, + NegateOpts: term.negateOpts, + }) + } + } + return termsList +} + +// Validate the protoField and convert it to an AstField. +// This includes ensuring that at least one of StrValue, JqValue, RegexValue is set, +// and Count is non zero. +// The validation that only one of StrValue, JqValue, RegexValue is set is done elsewhere. +func (f *protoField) ToField(nOpts *AstNegateOptsT) (AstFieldT, error) { + + t := AstFieldT{ + Count: f.Count, + Field: f.Field, + Extracts: f.Extract, + NegateOpts: nOpts, + } + + if t.Count == 0 { + t.Count = 1 + } + + switch { + case f.StrValue != "": + t.TermValue = match.TermT{ + Type: match.TermRaw, + Value: f.StrValue, + } + + case f.JqValue != "": + t.TermValue = match.TermT{ + Type: match.TermJqJson, + Value: f.JqValue, + } + + case f.RegexValue != "": + t.TermValue = match.TermT{ + Type: match.TermRegex, + Value: f.RegexValue, + } + + default: + return AstFieldT{}, ErrBadField + } + + return t, nil +} + +func (f *protoField) MustField(nOpts *AstNegateOptsT) AstFieldT { + field, err := f.ToField(nOpts) + if err != nil { + panic(fmt.Sprintf("invalid protoField: %v", err)) + } + return field +} + +// Validate the protoField to ensure it has a valid configuration. +// One of StrValue, JqValue, RegexValue must be set. +func (f *protoField) validate() error { + + var cnt int + if f.StrValue != "" { + cnt++ + } + if f.JqValue != "" { + cnt++ + } + if f.RegexValue != "" { + cnt++ + } + + switch cnt { + case 1: + return nil + case 0: + return fmt.Errorf("%w: one of ['%s','%s','%s'] must be set", ErrBadField, kwValue, kwJq, kwRegex) + default: + return fmt.Errorf("%w: only one of ['%s','%s','%s'] can be set", ErrBadField, kwValue, kwJq, kwRegex) + } +} diff --git a/pkg/ast/proto_test.go b/pkg/ast/proto_test.go new file mode 100644 index 0000000..44fc447 --- /dev/null +++ b/pkg/ast/proto_test.go @@ -0,0 +1,194 @@ +package ast + +import ( + "reflect" + "testing" + + "github.com/prequel-dev/prequel-logmatch/pkg/match" +) + +func TestProtoField_ToField(t *testing.T) { + tests := []struct { + name string + field protoField + nOpts *AstNegateOptsT + want AstFieldT + wantErr bool + }{ + { + name: "StrValue set", + field: protoField{ + Field: "foo", + StrValue: "bar", + Count: 2, + }, + nOpts: &AstNegateOptsT{Anchor: 1}, + want: AstFieldT{ + Field: "foo", + Count: 2, + TermValue: match.TermT{Type: match.TermRaw, Value: "bar"}, + NegateOpts: &AstNegateOptsT{Anchor: 1}, + }, + wantErr: false, + }, + { + name: "JqValue set", + field: protoField{ + Field: "foo", + JqValue: ".baz", + }, + nOpts: nil, + want: AstFieldT{ + Field: "foo", + Count: 1, + TermValue: match.TermT{Type: match.TermJqJson, Value: ".baz"}, + NegateOpts: nil, + }, + wantErr: false, + }, + { + name: "RegexValue set", + field: protoField{ + Field: "foo", + RegexValue: "re.*", + }, + nOpts: nil, + want: AstFieldT{ + Field: "foo", + Count: 1, + TermValue: match.TermT{Type: match.TermRegex, Value: "re.*"}, + NegateOpts: nil, + }, + wantErr: false, + }, + { + name: "none set", + field: protoField{Field: "foo"}, + nOpts: nil, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.field.ToField(tt.nOpts) + if (err != nil) != tt.wantErr { + t.Errorf("ToField() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !tt.wantErr && !reflect.DeepEqual(got, tt.want) { + t.Errorf("ToField() = %+v, want %+v", got, tt.want) + } + }) + } +} + +func TestProtoField_MustField(t *testing.T) { + defer func() { + if r := recover(); r == nil { + t.Errorf("MustField() did not panic on error") + } + }() + // This should panic because no value is set + f := protoField{Field: "foo"} + _ = f.MustField(nil) +} + +func TestProtoField_validate(t *testing.T) { + tests := []struct { + name string + field protoField + wantErr bool + }{ + { + name: "StrValue only", + field: protoField{StrValue: "a"}, + wantErr: false, + }, + { + name: "JqValue only", + field: protoField{JqValue: ".jq"}, + wantErr: false, + }, + { + name: "RegexValue only", + field: protoField{RegexValue: "re"}, + wantErr: false, + }, + { + name: "none set", + field: protoField{}, + wantErr: true, + }, + { + name: "multiple set", + field: protoField{StrValue: "a", JqValue: ".jq"}, + wantErr: true, + }, + { + name: "all set", + field: protoField{StrValue: "a", JqValue: ".jq", RegexValue: "re"}, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.field.validate() + if (err != nil) != tt.wantErr { + t.Errorf("validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestProtoTerm_count(t *testing.T) { + tests := []struct { + name string + term protoTerm + want uint64 + }{ + { + name: "field with count", + term: protoTerm{field: &protoField{Count: 5}}, + want: 5, + }, + { + name: "no field", + term: protoTerm{field: nil}, + want: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.term.count() + if got != tt.want { + t.Errorf("count() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestProtoTermsToAstFields(t *testing.T) { + terms := []*protoTerm{ + {field: &protoField{Field: "foo", StrValue: "bar"}}, + {field: nil}, + } + fields := protoTermsToAstFields(terms) + if len(fields) != 1 || fields[0].Field != "foo" { + t.Errorf("protoTermsToAstFields() = %+v, want field 'foo'", fields) + } +} + +func TestProtoTermsToAstTerms(t *testing.T) { + child := &AstMatchLeafT{} + terms := []*protoTerm{ + {child: child, negateOpts: &AstNegateOptsT{Anchor: 2}}, + {child: nil}, + } + astTerms := protoTermsToAstTerms(terms) + if len(astTerms) != 1 || astTerms[0].Term != child || astTerms[0].NegateOpts.Anchor != 2 { + t.Errorf("protoTermsToAstTerms() = %+v, want child and NegateOpts", astTerms) + } +} diff --git a/pkg/ast/state.go b/pkg/ast/state.go new file mode 100644 index 0000000..35769b7 --- /dev/null +++ b/pkg/ast/state.go @@ -0,0 +1,71 @@ +package ast + +// State struct for tracking rule metadata, origin count, unique ID generation, depth, and rank during parsing. +// This is passed through recursive calls to ensure consistent state management and error reporting. + +type ruleState struct { + meta *AstMetadataT + addr *AstNodeAddressT + origin *int + idCounter *uint32 + rank uint32 +} + +func newRuleState(meta *AstMetadataT) ruleState { + var ( + originCnt int + idCounter uint32 + ) + return ruleState{ + meta: meta, + origin: &originCnt, + idCounter: &idCounter, + } +} + +func (s ruleState) incOrigin() int { + *s.origin++ + return *s.origin +} + +func (s ruleState) getOrigin() int { + return *s.origin +} + +func (s ruleState) incRank() ruleState { + s.rank++ + return s +} + +func (s ruleState) setRank(r uint32) ruleState { + s.rank = r + return s +} + +func (s ruleState) nextId() uint32 { + id := *s.idCounter + *s.idCounter++ + return id +} + +func (s ruleState) pushNode(ty AstNodeType) ruleState { + var ( + depth uint32 + reserveId = s.nextId() + ) + + if s.addr != nil { + depth = s.addr.Depth + 1 + } + + s.addr = &AstNodeAddressT{ + Type: ty, + RuleId: s.meta.Id, + RuleHash: s.meta.Hash, + Rank: s.rank, + Depth: depth, + NodeId: reserveId, + } + + return s +} diff --git a/pkg/ast/walk.go b/pkg/ast/walk.go new file mode 100644 index 0000000..ab3abbe --- /dev/null +++ b/pkg/ast/walk.go @@ -0,0 +1,71 @@ +package ast + +import "fmt" + +type WalkFunc func(node AstNode, negateOpts *AstNegateOptsT) error + +// Walk rule tree depth first, calling the provided function on each node. +// The negateOpts parameter will be non-nil if the node is being visited as part of a negation, +// and will contain the negate options for that negation. + +func (r AstRuleT) Walk(fn WalkFunc) error { + return _walk(r.Root, nil, fn) +} + +func _walk(node AstNode, negateOpts *AstNegateOptsT, fn WalkFunc) error { + + switch n := node.(type) { + + case *AstInnerNodeT: + return _walkInnerNode(n, negateOpts, fn) + + case *AstScriptT: + return _walkScriptNode(n, negateOpts, fn) + + case *AstMatchLeafT, *AstPromT: + if err := fn(node, negateOpts); err != nil { + return err + } + + default: + return fmt.Errorf("%w: %T", ErrUnknownNodeType, node) + } + + return nil +} + +func _walkInnerNode(node *AstInnerNodeT, negateOpts *AstNegateOptsT, fn WalkFunc) error { + + if err := fn(node, negateOpts); err != nil { + return err + } + + for _, term := range node.Terms { + if err := _walk(term.Term, nil, fn); err != nil { + return err + } + } + + for _, term := range node.Negate { + termNegateOpts := term.NegateOpts + if termNegateOpts == nil { + // Stub as an indicator that we're in a negation context, even if there are no options. + termNegateOpts = &AstNegateOptsT{} + } + if err := _walk(term.Term, termNegateOpts, fn); err != nil { + return err + } + } + + return nil +} + +func _walkScriptNode(node *AstScriptT, negateOpts *AstNegateOptsT, fn WalkFunc) error { + + if err := fn(node, negateOpts); err != nil { + return err + } + + // No negate options allowed on a script node. + return _walk(node.Input, nil, fn) +} diff --git a/pkg/ast/walk_test.go b/pkg/ast/walk_test.go new file mode 100644 index 0000000..12dea8a --- /dev/null +++ b/pkg/ast/walk_test.go @@ -0,0 +1,154 @@ +package ast + +import ( + "errors" + "testing" +) + +func TestAstRuleT_Walk_VisitsAllNodes(t *testing.T) { + // Build a tree: + // InnerNode + // - Terms: [Leaf1, Script] + // - Negate: [Leaf2 (with NegateOpts)] + leaf1 := &AstMatchLeafT{baseAst: baseAst{scope: AstScopeNode}} + leaf2 := &AstMatchLeafT{baseAst: baseAst{scope: AstScopeNode}} + script := &AstScriptT{baseAst: baseAst{scope: AstScopeNode}, Input: leaf2} + inner := &AstInnerNodeT{ + baseAst: baseAst{scope: AstScopeCluster}, + Terms: []AstTermT{ + {Term: leaf1}, + {Term: script}, + }, + Negate: []AstTermT{ + {Term: leaf2, NegateOpts: &AstNegateOptsT{Window: 1}}, + }, + } + rule := AstRuleT{Root: inner} + + var visited []AstNode + var negated []AstNode + err := rule.Walk(func(node AstNode, nopts *AstNegateOptsT) error { + visited = append(visited, node) + if nopts != nil { + negated = append(negated, node) + } + return nil + }) + if err != nil { + t.Fatalf("Walk returned error: %v", err) + } + // Should visit inner, leaf1, script, leaf2 (twice: once as script input, once as negate) + if len(visited) != 5 { + t.Errorf("expected 5 nodes visited, got %d", len(visited)) + } + if len(negated) == 0 { + t.Errorf("expected at least one node visited with negateOpts") + } +} + +func TestAstRuleT_Walk_ErrorPropagation(t *testing.T) { + leaf := &AstMatchLeafT{baseAst: baseAst{scope: AstScopeNode}} + rule := AstRuleT{Root: leaf} + myErr := errors.New("fail") + err := rule.Walk(func(node AstNode, nopts *AstNegateOptsT) error { + return myErr + }) + if !errors.Is(err, myErr) { + t.Errorf("Walk did not propagate error, got %v", err) + } +} + +type unknownNode struct{} + +func (unknownNode) Type() AstNodeType { return AstNodeType(99) } +func (unknownNode) Scope() AstScopeT { return AstScopeT(99) } +func (unknownNode) Address() AstNodeAddressT { return AstNodeAddressT{} } +func (unknownNode) Parent() *AstNodeAddressT { return nil } + +func TestAstRuleT_Walk_UnknownNodeType(t *testing.T) { + rule := AstRuleT{Root: unknownNode{}} + err := rule.Walk(func(node AstNode, nopts *AstNegateOptsT) error { return nil }) + if err == nil || !errors.Is(err, ErrUnknownNodeType) { + t.Errorf("Walk did not return ErrUnknownNodeType for unknown node, got %v", err) + } +} + +func Test_walkInnerNode_ErrorPropagation(t *testing.T) { + leaf := &AstMatchLeafT{baseAst: baseAst{scope: AstScopeNode}} + inner := &AstInnerNodeT{ + baseAst: baseAst{scope: AstScopeCluster}, + Terms: []AstTermT{ + {Term: leaf}, + }, + Negate: []AstTermT{ + {Term: leaf, NegateOpts: &AstNegateOptsT{}}, + }, + } + myErr := errors.New("inner error") + // Error on the node itself + called := 0 + err := _walkInnerNode(inner, nil, func(node AstNode, nopts *AstNegateOptsT) error { + called++ + if called == 1 { + return myErr + } + return nil + }) + if !errors.Is(err, myErr) { + t.Errorf("_walkInnerNode did not propagate error from fn(node), got %v", err) + } + // Error on term + called = 0 + err = _walkInnerNode(inner, nil, func(node AstNode, nopts *AstNegateOptsT) error { + called++ + if called == 2 { + return myErr + } + return nil + }) + if !errors.Is(err, myErr) { + t.Errorf("_walkInnerNode did not propagate error from term, got %v", err) + } + // Error on negate + called = 0 + err = _walkInnerNode(inner, nil, func(node AstNode, nopts *AstNegateOptsT) error { + called++ + if called == 3 { + return myErr + } + return nil + }) + if !errors.Is(err, myErr) { + t.Errorf("_walkInnerNode did not propagate error from negate, got %v", err) + } +} + +func Test_walkScriptNode_ErrorPropagation(t *testing.T) { + leaf := &AstMatchLeafT{baseAst: baseAst{scope: AstScopeNode}} + script := &AstScriptT{baseAst: baseAst{scope: AstScopeNode}, Input: leaf} + myErr := errors.New("script error") + // Error on the script node itself + called := 0 + err := _walkScriptNode(script, nil, func(node AstNode, nopts *AstNegateOptsT) error { + called++ + if called == 1 { + return myErr + } + return nil + }) + if !errors.Is(err, myErr) { + t.Errorf("_walkScriptNode did not propagate error from fn(node), got %v", err) + } + // Error on the input node + called = 0 + err = _walkScriptNode(script, nil, func(node AstNode, nopts *AstNegateOptsT) error { + called++ + if called == 2 { + return myErr + } + return nil + }) + if !errors.Is(err, myErr) { + t.Errorf("_walkScriptNode did not propagate error from input, got %v", err) + } +} diff --git a/pkg/compiler/callbacks.go b/pkg/compiler/callbacks.go index cb290b9..fae71bf 100644 --- a/pkg/compiler/callbacks.go +++ b/pkg/compiler/callbacks.go @@ -2,30 +2,18 @@ package compiler import ( "context" - "errors" "github.com/prequel-dev/prequel-compiler/pkg/ast" - lm "github.com/prequel-dev/prequel-logmatch/pkg/match" - "github.com/rs/zerolog/log" -) - -var ( - ErrExpectedReteMatcher = errors.New("expected rete matcher") - ErrExpectedJsonMatcher = errors.New("expected jq json matcher") - ErrExpectedLogMatcher = errors.New("expected log matcher") - ErrExpectedCbDetect = errors.New("expected detect callback") - ErrInvalidCbArgs = errors.New("invalid callback arguments") - ErrNotFound = errors.New("not found") ) type MatchParamsT struct { - Address *ast.AstNodeAddressT + Address ast.AstNodeAddressT ParentAddress *ast.AstNodeAddressT Origin bool } type AssertParamsT struct { - Address *ast.AstNodeAddressT + Address ast.AstNodeAddressT } type CallbackT func(ctx context.Context, param any) error @@ -35,76 +23,10 @@ type RuntimeI interface { NewCbAssert(params AssertParamsT) CallbackT } -func GetJqMatcher(obj *ObjT) (lm.MatchFunc, error) { - var ( - m lm.MatchFunc - ok bool - ) - - log.Info().Type("object", obj.Object).Msg("Getting jq matcher") - if m, ok = obj.Object.(lm.MatchFunc); !ok { - return nil, ErrExpectedJsonMatcher - } - - return m, nil -} - -func GetLogInverseSeqMatcher(obj *ObjT) (*lm.InverseSeq, error) { - var ( - m *lm.InverseSeq - ok bool - ) - - if m, ok = obj.Object.(*lm.InverseSeq); !ok { - return nil, ErrExpectedLogMatcher - } - - return m, nil -} - -func GetLogSeqMatcher(obj *ObjT) (*lm.MatchSeq, error) { - var ( - m *lm.MatchSeq - ok bool - ) - - if m, ok = obj.Object.(*lm.MatchSeq); !ok { - return nil, ErrExpectedLogMatcher - } - - return m, nil -} - -func GetLogSingleMatcher(obj *ObjT) (*lm.MatchSingle, error) { - var ( - m *lm.MatchSingle - ok bool - ) - - log.Info().Type("object", obj.Object).Msg("Getting log single matcher") - - if m, ok = obj.Object.(*lm.MatchSingle); !ok { - return nil, ErrExpectedLogMatcher +func AssertObject[T any](obj *ObjT) (*T, error) { + m, ok := obj.Object.(*T) + if !ok { + return nil, ErrObjectTypeAssertion } - return m, nil } - -// ----- -type NoopRuntime struct{} - -func NewNoopRuntime() *NoopRuntime { - return &NoopRuntime{} -} - -func (f *NoopRuntime) NewCbMatch(params MatchParamsT) CallbackT { - return func(ctx context.Context, param any) error { - return nil - } -} - -func (f *NoopRuntime) NewCbAssert(params AssertParamsT) CallbackT { - return func(ctx context.Context, param any) error { - return nil - } -} diff --git a/pkg/compiler/compile_test.go b/pkg/compiler/compile_test.go new file mode 100644 index 0000000..74fa9b0 --- /dev/null +++ b/pkg/compiler/compile_test.go @@ -0,0 +1,40 @@ +package compiler + +import ( + "testing" + + "github.com/prequel-dev/prequel-compiler/pkg/ast" +) + +func TestCompile(t *testing.T) { + rule := `rules: + - cre: + id: set-1x1 + tags: + - test1 + - test2 + - test3 + metadata: + id: eeJwJiWQa9TyH3qTYYSZM9 + hash: 9GJSdx4smGJeJCdiw6tiK5 + rule: + set: + window: 5m + event: + source: nginx.access.log + origin: true + match: + - value: "test" + - value: "knock" + negate: + - value: "nope" + anchor: 1 +` + + xx, err := Compile([]byte(rule), ast.AstScopeNode) + if err != nil { + t.Fatal(err) + } + + t.Logf("%+v\n", xx) +} diff --git a/pkg/compiler/compiler.go b/pkg/compiler/compiler.go index 97b3d7c..f52c782 100644 --- a/pkg/compiler/compiler.go +++ b/pkg/compiler/compiler.go @@ -2,65 +2,52 @@ package compiler import ( "errors" + "fmt" "sort" "github.com/prequel-dev/prequel-compiler/pkg/ast" - "github.com/prequel-dev/prequel-compiler/pkg/parser" - "github.com/prequel-dev/prequel-compiler/pkg/schema" "github.com/rs/zerolog/log" ) -var ( - ErrUnsupportedMatcher = errors.New("unsupported matcher") - ErrUnsupportedScope = errors.New("unsupported scope") - ErrInvalidMatcher = errors.New("invalid matcher") -) - -var ( - defaultPlugin = NewDefaultPlugin() - defaultRuntime = &NoopRuntime{} -) - type ObjsT []*ObjT -type ObjTypeT string +type ObjTypeT int const ( - ObjTypeMatcher ObjTypeT = "match" - ObjTypeAssert ObjTypeT = "assert" + ObjTypeMatcher ObjTypeT = iota + ObjTypeAssert ) func (o ObjTypeT) String() string { - return string(o) + switch o { + case ObjTypeMatcher: + return "matcher" + case ObjTypeAssert: + return "assert" + default: + return "unknown" + } } type ObjT struct { - RuleId string `json:"rule_id"` - Address *ast.AstNodeAddressT `json:"address"` - ParentAddress *ast.AstNodeAddressT `json:"parent_address"` - Scope string `json:"scope"` - AbstractType schema.NodeTypeT `json:"abstract_type"` - ObjectType ObjTypeT `json:"object_type"` - Event ast.AstEventT `json:"event"` - Object any `json:"object"` - Cb CallbackT `json:"cb"` + Address ast.AstNodeAddressT + ParentAddress *ast.AstNodeAddressT + Scope ast.AstScopeT + AbstractType ast.AstNodeType + ObjectType ObjTypeT + Event ast.AstEventT + Object any + Cb CallbackT } type compilerOptsT struct { - debugTree string - runtime RuntimeI - plugins map[string]PluginI + runtime RuntimeI + plugins map[ast.AstScopeT]PluginI } type CompilerOptT func(*compilerOptsT) type PluginI interface { - Compile(runtime RuntimeI, node *ast.AstNodeT) (ObjsT, error) -} - -func WithDebugTree(path string) CompilerOptT { - return func(o *compilerOptsT) { - o.debugTree = path - } + Compile(runtime RuntimeI, node ast.AstNode) (ObjsT, error) } func WithRuntime(cb RuntimeI) CompilerOptT { @@ -69,15 +56,16 @@ func WithRuntime(cb RuntimeI) CompilerOptT { } } -func WithPlugin(scope string, plugin PluginI) CompilerOptT { +func WithPlugin(scope ast.AstScopeT, plugin PluginI) CompilerOptT { return func(o *compilerOptsT) { o.plugins[scope] = plugin } } func parseOpts(opts []CompilerOptT) compilerOptsT { + o := compilerOptsT{ - plugins: map[string]PluginI{schema.ScopeDefault: defaultPlugin}, + plugins: map[ast.AstScopeT]PluginI{ast.AstScopeNode: defaultPlugin}, runtime: defaultRuntime, } for _, opt := range opts { @@ -86,99 +74,61 @@ func parseOpts(opts []CompilerOptT) compilerOptsT { return o } -func traverseTree(node *ast.AstNodeT, scope string, callback func(node *ast.AstNodeT) error) error { - for _, child := range node.Children { - if err := traverseTree(child, scope, callback); err != nil { - return err - } - } - return callback(node) -} - -func NewObj(node *ast.AstNodeT, objType ObjTypeT) *ObjT { - return &ObjT{ - RuleId: node.Metadata.RuleId, - Address: node.Metadata.Address, - ParentAddress: node.Metadata.ParentAddress, - Scope: node.Metadata.Scope, - AbstractType: node.Metadata.Type, - ObjectType: objType, - } -} - -// Should we sort by object type? -func sortObjs(items []*ObjT, t schema.NodeTypeT) { - sort.SliceStable(items, func(i, j int) bool { - if items[i].AbstractType == t && items[j].AbstractType != t { - return true - } - if items[j].AbstractType == t && items[i].AbstractType != t { - return false - } - return false - }) -} - -func CompileTree(pt *parser.TreeT, scope string, opts ...CompilerOptT) (ObjsT, error) { - - var ( - err error - o = parseOpts(opts) - tree *ast.AstT - ) +func Compile(data []byte, scope ast.AstScopeT, opts ...CompilerOptT) (ObjsT, error) { - if tree, err = ast.BuildTree(pt); err != nil { + rules, err := ast.ParseRules(data) + if err != nil { return nil, err } - if o.debugTree != "" { - if err = ast.DrawTree(tree, o.debugTree); err != nil { - return nil, err - } - } + return CompileRules(rules, scope, opts...) +} - return compile(o, tree, scope) +func CompileRule(rule ast.AstRuleT, scope ast.AstScopeT, opts ...CompilerOptT) (ObjsT, error) { + o := parseOpts(opts) + return compileRule(o, rule, scope) } -func CompileAst(tree *ast.AstT, scope string, opts ...CompilerOptT) (ObjsT, error) { +func CompileRules(rules []ast.AstRuleT, scope ast.AstScopeT, opts ...CompilerOptT) (ObjsT, error) { + o := parseOpts(opts) + var ( - o = parseOpts(opts) + outObjs ObjsT + errList []error ) - if o.debugTree != "" { - if err := ast.DrawTree(tree, o.debugTree); err != nil { - return nil, err + for _, rule := range rules { + objs, err := compileRule(o, rule, scope) + if err != nil { + errList = append(errList, err) + } else { + outObjs = append(outObjs, objs...) } } - return compile(o, tree, scope) + return outObjs, errors.Join(errList...) } -func compile(o compilerOptsT, tree *ast.AstT, scope string) (ObjsT, error) { +func compileRule(o compilerOptsT, rule ast.AstRuleT, scope ast.AstScopeT) (ObjsT, error) { var ( - err error outObjs ObjsT ) - compile := func(node *ast.AstNodeT) error { + compile := func(node ast.AstNode, _ *ast.AstNegateOptsT) error { - if node.Metadata.Scope != scope { + if node.Scope() != scope { return nil } plugin, ok := o.plugins[scope] if !ok { - log.Error().Str("scope", scope).Msg("No plugin found") - return ErrUnsupportedScope + log.Error().Str("scope", scope.String()).Msg("No plugin found") + return fmt.Errorf("%w: %s", ErrUnsupportedScope, scope.String()) } objs, err := plugin.Compile(o.runtime, node) if err != nil { - log.Error(). - Err(err). - Str("scope", scope). - Msg("Failed to compile") return err } @@ -187,42 +137,36 @@ func compile(o compilerOptsT, tree *ast.AstT, scope string) (ObjsT, error) { return nil } - for _, node := range tree.Nodes { - if err = traverseTree(node, scope, compile); err != nil { - return nil, err - } + if err := rule.Walk(compile); err != nil { + return nil, err } - sortObjs(outObjs, schema.NodeTypeSeq) - sortObjs(outObjs, schema.NodeTypeSet) - - for _, obj := range outObjs { - log.Debug(). - Str("abstract_type", obj.AbstractType.String()). - Str("abstract_address", obj.Address.String()). - Str("object_type", obj.ObjectType.String()). - Msg("Compiled object") - } + sortObjs(outObjs, ast.AstNodeTypeSeq) + sortObjs(outObjs, ast.AstNodeTypeSet) return outObjs, nil } -func Compile(data []byte, scope string, opts ...CompilerOptT) (ObjsT, error) { - var ( - tree *ast.AstT - o = parseOpts(opts) - err error - ) +func NewObj(node ast.AstNode, objType ObjTypeT) *ObjT { - if tree, err = ast.Build(data); err != nil { - return nil, err + return &ObjT{ + Address: node.Address(), + ParentAddress: node.Parent(), + Scope: node.Scope(), + AbstractType: node.Type(), + ObjectType: objType, } +} - if o.debugTree != "" { - if err = ast.DrawTree(tree, o.debugTree); err != nil { - return nil, err +// Should we sort by object type? +func sortObjs(items []*ObjT, t ast.AstNodeType) { + sort.SliceStable(items, func(i, j int) bool { + if items[i].AbstractType == t && items[j].AbstractType != t { + return true } - } - - return compile(o, tree, scope) + if items[j].AbstractType == t && items[i].AbstractType != t { + return false + } + return false + }) } diff --git a/pkg/compiler/compiler_log.go b/pkg/compiler/compiler_log.go index 16ff291..0c9100c 100644 --- a/pkg/compiler/compiler_log.go +++ b/pkg/compiler/compiler_log.go @@ -1,21 +1,13 @@ package compiler import ( - "errors" + "fmt" "github.com/prequel-dev/prequel-compiler/pkg/ast" - "github.com/prequel-dev/prequel-compiler/pkg/schema" "github.com/prequel-dev/prequel-logmatch/pkg/match" "github.com/rs/zerolog/log" ) -var ( - ErrUnsupportedNodeType = errors.New("unsupported node type") - ErrUnsupportedEventType = errors.New("unsupported event type") - ErrSequenceSingleMatch = errors.New("sequence with single match (use set instead)") - ErrNoFields = errors.New("no fields") -) - func toLogResets(terms []ast.AstFieldT) []match.ResetT { resets := make([]match.ResetT, 0, len(terms)) for _, term := range terms { @@ -43,112 +35,92 @@ func toLogResets(terms []ast.AstFieldT) []match.ResetT { func toLogTerms(fields []ast.AstFieldT) []match.TermT { terms := make([]match.TermT, 0, len(fields)) for _, field := range fields { - terms = append(terms, field.TermValue) + // match interface does not yet support explicit counts, do dupe. + // TODO: Revise when log matcher supports counts. + cnt := max(field.Count, 1) + for range cnt { + terms = append(terms, field.TermValue) + } } return terms } -func ObjLogMatcher(runtime RuntimeI, node *ast.AstNodeT) (*ObjT, error) { +func ObjLogMatcher(runtime RuntimeI, node *ast.AstMatchLeafT) (*ObjT, error) { var ( - obj = NewObj(node, ObjTypeMatcher) - lm *ast.AstLogMatcherT - ok bool err error + obj = NewObj(node, ObjTypeMatcher) ) - if lm, ok = node.Object.(*ast.AstLogMatcherT); !ok { - log.Error().Interface("matcher", node.Object).Msg("Failed to compile log matcher") - return nil, ErrInvalidMatcher - } - - obj.Event.Origin = lm.Event.Origin - obj.Event.Source = lm.Event.Source + obj.Event.Origin = node.Event.Origin + obj.Event.Source = node.Event.Source params := MatchParamsT{ - Address: node.Metadata.Address, - ParentAddress: node.Metadata.ParentAddress, - Origin: lm.Event.Origin, + Address: node.Address(), + ParentAddress: node.Parent(), + Origin: node.Event.Origin, } obj.Cb = runtime.NewCbMatch(params) - switch node.Metadata.Type { - case schema.NodeTypeLogSeq: - if obj.Object, err = makeLogSeqObjects(lm, node.Metadata.NegIdx); err != nil { + switch node.Type() { + case ast.AstNodeTypeLogSeq: + if obj.Object, err = makeLogSeqObjects(node); err != nil { return nil, err } - case schema.NodeTypeLogSet: + case ast.AstNodeTypeLogSet: - if obj.Object, err = makeLogSetObjects(lm, node.Metadata.NegIdx); err != nil { + if obj.Object, err = makeLogSetObjects(node); err != nil { return nil, err } default: - log.Error().Type("node_type", node.Metadata.Type).Msg("Unsupported node type") - return nil, ErrUnsupportedNodeType + return nil, fmt.Errorf("%w: %s", ErrUnsupportedNodeType, node.Type()) } return obj, nil } -func makeLogSeqObjects(lm *ast.AstLogMatcherT, negIdx int) (any, error) { +func makeLogSeqObjects(node *ast.AstMatchLeafT) (any, error) { - var ( - obj any - err error - ) + switch { + case len(node.Negate) > 0: + return match.NewInverseSeq( + node.Window.Nanoseconds(), + toLogTerms(node.Terms), + toLogResets(node.Negate), + ) - if negIdx > 0 { - log.Trace().Any("terms", toLogTerms(lm.Match)).Msg("Creating inverse match sequence") - if obj, err = match.NewInverseSeq(lm.Window.Nanoseconds(), toLogTerms(lm.Match), toLogResets(lm.Negate)); err != nil { - log.Error().Err(err).Msg("Failed to create inverse match sequence") - return nil, err - } - } else { - if len(lm.Match) == 1 { - log.Error().Msg("Sequence with single match (use set instead)") - return nil, ErrSequenceSingleMatch - } else { - log.Debug().Any("terms", toLogTerms(lm.Match)).Msg("Creating match sequence") - if obj, err = match.NewMatchSeq(lm.Window.Nanoseconds(), toLogTerms(lm.Match)...); err != nil { - log.Error().Err(err).Msg("Failed to create match sequence") - return nil, err - } - } - } + // A sequence with a single term and no negation should be specified as a set, + // irregardless of the count. It is not a sequence of events, but a single event + // that may occur multiple times (on count > 1). + case len(node.Terms) == 1: + return nil, ErrSequenceSingleMatch - return obj, nil + default: + return match.NewMatchSeq(node.Window.Nanoseconds(), toLogTerms(node.Terms)...) + } } -func makeLogSetObjects(lm *ast.AstLogMatcherT, negIdx int) (any, error) { +func makeLogSetObjects(node *ast.AstMatchLeafT) (any, error) { - var ( - err error - obj any - ) + // Expand the terms before checking length as toLogs will + // expand terms with counts > 1 into multiple terms, + // which may affect whether we can use a single match or need a set. + logTrms := toLogTerms(node.Terms) - if negIdx > 0 { - log.Debug().Any("terms", toLogTerms(lm.Match)).Msg("Creating inverse match set") - if obj, err = match.NewInverseSet(lm.Window.Nanoseconds(), toLogTerms(lm.Match), toLogResets(lm.Negate)); err != nil { - log.Error().Err(err).Msg("Failed to create inverse match set") - return nil, err - } - } else { - if len(lm.Match) == 1 { - log.Debug().Any("term", toLogTerms(lm.Match)[0]).Msg("Creating match single") - if obj, err = match.NewMatchSingle(toLogTerms(lm.Match)[0]); err != nil { - log.Error().Err(err).Msg("Failed to create match single") - return nil, err - } - } else { - log.Debug().Any("terms", toLogTerms(lm.Match)).Msg("Creating match set") - if obj, err = match.NewMatchSet(lm.Window.Nanoseconds(), toLogTerms(lm.Match)...); err != nil { - log.Error().Err(err).Msg("Failed to create match set") - return nil, err - } - } - } + switch { + case len(node.Negate) > 0: + return match.NewInverseSet( + node.Window.Nanoseconds(), + logTrms, + toLogResets(node.Negate), + ) - return obj, nil + case len(logTrms) == 1: + return match.NewMatchSingle(logTrms[0]) + + default: + return match.NewMatchSet(node.Window.Nanoseconds(), logTrms...) + } } diff --git a/pkg/compiler/defaults.go b/pkg/compiler/defaults.go new file mode 100644 index 0000000..96a1714 --- /dev/null +++ b/pkg/compiler/defaults.go @@ -0,0 +1,55 @@ +package compiler + +import ( + "context" + "fmt" + + "github.com/prequel-dev/prequel-compiler/pkg/ast" +) + +var ( + defaultPlugin = NewDefaultPlugin() + defaultRuntime = NewNoopRuntime() +) + +// ----- +type NoopRuntime struct{} + +func NewNoopRuntime() *NoopRuntime { + return &NoopRuntime{} +} + +func (f *NoopRuntime) NewCbMatch(params MatchParamsT) CallbackT { + return func(ctx context.Context, param any) error { + return nil + } +} + +func (f *NoopRuntime) NewCbAssert(params AssertParamsT) CallbackT { + return func(ctx context.Context, param any) error { + return nil + } +} + +// ----- + +type DefaultPlugin struct{} + +func NewDefaultPlugin() *DefaultPlugin { + return &DefaultPlugin{} +} + +func (p *DefaultPlugin) Compile(runtime RuntimeI, node ast.AstNode) (ObjsT, error) { + + match, ok := node.(*ast.AstMatchLeafT) + if !ok { + return nil, fmt.Errorf("%w: %T", ErrUnsupportedAstType, node) + } + + obj, err := ObjLogMatcher(runtime, match) + if err != nil { + return nil, err + } + + return ObjsT{obj}, nil +} diff --git a/pkg/compiler/error.go b/pkg/compiler/error.go new file mode 100644 index 0000000..7784a77 --- /dev/null +++ b/pkg/compiler/error.go @@ -0,0 +1,12 @@ +package compiler + +import "errors" + +var ( + ErrNoFields = errors.New("no fields") + ErrUnsupportedNodeType = errors.New("unsupported node type") + ErrUnsupportedAstType = errors.New("unsupported AST node type") + ErrUnsupportedScope = errors.New("unsupported scope") + ErrSequenceSingleMatch = errors.New("sequence with single match (use set instead)") + ErrObjectTypeAssertion = errors.New("object type assertion failed") +) diff --git a/pkg/compiler/plugin.go b/pkg/compiler/plugin.go deleted file mode 100644 index 8d01d55..0000000 --- a/pkg/compiler/plugin.go +++ /dev/null @@ -1,40 +0,0 @@ -package compiler - -import ( - "github.com/prequel-dev/prequel-compiler/pkg/ast" - "github.com/prequel-dev/prequel-compiler/pkg/schema" - "github.com/rs/zerolog/log" -) - -type DefaultPlugin struct{} - -func NewDefaultPlugin() *DefaultPlugin { - return &DefaultPlugin{} -} - -func (p *DefaultPlugin) Compile(runtime RuntimeI, node *ast.AstNodeT) (ObjsT, error) { - - var ( - objs = make(ObjsT, 0) - obj *ObjT - err error - ) - - switch node.Metadata.Type { - case schema.NodeTypeLogSeq, schema.NodeTypeLogSet: - if obj, err = ObjLogMatcher(runtime, node); err != nil { - log.Error().Err(err).Str("scope", node.Metadata.Scope).Msg("Failed to compile matchers") - return nil, err - } - default: - log.Error(). - Interface("node_type", node.Metadata.Type). - Interface("node", node). - Msg("Unsupported node type") - return nil, ErrUnsupportedNodeType - } - - objs = append(objs, obj) - - return objs, nil -} diff --git a/pkg/datasrc/parse.go b/pkg/datasrc/parse.go index 901fa92..23278f4 100644 --- a/pkg/datasrc/parse.go +++ b/pkg/datasrc/parse.go @@ -4,7 +4,7 @@ import ( "os" "time" - "gopkg.in/yaml.v3" + "github.com/goccy/go-yaml" ) // version: 0.0.1 diff --git a/pkg/parser/parse.go b/pkg/parser/parse.go deleted file mode 100644 index e88de5a..0000000 --- a/pkg/parser/parse.go +++ /dev/null @@ -1,214 +0,0 @@ -package parser - -import ( - "gopkg.in/yaml.v3" -) - -// Note that we prefer lower camel case like Kubernetes -// Also, have to keep the JSON tags although we are using YAML. -// The hash function uses JSON serialization, so the JSON tags are required to ensure consistent field names for hashing. - -const ( - docRules = "rules" - docRule = "rule" - docSeq = "sequence" - docSet = "set" - docOrder = "order" - docWindow = "window" - docMatch = "match" - docNegate = "negate" - docTerms = "terms" - docSection = "section" - docVersion = "version" -) - -type ParseRuleT struct { - Metadata ParseRuleMetadataT `yaml:"metadata,omitempty" json:"metadata,omitempty"` - Cre ParseCreT `yaml:"cre,omitempty" json:"cre,omitempty"` - Rule ParseRuleDataT `yaml:"rule,omitempty" json:"rule,omitempty"` -} - -type ParseRuleMetadataT struct { - Name string `yaml:"name,omitempty" json:"name,omitempty"` - Id string `yaml:"id,omitempty" json:"id,omitempty"` - Hash string `yaml:"hash,omitempty" json:"hash,omitempty"` - Gen uint `yaml:"generation" json:"generation"` - Kind string `yaml:"kind,omitempty" json:"kind,omitempty"` - Version string `yaml:"version,omitempty" json:"version,omitempty"` -} - -type ParseRuleDataT struct { - Sequence *ParseSequenceT `yaml:"sequence,omitempty"` - Set *ParseSetT `yaml:"set,omitempty"` -} - -type ParseApplicationT struct { - Name string `yaml:"name,omitempty" json:"name,omitempty"` - ProcessName string `yaml:"processName,omitempty" json:"process_name,omitempty"` - ProcessPath string `yaml:"processPath,omitempty" json:"process_path,omitempty"` - ContainerName string `yaml:"containerName,omitempty" json:"container_name,omitempty"` - ImageUrl string `yaml:"imageUrl,omitempty" json:"image_url,omitempty"` - RepoUrl string `yaml:"repoUrl,omitempty" json:"repo_url,omitempty"` - Version string `yaml:"version,omitempty" json:"version,omitempty"` -} - -const ( - SeverityCritical = 0 - SeverityHigh = 1 - SeverityMedium = 2 - SeverityLow = 3 - SeverityInfo = 4 -) - -type ParseCreT struct { - Id string `yaml:"id,omitempty" json:"id,omitempty"` - Severity uint `yaml:"severity" json:"severity"` - Title string `yaml:"title,omitempty" json:"title,omitempty"` - Category string `yaml:"category,omitempty" json:"category,omitempty"` - Tags []string `yaml:"tags,omitempty" json:"tags,omitempty"` - Author string `yaml:"author,omitempty" json:"author,omitempty"` - Description string `yaml:"description,omitempty" json:"description,omitempty"` - Impact string `yaml:"impact,omitempty" json:"impact,omitempty"` - ImpactScore uint `yaml:"impactScore,omitempty" json:"impact_score,omitempty"` - Cause string `yaml:"cause,omitempty" json:"cause,omitempty"` - Mitigation string `yaml:"mitigation,omitempty" json:"mitigation,omitempty"` - MitigationScore uint `yaml:"mitigationScore,omitempty" json:"mitigation_score,omitempty"` - References []string `yaml:"references,omitempty" json:"references,omitempty"` - Reports uint `yaml:"reports,omitempty" json:"reports,omitempty"` - Applications []ParseApplicationT `yaml:"applications,omitempty" json:"applications,omitempty"` -} - -type ParseSequenceT struct { - Window string `yaml:"window"` - Correlations []string `yaml:"correlations,omitempty"` - Event *ParseEventT `yaml:"event,omitempty"` - Origin bool `yaml:"origin,omitempty"` - Order []ParseTermT `yaml:"order,omitempty"` - Negate []ParseTermT `yaml:"negate,omitempty"` -} - -type ParseNegateOptsT struct { - Window string `yaml:"window,omitempty"` - Slide string `yaml:"slide,omitempty"` - Anchor uint32 `yaml:"anchor,omitempty"` - Absolute bool `yaml:"absolute,omitempty"` -} - -type ParseSetT struct { - Window string `yaml:"window,omitempty"` - Correlations []string `yaml:"correlations,omitempty"` - Event *ParseEventT `yaml:"event,omitempty"` - Match []ParseTermT `yaml:"match,omitempty"` - Negate []ParseTermT `yaml:"negate,omitempty"` -} - -type ParseExtractT struct { - Name string `yaml:"name"` - JqValue string `yaml:"jq,omitempty"` - RegexValue string `yaml:"regex,omitempty"` -} - -type ParsePromQL struct { - Expr string `yaml:"expr"` - Interval string `yaml:"interval,omitempty"` - For string `yaml:"for,omitempty"` - Event *ParseEventT `yaml:"event,omitempty"` -} - -type ParseScriptT struct { - Code string `yaml:"code"` - Language string `yaml:"language,omitempty"` // Assumes 'lua' if empty - Timeout string `yaml:"timeout,omitempty"` // Uses default if empty; expects duration string - Input *ParseTermT `yaml:"input"` // Required input -} - -type ParseEventT struct { - Source string `yaml:"source"` - Origin bool `yaml:"origin,omitempty" json:"origin,omitempty"` -} - -type ParseTermT struct { - Field string `yaml:"field,omitempty"` - StrValue string `yaml:"value,omitempty"` - JqValue string `yaml:"jq,omitempty"` - RegexValue string `yaml:"regex,omitempty"` - Count int `yaml:"count,omitempty"` - Set *ParseSetT `yaml:"set,omitempty"` - Sequence *ParseSequenceT `yaml:"sequence,omitempty"` - NegateOpts *ParseNegateOptsT `yaml:",inline,omitempty"` - PromQL *ParsePromQL `yaml:"promql,omitempty"` - Script *ParseScriptT `yaml:"script,omitempty"` - Extract []ParseExtractT `yaml:"extract,omitempty"` -} - -func (o *ParseTermT) UnmarshalYAML(unmarshal func(any) error) error { - - // Try to unmarshal as a raw string first. - // If that fails, unmarshal as a struct. - // This allows for a shorthand syntax for simple match terms. - var str string - if err := unmarshal(&str); err == nil { - o.StrValue = str - return nil - } - - var temp struct { - Field string `yaml:"field"` - StrValue string `yaml:"value"` - JqValue string `yaml:"jq"` - RegexValue string `yaml:"regex"` - Count int `yaml:"count"` - Set *ParseSetT `yaml:"set"` - Sequence *ParseSequenceT `yaml:"sequence"` - NegateOpts *ParseNegateOptsT `yaml:",inline"` - ParsePromQL *ParsePromQL `yaml:"promql"` - Script *ParseScriptT `yaml:"script"` - Extract []ParseExtractT `yaml:"extract"` - } - if err := unmarshal(&temp); err != nil { - return err - } - o.Field = temp.Field - o.StrValue = temp.StrValue - o.JqValue = temp.JqValue - o.RegexValue = temp.RegexValue - o.Count = temp.Count - o.Set = temp.Set - o.Sequence = temp.Sequence - o.NegateOpts = temp.NegateOpts - o.PromQL = temp.ParsePromQL - o.Script = temp.Script - o.Extract = temp.Extract - return nil -} - -func RootNode(data []byte) (*yaml.Node, error) { - var root yaml.Node - if err := yaml.Unmarshal(data, &root); err != nil { - return nil, err - } - return &root, nil -} - -type RulesT struct { - Rules []ParseRuleT `yaml:"rules"` - Root *yaml.Node `yaml:"-"` - TermsT map[string]ParseTermT `yaml:"terms,omitempty"` - TermsY map[string]*yaml.Node `yaml:"-"` -} - -func _parse(data []byte) (*RulesT, *yaml.Node, error) { - - root, err := RootNode(data) - if err != nil { - return nil, nil, err - } - - var rules RulesT - if err := root.Decode(&rules); err != nil { - return nil, nil, err - - } - - return &rules, root, nil -} diff --git a/pkg/parser/parse_test.go b/pkg/parser/parse_test.go deleted file mode 100644 index f6e70ad..0000000 --- a/pkg/parser/parse_test.go +++ /dev/null @@ -1,404 +0,0 @@ -package parser - -import ( - "errors" - "fmt" - "os" - "path/filepath" - "reflect" - "testing" - - "github.com/prequel-dev/prequel-compiler/pkg/pqerr" - "github.com/prequel-dev/prequel-compiler/pkg/testdata" - "github.com/rs/zerolog/log" -) - -// traverses the tree and collects node types in DFS pre-order (root, then children) -func gatherNodeTypes(node any, out *[]string) { - if node == nil { - return - } - - if n, ok := node.(*NodeT); ok { - *out = append(*out, n.Metadata.Type.String()) - for _, child := range n.Children { - gatherNodeTypes(child, out) - } - } -} - -// traverses the tree and collects node negative indexes in DFS pre-order (root, then children) -func gatherNodeNegativeIndexes(node any, out *[]int) { - if node == nil { - return - } - - if n, ok := node.(*NodeT); ok { - *out = append(*out, n.NegIdx) - for _, child := range n.Children { - gatherNodeNegativeIndexes(child, out) - } - } -} - -func TestParseSuccess(t *testing.T) { - - var opts = []ParseOptT{WithGenIds()} - - var tests = map[string]struct { - rule string - expectedNodeTypes []string - expectedNegIndexes []int - }{ - "Success_Simple1": { - rule: testdata.TestSuccessSimpleRule1, - expectedNodeTypes: []string{"log_seq"}, - expectedNegIndexes: []int{-1}, - }, - "Success_Complex2": { - rule: testdata.TestSuccessComplexRule2, - expectedNodeTypes: []string{"machine_seq", "log_seq", "log_set", "machine_seq", "log_seq", "log_set", "log_set"}, - expectedNegIndexes: []int{-1, 2, 2, -1, -1, -1, -1}, - }, - "Success_MissingRuleId": { - rule: testdata.TestFailMissingRuleIdRule, - expectedNodeTypes: []string{"log_set"}, - expectedNegIndexes: []int{-1}, - }, - "Success_MissingRuleHash": { - rule: testdata.TestFailMissingRuleHashRule, - expectedNodeTypes: []string{"log_set"}, - expectedNegIndexes: []int{-1}, - }, - "Success_PromQL": { - rule: testdata.TestSuccessSimplePromQL, - expectedNodeTypes: []string{"machine_set", "promql", "log_set"}, - expectedNegIndexes: []int{-1, -1, -1}, - }, - "Success_ChildScript": { - rule: testdata.TestSuccessChildScript, - expectedNodeTypes: []string{"machine_seq", "script", "log_seq", "log_set"}, - expectedNegIndexes: []int{-1, -1, -1, -1}, - }, - "Success_ChildScriptMultipleInputs": { - rule: testdata.TestSuccessChildScriptMultipleInputs, - expectedNodeTypes: []string{"machine_set", "script", "machine_seq", "log_seq", "log_set"}, - expectedNegIndexes: []int{-1, -1, -1, -1, -1}, - }, - "Success_ChildScriptPromQLInput": { - rule: testdata.TestSuccessChildScriptPromQLInput, - expectedNodeTypes: []string{"machine_set", "script", "promql"}, - expectedNegIndexes: []int{-1, -1, -1}, - }, - } - - for name, test := range tests { - t.Run(name, func(t *testing.T) { - tree, err := Parse([]byte(test.rule), opts...) - if err != nil { - t.Fatalf("Error parsing rule: %v", err) - } - - if len(tree.Nodes) != 1 { - t.Fatalf("Expected 1 root node, got %d", len(tree.Nodes)) - } - - var actualNodes []string - gatherNodeTypes(tree.Nodes[0], &actualNodes) - - if !reflect.DeepEqual(actualNodes, test.expectedNodeTypes) { - t.Errorf("gathered types = %v, want %v", actualNodes, test.expectedNodeTypes) - } - - var actualNegIndexes []int - gatherNodeNegativeIndexes(tree.Nodes[0], &actualNegIndexes) - - if !reflect.DeepEqual(actualNegIndexes, test.expectedNegIndexes) { - t.Errorf("gathered neg indexes = %v, want %v", actualNegIndexes, test.expectedNegIndexes) - } - }) - } -} - -func TestSuccessExamples(t *testing.T) { - - var opts = []ParseOptT{WithGenIds()} - - rules, err := filepath.Glob(filepath.Join("../testdata", "success_examples", "*.yaml")) - if err != nil { - t.Fatalf("Error finding CRE test files: %v", err) - } - - for _, rule := range rules { - - // Read the test file - testData, err := os.ReadFile(rule) - if err != nil { - t.Fatalf("Error reading test file %s: %v", rule, err) - } - - _, err = Parse(testData, opts...) - if err != nil { - t.Fatalf("Error parsing rule %s: %v", rule, err) - } - } -} - -func TestParseFail(t *testing.T) { - - var opts = []ParseOptT{} - - var tests = map[string]struct { - rule string - line int - col int - err error - }{ - "Fail_Typo": { - rule: testdata.TestFailTypo, - line: 16, - col: 11, - err: ErrTermNotFound, - }, - "Fail_MissingOrder": { - rule: testdata.TestFailMissingOrder, - line: 12, - col: 9, - err: ErrMissingOrder, - }, - "Fail_MissingMatch": { - rule: testdata.TestFailMissingMatch, - line: 12, - col: 9, - err: ErrMissingMatch, - }, - "Fail_InvalidWindow": { - rule: testdata.TestFailInvalidWindow, - line: 12, - col: 17, - err: ErrInvalidWindow, - }, - "Fail_UnsupportedRule": { - rule: testdata.TestFailUnsupportedRule, - line: 11, - col: 7, - err: ErrNotSupported, - }, - "Fail_TermsSyntaxError": { - rule: testdata.TestFailTermsSyntaxError1, - line: 34, - col: 7, - err: ErrMissingMatch, - }, - "Fail_TermsSyntaxError2": { - rule: testdata.TestFailTermsSyntaxError2, - line: 36, - col: 15, - err: ErrInvalidWindow, - }, - "Fail_MissingCreId": { - rule: testdata.TestFailMissingCreRule, - line: 10, - col: 7, - err: ErrMissingCreId, - }, - "Fail_MissingRuleId": { - rule: testdata.TestFailMissingRuleIdRule, - line: 10, - col: 7, - err: ErrMissingRuleId, - }, - "Fail_MissingRuleHash": { - rule: testdata.TestFailMissingRuleHashRule, - line: 10, - col: 7, - err: ErrMissingRuleHash, - }, - "Fail_BadRuleId": { - rule: testdata.TestFailBadRuleIdRule, - line: 11, - col: 7, - err: ErrInvalidRuleId, - }, - "Fail_BadCreId": { - rule: testdata.TestFailBadCreIdRule, - line: 11, - col: 7, - err: ErrInvalidCreId, - }, - "Fail_BadRuleHash": { - rule: testdata.TestFailBadRuleHashRule, - line: 11, - col: 7, - err: ErrInvalidRuleHash, - }, - "Fail_ScriptRoot": { - rule: testdata.TestFailScriptRoot, - line: 10, - col: 7, - err: ErrNotSupported, - }, - "Fail_ScriptNoInput": { - rule: testdata.TestFailScriptNoInput, - line: 11, - col: 9, - err: ErrMissingInput, - }, - } - - for name, test := range tests { - t.Run(name, func(t *testing.T) { - _, err := Parse([]byte(test.rule), opts...) - if err == nil { - t.Fatalf("Expected error parsing rule") - } - - if !errors.Is(err, test.err) { - log.Info().Type("err_type", err).Msg("error") - t.Errorf("Expected error %v, got %v", test.err, err) - } - - if pos, ok := pqerr.PosOf(err); ok { - if pos.Line != test.line { - t.Errorf("Expected error position line=%d, got line=%d", test.line, pos.Line) - } - if pos.Col != test.col { - t.Errorf("Expected error position col=%d, got col=%d", test.col, pos.Col) - } - } else { - DumpErrorChain(err) - t.Errorf("Expected wrapped pqerr error %v, got %v", test.err, err) - } - }) - } -} - -const stableRuleYaml = ` -rules: - - cre: - id: PREQUEL-2026-0004 - severity: 3 - title: ArgoCD Excessive Syncs - category: argocd-problems - author: Prequel - description: | - ArgoCD Reconciliation Storm - tags: - - argocd - - sync-loop - - prequel-v0.14+ - mitigation: - Remove "CreateNamespace=true" from applications involved in the sync loop reconciliation storm. - impact: | - The ArgoCD applications are in a sync loop, which means that they are being synced more than once per minute. This increases the load on the ArgoCD server and the Kubernetes cluster. - mitigationScore: 3 - impactScore: 4 - references: - - https://github.com/argoproj/argo-cd/issues/14666#issuecomment-1715538502 - - https://argo-cd.readthedocs.io/en/stable/operator-manual/reconcile/ - applications: - - name: "argocd" - processName: "argocd-application-controller" - processPath: "/app/argocd/argocd-application-controller" - containerName: "argocd-application-controller" - imageUrl: "quay.io/argoproj/argocd:v2.7.5" - repoUrl: "https://github.com/argoproj/argo-cd" - - metadata: - kind: custom - id: NRdyR6FoTTsziQRVrxFMv5 - gen: 1 - rule: - set: - event: - source: cre.kubernetes - correlations: - - appNamespace - - appName - window: 1200s - match: - - jq: | - (.message | test("Initiated automated sync to '.*'")) - and (.source.component == "argocd-application-controller") - extract: - - name: appNamespace - jq: .involvedObject.namespace - - name: appName - jq: .involvedObject.name - count: 3 - - jq: | - (.message | test("(Partial s|S)ync operation to .* succeeded")) - and (.source.component == "argocd-application-controller") - extract: - - name: appNamespace - jq: .involvedObject.namespace - - name: appName - jq: .involvedObject.name - count: 3 - - jq: | - (.message | test("Updated sync status: Synced -> OutOfSync")) - and (.source.component == "argocd-application-controller") - extract: - - name: appNamespace - jq: .involvedObject.namespace - - name: appName - jq: .involvedObject.name - count: 3 - - jq: | - (.message | test("Updated sync status: OutOfSync -> Synced")) - and (.source.component == "argocd-application-controller") - extract: - - name: appNamespace - jq: .involvedObject.namespace - - name: appName - jq: .involvedObject.name - count: 3 -` - -func TestStableHashStability(t *testing.T) { - // Use a stable rule from above for test. - ruleYaml := stableRuleYaml - - // Unmarshal YAML to ParseRuleT - rules, err := Unmarshal([]byte(ruleYaml)) - if err != nil { - t.Fatalf("Failed to unmarshal rule: %v", err) - } - if len(rules.Rules) == 0 { - t.Fatalf("No rules found in testdata") - } - rule := rules.Rules[0] - - // Compute stable hash - hash1, err := StableHash(rule) - if err != nil { - t.Fatalf("Failed to compute stable hash: %v", err) - } - - // Modify non-semantic metadata fields - rule.Metadata.Version = "v2.0.0" - rule.Metadata.Gen = 42 - - // Compute stable hash again - hash2, err := StableHash(rule) - if err != nil { - t.Fatalf("Failed to compute stable hash after metadata change: %v", err) - } - - if hash1 != hash2 { - t.Errorf("StableHash changed after non-semantic metadata update: %s != %s", hash1, hash2) - } - - if hash1 != "QFr5UWZMni8KYe4B7FkYg64p8CaRr6yeuynwDfPXjDj" { - t.Errorf("StableHash value changed unexpectedly: got %s, want %s", hash1, "QFr5UWZMni8KYe4B7FkYg64p8CaRr6yeuynwDfPXjDj") - } -} - -func DumpErrorChain(err error) { - i := 0 - for err != nil { - fmt.Printf("#%d %T %q\n", i, err, err.Error()) - i++ - err = errors.Unwrap(err) - } -} diff --git a/pkg/parser/tree.go b/pkg/parser/tree.go deleted file mode 100644 index 4721fdf..0000000 --- a/pkg/parser/tree.go +++ /dev/null @@ -1,1253 +0,0 @@ -package parser - -import ( - "crypto/sha1" - "crypto/sha256" - "encoding/json" - "errors" - "fmt" - "io" - "regexp" - "time" - - "github.com/btcsuite/btcutil/base58" - - "github.com/prequel-dev/prequel-compiler/pkg/pqerr" - "github.com/prequel-dev/prequel-compiler/pkg/schema" - "github.com/rs/zerolog/log" - "gopkg.in/yaml.v3" -) - -var ( - ErrRuleNotFound = errors.New("rule not found") - ErrRuleRootNotFound = errors.New("missing rule section") - ErrNotSupported = errors.New("not supported") - ErrTermNotFound = errors.New("term not found") - ErrMissingOrder = errors.New("'sequence' missing 'order'") - ErrMissingMatch = errors.New("'set' missing 'match'") - ErrMissingInput = errors.New("'script' missing 'input'") - ErrInputType = errors.New("invalid 'script' input type") - ErrInvalidWindow = errors.New("invalid 'window'") - ErrTermsMapping = errors.New("'terms' must be a mapping") - ErrDuplicateTerm = errors.New("duplicate term name") - ErrMissingRuleId = errors.New("missing rule id") - ErrMissingRuleHash = errors.New("missing rule hash") - ErrMissingCreId = errors.New("missing cre id") - ErrInvalidCreId = errors.New("invalid cre id") - ErrInvalidRuleId = errors.New("invalid rule id (must be base58)") - ErrInvalidRuleHash = errors.New("invalid rule hash (must be base58)") - ErrExtractName = errors.New("invalid extract name (alphanumeric and underscores only)") - ErrInnerEvent = errors.New("invalid event on inner node") - ErrScriptLanguage = errors.New("invalid script language") -) - -var ( - validCreIdRegex = regexp.MustCompile(`^[A-Za-z0-9-]{4,}$`) - validBase58IdRegex = regexp.MustCompile(`^[1-9A-Za-z]{12,}$`) - validateExtractName = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_]*$`) -) - -type TreeT struct { - Nodes []*NodeT `json:"nodes"` -} - -type EventT struct { - Origin bool `json:"origin"` - Source string `json:"source"` -} - -type NodeMetadataT struct { - RuleHash string `json:"rule_hash"` - RuleId string `json:"rule_id"` - CreId string `json:"cre_id"` - Window time.Duration `json:"window"` - Event *EventT `json:"event"` - Type schema.NodeTypeT `json:"type"` - Correlations []string `json:"correlations"` - NegateOpts *NegateOptsT `json:"negate_opts"` - Pos pqerr.Pos `json:"pos"` -} - -type NodeT struct { - Metadata NodeMetadataT `json:"metadata"` - NegIdx int `json:"neg_idx"` - Children []any `json:"children"` -} - -type NegateOptsT struct { - Window time.Duration `json:"window"` - Slide time.Duration `json:"slide"` - Anchor uint32 `json:"anchor"` - Absolute bool `json:"absolute"` -} - -type ExtractT struct { - Name string `json:"name"` - JqValue string `json:"jq_value,omitempty"` - RegexValue string `json:"regex_value,omitempty"` -} - -type FieldT struct { - Field string `json:"field"` - StrValue string `json:"value"` - JqValue string `json:"jq_value"` - RegexValue string `json:"regex_value"` - Count int `json:"count"` - NegateOpts *NegateOptsT `json:"negate"` - Extract []ExtractT `json:"extract,omitempty"` -} - -type TermsT struct { - Fields []FieldT `json:"fields"` -} - -type MatcherT struct { - Match TermsT `json:"match"` - Negate TermsT `json:"negate"` - Window time.Duration `json:"window"` -} - -type PromQLT struct { - Expr string `json:"expr"` - For *time.Duration `json:"for,omitempty"` - Interval *time.Duration `json:"interval,omitempty"` -} - -type ScriptT struct { - Code string `json:"code"` - Language string `json:"language,omitempty"` - Timeout *time.Duration `json:"timeout,omitempty"` -} - -// Hooks exposed to avoid importing dependencies in compiler. -var PromQLValidator = func(expr string) error { return nil } // PromQLValidator validates a PromQL expression. -var LuaValidator = func(code string) error { return nil } // LuaValidator validates Lua script syntax. - -func newEvent(t *ParseEventT) *EventT { - return &EventT{ - Source: t.Source, - Origin: t.Origin, - } -} - -func isValidBase58Id(s string) bool { - return validBase58IdRegex.MatchString(s) -} - -func isValidCreId(s string) bool { - return validCreIdRegex.MatchString(s) -} - -func isValidExtractName(s string) bool { - return validateExtractName.MatchString(s) -} - -func initNode(ruleId, ruleHash string, creId string, yn *yaml.Node) (*NodeT, error) { - - if ruleId == "" { - return nil, ErrMissingRuleId - } - - if !isValidBase58Id(ruleId) { - return nil, ErrInvalidRuleId - } - - if ruleHash == "" { - return nil, ErrMissingRuleHash - } - - if !isValidBase58Id(ruleHash) { - return nil, ErrInvalidRuleHash - } - - if creId == "" { - return nil, ErrMissingCreId - } - - if !isValidCreId(creId) { - return nil, ErrInvalidCreId - } - - return &NodeT{ - Metadata: NodeMetadataT{ - RuleId: ruleId, - RuleHash: ruleHash, - CreId: creId, - Pos: pqerr.Pos{Line: yn.Line, Col: yn.Column}, - }, - NegIdx: -1, - Children: make([]any, 0), - }, nil -} - -func assignNodeSeq(node *NodeT, seq *ParseSequenceT) error { - - if seq.Event == nil { - node.Metadata.Type = schema.NodeTypeSeq - return nil - } - - // Propagate the event - node.Metadata.Event = newEvent(seq.Event) - - switch { - case node.IsPromNode(): - node.Metadata.Type = schema.NodeTypePromQL - case !node.IsMatcherNode(): - return ErrInnerEvent - default: - node.Metadata.Type = schema.NodeTypeLogSeq - } - - return nil -} - -func assignNodeSet(node *NodeT, set *ParseSetT) error { - - if set.Event == nil { - node.Metadata.Type = schema.NodeTypeSet - return nil - } - - // Propagate the event - node.Metadata.Event = newEvent(set.Event) - - switch { - case node.IsPromNode(): - node.Metadata.Type = schema.NodeTypePromQL - case !node.IsMatcherNode(): - return ErrInnerEvent - default: - node.Metadata.Type = schema.NodeTypeLogSet - } - - return nil -} - -func (node *NodeT) IsMatcherNode() bool { - if len(node.Children) == 0 { - return false - } - - allMatcher := true - - for _, child := range node.Children { - if _, ok := child.(*MatcherT); !ok { - allMatcher = false - break - } - } - - return allMatcher -} - -func (node *NodeT) IsPromNode() bool { - if len(node.Children) == 0 { - return false - } - - allPromQL := true - for _, child := range node.Children { - if _, ok := child.(*PromQLT); !ok { - allPromQL = false - break - } - } - - return allPromQL -} - -func (node *NodeT) IsScriptNode() bool { - if len(node.Children) != 2 { - return false - } - - // Expect first child to be a script definition and second child to be undefined term. - _, ok := node.Children[0].(*ScriptT) - - return ok -} - -func seqNodeProps(node *NodeT, seq *ParseSequenceT, order bool, yn *yaml.Node) error { - - if !order { - return node.WrapError(ErrMissingOrder) - } - - if err := assignNodeSeq(node, seq); err != nil { - return err - } - - if seq.Window != "" { - var err error - - if winNode, ok := findChild(yn, docWindow); ok { - node.Metadata.Pos = pqerr.Pos{Line: winNode.Line, Col: winNode.Column} - } - - if node.Metadata.Window, err = time.ParseDuration(seq.Window); err != nil { - return node.WrapError(ErrInvalidWindow) - } - } - - if seq.Correlations != nil { - node.Metadata.Correlations = seq.Correlations - } - - return nil -} - -func setNodeProps(node *NodeT, set *ParseSetT, match bool, yn *yaml.Node) error { - - if !match { - return node.WrapError(ErrMissingMatch) - } - - if err := assignNodeSet(node, set); err != nil { - return err - } - - if set.Window != "" { - var err error - - if winNode, ok := findChild(yn, docWindow); ok { - node.Metadata.Pos = pqerr.Pos{Line: winNode.Line, Col: winNode.Column} - } - - if node.Metadata.Window, err = time.ParseDuration(set.Window); err != nil { - return node.WrapError(ErrInvalidWindow) - } - } - - if set.Correlations != nil { - node.Metadata.Correlations = set.Correlations - } - - return nil -} - -func buildTree(termsT map[string]ParseTermT, r ParseRuleT, ruleNode *yaml.Node, termsY map[string]*yaml.Node) (*NodeT, error) { - - var ( - root *NodeT - n *yaml.Node - ok bool - err error - ) - - n, ok = findChild(ruleNode, docRule) - if !ok { - return nil, pqerr.Wrap( - pqerr.Pos{Line: ruleNode.Line, Col: ruleNode.Column}, - r.Metadata.Id, - r.Metadata.Hash, - r.Cre.Id, - ErrRuleRootNotFound, - ) - } - - switch { - case r.Rule.Sequence != nil: - seqNode, _ := findChild(n, docSeq) - root, err = initNode(r.Metadata.Id, r.Metadata.Hash, r.Cre.Id, seqNode) - if err != nil { - return nil, pqerr.Wrap( - pqerr.Pos{Line: n.Line, Col: n.Column}, - r.Metadata.Id, - r.Metadata.Hash, - r.Cre.Id, - err, - ) - } - return buildSequenceTree(root, termsT, r, seqNode, termsY) - case r.Rule.Set != nil: - setNode, _ := findChild(n, docSet) - root, err = initNode(r.Metadata.Id, r.Metadata.Hash, r.Cre.Id, setNode) - if err != nil { - return nil, pqerr.Wrap( - pqerr.Pos{Line: n.Line, Col: n.Column}, - r.Metadata.Id, - r.Metadata.Hash, - r.Cre.Id, - err, - ) - } - return buildSetTree(root, termsT, r, setNode, termsY) - default: - return nil, pqerr.Wrap( - pqerr.Pos{Line: n.Line, Col: n.Column}, - r.Metadata.Id, - r.Metadata.Hash, - r.Cre.Id, - ErrNotSupported, - ) - } -} - -// buildSequenceTree processes a rule with a Sequence definition. -func buildSequenceTree(root *NodeT, termsT map[string]ParseTermT, r ParseRuleT, ruleNode *yaml.Node, termsY map[string]*yaml.Node) (*NodeT, error) { - - var ( - seq = r.Rule.Sequence - orderYn *yaml.Node - negateYn *yaml.Node - ok bool - ) - - orderYn, ok = findChild(ruleNode, docOrder) - if !ok { - return nil, pqerr.Wrap( - pqerr.Pos{Line: ruleNode.Line, Col: ruleNode.Column}, - r.Metadata.Id, - r.Metadata.Hash, - r.Cre.Id, - ErrMissingOrder, - ) - } - - // Negate is optional - negateYn, _ = findChild(ruleNode, docNegate) - - // Build positive children from seq.Order (non-negated) - // Build negative children from seq.Negate (negated) - pos, neg, err := buildChildrenGroups(root, termsT, seq.Order, seq.Negate, orderYn, negateYn, termsY) - if err != nil { - return nil, err - } - - // Order positive first, then negatives - root.Children = append(root.Children, pos...) - root.Children = append(root.Children, neg...) - if len(neg) > 0 { - root.NegIdx = len(pos) - } - - // Apply sequence-specific node properties - if err := seqNodeProps(root, seq, seq.Order != nil, orderYn); err != nil { - return nil, err - } - - return root, nil -} - -// buildSetTree processes a rule with a Set definition. -func buildSetTree(root *NodeT, termsT map[string]ParseTermT, r ParseRuleT, ruleNode *yaml.Node, termsY map[string]*yaml.Node) (*NodeT, error) { - - var ( - set = r.Rule.Set - matchYn *yaml.Node - negateYn *yaml.Node - ok bool - ) - - matchYn, ok = findChild(ruleNode, docMatch) - if !ok { - return nil, pqerr.Wrap( - pqerr.Pos{Line: ruleNode.Line, Col: ruleNode.Column}, - r.Metadata.Id, - r.Metadata.Hash, - r.Cre.Id, - ErrMissingMatch, - ) - } - - // Negate is optional - negateYn, _ = findChild(ruleNode, docNegate) - - pos, neg, err := buildChildrenGroups(root, termsT, set.Match, set.Negate, matchYn, negateYn, termsY) - if err != nil { - return nil, err - } - - // Order positive first, then negatives - root.Children = append(root.Children, pos...) - root.Children = append(root.Children, neg...) - if len(neg) > 0 { - root.NegIdx = len(pos) - } - - // Apply set-specific node properties - if err := setNodeProps(root, set, set.Match != nil, ruleNode); err != nil { - return nil, err - } - - return root, nil -} - -// buildChildrenGroups is a helper for building positive/negative children -// in a single pass. The boolean flags specify whether each slice -// is being treated as negated or not. -func buildChildrenGroups(root *NodeT, termsT map[string]ParseTermT, matches, negates []ParseTermT, orderYn, negateYn *yaml.Node, termsY map[string]*yaml.Node) (pos []any, neg []any, err error) { - - if len(matches) > 0 { - - cPos, err := buildChildren(root, termsT, matches, false, orderYn, termsY) - if err != nil { - return nil, nil, err - } - pos = append(pos, cPos...) - } - - if len(negates) > 0 { - cNeg, err := buildChildren(root, termsT, negates, true, negateYn, termsY) - if err != nil { - return nil, nil, err - } - // If double-negatives or other logic is needed, adjust the append here - neg = append(neg, cNeg...) - } - - return pos, neg, nil -} - -func buildChildren(parent *NodeT, tm map[string]ParseTermT, terms []ParseTermT, parentNegate bool, yn *yaml.Node, termsY map[string]*yaml.Node) ([]any, error) { - var ( - children = make([]any, 0) - ) - - for _, term := range terms { - var ( - t = term - n = yn - ) - - if term.StrValue != "" { - // If the term is not found in the terms map, then use as str value - if resolvedTerm, ok := tm[term.StrValue]; ok { - t = resolvedTerm - if n, ok = termsY[term.StrValue]; !ok { - return nil, parent.WrapError(ErrTermNotFound) - } - - if term.NegateOpts != nil { - t.NegateOpts = term.NegateOpts - } - } - } - - if node, err := nodeFromTerm(parent, tm, t, parentNegate, n, termsY); err != nil { - return nil, err - } else { - children = append(children, node) - } - - } - - return children, nil -} - -func nodeFromSeq(parent *NodeT, termsT map[string]ParseTermT, term ParseTermT, yn *yaml.Node, termsY map[string]*yaml.Node) (node *NodeT, err error) { - - n, ok := findChild(yn, docSeq) - if !ok { - n = yn - } - - node, err = buildSequenceNode(parent, termsT, term.Sequence, n, termsY) - if err != nil { - return - } - - if term.NegateOpts == nil { - return - } - - opts, err := negateOpts(term) - if err != nil { - return - } - node.Metadata.NegateOpts = opts - - return -} - -func nodeFromSet(parent *NodeT, termsT map[string]ParseTermT, term ParseTermT, yn *yaml.Node, termsY map[string]*yaml.Node) (node *NodeT, err error) { - - n, ok := findChild(yn, docSet) - if !ok { - n = yn - } - - node, err = buildSetNode(parent, termsT, term.Set, n, termsY) - if err != nil { - return - } - - if term.NegateOpts == nil { - return - } - - opts, err := negateOpts(term) - if err != nil { - return - } - node.Metadata.NegateOpts = opts - - return -} - -func nodeFromTerm(parent *NodeT, termsT map[string]ParseTermT, term ParseTermT, parentNegate bool, yn *yaml.Node, termsY map[string]*yaml.Node) (v any, err error) { - - switch { - case term.Sequence != nil: - v, err = nodeFromSeq(parent, termsT, term, yn, termsY) - - case term.Set != nil: - v, err = nodeFromSet(parent, termsT, term, yn, termsY) - - case term.PromQL != nil: - return nodeFromProm(parent, term, yn) - - case term.Script != nil: - return nodeFromScript(parent, termsT, term, yn, termsY) - - case term.StrValue != "" || term.JqValue != "" || term.RegexValue != "": - return parseValue(term, parentNegate) - - default: - parent.Metadata.Pos = pqerr.Pos{Line: yn.Line, Col: yn.Column} - return nil, parent.WrapError(ErrTermNotFound) - } - - return -} - -func extractTerms(terms []ParseExtractT) ([]ExtractT, error) { - var extracts []ExtractT - for _, term := range terms { - - if !isValidExtractName(term.Name) { - return nil, ErrExtractName - } - - extracts = append(extracts, ExtractT{ - Name: term.Name, - JqValue: term.JqValue, - RegexValue: term.RegexValue, - }) - } - return extracts, nil -} - -func negateOpts(term ParseTermT) (*NegateOptsT, error) { - var ( - opts = &NegateOptsT{} - err error - ) - - if term.NegateOpts.Window != "" { - if opts.Window, err = time.ParseDuration(term.NegateOpts.Window); err != nil { - return nil, err - } - } - - if term.NegateOpts.Slide != "" { - if opts.Slide, err = time.ParseDuration(term.NegateOpts.Slide); err != nil { - return nil, err - } - } - - opts.Anchor = term.NegateOpts.Anchor - opts.Absolute = term.NegateOpts.Absolute - - return opts, nil -} - -func buildSequenceNode(parent *NodeT, termsT map[string]ParseTermT, seq *ParseSequenceT, yn *yaml.Node, termsY map[string]*yaml.Node) (*NodeT, error) { - node, err := initNode(parent.Metadata.RuleId, parent.Metadata.RuleHash, parent.Metadata.CreId, yn) - if err != nil { - return nil, parent.WrapError(err) - } - - pos, neg, err := buildPosNegChildren(node, termsT, seq.Order, seq.Negate, yn, termsY) - if err != nil { - return nil, err - } - - node.Children = append(node.Children, pos...) - node.Children = append(node.Children, neg...) - if len(neg) > 0 { - node.NegIdx = len(pos) - } - - // Apply sequence-specific node properties - if err := seqNodeProps(node, seq, seq.Order != nil, yn); err != nil { - return nil, err - } - - return node, nil -} - -func buildSetNode(parent *NodeT, termsT map[string]ParseTermT, set *ParseSetT, yn *yaml.Node, termsY map[string]*yaml.Node) (*NodeT, error) { - node, err := initNode(parent.Metadata.RuleId, parent.Metadata.RuleHash, parent.Metadata.CreId, yn) - if err != nil { - return nil, parent.WrapError(err) - } - - pos, neg, err := buildPosNegChildren(node, termsT, set.Match, set.Negate, yn, termsY) - if err != nil { - return nil, err - } - - node.Children = append(node.Children, pos...) - node.Children = append(node.Children, neg...) - if len(neg) > 0 { - node.NegIdx = len(pos) - } - - // Apply set-specific node properties - if err := setNodeProps(node, set, set.Match != nil, yn); err != nil { - return nil, err - } - - return node, nil -} - -// buildPosNegChildren is a helper for building -// positive and negative children across Sequence and Set -func buildPosNegChildren(node *NodeT, termsT map[string]ParseTermT, matches, negates []ParseTermT, yn *yaml.Node, termsY map[string]*yaml.Node) (pos []any, neg []any, err error) { - - pos, neg = []any{}, []any{} - - if len(matches) > 0 { - cPos, err := buildChildren(node, termsT, matches, false, yn, termsY) - if err != nil { - return nil, nil, err - } - pos = append(pos, cPos...) - } - - if len(negates) > 0 { - cNeg, err := buildChildren(node, termsT, negates, true, yn, termsY) - if err != nil { - return nil, nil, err - } - neg = append(neg, cNeg...) - } - - return pos, neg, nil -} - -func nodeFromProm(parent *NodeT, term ParseTermT, yn *yaml.Node) (*NodeT, error) { - - var interval *time.Duration - if term.PromQL.Interval != "" { - dur, err := time.ParseDuration(term.PromQL.Interval) - if err != nil { - return nil, err - } - interval = &dur - } - - var forDuration *time.Duration - if term.PromQL.For != "" { - dur, err := time.ParseDuration(term.PromQL.For) - if err != nil { - return nil, err - } - forDuration = &dur - } - - if err := PromQLValidator(term.PromQL.Expr); err != nil { - return nil, err - } - - node, err := initNode(parent.Metadata.RuleId, parent.Metadata.RuleHash, parent.Metadata.CreId, yn) - if err != nil { - return nil, parent.WrapError(err) - } - - node.Metadata.Type = schema.NodeTypePromQL - - // Propagate the event - if term.PromQL.Event != nil { - node.Metadata.Event = newEvent(term.PromQL.Event) - } - - node.Children = append(node.Children, &PromQLT{ - Expr: term.PromQL.Expr, - For: forDuration, - Interval: interval, - }) - - return node, nil -} - -// Script nodes are internal nodes with one input node. -// The first child in the resultant NodeT is always the script definition, the following child is the input node. -// The input node can be a matcher, promql, or another script node, but not a value term since values cannot be inputs to scripts. -func nodeFromScript(parent *NodeT, termsT map[string]ParseTermT, term ParseTermT, yn *yaml.Node, termsY map[string]*yaml.Node) (*NodeT, error) { - - var timeout *time.Duration - if term.Script.Timeout != "" { - dur, err := time.ParseDuration(term.Script.Timeout) - if err != nil { - return nil, err - } - timeout = &dur - } - - switch term.Script.Language { - case "", "lua": - if err := LuaValidator(term.Script.Code); err != nil { - return nil, err - } - default: - return nil, parent.WrapError(ErrScriptLanguage) - } - - // Create the script node with metadata from the parent rule. - node, err := initNode(parent.Metadata.RuleId, parent.Metadata.RuleHash, parent.Metadata.CreId, yn) - if err != nil { - return nil, parent.WrapError(err) - } - - // Script node requires one input. - // The input can be a sequence, a set, or a promql term, but not a value term since values cannot be inputs to scripts. - if term.Script.Input == nil { - return nil, parent.WrapError(ErrMissingInput) - } - - // Validator function: only allow terms that could be an input - allowTerm := func(t ParseTermT) bool { - switch { - case t.Sequence != nil: - case t.Set != nil: - case t.PromQL != nil: - case t.Script != nil: - default: - return false - } - return true - } - - // Validate that input is of an allowed type - if !allowTerm(*term.Script.Input) { - return nil, parent.WrapError(ErrInputType) - } - - childNode, err := nodeFromTerm(node, termsT, *term.Script.Input, false, yn, termsY) - switch { - case err != nil: - return nil, err - case childNode == nil: - return nil, parent.WrapError(ErrMissingInput) - } - - // Assign the script node type - node.Metadata.Type = schema.NodeTypeScript - - // Append the script definition as the first child, followed by the input. - node.Children = append(node.Children, &ScriptT{ - Code: term.Script.Code, - Language: term.Script.Language, - Timeout: timeout, - }, childNode) - - return node, nil -} - -func parseValue(term ParseTermT, negate bool) (*MatcherT, error) { - - var ( - err error - matcher = &MatcherT{} - ) - - switch negate { - case false: - var extracts []ExtractT - if len(term.Extract) > 0 { - if extracts, err = extractTerms(term.Extract); err != nil { - return nil, err - } - } - - matcher.Match.Fields = append(matcher.Match.Fields, FieldT{ - Field: term.Field, - StrValue: term.StrValue, - JqValue: term.JqValue, - RegexValue: term.RegexValue, - Count: term.Count, - Extract: extracts, - }) - case true: - - var ( - opts *NegateOptsT - ) - - if term.NegateOpts != nil { - if opts, err = negateOpts(term); err != nil { - return nil, err - } - } - - matcher.Negate.Fields = append(matcher.Negate.Fields, FieldT{ - Field: term.Field, - StrValue: term.StrValue, - JqValue: term.JqValue, - RegexValue: term.RegexValue, - Count: term.Count, - NegateOpts: opts, - }) - } - - return matcher, nil -} - -func ParseCres(data []byte) (map[string]ParseCreT, error) { - - cfg, _, err := _parse(data) - if err != nil { - return nil, err - } - - cres := make(map[string]ParseCreT, len(cfg.Rules)) - for _, rule := range cfg.Rules { - cres[rule.Metadata.Hash] = rule.Cre - } - - return cres, nil -} - -func Parse(data []byte, opts ...ParseOptT) (*TreeT, error) { - - var ( - config *RulesT - err error - ) - - if config, err = Unmarshal(data); err != nil { - return nil, err - } - - return ParseRules(config, opts) -} - -func Unmarshal(data []byte) (*RulesT, error) { - - cfg, root, err := _parse(data) - if err != nil { - return nil, err - } - - docMap := root.Content[0] - - var ok bool - if cfg.Root, ok = findChild(docMap, docRules); !ok { - return nil, errors.New("rules not found") - } - - if termsNode, ok := findChild(docMap, docTerms); ok { - cfg.TermsY = collectTermsY(termsNode) - } - - return cfg, nil -} - -func Hash(h string) string { - hash := sha1.Sum([]byte(h)) - return base58.Encode(hash[:]) -} - -// HashRule to provide a unique identity for the rule. -// The hash is based on the rule's content, excluding previous hash calculations. - -func HashRule(rule ParseRuleT) (string, error) { - rule.Metadata.Hash = "" // Hash is what we are generating here, not semantically important - return _hashRule(rule) -} - -// StableHash to provide a unique stable identity for the rule. It can be used for dupe detection. -// The hash is based on the rule's content, excluding metadata that is not semantically important. - -func StableHash(rule ParseRuleT) (string, error) { - - // Strip out versioning metadata before calculating the stable hash. - // The versioning metadata is not semantically important for the rule's content, - // so we can safely ignore it for the purpose of hashing. - // This is important to ensure that the hash remains consistent across changes - // that do not affect the rule's content, such as version bumps or metadata changes. - - // The field rule.Metadata.Id is considered part of the rules identity and should be included in the stable hash. - // Rules can change over time having the following properties: - // - Metadata.Id: Unique identifier for the rule, which is immutable for the lifetime of the rule. - // - Metadata.Hash: A hash of the rule's content, which is regenerated on every semantic change. - // - Metadata.Version: A version string that *should* be incremented on changes, but is not semantically important. - // - Metadata.Gen: A generation counter that is incremented on every change, but is not semantically important. - - rule.Metadata.Gen = 0 // Gen is bumped on every semantic change, so we don't want it in the hash - rule.Metadata.Version = "" // Version may be bumped on change, also not semantically important - return HashRule(rule) -} - -func _hashRule(rule ParseRuleT) (string, error) { - // json.Marshal to produce deterministic output - jsonBytes, err := json.Marshal(rule) - if err != nil { - return "", err - } - - hash := sha256.Sum256(jsonBytes) - - return base58.Encode(hash[:]), nil -} - -func parseRules(rules []ParseRuleT, termsT map[string]ParseTermT, rulesRoot *yaml.Node, termsY map[string]*yaml.Node, opts ...ParseOptT) (*TreeT, error) { - - var ( - o = parseOpts(opts...) - tree = &TreeT{ - Nodes: make([]*NodeT, 0), - } - ) - - for i, rule := range rules { - var ( - node *NodeT - ruleNode *yaml.Node - ok bool - err error - ) - - if ruleNode, ok = seqItem(rulesRoot, i); !ok { - log.Error(). - Int("index", i). - Msg("Rule not found") - return nil, ErrRuleNotFound - } - - if o.genIds { - if rule.Metadata.Id == "" { - rule.Metadata.Id = Hash(rule.Cre.Id) - log.Warn(). - Str("rule.Metadata.Id", rule.Metadata.Id). - Str("rule.Cre.Id", rule.Cre.Id). - Msg("Rule id is empty, generating from cre id") - } - if rule.Metadata.Hash == "" { - if rule.Metadata.Hash, err = HashRule(rule); err != nil { - return nil, err - } - log.Warn(). - Str("rule.Cre.Id", rule.Cre.Id). - Str("rule.Metadata.Id", rule.Metadata.Id). - Str("rule.Metadata.Hash", rule.Metadata.Hash). - Msg("Rule hash is empty, generating from rule data") - } - } - - if node, err = buildTree(termsT, rule, ruleNode, termsY); err != nil { - return nil, err - } - - tree.Nodes = append(tree.Nodes, node) - } - - return tree, nil -} - -func ParseRules(config *RulesT, opts []ParseOptT) (*TreeT, error) { - return parseRules(config.Rules, config.TermsT, config.Root, config.TermsY, opts...) -} - -func findChild(n *yaml.Node, key string) (*yaml.Node, bool) { - if n == nil || n.Kind != yaml.MappingNode { - return nil, false - } - for i := 0; i < len(n.Content); i += 2 { - k, v := n.Content[i], n.Content[i+1] - if k.Value == key { - return v, true - } - } - return nil, false -} - -func seqItem(seq *yaml.Node, idx int) (*yaml.Node, bool) { - if seq == nil || seq.Kind != yaml.SequenceNode || idx < 0 || - idx >= len(seq.Content) { - return nil, false - } - return seq.Content[idx], true -} - -func collectTermsY(doc *yaml.Node) map[string]*yaml.Node { - termsY := make(map[string]*yaml.Node) - if doc == nil || doc.Kind != yaml.MappingNode { - return termsY - } - for i := 0; i < len(doc.Content); i += 2 { - key := doc.Content[i] // scalar - termsY[key.Value] = doc.Content[i+1] - } - return termsY -} - -func (n *NodeT) WrapError(err error) error { - return pqerr.Wrap( - pqerr.Pos{Line: n.Metadata.Pos.Line, Col: n.Metadata.Pos.Col}, - n.Metadata.RuleId, - n.Metadata.RuleHash, - n.Metadata.CreId, err) -} - -type ParseOptT func(*parseOptsT) - -func WithGenIds() func(*parseOptsT) { - return func(o *parseOptsT) { - o.genIds = true - } -} - -type parseOptsT struct { - genIds bool -} - -func parseOpts(opts ...ParseOptT) *parseOptsT { - o := &parseOptsT{} - for _, opt := range opts { - opt(o) - } - - return o -} - -func Read(rdr io.Reader, opts ...ParseOptT) (*RulesT, error) { - var ( - allRules = &RulesT{ - Rules: make([]ParseRuleT, 0), - TermsT: make(map[string]ParseTermT), - TermsY: make(map[string]*yaml.Node), - } - root *yaml.Node - dupes = make(map[string]struct{}) - decoder *yaml.Decoder - o = parseOpts(opts...) - ok bool - ) - - decoder = yaml.NewDecoder(rdr) - -LOOP: - for { - // 1) grab the raw document (with positions) --------------------------- - var doc yaml.Node - if err := decoder.Decode(&doc); err != nil { - switch err { - case io.EOF: - break LOOP - default: - log.Error().Err(err).Msg("fail yaml decode") - return nil, err - } - } - if len(doc.Content) == 0 { // empty document ("---\n") - continue - } - - root = doc.Content[0] - - if sec, ok := findChild(root, docSection); ok { // key “section” exists? - if sec.Kind == yaml.ScalarNode && sec.Value == docVersion { - // Entire document is a version footer: ignore it and move on - continue - } - } - - allRules.Root, ok = findChild(root, docRules) - if !ok { - return nil, errors.New("rules not found") - } - - // 2) walk keys in that mapping --------------------------------------- - for i := 0; i < len(root.Content); i += 2 { - kNode, vNode := root.Content[i], root.Content[i+1] - switch kNode.Value { - - case "rules": - var rules []ParseRuleT - if err := vNode.Decode(&rules); err != nil { - return nil, err - } - if !o.genIds { - if err := checkDuplicates(rules, dupes); err != nil { - return nil, err - } - } - allRules.Rules = append(allRules.Rules, rules...) - - case "terms": - - termsTNew, termsYNew, err := parseTermsNode(vNode) // vNode is *yaml.Node for this block - if err != nil { - return nil, err - } - - if allRules.TermsT == nil { - allRules.TermsT = make(map[string]ParseTermT) - } - - if err := mergeTerms(allRules.TermsT, allRules.TermsY, termsTNew, termsYNew); err != nil { - return nil, err - } - default: - // unknown section – ignore or warn - } - } - } - - return allRules, nil -} - -func mergeTerms(dst map[string]ParseTermT, dstPos map[string]*yaml.Node, src map[string]ParseTermT, srcPos map[string]*yaml.Node) error { - for k, v := range src { - if _, dup := dst[k]; dup { - return ErrDuplicateTerm - } - dst[k] = v - dstPos[k] = srcPos[k] - } - return nil -} - -func checkDuplicates(rules []ParseRuleT, seen map[string]struct{}) error { - for _, r := range rules { - for _, id := range []string{r.Metadata.Hash, r.Metadata.Id, r.Cre.Id} { - if _, dup := seen[id]; dup { - return fmt.Errorf("duplicate id=%s (cre=%s)", id, r.Cre.Id) - } - seen[id] = struct{}{} - } - } - return nil -} - -func parseTermsNode(n *yaml.Node) (map[string]ParseTermT, map[string]*yaml.Node, error) { - var m = make(map[string]ParseTermT) - var p = make(map[string]*yaml.Node) - - if n.Kind != yaml.MappingNode { - log.Error().Msg("terms node is not a mapping") - return nil, nil, ErrTermsMapping - } - - for i := 0; i < len(n.Content); i += 2 { - kNode, vNode := n.Content[i], n.Content[i+1] - - if _, dup := m[kNode.Value]; dup { - return nil, nil, ErrDuplicateTerm - } - - var t ParseTermT - if err := vNode.Decode(&t); err != nil { - return nil, nil, err - } - - m[kNode.Value] = t - p[kNode.Value] = vNode - } - - return m, p, nil -} diff --git a/pkg/schema/schema.go b/pkg/schema/schema.go deleted file mode 100644 index 8845a20..0000000 --- a/pkg/schema/schema.go +++ /dev/null @@ -1,23 +0,0 @@ -package schema - -const ( - ScopeOrganization = "organization" - ScopeCluster = "cluster" - ScopeNode = "node" - ScopeDefault = "default" -) - -type NodeTypeT string - -const ( - NodeTypeSeq NodeTypeT = "machine_seq" - NodeTypeSet NodeTypeT = "machine_set" - NodeTypeLogSeq NodeTypeT = "log_seq" - NodeTypeLogSet NodeTypeT = "log_set" - NodeTypePromQL NodeTypeT = "promql" - NodeTypeScript NodeTypeT = "script" -) - -func (t NodeTypeT) String() string { - return string(t) -} diff --git a/pkg/testdata/rules.go b/pkg/testdata/rules.go index da55f61..7760325 100644 --- a/pkg/testdata/rules.go +++ b/pkg/testdata/rules.go @@ -8,13 +8,14 @@ rules: metadata: id: "J7uRQTGpGMyL1iFpssnBeS" hash: "rdJLgqYgkEp8jg8Qks1qiq" - generation: 1 + gen: 1 rule: - sequence: + set: window: 10s event: source: kafka - order: + origin: true + match: - value: "io.vertx.core.VertxException: Thread blocked" count: 3 ` @@ -27,65 +28,58 @@ rules: metadata: id: "J7uRQTGpGMyL1iFpssnBeS" hash: "rdJLgqYgkEp8jg8Qks1qiq" - generation: 1 + gen: 1 rule: sequence: window: 30s correlations: - hostname order: - - term1 - - term2 - - term3 -terms: - term1: - sequence: - window: 10s - event: - source: rabbitmq - origin: true - order: - - value: Discarding message - count: 10 - - Mnesia overloaded - negate: - - SIGTERM - term2: - set: - window: 1s - event: - source: k8s - match: - - field: "reason" - value: "Killing" - - field: "reason" - value: "NodeShutdown" - negate: - - SIGTERM - term3: - sequence: - window: 5s - correlations: - - hostname - order: - - sequence: - window: 1s - event: - source: nginx - order: - - error message - - shutdown - - set: - event: - source: nginx - match: - - 90% - - set: - event: - source: k8s - match: - - field: "reason" - value: "Killing" + - sequence: + window: 10s + event: + source: rabbitmq + origin: true + order: + - value: Discarding message + count: 10 + - Mnesia overloaded + negate: + - SIGTERM + - set: + window: 1s + event: + source: k8s + match: + - field: "reason" + value: "Killing" + - field: "reason" + value: "NodeShutdown" + negate: + - SIGTERM + - sequence: + window: 5s + correlations: + - hostname + order: + - sequence: + window: 1s + event: + source: nginx + order: + - error message + - shutdown + - set: + event: + source: nginx + match: + - 90% + - set: + event: + source: k8s + match: + - field: "reason" + value: "Killing" ` var TestSuccessComplexRule3 = ` @@ -101,28 +95,23 @@ rules: correlations: - hostname order: - - term1 - - term2 -terms: - term1: - sequence: - window: 10s - event: - source: rabbitmq - origin: true - order: - - value: Discarding message - count: 10 - - Mnesia overloaded - negate: - - SIGTERM - term2: - set: - event: - source: k8s - match: - - field: "reason" - value: "Killing" + - sequence: + window: 10s + event: + source: rabbitmq + origin: true + order: + - value: Discarding message + count: 10 + - Mnesia overloaded + negate: + - SIGTERM + - set: + event: + source: k8s + match: + - field: "reason" + value: "Killing" ` var TestSuccessComplexRule4 = ` @@ -138,121 +127,70 @@ rules: correlations: - hostname order: - - term1 - - term2 - - term4 + - sequence: + window: 10s + event: + source: rabbitmq + origin: true + order: + - value: Discarding message + count: 10 + - Mnesia overloaded + negate: + - SIGTERM + - sequence: + window: 5s + correlations: + - container_id + order: + - sequence: + window: 1s + event: + source: nginx + order: + - error message + - shutdown + - set: + event: + source: nginx + match: + - 90% + - set: + event: + source: k8s + match: + - field: "reason" + value: "Killing" + - sequence: + window: 5s + correlations: + - container_id + order: + - sequence: + window: 1s + event: + source: nginx + order: + - error message + - shutdown + - set: + event: + source: nginx + match: + - 90% + - set: + event: + source: k8s + match: + - field: "reason" + value: "Killing" negate: - - term3 - -terms: - term1: - sequence: - window: 10s - event: - source: rabbitmq - origin: true - order: - - value: Discarding message - count: 10 - - Mnesia overloaded - negate: - - SIGTERM - - term2: - sequence: - window: 5s - correlations: - - container_id - order: - - sequence: - window: 1s - event: - source: nginx - order: - - error message - - shutdown - - set: - event: - source: nginx - match: - - 90% - - set: - event: - source: k8s - match: - - field: "reason" - value: "Killing" - term4: - sequence: - window: 5s - correlations: - - container_id - order: - - sequence: - window: 1s - event: - source: nginx - order: - - error message - - shutdown - - set: - event: - source: nginx - match: - - 90% - - set: - event: - source: k8s - match: - - field: "reason" - value: "Killing" - term3: - set: - event: - source: k8s - match: - - field: "reason" - value: "NodeShutdown" -` - -var TestSuccessComplexRule5 = ` -rules: - - cre: - id: TestSuccessComplexRule5 - severity: 1 - metadata: - id: "J7uRQTGpGMyL1iFpssnBeS" - hash: "rdJLgqYgkEp8jg8Qks1qiq" - generation: 1 - rule: - sequence: - window: 30s - correlations: - - hostname - order: - - term1 - - term2 -terms: - term1: - sequence: - window: 10s - event: - src: log - origin: true - imageUrl: "*rabbitmq*" - order: - - value: Discarding message - count: 10 - - Mnesia overloaded - negate: - - SIGTERM - term2: - set: - event: - src: k8s - match: - - field: "reason" - value: "Killing" + - set: + event: + source: k8s + match: + - field: "reason" + value: "NodeShutdown" ` var TestSuccessNegateOptions1 = ` @@ -262,13 +200,14 @@ rules: metadata: id: "J7uRQTGpGMyL1iFpssnBeS" hash: "rdJLgqYgkEp8jg8Qks1qiq" - generation: 1 + gen: 1 rule: - sequence: + set: window: 10s event: source: kafka - order: + origin: true + match: - value: "io.vertx.core.VertxException: Thread blocked" count: 3 negate: @@ -276,12 +215,12 @@ rules: window: 10s slide: 1s anchor: 0 - abs: true + absolute: true - value: "SIGKILL" window: 10s slide: 1s anchor: 0 - abs: true + absolute: true ` var TestSuccessNegateOptions2 = ` @@ -291,49 +230,40 @@ rules: metadata: id: "J7uRQTGpGMyL1iFpssnBeS" hash: "rdJLgqYgkEp8jg8Qks1qiq" - generation: 1 + gen: 1 rule: sequence: window: 30s correlations: - hostname order: - - term1 - - term2 + - sequence: + window: 10s + event: + source: log + origin: true + order: + - value: Discarding message + count: 10 + - Mnesia overloaded + negate: + - SIGTERM + - set: + event: + source: k8s + match: + - field: "reason" + value: "Killing" negate: - - value: term3 + - set: + event: + source: log + match: + - value: "Killing" window: 10s slide: 1s anchor: 0 - abs: true - -terms: - term1: - sequence: - window: 10s - event: - source: log - origin: true - image_url: "*rabbitmq*" - order: - - value: Discarding message - count: 10 - - Mnesia overloaded - negate: - - SIGTERM - term2: - set: - event: - source: k8s - match: - - field: "reason" - value: "Killing" - term3: - set: - event: - source: log - match: - - value: "Killing" + absolute: true ` var TestSuccessSimpleExtraction = ` @@ -343,12 +273,13 @@ rules: metadata: id: "J7uRQTGpGMyL1iFpssnBeS" hash: "rdJLgqYgkEp8jg8Qks1qiq" - generation: 1 + gen: 1 rule: sequence: window: 30s event: source: log + origin: true correlations: - corr1 order: @@ -370,7 +301,7 @@ rules: metadata: id: "J7uRQTGpGMyL1iFpssnBeS" hash: "rdJLgqYgkEp8jg8Qks1qiq" - generation: 1 + gen: 1 rule: script: code: | @@ -387,7 +318,7 @@ rules: metadata: id: "J7uRQTGpGMyL1iFpssnBeS" hash: "rdJLgqYgkEp8jg8Qks1qiq" - generation: 1 + gen: 1 rule: set: match: @@ -402,7 +333,7 @@ rules: metadata: id: "J7uRQTGpGMyL1iFpssnBeS" hash: "rdJLgqYgkEp8jg8Qks1qiq" - generation: 1 + gen: 1 rule: sequence: window: 30s @@ -433,7 +364,7 @@ rules: metadata: id: J7uRQTGpGMyL1iFpssnBeS hash: rdJLgqYgkEp8jg8Qks1qiq - generation: 1 + gen: 1 rule: set: match: @@ -467,7 +398,7 @@ rules: metadata: id: "J7uRQTGpGMyL1iFpssnBeS" hash: "rdJLgqYgkEp8jg8Qks1qiq" - generation: 1 + gen: 1 rule: set: window: 30s @@ -492,7 +423,7 @@ rules: metadata: id: "J7uRQTGpGMyL1iFpssnBeS" hash: "rdJLgqYgkEp8jg8Qks1qiq" - generation: 1 + gen: 1 rule: sequence: window: 10s @@ -510,7 +441,7 @@ rules: metadata: id: "J7uRQTGpGMyL1iFpssnBeS" hash: "rdJLgqYgkEp8jg8Qks1qiq" - generation: 1 + gen: 1 rule: sequence: window: 10s @@ -528,7 +459,7 @@ rules: metadata: id: "J7uRQTGpGMyL1iFpssnBeS" hash: "rdJLgqYgkEp8jg8Qks1qiq" - generation: 1 + gen: 1 rule: set: window: 10s @@ -546,7 +477,7 @@ rules: metadata: id: "J7uRQTGpGMyL1iFpssnBeS" hash: "rdJLgqYgkEp8jg8Qks1qiq" - generation: 1 + gen: 1 rule: set: window: 10d # invalid window @@ -564,7 +495,7 @@ rules: metadata: id: "J7uRQTGpGMyL1iFpssnBeS" hash: "rdJLgqYgkEp8jg8Qks1qiq" - generation: 1 + gen: 1 rule: superduperset: # unsupported rule type window: 10s @@ -581,60 +512,53 @@ rules: metadata: id: "J7uRQTGpGMyL1iFpssnBeS" hash: "rdJLgqYgkEp8jg8Qks1qiq" - generation: 1 + gen: 1 rule: sequence: window: 30s correlations: - hostname order: - - term1 - - term2 - - term3 -terms: - term1: - sequence: - window: 10s - event: - source: rabbitmq - origin: true - order: - - value: Discarding message - count: 10 - - Mnesia overloaded - negate: - - SIGTERM - term2: - set: - event: - source: k8s - negate: - - field: "reason" - value: "NodeShutdown" - term3: - sequence: - window: 5s - correlations: - - container_id - order: - - sequence: - window: 1s - event: - source: nginx - order: - - error message - - shutdown - - set: - event: - source: nginx - match: - - 90% - - set: - event: - source: k8s - match: - - field: "reason" - value: "Killing" + - sequence: + window: 10s + event: + source: rabbitmq + origin: true + order: + - value: Discarding message + count: 10 + - Mnesia overloaded + negate: + - SIGTERM + - set: + event: + source: k8s + negate: + - field: "reason" + value: "NodeShutdown" + - sequence: + window: 5s + correlations: + - container_id + order: + - sequence: + window: 1s + event: + source: nginx + order: + - error message + - shutdown + - set: + event: + source: nginx + match: + - 90% + - set: + event: + source: k8s + match: + - field: "reason" + value: "Killing" ` var TestFailNegativeCondition1 = ` # Line 1 starts here @@ -644,63 +568,57 @@ rules: metadata: id: "J7uRQTGpGMyL1iFpssnBeS" hash: "rdJLgqYgkEp8jg8Qks1qiq" - generation: 1 + gen: 1 rule: sequence: window: 30s correlations: - hostname order: - - term1 - - term2 - - term3 -terms: - term1: - sequence: - window: 10s - event: - src: log - origin: true - imageUrl: "*rabbitmq*" - order: - - value: Discarding message - count: 10 - - Mnesia overloaded - negate: - - SIGTERM - term2: - set: - event: - src: k8s - negate: - - field: "reason" - value: "NodeShutdown" - term3: - sequence: - window: 5s - correlations: - - container_id - order: - - sequence: - window: 1s - event: - src: log - containerName: nginx - order: - - error message - - shutdown - - set: - event: - src: log - containerName: nginx - match: - - 90% - - set: - event: - src: k8s - match: - - field: "reason" - value: "Killing" + - sequence: + window: 10s + event: + source: log + origin: true + imageUrl: "*rabbitmq*" + order: + - value: Discarding message + count: 10 + - Mnesia overloaded + negate: + - SIGTERM + - set: + event: + source: k8s + negate: + - field: "reason" + value: "NodeShutdown" + - sequence: + window: 5s + correlations: + - container_id + order: + - sequence: + window: 1s + event: + source: log + containerName: nginx + order: + - error message + - shutdown + - set: + event: + source: log + containerName: nginx + match: + - 90% + - set: + event: + source: k8s + match: + - field: "reason" + value: "Killing" + ` var TestFailNegativeCondition2 = ` # Line 1 starts here @@ -710,36 +628,31 @@ rules: metadata: id: "J7uRQTGpGMyL1iFpssnBeS" hash: "rdJLgqYgkEp8jg8Qks1qiq" - generation: 1 + gen: 1 rule: sequence: window: 30s correlations: - hostname order: - - term1 - - term2 -terms: - term1: - sequence: - window: 10s - event: - src: log - origin: true - imageUrl: "*rabbitmq*" - order: - - value: Discarding message - count: 10 - - Mnesia overloaded - negate: - - SIGTERM - term2: - set: - event: - src: k8s - negate: - - field: "reason" - value: "Killing" + - sequence: + window: 10s + event: + source: log + origin: true + imageUrl: "*rabbitmq*" + order: + - value: Discarding message + count: 10 + - Mnesia overloaded + negate: + - SIGTERM + - set: + event: + source: k8s + negate: + - field: "reason" + value: "Killing" ` var TestFailNegateOptions3 = ` # Line 1 starts here @@ -749,48 +662,40 @@ rules: metadata: id: "J7uRQTGpGMyL1iFpssnBeS" hash: "rdJLgqYgkEp8jg8Qks1qiq" - generation: 1 + gen: 1 rule: sequence: window: 30s correlations: - hostname order: - - term1 - - term2 - - term3 - -terms: - term1: - sequence: - window: 10s - event: - source: rabbitmq - origin: true - order: - - value: Discarding message - count: 10 - - Mnesia overloaded - negate: - - SIGTERM - term2: - set: - event: - source: k8s - match: - - field: "reason" - value: "Killing" - term3: - set: - event: - source: k8s - negate: - - field: "reason" - value: "Killing" - window: 10s - slide: 1s - anchor: 0 - abs: true + - sequence: + window: 10s + event: + source: rabbitmq + origin: true + order: + - value: Discarding message + count: 10 + - Mnesia overloaded + negate: + - SIGTERM + - set: + event: + source: k8s + match: + - field: "reason" + value: "Killing" + - set: + event: + source: k8s + negate: + - field: "reason" + value: "Killing" + window: 10s + slide: 1s + anchor: 0 + abs: true ` var TestFailNegateOptions4 = ` # Line 1 starts here @@ -800,49 +705,41 @@ rules: metadata: id: "J7uRQTGpGMyL1iFpssnBeS" hash: "rdJLgqYgkEp8jg8Qks1qiq" - generation: 1 + gen: 1 rule: sequence: window: 30s correlations: - hostname order: - - term1 - - term2 + - sequence: + window: 10s + event: + source: rabbitmq + origin: true + order: + - value: Discarding message + count: 10 + - Mnesia overloaded + negate: + - SIGTERM + - set: + event: + source: k8s + match: + - field: "reason" + value: "Killing" negate: - - term3 - -terms: - term1: - sequence: - window: 10s - event: - source: rabbitmq - origin: true - order: - - value: Discarding message - count: 10 - - Mnesia overloaded - negate: - - SIGTERM - term2: - set: - event: - source: k8s - match: - - field: "reason" - value: "Killing" - term3: - set: - event: - source: k8s - negate: - - field: "reason" - value: "Killing" - window: 10s - slide: 1s - anchor: 0 - abs: true + - set: + event: + source: k8s + negate: + - field: "reason" + value: "Killing" + window: 10s + slide: 1s + anchor: 0 + abs: true ` var TestFailTermsSyntaxError1 = ` # Line 1 starts here @@ -852,48 +749,40 @@ rules: metadata: id: "J7uRQTGpGMyL1iFpssnBeS" hash: "rdJLgqYgkEp8jg8Qks1qiq" - generation: 1 + gen: 1 rule: sequence: window: 30s correlations: - hostname order: - - term1 - - term2 - - term3 - -terms: - term1: - sequence: - window: 10s - event: - source: rabbitmq - origin: true - order: - - value: Discarding message - count: 10 - - Mnesia overloaded - negate: - - SIGTERM - term2: - set: - event: - source: k8s - moooch: - - field: "reason" - value: "Killing" - term3: - set: - event: - source: k8s - negate: - - field: "reason" - value: "Killing" - window: 10s - slide: 1s - anchor: 0 - abs: true + - sequence: + window: 10s + event: + source: rabbitmq + origin: true + order: + - value: Discarding message + count: 10 + - Mnesia overloaded + negate: + - SIGTERM + - set: + event: + source: k8s + moooch: + - field: "reason" + value: "Killing" + - set: + event: + source: k8s + negate: + - field: "reason" + value: "Killing" + window: 10s + slide: 1s + anchor: 0 + abs: true ` var TestFailTermsSyntaxError2 = ` # Line 1 starts here @@ -903,49 +792,41 @@ rules: metadata: id: "J7uRQTGpGMyL1iFpssnBeS" hash: "rdJLgqYgkEp8jg8Qks1qiq" - generation: 1 + gen: 1 rule: sequence: window: 30s correlations: - hostname order: - - term1 - - term2 - - term3 - -terms: - term1: - sequence: - window: 10s - event: - source: rabbitmq - origin: true - order: - - value: Discarding message - count: 10 - - Mnesia overloaded - negate: - - SIGTERM - term2: - set: - event: - source: k8s - window: 10d - match: - - field: "reason" - value: "Killing" - term3: - set: - event: - source: k8s - negate: - - field: "reason" - value: "Killing" - window: 10s - slide: 1s - anchor: 0 - abs: true + - sequence: + window: 10s + event: + source: rabbitmq + origin: true + order: + - value: Discarding message + count: 10 + - Mnesia overloaded + negate: + - SIGTERM + - set: + event: + source: k8s + window: 10d + match: + - field: "reason" + value: "Killing" + - set: + event: + source: k8s + negate: + - field: "reason" + value: "Killing" + window: 10s + slide: 1s + anchor: 0 + abs: true ` var TestFailTermsSemanticError1 = ` # Line 1 starts here @@ -955,49 +836,37 @@ rules: metadata: id: "J7uRQTGpGMyL1iFpssnBeS" hash: "rdJLgqYgkEp8jg8Qks1qiq" - generation: 1 + gen: 1 rule: sequence: window: 30s correlations: - hostname order: - - term1 - - term2 - - term3 - -terms: - term1: - sequence: - window: 10s - event: - source: rabbitmq - origin: true - order: - - value: Discarding message - count: 10 - - Mnesia overloaded - negate: - - SIGTERM - term2: - sequence: - event: - source: k8s - window: 1s - order: - - field: "reason" - value: "Killing" - term3: - set: - event: - source: k8s - match: - - field: "reason" - value: "Killing" - window: 10s - slide: 1s - anchor: 0 - abs: true + - sequence: + window: 10s + event: + source: rabbitmq + origin: true + order: + - value: Discarding message + count: 10 + - Mnesia overloaded + negate: + - SIGTERM + - sequence: + event: + source: k8s + window: 1s + order: + - field: "reason" + value: "Killing" + - set: + event: + source: k8s + match: + - field: "reason" + value: "Killing" ` var TestFailTermsSemanticError2 = ` # Line 1 starts here @@ -1007,7 +876,7 @@ rules: metadata: id: "J7uRQTGpGMyL1iFpssnBeS" hash: "rdJLgqYgkEp8jg8Qks1qiq" - generation: 1 + gen: 1 rule: sequence: window: 0s @@ -1015,6 +884,7 @@ rules: - hostname order: - term1 + - term2 ` var TestFailTermsSemanticError3 = ` # Line 1 starts here @@ -1024,7 +894,7 @@ rules: metadata: id: "J7uRQTGpGMyL1iFpssnBeS" hash: "rdJLgqYgkEp8jg8Qks1qiq" - generation: 1 + gen: 1 rule: set: correlations: @@ -1045,7 +915,7 @@ rules: metadata: id: "J7uRQTGpGMyL1iFpssnBeS" hash: "rdJLgqYgkEp8jg8Qks1qiq" - generation: 1 + gen: 1 rule: set: correlations: @@ -1066,7 +936,7 @@ rules: metadata: id: "J7uRQTGpGMyL1iFpssnBeS" hash: "rdJLgqYgkEp8jg8Qks1qiq" - generation: 1 + gen: 1 rule: set: event: @@ -1088,7 +958,7 @@ rules: metadata: id: "J7uRQTGpGMyL1iFpssnBeS" hash: "rdJLgqYgkEp8jg8Qks1qiq" - generation: 1 + gen: 1 rule: set: event: @@ -1105,7 +975,7 @@ rules: metadata: id: "J7uRQTGpGMyL1iFpssnBeS" hash: "rdJLgqYgkEp8jg8Qks1qiq" - generation: 1 + gen: 1 rule: set: window: 10s @@ -1122,12 +992,13 @@ rules: severity: 1 metadata: hash: "rdJLgqYgkEp8jg8Qks1qiq" - generation: 1 + gen: 1 rule: set: window: 10s event: source: kafka + origin: true match: - regex: "io.vertx.core.VertxException: Thread blocked" ` @@ -1139,7 +1010,7 @@ rules: severity: 1 metadata: id: "J7uRQTGpGMyL1iFpssnBeS" - generation: 1 + gen: 1 rule: set: window: 10s @@ -1157,7 +1028,7 @@ rules: metadata: id: "J7uRQTGpGMyL1iFpssnBeS" hash: "rdJLgqYgkEp8jg8Qks1qiq" - generation: 1 + gen: 1 rule: set: window: 10s @@ -1175,7 +1046,7 @@ rules: metadata: id: "zzzzzz zzzzzz zzzzzz zzzzzz" hash: "rdJLgqYgkEp8jg8Qks1qiq" - generation: 1 + gen: 1 rule: set: window: 10s @@ -1193,7 +1064,7 @@ rules: metadata: id: "J7uRQTGpGMyL1iFpssnBeS" hash: "asdfas asdf a" - generation: 1 + gen: 1 rule: set: window: 10s @@ -1210,7 +1081,7 @@ rules: metadata: id: "J7uRQTGpGMyL1iFpssnBeS" hash: "rdJLgqYgkEp8jg8Qks1qiq" - generation: 1 + gen: 1 rule: set: window: 50s @@ -1235,7 +1106,7 @@ rules: metadata: id: "J7uRQTGpGMyL1iFpssnB3S" hash: "rdJLgqYgkEp8jg8Qks1qqq" - generation: 1 + gen: 1 rule: set: window: 50s @@ -1253,3 +1124,61 @@ rules: match: - regex: "io.vertx.core.VertxException: Thread blocked" ` + +var TestFailMissingWindow = ` +rules: + - cre: + id: TestMissingWindow + metadata: + id: "J7uRQTGpGMyL1iFpssnB3S" + hash: "rdJLgqYgkEp8jg8Qks1qqq" + rule: + set: + event: + source: kafka + match: + - raw1 + - raw2 +` + +var TestFailMaxDepthExceeded = ` +rules: + - cre: + id: TestFailMaxDepthExceeded + metadata: + id: "J7uRQTGpGMyL1iFpssnB3S" + hash: "rdJLgqYgkEp8jg8Qks1qqq" + rule: + set: + match: + - set: + match: + - set: + window: 10s + event: + source: kafka + origin: true + match: + - raw1 + - raw2 +` + +var TestFailMaxRankExceeded = ` +rules: + - cre: + id: TestFailMaxRankExceeded + metadata: + id: "J7uRQTGpGMyL1iFpssnB3S" + hash: "rdJLgqYgkEp8jg8Qks1qqq" + rule: + set: + window: 10s + event: + source: kafka + origin: true + match: + - rank0 + - rank1 + - rank2 + - rank3 +` diff --git a/pkg/testdata/success_examples/00-rules-document-example.yaml b/pkg/testdata/success_examples/00-rules-document-example.yaml index 48c55ae..c779c22 100644 --- a/pkg/testdata/success_examples/00-rules-document-example.yaml +++ b/pkg/testdata/success_examples/00-rules-document-example.yaml @@ -9,6 +9,7 @@ rules: window: 5s event: source: cre.log.kafka + origin: true match: - commonExpression1 - "this is another match" @@ -22,14 +23,9 @@ rules: window: 10s event: source: cre.log.kafka + origin: true order: - regex: "foo(.+)bar" - - commonExpression1 - - commonExpression2 + - regex: "bo(.+)r" + - value: "some other match" -# Optional terms section for reusing common conditions -terms: - commonExpression1: - regex: "bo(.+)r" - commonExpression2: - value: "some other match" diff --git a/pkg/testdata/success_examples/01-set-single-example.yaml b/pkg/testdata/success_examples/01-set-single-example.yaml index 8d0dda8..9afe1d8 100644 --- a/pkg/testdata/success_examples/01-set-single-example.yaml +++ b/pkg/testdata/success_examples/01-set-single-example.yaml @@ -8,5 +8,6 @@ rules: set: event: source: kafka + origin: true match: - "foo(.+)bar" diff --git a/pkg/testdata/success_examples/02-set-multiple-example-bad-window.yaml b/pkg/testdata/success_examples/02-set-multiple-example-bad-window.yaml index 1bef379..802dcd0 100644 --- a/pkg/testdata/success_examples/02-set-multiple-example-bad-window.yaml +++ b/pkg/testdata/success_examples/02-set-multiple-example-bad-window.yaml @@ -9,6 +9,7 @@ rules: window: 1s event: source: cre.log.kafka + origin: true match: - value: "test" - regex: "foo(.+)bar" diff --git a/pkg/testdata/success_examples/02-set-multiple-example-good-window.yaml b/pkg/testdata/success_examples/02-set-multiple-example-good-window.yaml index d9b4bcb..4ff50d2 100644 --- a/pkg/testdata/success_examples/02-set-multiple-example-good-window.yaml +++ b/pkg/testdata/success_examples/02-set-multiple-example-good-window.yaml @@ -9,6 +9,7 @@ rules: window: 10s event: source: cre.log.kafka + origin: true match: - value: "test" - regex: "foo(.+)bar" diff --git a/pkg/testdata/success_examples/03-set-negative-example.yaml b/pkg/testdata/success_examples/03-set-negative-example.yaml index c8361ee..70f0080 100644 --- a/pkg/testdata/success_examples/03-set-negative-example.yaml +++ b/pkg/testdata/success_examples/03-set-negative-example.yaml @@ -9,6 +9,7 @@ rules: window: 10s event: source: cre.log.kafka + origin: true match: - value: "test" - regex: "foo(.+)bar" diff --git a/pkg/testdata/success_examples/04-set-1x1-example.yaml b/pkg/testdata/success_examples/04-set-1x1-example.yaml index 738c036..3121720 100644 --- a/pkg/testdata/success_examples/04-set-1x1-example.yaml +++ b/pkg/testdata/success_examples/04-set-1x1-example.yaml @@ -8,6 +8,7 @@ rules: set: event: source: cre.log.kafka + origin: true match: - value: "test" negate: diff --git a/pkg/testdata/success_examples/08-sequence-example-bad-window.yaml b/pkg/testdata/success_examples/08-sequence-example-bad-window.yaml index 747a89a..7dd55a1 100644 --- a/pkg/testdata/success_examples/08-sequence-example-bad-window.yaml +++ b/pkg/testdata/success_examples/08-sequence-example-bad-window.yaml @@ -8,6 +8,7 @@ rules: sequence: event: source: cre.log.kafka + origin: true window: 1s order: - regex: "foo(.+)bar" diff --git a/pkg/testdata/success_examples/08-sequence-example-good-window.yaml b/pkg/testdata/success_examples/08-sequence-example-good-window.yaml index fa7c160..be17621 100644 --- a/pkg/testdata/success_examples/08-sequence-example-good-window.yaml +++ b/pkg/testdata/success_examples/08-sequence-example-good-window.yaml @@ -8,6 +8,7 @@ rules: sequence: event: source: cre.log.kafka + origin: true window: 10s order: - regex: "foo(.+)bar" diff --git a/pkg/testdata/success_examples/09-sequence-negate-example.yaml b/pkg/testdata/success_examples/09-sequence-negate-example.yaml index 7e43ef4..59b6e05 100644 --- a/pkg/testdata/success_examples/09-sequence-negate-example.yaml +++ b/pkg/testdata/success_examples/09-sequence-negate-example.yaml @@ -8,6 +8,7 @@ rules: sequence: event: source: cre.log.kafka + origin: true window: 10s order: - regex: "foo(.+)bar" diff --git a/pkg/testdata/success_examples/13-string-example.yaml b/pkg/testdata/success_examples/13-string-example.yaml index d6444e9..06989ba 100644 --- a/pkg/testdata/success_examples/13-string-example.yaml +++ b/pkg/testdata/success_examples/13-string-example.yaml @@ -8,5 +8,6 @@ rules: set: event: source: cre.log.kafka + origin: true match: - value: "[emerg] 1655#1655: still could not bind()" diff --git a/pkg/testdata/success_examples/14-string-example.yaml b/pkg/testdata/success_examples/14-string-example.yaml index a46ab1d..f5fae44 100644 --- a/pkg/testdata/success_examples/14-string-example.yaml +++ b/pkg/testdata/success_examples/14-string-example.yaml @@ -8,5 +8,6 @@ rules: set: event: source: cre.log.kafka + origin: true match: - "[emerg] 1655#1655: still could not bind()" diff --git a/pkg/testdata/success_examples/15-regex-example.yaml b/pkg/testdata/success_examples/15-regex-example.yaml index ab734d9..912f6f9 100644 --- a/pkg/testdata/success_examples/15-regex-example.yaml +++ b/pkg/testdata/success_examples/15-regex-example.yaml @@ -8,5 +8,6 @@ rules: set: event: source: cre.log.kafka + origin: true match: - regex: "emerg(.+)still could not bind()" diff --git a/pkg/testdata/success_examples/16-regex-example.yaml b/pkg/testdata/success_examples/16-regex-example.yaml index 9af7cfc..a6fe37b 100644 --- a/pkg/testdata/success_examples/16-regex-example.yaml +++ b/pkg/testdata/success_examples/16-regex-example.yaml @@ -8,5 +8,6 @@ rules: set: event: source: cre.log.kafka + origin: true match: - regex: "\\[emerg] (.+) still could not bind()" diff --git a/pkg/testdata/success_examples/17-jq-example.yaml b/pkg/testdata/success_examples/17-jq-example.yaml index 8551184..61936b1 100644 --- a/pkg/testdata/success_examples/17-jq-example.yaml +++ b/pkg/testdata/success_examples/17-jq-example.yaml @@ -8,5 +8,6 @@ rules: set: event: source: cre.log.kafka + origin: true match: - jq: ".event == \"worker_process_crash\"" diff --git a/pkg/testdata/success_examples/18-jq-example.yaml b/pkg/testdata/success_examples/18-jq-example.yaml index 18c1dea..66753b7 100644 --- a/pkg/testdata/success_examples/18-jq-example.yaml +++ b/pkg/testdata/success_examples/18-jq-example.yaml @@ -8,5 +8,6 @@ rules: set: event: source: cre.log.kafka + origin: true match: - jq: "select(.event == \"worker_process_crash\" and .level == \"error\")" diff --git a/pkg/testdata/success_examples/19-bad-literal-block-example.yaml b/pkg/testdata/success_examples/19-bad-literal-block-example.yaml index 44c9e4a..2e6bc33 100644 --- a/pkg/testdata/success_examples/19-bad-literal-block-example.yaml +++ b/pkg/testdata/success_examples/19-bad-literal-block-example.yaml @@ -8,6 +8,7 @@ rules: set: event: source: cre.log.kafka + origin: true match: - regex: | "\\[emerg] (.+) still could not bind()" diff --git a/pkg/testdata/success_examples/20-bad-regex-example.yaml b/pkg/testdata/success_examples/20-bad-regex-example.yaml index ed95765..6fcfb9c 100644 --- a/pkg/testdata/success_examples/20-bad-regex-example.yaml +++ b/pkg/testdata/success_examples/20-bad-regex-example.yaml @@ -8,5 +8,6 @@ rules: set: event: source: cre.log.kafka + origin: true match: - "\\[emerg] (.+) still could not bind()" # use regex: instead diff --git a/pkg/testdata/success_examples/21-negative-example.yaml b/pkg/testdata/success_examples/21-negative-example.yaml index 6a8a8aa..16badf2 100644 --- a/pkg/testdata/success_examples/21-negative-example.yaml +++ b/pkg/testdata/success_examples/21-negative-example.yaml @@ -8,6 +8,7 @@ rules: sequence: event: source: cre.log.kafka + origin: true window: 10s order: - regex: "foo(.+)bar" diff --git a/pkg/testdata/success_examples/24-multiple-negatives.yaml b/pkg/testdata/success_examples/24-multiple-negatives.yaml index 80e5892..d84c7f1 100644 --- a/pkg/testdata/success_examples/24-multiple-negatives.yaml +++ b/pkg/testdata/success_examples/24-multiple-negatives.yaml @@ -8,6 +8,7 @@ rules: sequence: event: source: cre.log.kafka + origin: true window: 10s order: - regex: "foo(.+)bar" diff --git a/pkg/testdata/success_examples/25-negate-options-1x1.yaml b/pkg/testdata/success_examples/25-negate-options-1x1.yaml index 2d269ac..e4cf019 100644 --- a/pkg/testdata/success_examples/25-negate-options-1x1.yaml +++ b/pkg/testdata/success_examples/25-negate-options-1x1.yaml @@ -8,6 +8,7 @@ rules: set: event: source: cre.log.kafka + origin: true match: - regex: "foo(.+)bar" negate: diff --git a/pkg/testdata/success_examples/26-negate-window.yaml b/pkg/testdata/success_examples/26-negate-window.yaml index 2f319e5..3e90748 100644 --- a/pkg/testdata/success_examples/26-negate-window.yaml +++ b/pkg/testdata/success_examples/26-negate-window.yaml @@ -8,6 +8,7 @@ rules: set: event: source: cre.log.kafka + origin: true match: - regex: "foo(.+)bar" negate: diff --git a/pkg/testdata/success_examples/27-negate-window-shorter.yaml b/pkg/testdata/success_examples/27-negate-window-shorter.yaml index 12ed701..6ac8dfd 100644 --- a/pkg/testdata/success_examples/27-negate-window-shorter.yaml +++ b/pkg/testdata/success_examples/27-negate-window-shorter.yaml @@ -8,6 +8,7 @@ rules: set: event: source: cre.log.kafka + origin: true window: 5s match: - regex: "foo(.+)bar" diff --git a/pkg/testdata/success_examples/27-negate-window.yaml b/pkg/testdata/success_examples/27-negate-window.yaml index 33b06b4..69b8727 100644 --- a/pkg/testdata/success_examples/27-negate-window.yaml +++ b/pkg/testdata/success_examples/27-negate-window.yaml @@ -8,6 +8,7 @@ rules: set: event: source: cre.log.kafka + origin: true window: 5s match: - regex: "foo(.+)bar" diff --git a/pkg/testdata/success_examples/28-negate-anchor-shorter.yaml b/pkg/testdata/success_examples/28-negate-anchor-shorter.yaml index 01b331b..8f22b65 100644 --- a/pkg/testdata/success_examples/28-negate-anchor-shorter.yaml +++ b/pkg/testdata/success_examples/28-negate-anchor-shorter.yaml @@ -8,6 +8,7 @@ rules: set: event: source: cre.log.kafka + origin: true window: 5s match: - regex: "foo(.+)bar" diff --git a/pkg/testdata/success_examples/28-negate-anchor.yaml b/pkg/testdata/success_examples/28-negate-anchor.yaml index 1b3f33d..0015110 100644 --- a/pkg/testdata/success_examples/28-negate-anchor.yaml +++ b/pkg/testdata/success_examples/28-negate-anchor.yaml @@ -8,6 +8,7 @@ rules: set: event: source: cre.log.kafka + origin: true window: 5s match: - regex: "foo(.+)bar" diff --git a/pkg/testdata/success_examples/29-negate-slide-anchor-1-window.yaml b/pkg/testdata/success_examples/29-negate-slide-anchor-1-window.yaml index cff5007..a3041a0 100644 --- a/pkg/testdata/success_examples/29-negate-slide-anchor-1-window.yaml +++ b/pkg/testdata/success_examples/29-negate-slide-anchor-1-window.yaml @@ -8,6 +8,7 @@ rules: set: event: source: cre.log.kafka + origin: true window: 5s match: - regex: "foo(.+)bar" diff --git a/pkg/testdata/success_examples/29-negate-slide-anchor-1.yaml b/pkg/testdata/success_examples/29-negate-slide-anchor-1.yaml index 59d54d7..a91ef70 100644 --- a/pkg/testdata/success_examples/29-negate-slide-anchor-1.yaml +++ b/pkg/testdata/success_examples/29-negate-slide-anchor-1.yaml @@ -8,6 +8,7 @@ rules: set: event: source: cre.log.kafka + origin: true window: 5s match: - regex: "foo(.+)bar" diff --git a/pkg/testdata/success_examples/29-negate-slide.yaml b/pkg/testdata/success_examples/29-negate-slide.yaml index 864d210..600f6c0 100644 --- a/pkg/testdata/success_examples/29-negate-slide.yaml +++ b/pkg/testdata/success_examples/29-negate-slide.yaml @@ -8,6 +8,7 @@ rules: set: event: source: cre.log.kafka + origin: true window: 5s match: - regex: "foo(.+)bar" diff --git a/pkg/testdata/success_examples/30-negate-absolute.yaml b/pkg/testdata/success_examples/30-negate-absolute.yaml index 60319e7..070bc20 100644 --- a/pkg/testdata/success_examples/30-negate-absolute.yaml +++ b/pkg/testdata/success_examples/30-negate-absolute.yaml @@ -8,6 +8,7 @@ rules: set: event: source: cre.log.kafka + origin: true window: 5s match: - regex: "foo(.+)bar" diff --git a/pkg/testdata/success_examples/41-nested.yaml b/pkg/testdata/success_examples/41-nested.yaml index 3ff624e..cfa6b16 100644 --- a/pkg/testdata/success_examples/41-nested.yaml +++ b/pkg/testdata/success_examples/41-nested.yaml @@ -10,52 +10,44 @@ rules: correlations: - hostname order: - - term1 - - term3 + - sequence: + window: 10s + event: + source: cre.log.rabbitmq + origin: true + order: + - value: Discarding message + count: 10 + - Mnesia overloaded + negate: + - SIGTERM + - sequence: + window: 5s + correlations: + - container_id + order: + - sequence: + window: 1s + event: + source: cre.log.nginx + order: + - error message + - shutdown + - set: + event: + source: cre.log.nginx + match: + - 90% + - set: + event: + source: cre.prequel.k8s + match: + - field: "reason" + value: "Killing" negate: - - term2 - -terms: - term1: - sequence: - window: 10s - event: - source: cre.log.rabbitmq - origin: true - order: - - value: Discarding message - count: 10 - - Mnesia overloaded - negate: - - SIGTERM - term2: - set: - event: - source: cre.prequel.k8s - match: - - field: "reason" - value: "NodeShutdown" - term3: - sequence: - window: 5s - correlations: - - container_id - order: - - sequence: - window: 1s - event: - source: cre.log.nginx - order: - - error message - - shutdown - - set: - event: - source: cre.log.nginx - match: - - 90% - - set: - event: - source: cre.prequel.k8s - match: - - field: "reason" - value: "Killing" + - set: + event: + source: cre.prequel.k8s + match: + - field: "reason" + value: "NodeShutdown" diff --git a/pkg/version/version.go b/pkg/version/version.go new file mode 100644 index 0000000..52425e8 --- /dev/null +++ b/pkg/version/version.go @@ -0,0 +1,63 @@ +package version + +import ( + "fmt" + + "github.com/Masterminds/semver/v3" +) + +// 'vers' is the semantic version of the prequel-compiler package. +// It tracks the interface and the functionality of the package, +// and should be updated whenever there are changes that +// modify the logic of the rules engine. + +// Note that this is not the same as the git commit version of the package. + +var ( + vers = "v0.1.0" + sver = semver.MustParse(vers) +) + +var ErrBadConstraint = fmt.Errorf("bad version constraint") + +type ErrVersionNotAllowed struct { + Version string + Expression string +} + +func (e ErrVersionNotAllowed) Error() string { + return fmt.Sprintf("compiler version %s does not satisfy version constraint: '%s'", e.Version, e.Expression) +} + +func (e ErrVersionNotAllowed) Is(target error) bool { + _, ok := target.(ErrVersionNotAllowed) + if !ok { + tt, ok := target.(*ErrVersionNotAllowed) + if !ok || tt == nil { + return false + } + } + return true +} + +func SemVer() string { + return vers +} + +// Evaluate version expression against the compiler version. Version expressions are of the form: +// - "0.1.0" (exact match) +// - "<0.2.0" (less than) +// - ">=0.1.0" (greater than or equal to) + +func AllowVersion(expression string) error { + constraint, err := semver.NewConstraint(expression) + if err != nil { + return fmt.Errorf("%w: %w", ErrBadConstraint, err) + } + + if !constraint.Check(sver) { + return ErrVersionNotAllowed{Expression: expression, Version: vers} + } + + return nil +} diff --git a/pkg/version/version_test.go b/pkg/version/version_test.go new file mode 100644 index 0000000..c1f4e6b --- /dev/null +++ b/pkg/version/version_test.go @@ -0,0 +1,92 @@ +package version + +import ( + "errors" + "testing" + + "github.com/Masterminds/semver/v3" +) + +func withVers(t *testing.T, tempVers string, fn func()) { + t.Helper() + + origVers := vers + origSver := sver + vers = tempVers + sver = semver.MustParse(tempVers) + defer func() { + vers = origVers + sver = origSver + }() + fn() +} + +func TestSemVer(t *testing.T) { + withVers(t, "v9.9.9", func() { + got := SemVer() + want := "v9.9.9" + if got != want { + t.Errorf("SemVer() = %q, want %q", got, want) + } + }) +} + +func TestAllowVersion_ExactMatch(t *testing.T) { + withVers(t, "v1.2.3", func() { + err := AllowVersion("=1.2.3") + if err != nil { + t.Errorf("AllowVersion(\"=1.2.3\") returned error: %v", err) + } + }) +} + +func TestAllowVersion_GreaterThan(t *testing.T) { + withVers(t, "v2.0.0", func() { + err := AllowVersion(">1.0.0") + if err != nil { + t.Errorf("AllowVersion(\">1.0.0\") returned error: %v", err) + } + }) +} + +func TestAllowVersion_LessThan(t *testing.T) { + withVers(t, "v1.0.0", func() { + err := AllowVersion("<2.0.0") + if err != nil { + t.Errorf("AllowVersion(\"<2.0.0\") returned error: %v", err) + } + }) +} + +func TestAllowVersion_NotAllowed(t *testing.T) { + withVers(t, "v1.0.0", func() { + err := AllowVersion(">2.0.0") + var vErr ErrVersionNotAllowed + if !errors.As(err, &vErr) { + t.Errorf("AllowVersion(>2.0.0) error = %v, want ErrVersionNotAllowed", err) + } + if vErr.Version != "v1.0.0" || vErr.Expression != ">2.0.0" { + t.Errorf("ErrVersionNotAllowed fields = %+v, want Version=v1.0.0, Expression=>2.0.0", vErr) + } + }) +} + +func TestAllowVersion_BadExpression(t *testing.T) { + withVers(t, "v1.0.0", func() { + err := AllowVersion("not-a-valid-expression") + if err == nil { + t.Errorf("AllowVersion with bad expression should return error") + } + }) +} + +func TestErrVersionNotAllowed_Error(t *testing.T) { + var ( + e = ErrVersionNotAllowed{Version: "v1.2.3", Expression: "<1.0.0"} + expect = "compiler version v1.2.3 does not satisfy version constraint: '<1.0.0'" + ) + if e.Error() != expect { + t.Errorf("ErrVersionNotAllowed.Error() = %q, want %q", e.Error(), expect) + } + +}