feat: Audit Trail Implementation

README.md

@@ -3,18 +3,19 @@
 ## Introduction
 
 This library provides definitions of structures used in the output of the THOR APT Forensic Scanner. These structures can be used for different use cases:
-- generate a schema for THOR JSON logs
+
+- generate schemas for THOR JSON logs and the THOR audit trail
 - convert JSON logs into text logs
 - parse JSON logs
 
 ## Versions
 
 There are three versions of the THOR log format:
 
- - v1: The original THOR log format, used up to and including THOR version 10.7. This is equivalent to the THOR text format, simply serialized as JSON.
- - v2: The format used in THOR version 10.7 with the `--jsonv2` flag. This format introduced a more structured approach to logging,
-   with subobjects for reasons, files, and other entities. It is largely open-ended and allows for custom fields.
- - v3: The format used in THOR 11 and later. This format is more strict and versioned, with a defined schema. It introduces the concept of _reportable objects_.
+- v1: The original THOR log format, used up to and including THOR version 10.7. This is equivalent to the THOR text format, simply serialized as JSON.
+- v2: The format used in THOR version 10.7 with the `--jsonv2` flag. This format introduced a more structured approach to logging,
+  with subobjects for reasons, files, and other entities. It is largely open-ended and allows for custom fields.
+- v3: The format used in THOR 11 and later. This format is more strict and versioned, with a defined schema. It introduces the concept of _reportable objects_.
 
 ## Parsing Events
 
@@ -39,25 +40,52 @@ This type determines how the object should be interpreted and what fields it con
 ### Event Types
 
 The object types contained in a THOR log are `THOR finding` and `THOR message`:
- - Findings are the results of THOR's analysis, such as detected threats or anomalies.
- - Messages are informational or status updates from THOR, such as progress updates.
+
+- Findings are the results of THOR's analysis, such as detected threats or anomalies.
+- Messages are informational or status updates from THOR, such as progress updates.
 
 Both findings and messages are together called _events_.
 
+### Audit Entry Types
+
+The THOR audit trail is a separate log that documents which objects a scan examined,
+regardless of whether THOR reported anything about them.
+The object types contained in this log are `THOR audit record` and `THOR audit message`:
+
+- Audit records document a single object that THOR observed, together with the timestamps known for
+  it, any indicators that matched on it, and its relations to other audit records.
+- Audit messages are the messages that THOR printed during the scan, in a less verbose form than the
+  `THOR message` events.
+
+Both audit records and audit messages are together called _audit entries_.
+
 ### Reportable Objects
 
-Findings may contain more objects, e.g. as a subject that they report. 
+Findings may contain more objects, e.g. as a subject that they report.
 Object types that can appear as subjects are called _reportable objects_.
 The most common reportable objects are:
+
 - `file`
 - `process`
 
 Reportable objects should contain only fields that relate directly to the object itself.
-E.g. when extracting a file from an archive, the file object should contain only fields 
+E.g. when extracting a file from an archive, the file object should contain only fields
 that relate to the file itself, not to the archive.
 The archive data will instead appear in the _context_ of the finding.
 
 ## Schema
 
-A schema for the version 3 format is attached to each release.
-It can also be generated using the `thorlog/jsonschema` package.
+There are two schemas for the version 3 format:
+
+- `thor-event.json` describes the events in a THOR log.
+- `thor-audit-entry.json` describes the entries in the THOR audit trail.
+
+The schemas are attached to each release.
+They can also be generated using the `thorlog/jsonschema` package.
+The generator takes the schema to generate as its only argument and must be run from within its directory:
+
+```sh
+cd thorlog/jsonschema
+go run . log > thor-event.json
+go run . audittrail > thor-audit-entry.json
+```

thorlog/jsonschema/generateschema.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/NextronSystems/jsonlog"
 	"github.com/NextronSystems/jsonlog/thorlog/v3"
+	_ "github.com/NextronSystems/jsonlog/thorlog/v3/audittrail"
 	"github.com/invopop/jsonschema"
 	orderedmap "github.com/wk8/go-ordered-map/v2"
 )
@@ -22,13 +23,14 @@ func makeObjectSchema() (mainEntry string, defs map[string]*jsonschema.Schema) {
 	var logObjectTypes []any
 	var reflector jsonschema.Reflector
 	reflector.AllowAdditionalProperties = true
+	// Walks subdirectories too, so this also covers audittrail.
 	err := reflector.AddGoComments("github.com/NextronSystems/jsonlog/thorlog/v3", "../v3")
 	if err != nil {
 		panic(err)
 	}
 	defs = map[string]*jsonschema.Schema{}
 
-	// Sort the object type names to have a stable output
+	// Sort the object type names to have a stable output.
 	var objectTypeNames = slices.Collect(maps.Keys(thorlog.LogObjectTypes))
 	slices.Sort(objectTypeNames)
 
@@ -90,26 +92,49 @@ func makeObjectSchema() (mainEntry string, defs map[string]*jsonschema.Schema) {
 }
 
 func main() {
-	logEventSchema := jsonschema.Schema{
-		Version:     jsonschema.Version,
-		ID:          "https://www.nextron-systems.com/schemas/thorlog/v3/thor-event.json",
-		Definitions: map[string]*jsonschema.Schema{},
-		Title:       "ThorEvent",
-		OneOf: []*jsonschema.Schema{
-			{
-				Ref: "#/$defs/Assessment",
+	var logEventSchema jsonschema.Schema
+	if len(os.Args) == 2 && os.Args[1] == "log" {
+		logEventSchema = jsonschema.Schema{
+			Version:     jsonschema.Version,
+			ID:          "https://www.nextron-systems.com/schemas/thorlog/v3/thor-event.json",
+			Definitions: map[string]*jsonschema.Schema{},
+			Title:       "ThorEvent",
+			OneOf: []*jsonschema.Schema{
+				{
+					Ref: "#/$defs/Assessment",
+				},
+				{
+					Ref: "#/$defs/Message",
+				},
 			},
-			{
-				Ref: "#/$defs/Message",
+		}
+	} else if len(os.Args) == 2 && os.Args[1] == "audittrail" {
+		logEventSchema = jsonschema.Schema{
+			Version:     jsonschema.Version,
+			ID:          "https://www.nextron-systems.com/schemas/thorlog/v3/thor-audit-entry.json",
+			Definitions: map[string]*jsonschema.Schema{},
+			Title:       "ThorAuditEntry",
+			OneOf: []*jsonschema.Schema{
+				{
+					Ref: "#/$defs/AuditMessage",
+				},
+				{
+					Ref: "#/$defs/AuditRecord",
+				},
 			},
-		},
+		}
+	} else {
+		fmt.Fprintf(os.Stderr, "Usage: %s (log|audittrail)\n", os.Args[0])
+		os.Exit(2)
 	}
+
 	entry, defs := makeObjectSchema()
 	for key, value := range defs {
 		logEventSchema.Definitions[key] = value
 	}
 
 	flatten(logEventSchema.Definitions[entry], logEventSchema.Definitions)
+	prune(&logEventSchema)
 
 	encoder := json.NewEncoder(os.Stdout)
 	encoder.SetIndent("", "  ")
@@ -144,3 +169,46 @@ func flatten(schema *jsonschema.Schema, definitions jsonschema.Definitions) {
 		flatten(subschema, definitions)
 	}
 }
+
+// prune removes all definitions that are not reachable from the root schema via $ref.
+func prune(root *jsonschema.Schema) {
+	reachable := map[string]bool{}
+	var visit func(schema *jsonschema.Schema)
+	visit = func(schema *jsonschema.Schema) {
+		// Most subschema fields are nil.
+		if schema == nil {
+			return
+		}
+		// Check !reachable[name] so we won't run into an endless loop
+		// and we only visit each definition once.
+		if name, ok := strings.CutPrefix(schema.Ref, "#/$defs/"); ok && !reachable[name] {
+			def, ok := root.Definitions[name]
+			if !ok {
+				panic("dangling reference " + schema.Ref)
+			}
+			// Mark the definition before the recursive call.
+			reachable[name] = true
+			visit(def)
+		}
+		// Definitions are not visited, since they are only reachable via $ref.
+		// Visit all applicators https://www.learnjsonschema.com/2020-12/applicator/
+		// and contentSchema, which could also contain references.
+		children := slices.Concat(
+			[]*jsonschema.Schema{
+				schema.Not, schema.If, schema.Then, schema.Else, schema.Items, schema.Contains,
+				schema.AdditionalProperties, schema.PropertyNames, schema.ContentSchema,
+			},
+			schema.AllOf, schema.AnyOf, schema.OneOf, schema.PrefixItems,
+			slices.Collect(maps.Values(schema.PatternProperties)),
+			slices.Collect(maps.Values(schema.DependentSchemas)),
+		)
+		for pair := schema.Properties.Oldest(); pair != nil; pair = pair.Next() {
+			children = append(children, pair.Value)
+		}
+		for _, child := range children {
+			visit(child)
+		}
+	}
+	visit(root)
+	maps.DeleteFunc(root.Definitions, func(name string, _ *jsonschema.Schema) bool { return !reachable[name] })
+}