feat: rework context

reference.go

@@ -258,7 +258,7 @@ func ConcatTextLabels(prefix string, label string) string {
 	if label == "" {
 		return prefix
 	}
-	if prefix == label {
+	if prefix == label || strings.HasPrefix(label, prefix+"_") { // Don't duplicate the prefix if it already exists
 		return label
 	}
 	return prefix + "_" + label

thorlog/parser/parser_test.go

@@ -130,7 +130,9 @@ func TestParseEvent(t *testing.T) {
 		},
 		{
 			"JsonV3Assessment",
-			`{"type":"THOR assessment","meta":{"time":"2024-09-24T14:18:46.190394329+02:00","level":"Alert","module":"Test","scan_id":"abdc","event_id":"abdas","hostname":"aserarsd"},"message":"This is a test assessment","subject":{"type":"file","path":"path/to/file"},"score":70,"reasons":[{"type":"reason","summary":"Reason 1","signature":{"score":70,"ref":null,"origin":"internal","kind":""},"matched":null}],"reason_count":0,"context":[{"object":{"type":"at job"},"relation":"","unique":false}],"log_version":"v3"}`,
+			`{"type":"THOR assessment","meta":{"time":"2024-09-24T14:18:46.190394329+02:00","level":"Alert","module":"Test","scan_id":"abdc","event_id":"abdas","hostname":"aserarsd"},
+"message":"This is a test assessment","subject":{"type":"file","path":"path/to/file"},"score":70,"reasons":[{"type":"reason","summary":"Reason 1","signature":{"score":70,"ref":null,"origin":"internal","kind":""},"matched":null}],"reason_count":0,
+"ancestors":[{"object":{"type":"at job"},"distance":1,"top_level":false}],"derivatives":[{"object":{"type":"at job"}}],"log_version":"v3"}`,
 			&thorlog.Assessment{
 				ObjectHeader: jsonlog.ObjectHeader{
 					Type: "THOR assessment",
@@ -164,7 +166,17 @@ func TestParseEvent(t *testing.T) {
 					},
 				},
 				ReasonCount: 0,
-				EventContext: thorlog.Context{
+				Ancestors: thorlog.Ancestors{
+					{
+						Object: &thorlog.AtJob{
+							ObjectHeader: jsonlog.ObjectHeader{
+								Type: "at job",
+							},
+						},
+						Distance: 1,
+					},
+				},
+				Derivatives: []thorlog.Derivative{
 					{
 						Object: &thorlog.AtJob{
 							ObjectHeader: jsonlog.ObjectHeader{

thorlog/v3/atjob.go

@@ -8,6 +8,7 @@ type AtJob struct {
 	jsonlog.ObjectHeader
 
 	Command string `json:"command" textlog:"command"`
+	Image   *File  `json:"image" textlog:"image,expand"`
 }
 
 const typeAtJob = "at job"

thorlog/v3/event.go

@@ -5,7 +5,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"reflect"
-	"strconv"
 	"strings"
 
 	"github.com/NextronSystems/jsonlog"
@@ -37,14 +36,13 @@ type Assessment struct {
 	Reasons []Reason `json:"reasons" textlog:",expand"`
 	// ReasonCount contains the total number of reasons (before any truncations).
 	ReasonCount int `json:"reason_count,omitempty" textlog:"reasons_count,omitempty"`
-	// EventContext contains other objects that may be relevant for an analyst and their relation to the
-	// Subject.
+	// Ancestors contains information about objects that are the subject's ancestors.
 	//
-	// To give an example: if the Subject is a file in a ZIP archive,
-	// the ZIP archive would be listed in the EventContext with a relation type of "derives from"
-	// and a relation name of "parent", indicating that the Subject derives from this object,
-	// which is its parent.
-	EventContext Context `json:"context" textlog:",expand" jsonschema:"nullable"`
+	// E.g. if the subject is a file in a nested ZIP, all the ZIPs (the topmost one and each nested one) are ancestors.
+	// Ancestors does not necessarily include information about all ancestors, but it will always include information about at least the topmost ancestor and the parent.
+	Ancestors Ancestors `json:"ancestors" textlog:",expand"`
+	// Derivatives contains information about objects that have been referenced in the subject.
+	Derivatives []Derivative `json:"derivatives" textlog:"file,expand"`
 	// Issues lists any problems that THOR encountered when trying to create a JSON struct for this assessment.
 	// This may include e.g. overly long fields that were truncated, fields that could not be rendered to JSON,
 	// or similar problems.
@@ -102,6 +100,16 @@ func (a *Assessment) UnmarshalJSON(data []byte) error {
 			a.Reasons[i].StringMatches[j].Field = jsonlog.NewReference(a.Subject, target)
 		}
 	}
+	for i := range a.Derivatives {
+		if a.Derivatives[i].Via == nil {
+			continue
+		}
+		target, err := jsonpointer.Resolve(a, a.Derivatives[i].Via.ToJsonPointer())
+		if err != nil {
+			return err
+		}
+		a.Derivatives[i].Via = jsonlog.NewReference(a, target)
+	}
 	for i := range a.Issues {
 		if a.Issues[i].Affected == nil {
 			continue
@@ -117,69 +125,95 @@ func (a *Assessment) UnmarshalJSON(data []byte) error {
 
 var _ common.Event = (*Assessment)(nil)
 
-type Context []ContextObject
-
-// ContextObject describes a relation of an object to another.
-type ContextObject struct {
-	Object ObservedObject `json:"object" textlog:",expand"`
-	// Relations describes how the object relates to the assessed subject.
-	// There may be multiple relations, e.g. if the object is both the parent and the topmost ancestor of the subject.
-	//
-	// Relations should be ordered by relevance, i.e. the most important relation should be first.
-	// Only the first (and most relevant) relation is used for text log formatting.
-	Relations []Relation `json:"relations" textlog:",expand" jsonschema:"minItems=1"`
+type Ancestors []Ancestor
+
+type Ancestor struct {
+	// Per describes the action that caused the ancestor to create its child.
+	Per string `json:"per"`
+	// TopLevel is true if the ancestor does not have a parent.
+	TopLevel bool `json:"top_level,omitempty"`
+	// Distance is the number of links between the subject and the ancestor (parent = 1, grandparent = 2, ...)
+	Distance int `json:"distance"`
+	// Object describes the ancestor.
+	Object ObservedObject `json:"object"`
 }
 
-type Relation struct {
-	Type   string `json:"relation_type"` // RelationType is used to specify the type of relation, e.g. "derives from" or "related to"
-	Name   string `json:"relation_name"` // RelationName is used to specify the name of the relation, e.g. "parent". It is optional.
-	Unique bool   `json:"unique"`        // Unique indicates whether the relation is unique, i.e. there can only be one object with this relation type / name in the context.
-}
-
-func (c *ContextObject) UnmarshalJSON(data []byte) error {
-	type plainContextObject ContextObject
-	var rawContextObject struct {
+func (a *Ancestor) UnmarshalJSON(data []byte) error {
+	type plainAncestor Ancestor
+	var rawAncestor struct {
 		Object EmbeddedObject `json:"object"`
-		plainContextObject
+		plainAncestor
 	}
-	if err := json.Unmarshal(data, &rawContextObject); err != nil {
+	if err := json.Unmarshal(data, &rawAncestor); err != nil {
 		return err
 	}
-	reportableObject, isReportable := rawContextObject.Object.Object.(ObservedObject)
-	if !isReportable {
-		return fmt.Errorf("object of type %q must implement the ObservedObject interface", rawContextObject.Object.Object.EmbeddedHeader().Type)
+	reportableObject, isReportable := rawAncestor.Object.Object.(ObservedObject)
+	if !isReportable && rawAncestor.Object.Object != nil {
+		return fmt.Errorf("object of type %q must implement the ObservedObject interface", rawAncestor.Object.Object.EmbeddedHeader().Type)
 	}
-	*c = ContextObject(rawContextObject.plainContextObject) // Copy the fields from rawContextObject to c
-	c.Object = reportableObject
+	*a = Ancestor(rawAncestor.plainAncestor) // Copy the fields from rawAncestor to a
+	a.Object = reportableObject
 	return nil
 }
 
-const omitInContext = "omitincontext"
-
-func (c Context) MarshalTextLog(t jsonlog.TextlogFormatter) jsonlog.TextlogEntry {
-	type objectsByRelation struct {
-		Relation Relation
-		Objects  []ContextObject
+func (a Ancestors) MarshalTextLog(t jsonlog.TextlogFormatter) jsonlog.TextlogEntry {
+	oldOmit := t.Omit
+	t.Omit = func(modifiers []string, value any) bool {
+		if slices.Contains(modifiers, omitInContext) {
+			return true // Omit fields that are marked with "omitincontext"
+		}
+		if oldOmit != nil {
+			return oldOmit(modifiers, value) // Call the original omit function if it exists
+		}
+		return false // Default behavior is to not omit any fields
 	}
-	var elementsByRelation []objectsByRelation
-	for _, element := range c {
-		var groupExists bool
-		if len(element.Relations) == 0 {
+	var result jsonlog.TextlogEntry
+	for _, ancestor := range a {
+		var prefix string
+		if ancestor.Distance == 1 {
+			prefix = "parent"
+		} else if ancestor.TopLevel {
+			prefix = "origin"
+		} else {
 			continue
 		}
-		// only use the first relation for textlog conversion
-		relation := element.Relations[0]
-		for i := range elementsByRelation {
-			if elementsByRelation[i].Relation == relation {
-				elementsByRelation[i].Objects = append(elementsByRelation[i].Objects, element)
-				groupExists = true
-				break
-			}
-		}
-		if !groupExists {
-			elementsByRelation = append(elementsByRelation, objectsByRelation{Relation: relation, Objects: []ContextObject{element}})
+		marshaledElement := t.Format(ancestor.Object)
+		for i := range marshaledElement {
+			marshaledElement[i].Key = jsonlog.ConcatTextLabels(strings.ToUpper(prefix), marshaledElement[i].Key)
 		}
+		result = append(result, marshaledElement...)
+	}
+	return result
+}
+
+type Derivative struct {
+	// Via is a reference to the field that contains a link to Object.
+	Via *jsonlog.Reference `json:"via"`
+	// Object is the object that is derived from the assessment's Subject.
+	Object ObservedObject `json:"object" textlog:",expand"`
+}
+
+const omitInContext = "omitincontext"
+
+func (d *Derivative) UnmarshalJSON(data []byte) error {
+	type plainDerivative Derivative
+	var unmarshalableDerivative struct {
+		Object EmbeddedObject `json:"object"`
+		plainDerivative
 	}
+	if err := json.Unmarshal(data, &unmarshalableDerivative); err != nil {
+		return err
+	}
+	observedObject, isObservedObject := unmarshalableDerivative.Object.Object.(ObservedObject)
+	if !isObservedObject && unmarshalableDerivative.Object.Object != nil {
+		return fmt.Errorf("object of type %q must implement the ObservedObject interface", unmarshalableDerivative.Object.Object.EmbeddedHeader().Type)
+	}
+	*d = Derivative(unmarshalableDerivative.plainDerivative)
+	d.Object = observedObject
+	return nil
+}
+
+func (d Derivative) MarshalTextLog(t jsonlog.TextlogFormatter) jsonlog.TextlogEntry {
 	oldOmit := t.Omit
 	t.Omit = func(modifiers []string, value any) bool {
 		if slices.Contains(modifiers, omitInContext) {
@@ -190,21 +224,8 @@ func (c Context) MarshalTextLog(t jsonlog.TextlogFormatter) jsonlog.TextlogEntry
 		}
 		return false // Default behavior is to not omit any fields
 	}
-
-	var result jsonlog.TextlogEntry
-	for _, group := range elementsByRelation {
-		for g, element := range group.Objects {
-			marshaledElement := t.Format(element)
-			for i := range marshaledElement {
-				marshaledElement[i].Key = jsonlog.ConcatTextLabels(strings.ToUpper(group.Relation.Name), marshaledElement[i].Key)
-				if !group.Relation.Unique {
-					marshaledElement[i].Key = jsonlog.ConcatTextLabels(marshaledElement[i].Key, strconv.Itoa(g+1))
-				}
-			}
-			result = append(result, marshaledElement...)
-		}
-	}
-	return result
+	type plainDerivative Derivative // Wrap this struct to not implement TextlogMarshaler
+	return t.Format(plainDerivative(d))
 }
 
 const typeAssessment = "THOR assessment"