diff --git a/reference.go b/reference.go index 00ffd67..6269854 100644 --- a/reference.go +++ b/reference.go @@ -258,7 +258,7 @@ func ConcatTextLabels(prefix string, label string) string { if label == "" { return prefix } - if prefix == label { + if prefix == label || strings.HasPrefix(label, prefix+"_") { // Don't duplicate the prefix if it already exists return label } return prefix + "_" + label diff --git a/thorlog/parser/parser_test.go b/thorlog/parser/parser_test.go index b03d71a..6be7d9f 100644 --- a/thorlog/parser/parser_test.go +++ b/thorlog/parser/parser_test.go @@ -130,7 +130,9 @@ func TestParseEvent(t *testing.T) { }, { "JsonV3Assessment", - `{"type":"THOR assessment","meta":{"time":"2024-09-24T14:18:46.190394329+02:00","level":"Alert","module":"Test","scan_id":"abdc","event_id":"abdas","hostname":"aserarsd"},"message":"This is a test assessment","subject":{"type":"file","path":"path/to/file"},"score":70,"reasons":[{"type":"reason","summary":"Reason 1","signature":{"score":70,"ref":null,"origin":"internal","kind":""},"matched":null}],"reason_count":0,"context":[{"object":{"type":"at job"},"relation":"","unique":false}],"log_version":"v3"}`, + `{"type":"THOR assessment","meta":{"time":"2024-09-24T14:18:46.190394329+02:00","level":"Alert","module":"Test","scan_id":"abdc","event_id":"abdas","hostname":"aserarsd"}, +"message":"This is a test assessment","subject":{"type":"file","path":"path/to/file"},"score":70,"reasons":[{"type":"reason","summary":"Reason 1","signature":{"score":70,"ref":null,"origin":"internal","kind":""},"matched":null}],"reason_count":0, +"ancestors":[{"object":{"type":"at job"},"distance":1,"top_level":false}],"derivatives":[{"object":{"type":"at job"}}],"log_version":"v3"}`, &thorlog.Assessment{ ObjectHeader: jsonlog.ObjectHeader{ Type: "THOR assessment", @@ -164,7 +166,17 @@ func TestParseEvent(t *testing.T) { }, }, ReasonCount: 0, - EventContext: thorlog.Context{ + Ancestors: thorlog.Ancestors{ + { + Object: &thorlog.AtJob{ + ObjectHeader: jsonlog.ObjectHeader{ + Type: "at job", + }, + }, + Distance: 1, + }, + }, + Derivatives: []thorlog.Derivative{ { Object: &thorlog.AtJob{ ObjectHeader: jsonlog.ObjectHeader{ diff --git a/thorlog/v3/atjob.go b/thorlog/v3/atjob.go index f85539f..7eb439d 100644 --- a/thorlog/v3/atjob.go +++ b/thorlog/v3/atjob.go @@ -8,6 +8,7 @@ type AtJob struct { jsonlog.ObjectHeader Command string `json:"command" textlog:"command"` + Image *File `json:"image" textlog:"image,expand"` } const typeAtJob = "at job" diff --git a/thorlog/v3/event.go b/thorlog/v3/event.go index a433c87..922f8db 100644 --- a/thorlog/v3/event.go +++ b/thorlog/v3/event.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "reflect" - "strconv" "strings" "github.com/NextronSystems/jsonlog" @@ -37,14 +36,13 @@ type Assessment struct { Reasons []Reason `json:"reasons" textlog:",expand"` // ReasonCount contains the total number of reasons (before any truncations). ReasonCount int `json:"reason_count,omitempty" textlog:"reasons_count,omitempty"` - // EventContext contains other objects that may be relevant for an analyst and their relation to the - // Subject. + // Ancestors contains information about objects that are the subject's ancestors. // - // To give an example: if the Subject is a file in a ZIP archive, - // the ZIP archive would be listed in the EventContext with a relation type of "derives from" - // and a relation name of "parent", indicating that the Subject derives from this object, - // which is its parent. - EventContext Context `json:"context" textlog:",expand" jsonschema:"nullable"` + // E.g. if the subject is a file in a nested ZIP, all the ZIPs (the topmost one and each nested one) are ancestors. + // Ancestors does not necessarily include information about all ancestors, but it will always include information about at least the topmost ancestor and the parent. + Ancestors Ancestors `json:"ancestors" textlog:",expand"` + // Derivatives contains information about objects that have been referenced in the subject. + Derivatives []Derivative `json:"derivatives" textlog:"file,expand"` // Issues lists any problems that THOR encountered when trying to create a JSON struct for this assessment. // This may include e.g. overly long fields that were truncated, fields that could not be rendered to JSON, // or similar problems. @@ -102,6 +100,16 @@ func (a *Assessment) UnmarshalJSON(data []byte) error { a.Reasons[i].StringMatches[j].Field = jsonlog.NewReference(a.Subject, target) } } + for i := range a.Derivatives { + if a.Derivatives[i].Via == nil { + continue + } + target, err := jsonpointer.Resolve(a, a.Derivatives[i].Via.ToJsonPointer()) + if err != nil { + return err + } + a.Derivatives[i].Via = jsonlog.NewReference(a, target) + } for i := range a.Issues { if a.Issues[i].Affected == nil { continue @@ -117,69 +125,95 @@ func (a *Assessment) UnmarshalJSON(data []byte) error { var _ common.Event = (*Assessment)(nil) -type Context []ContextObject - -// ContextObject describes a relation of an object to another. -type ContextObject struct { - Object ObservedObject `json:"object" textlog:",expand"` - // Relations describes how the object relates to the assessed subject. - // There may be multiple relations, e.g. if the object is both the parent and the topmost ancestor of the subject. - // - // Relations should be ordered by relevance, i.e. the most important relation should be first. - // Only the first (and most relevant) relation is used for text log formatting. - Relations []Relation `json:"relations" textlog:",expand" jsonschema:"minItems=1"` +type Ancestors []Ancestor + +type Ancestor struct { + // Per describes the action that caused the ancestor to create its child. + Per string `json:"per"` + // TopLevel is true if the ancestor does not have a parent. + TopLevel bool `json:"top_level,omitempty"` + // Distance is the number of links between the subject and the ancestor (parent = 1, grandparent = 2, ...) + Distance int `json:"distance"` + // Object describes the ancestor. + Object ObservedObject `json:"object"` } -type Relation struct { - Type string `json:"relation_type"` // RelationType is used to specify the type of relation, e.g. "derives from" or "related to" - Name string `json:"relation_name"` // RelationName is used to specify the name of the relation, e.g. "parent". It is optional. - Unique bool `json:"unique"` // Unique indicates whether the relation is unique, i.e. there can only be one object with this relation type / name in the context. -} - -func (c *ContextObject) UnmarshalJSON(data []byte) error { - type plainContextObject ContextObject - var rawContextObject struct { +func (a *Ancestor) UnmarshalJSON(data []byte) error { + type plainAncestor Ancestor + var rawAncestor struct { Object EmbeddedObject `json:"object"` - plainContextObject + plainAncestor } - if err := json.Unmarshal(data, &rawContextObject); err != nil { + if err := json.Unmarshal(data, &rawAncestor); err != nil { return err } - reportableObject, isReportable := rawContextObject.Object.Object.(ObservedObject) - if !isReportable { - return fmt.Errorf("object of type %q must implement the ObservedObject interface", rawContextObject.Object.Object.EmbeddedHeader().Type) + reportableObject, isReportable := rawAncestor.Object.Object.(ObservedObject) + if !isReportable && rawAncestor.Object.Object != nil { + return fmt.Errorf("object of type %q must implement the ObservedObject interface", rawAncestor.Object.Object.EmbeddedHeader().Type) } - *c = ContextObject(rawContextObject.plainContextObject) // Copy the fields from rawContextObject to c - c.Object = reportableObject + *a = Ancestor(rawAncestor.plainAncestor) // Copy the fields from rawAncestor to a + a.Object = reportableObject return nil } -const omitInContext = "omitincontext" - -func (c Context) MarshalTextLog(t jsonlog.TextlogFormatter) jsonlog.TextlogEntry { - type objectsByRelation struct { - Relation Relation - Objects []ContextObject +func (a Ancestors) MarshalTextLog(t jsonlog.TextlogFormatter) jsonlog.TextlogEntry { + oldOmit := t.Omit + t.Omit = func(modifiers []string, value any) bool { + if slices.Contains(modifiers, omitInContext) { + return true // Omit fields that are marked with "omitincontext" + } + if oldOmit != nil { + return oldOmit(modifiers, value) // Call the original omit function if it exists + } + return false // Default behavior is to not omit any fields } - var elementsByRelation []objectsByRelation - for _, element := range c { - var groupExists bool - if len(element.Relations) == 0 { + var result jsonlog.TextlogEntry + for _, ancestor := range a { + var prefix string + if ancestor.Distance == 1 { + prefix = "parent" + } else if ancestor.TopLevel { + prefix = "origin" + } else { continue } - // only use the first relation for textlog conversion - relation := element.Relations[0] - for i := range elementsByRelation { - if elementsByRelation[i].Relation == relation { - elementsByRelation[i].Objects = append(elementsByRelation[i].Objects, element) - groupExists = true - break - } - } - if !groupExists { - elementsByRelation = append(elementsByRelation, objectsByRelation{Relation: relation, Objects: []ContextObject{element}}) + marshaledElement := t.Format(ancestor.Object) + for i := range marshaledElement { + marshaledElement[i].Key = jsonlog.ConcatTextLabels(strings.ToUpper(prefix), marshaledElement[i].Key) } + result = append(result, marshaledElement...) + } + return result +} + +type Derivative struct { + // Via is a reference to the field that contains a link to Object. + Via *jsonlog.Reference `json:"via"` + // Object is the object that is derived from the assessment's Subject. + Object ObservedObject `json:"object" textlog:",expand"` +} + +const omitInContext = "omitincontext" + +func (d *Derivative) UnmarshalJSON(data []byte) error { + type plainDerivative Derivative + var unmarshalableDerivative struct { + Object EmbeddedObject `json:"object"` + plainDerivative } + if err := json.Unmarshal(data, &unmarshalableDerivative); err != nil { + return err + } + observedObject, isObservedObject := unmarshalableDerivative.Object.Object.(ObservedObject) + if !isObservedObject && unmarshalableDerivative.Object.Object != nil { + return fmt.Errorf("object of type %q must implement the ObservedObject interface", unmarshalableDerivative.Object.Object.EmbeddedHeader().Type) + } + *d = Derivative(unmarshalableDerivative.plainDerivative) + d.Object = observedObject + return nil +} + +func (d Derivative) MarshalTextLog(t jsonlog.TextlogFormatter) jsonlog.TextlogEntry { oldOmit := t.Omit t.Omit = func(modifiers []string, value any) bool { if slices.Contains(modifiers, omitInContext) { @@ -190,21 +224,8 @@ func (c Context) MarshalTextLog(t jsonlog.TextlogFormatter) jsonlog.TextlogEntry } return false // Default behavior is to not omit any fields } - - var result jsonlog.TextlogEntry - for _, group := range elementsByRelation { - for g, element := range group.Objects { - marshaledElement := t.Format(element) - for i := range marshaledElement { - marshaledElement[i].Key = jsonlog.ConcatTextLabels(strings.ToUpper(group.Relation.Name), marshaledElement[i].Key) - if !group.Relation.Unique { - marshaledElement[i].Key = jsonlog.ConcatTextLabels(marshaledElement[i].Key, strconv.Itoa(g+1)) - } - } - result = append(result, marshaledElement...) - } - } - return result + type plainDerivative Derivative // Wrap this struct to not implement TextlogMarshaler + return t.Format(plainDerivative(d)) } const typeAssessment = "THOR assessment" diff --git a/thorlog/v3/event_test.go b/thorlog/v3/event_test.go index b41577e..b532792 100644 --- a/thorlog/v3/event_test.go +++ b/thorlog/v3/event_test.go @@ -11,87 +11,76 @@ import ( "github.com/NextronSystems/jsonlog/thorlog/common" ) -func TestContext_MarshalTextLog(t *testing.T) { +func TestAncestors_MarshalTextLog(t *testing.T) { tests := []struct { name string - c *Context + c Ancestors want string }{ { - name: "empty context", - c: &Context{}, + name: "empty ancestors", + c: nil, want: "", }, { - name: "context with unique related object", - c: &Context{ + name: "single ancestor", + c: Ancestors{ { - Object: NewFile("path/to/file"), - Relations: []Relation{{ - Name: "file", - Unique: true, - }}, + Object: NewFile("path/to/file"), + Distance: 1, + TopLevel: true, }, }, - want: "FILE: path/to/file", + want: "PARENT_FILE: path/to/file", }, { - name: "context with related object group", - c: &Context{ + name: "two ancestors", + c: Ancestors{ { - Object: NewFile("path/to/file"), - Relations: []Relation{{ - Name: "file", - Unique: false, - }}, + Object: NewFile("path/to/file"), + Distance: 1, }, { - Object: NewFile("path/to/otherfile"), - Relations: []Relation{{ - Name: "file", - Unique: false, - }}, + Object: NewFile("path/to/otherfile"), + Distance: 2, + TopLevel: true, }, }, - want: "FILE_1: path/to/file FILE_2: path/to/otherfile", + want: "PARENT_FILE: path/to/file ORIGIN_FILE: path/to/otherfile", }, { - name: "context with different related objects", - c: &Context{ + name: "ancestors with middle ancestor invisible", + c: Ancestors{ { - Object: NewFile("path/to/file"), - Relations: []Relation{{ - Name: "file", - Unique: false, - }}, + Object: NewFile("path/to/file"), + Distance: 1, }, { - Object: NewFile("path/to/otherfile"), - Relations: []Relation{{ - Name: "archive", - Unique: true, - }}, + Object: NewFile("path/to/otherfile"), + Distance: 3, + TopLevel: true, }, }, - want: "FILE_1: path/to/file ARCHIVE_FILE: path/to/otherfile", + want: "PARENT_FILE: path/to/file ORIGIN_FILE: path/to/otherfile", }, { - name: "context with object related in two ways", - c: &Context{ + name: "three ancestors", + c: Ancestors{ + { + Object: NewFile("path/to/file"), + Distance: 1, + }, { - Object: NewFile("path/to/file"), - Relations: []Relation{{ - Name: "parent", - Type: "derived from", - Unique: true, - }, { - Name: "origin", - Type: "derived from", - Unique: true, - }}, + Object: NewFile("path/to/middlefile"), + Distance: 2, + }, + { + Object: NewFile("path/to/otherfile"), + Distance: 3, + TopLevel: true, }, }, - want: "PARENT_FILE: path/to/file", + want: "PARENT_FILE: path/to/file ORIGIN_FILE: path/to/otherfile", }, } var formatter jsonlog.TextlogFormatter @@ -130,12 +119,15 @@ func TestAssessment_UnmarshalJSON(t *testing.T) { }, Text: "This is a test assessment", Subject: NewFile("path/to/file"), - EventContext: Context{ + Ancestors: Ancestors{ + { + Object: NewAtJob(), + Distance: 1, + }, + }, + Derivatives: []Derivative{ { - Object: NewAtJob(), - Relations: []Relation{{ - Type: "related to", - }}, + Object: NewAuditLogEntry(), }, }, Reasons: []Reason{ @@ -170,3 +162,14 @@ func TestAssessment_UnmarshalIssue(t *testing.T) { t.Fatalf("Failed to unmarshal assessment: %v", err) } } + +func TestUnmarshalNilObject(t *testing.T) { + for _, data := range []string{`{}`, `{"object":null}`} { + if err := json.Unmarshal([]byte(data), &Ancestor{}); err != nil { + t.Errorf("ancestor %s: %v", data, err) + } + if err := json.Unmarshal([]byte(data), &Derivative{}); err != nil { + t.Errorf("derivative %s: %v", data, err) + } + } +} diff --git a/thorlog/v3/hostinfo.go b/thorlog/v3/hostinfo.go index b93abf2..96fdc73 100644 --- a/thorlog/v3/hostinfo.go +++ b/thorlog/v3/hostinfo.go @@ -51,11 +51,11 @@ func (h *HostInfo) UnmarshalJSON(data []byte) error { return err } *h = HostInfo(unmarshalableInfo.hostInfoClone) - if platformInfo, isPlatformInfo := unmarshalableInfo.Platform.Object.(PlatformInfo); isPlatformInfo { - h.Platform = platformInfo - } else { + platformInfo, isPlatformInfo := unmarshalableInfo.Platform.Object.(PlatformInfo) + if !isPlatformInfo && unmarshalableInfo.Platform.Object != nil { return fmt.Errorf("platform information has invalid type %s", unmarshalableInfo.Platform.Object.EmbeddedHeader().Type) } + h.Platform = platformInfo return nil } diff --git a/thorlog/v3/hostinfo_test.go b/thorlog/v3/hostinfo_test.go new file mode 100644 index 0000000..e7a559f --- /dev/null +++ b/thorlog/v3/hostinfo_test.go @@ -0,0 +1,14 @@ +package thorlog + +import ( + "encoding/json" + "testing" +) + +func TestHostInfo_UnmarshalNilPlatform(t *testing.T) { + for _, data := range []string{`{}`, `{"platform":null}`} { + if err := json.Unmarshal([]byte(data), &HostInfo{}); err != nil { + t.Errorf("%s: %v", data, err) + } + } +} diff --git a/thorlog/v3/scheduledtask.go b/thorlog/v3/scheduledtask.go index b57594f..c517924 100644 --- a/thorlog/v3/scheduledtask.go +++ b/thorlog/v3/scheduledtask.go @@ -6,6 +6,11 @@ import ( "github.com/NextronSystems/jsonlog" ) +type ScheduledTaskCommand struct { + Image *File `json:"image" textlog:"image,expand"` + Command string `json:"command" textlog:"command"` +} + // ScheduledTask describes a Windows Scheduled Task. // // See also the Microsoft documentation at https://learn.microsoft.com/en-us/windows/win32/taskschd/task-scheduler-reference @@ -19,7 +24,7 @@ type ScheduledTask struct { Path string `json:"path" textlog:"path"` // Commands executed when this scheduled task activates. Commands each include both image and arguments. - Commands StringList `json:"commands" textlog:"command,omitempty"` + Commands []ScheduledTaskCommand `json:"commands" textlog:",expand"` // COM Handlers (as GUIDs) invoked when this scheduled task activates. ComHandlers StringList `json:"com_handlers,omitempty" textlog:"com_handler,expand,omitempty"` diff --git a/thorlog/v3/wer.go b/thorlog/v3/wer.go index 65113eb..74f06ae 100644 --- a/thorlog/v3/wer.go +++ b/thorlog/v3/wer.go @@ -15,7 +15,7 @@ import ( // (WerReportCreate()) and the WER_REPORT_UI enumeration that holds additional // error details if present. // -// [1] https://learn.microsoft.com/en-us/windows/win32/api/werapi/ns-werapi-wer_report_information +// [1] https://learn.microsoft.com/en-us/windows/win32/api/werapi/ns-werapi-wer_report_information // [2] https://learn.microsoft.com/en-us/windows/win32/api/werapi/ . type WERCrashReport struct { ReportType WERReportType `json:"type" textlog:"reporttype"`