diff --git a/thorlog/common/event.go b/thorlog/common/event.go index 8f2bbad..0b8be81 100644 --- a/thorlog/common/event.go +++ b/thorlog/common/event.go @@ -23,12 +23,16 @@ const ( // These fields are marked with the `textlog:"-"` tag to prevent them from being // included in the event body. type LogEventMetadata struct { - Time time.Time `json:"time" textlog:"-"` - Lvl LogLevel `json:"level" textlog:"-"` - Mod string `json:"module" textlog:"module"` - ScanID string `json:"scan_id" textlog:"scanid,omitempty"` - GenID string `json:"event_id" textlog:"uid,omitempty"` - Source string `json:"hostname" textlog:"-"` + Time time.Time `json:"time" textlog:"-"` + Lvl LogLevel `json:"level" textlog:"-"` + Mod string `json:"module" textlog:"module"` + // The ID of the scan where this event was created. + ScanID string `json:"scan_id" textlog:"scanid,omitempty"` + // A unique ID for this finding. + // The ID is transient and the same element may have different IDs across multiple scans. + GenID string `json:"event_id,omitempty" textlog:"uid,omitempty"` + // The hostname of the machine where this event was generated. + Source string `json:"hostname" textlog:"-"` } // Event describes the basic information of a THOR event that is available in all versions. diff --git a/thorlog/v3/event.go b/thorlog/v3/event.go index 3d2f2af..a3bf00c 100644 --- a/thorlog/v3/event.go +++ b/thorlog/v3/event.go @@ -14,19 +14,46 @@ import ( "golang.org/x/exp/slices" ) +// Finding is a summary of a Subject's analysis by THOR. +// This object is usually, but not necessarily suspicious; the +// severity can be seen in the Score, and beyond that the +// Reasons contain further information on why this Subject is +// considered suspicious. type Finding struct { jsonlog.ObjectHeader - Meta LogEventMetadata `json:"meta" textlog:",expand"` - Text string `json:"message" textlog:"message"` - Subject ReportableObject `json:"subject" textlog:",expand"` - Score int64 `json:"score" textlog:"score"` - Reasons []Reason `json:"reasons" textlog:",expand"` - ReasonCount int `json:"reason_count,omitempty" textlog:"reasons_count,omitempty"` - EventContext Context `json:"context" textlog:",expand" jsonschema:"nullable"` - Issues []Issue `json:"issues,omitempty" textlog:"-"` - LogVersion common.Version `json:"log_version"` + Meta LogEventMetadata `json:"meta" textlog:",expand"` + // Text is the message THOR printed for this finding. + // This is usually a summary based on this finding's subject and level. + Text string `json:"message" textlog:"message"` + // Subject is the object analysed by THOR. + Subject ReportableObject `json:"subject" textlog:",expand"` + // Score is a metric that combines severity and certainty. The score is always in a range of 0 to 100; + // 0 indicates that the analysis found no suspicious indicators, whereas 100 indicates very high + // severity and certainty. + Score int64 `json:"score" textlog:"score"` + // Reasons describes the indicators that contributed to the score. + // This list is not necessarily comprehensive; THOR may cut off all reasons after the first few. + // If this is the case, an Issue with category IssueCategoryTruncated pointing to this field will be present. + Reasons []Reason `json:"reasons" textlog:",expand"` + // ReasonCount contains the total number of reasons (before any truncations). + ReasonCount int `json:"reason_count,omitempty" textlog:"reasons_count,omitempty"` + // EventContext contains other objects that may be relevant for an analyst and their relation to the + // Subject. + // + // To give an example: if the Subject is a file in a ZIP archive, + // the ZIP archive would be listed in the EventContext with a relation type of "derives from" + // and a relation name of "parent", indicating that the Subject derives from this object, + // which is its parent. + EventContext Context `json:"context" textlog:",expand" jsonschema:"nullable"` + // Issues lists any problems that THOR encountered when trying to create a Finding for this analysis. + // This may include e.g. overly long fields that were truncated, fields that could not be rendered to JSON, + // or similar problems. + Issues []Issue `json:"issues,omitempty" textlog:"-"` + // LogVersion describes the jsonlog version that this event was created with. + LogVersion common.Version `json:"log_version"` } +// ReportableObject can be any object type that THOR analyses, e.g. File or Process. type ReportableObject interface { reportable() jsonlog.Object @@ -92,6 +119,7 @@ var _ common.Event = (*Finding)(nil) type Context []ContextObject +// ContextObject describes a relation of an object to another. type ContextObject struct { Object ReportableObject `json:"object" textlog:",expand"` RelationType string `json:"relation_type"` // RelationType is used to specify the type of relation, e.g. "derives from" or "related to" @@ -176,12 +204,18 @@ func NewFinding(subject ReportableObject, message string) *Finding { } } +// Message describes a THOR message printed during the scan. +// Unlike Finding, this does not describe an analysis' result, +// but rather something about the scan itself (e.g. how many IOCs were loaded). type Message struct { jsonlog.ObjectHeader - Meta LogEventMetadata `json:"meta" textlog:",expand"` - Text string `json:"message" textlog:"message"` - Fields MessageFields `json:"fields" textlog:",expand" jsonschema:"nullable"` - LogVersion common.Version `json:"log_version"` + Meta LogEventMetadata `json:"meta" textlog:",expand"` + // Text is the message that was logged. + Text string `json:"message" textlog:"message"` + // Fields contains additional structured fields that were logged. These + // contain details about the Text displayed. + Fields MessageFields `json:"fields" textlog:",expand" jsonschema:"nullable"` + LogVersion common.Version `json:"log_version"` } func (m *Message) Message() string { diff --git a/thorlog/v3/event_test.go b/thorlog/v3/event_test.go index f229773..65e03e7 100644 --- a/thorlog/v3/event_test.go +++ b/thorlog/v3/event_test.go @@ -134,7 +134,7 @@ func TestFinding_UnmarshalJSON(t *testing.T) { } func TestFinding_UnmarshalIssue(t *testing.T) { - finding := `{"type":"THOR finding","meta":{"time":"2025-07-01T12:05:12.993789131+02:00","level":"Info","module":"ProcessCheck","scan_id":"S-pSxgCmyvvfs","event_id":"","hostname":"dummy"},"message":"process found","subject":{"type":"process","pid":502168,"name":"chromium","command":"/usr/lib/chromium/chromium","owner":"owner","image":{"type":"file","path":"/usr/lib/chromium/chromium","exists":"yes","extension":"","magic_header":"ELF","hashes":{"md5":"fc04ee20f064adc18e370c22512e268e","sha1":"2c8b7d05d25e04db9c169ce85e8e8f84321ef0c8","sha256":"0cf1727aa8dc3995d5aa103001f656b8ee8a1b3ffbc6d8664c5ad95cf225771f"},"first_bytes":{"hex":"7f454c4602010100000000000000000003003e00","ascii":"ELF\u003e"},"file_times":{"modified":"2025-06-25T19:45:43+02:00","accessed":"2025-07-01T08:46:56.750309598+02:00","changed":"2025-06-26T08:39:59.980605063+02:00"},"size":252546120,"permissions":{"type":"Unix permissions","owner":"root","group":"root","permissions":{"user":{"readable":true,"writable":true,"executable":true},"group":{"readable":true,"writable":false,"executable":true},"world":{"readable":true,"writable":false,"executable":true}}}},"parent_info":{"pid":9011,"exe":"/usr/lib/chromium/chromium","command":"/usr/lib/chromium/chromium"},"tree":["/usr/lib/chromium/chromium","/usr/lib/chromium/chromium"],"created":"2025-07-01T12:00:05+02:00","session":"","listen_ports":null,"connections":[]},"score":0,"reasons":null,"reason_count":0,"context":null,"issues":[{"affected":"/subject/sections","category":"truncated","description":"Removed some sections from process memory (originally 638)"}],"log_version":"v3.0.0"}` + finding := `{"type":"THOR finding","meta":{"time":"2025-07-01T12:05:12.993789131+02:00","level":"Info","module":"ProcessCheck","scan_id":"S-pSxgCmyvvfs","event_id":"","hostname":"dummy"},"message":"process found","subject":{"type":"process","pid":502168,"name":"chromium","command":"/usr/lib/chromium/chromium","owner":"owner","image":{"type":"file","path":"/usr/lib/chromium/chromium","exists":"yes","extension":"","magic_header":"ELF","hashes":{"md5":"fc04ee20f064adc18e370c22512e268e","sha1":"2c8b7d05d25e04db9c169ce85e8e8f84321ef0c8","sha256":"0cf1727aa8dc3995d5aa103001f656b8ee8a1b3ffbc6d8664c5ad95cf225771f"},"first_bytes":{"hex":"7f454c4602010100000000000000000003003e00","ascii":"ELF\u003e"},"file_times":{"modified":"2025-06-25T19:45:43+02:00","accessed":"2025-07-01T08:46:56.750309598+02:00","changed":"2025-06-26T08:39:59.980605063+02:00"},"size":252546120,"permissions":{"type":"Unix permissions","owner":"root","group":"root","mask":{"user":{"readable":true,"writable":true,"executable":true},"group":{"readable":true,"writable":false,"executable":true},"world":{"readable":true,"writable":false,"executable":true}}}},"parent_info":{"pid":9011,"exe":"/usr/lib/chromium/chromium","command":"/usr/lib/chromium/chromium"},"tree":["/usr/lib/chromium/chromium","/usr/lib/chromium/chromium"],"created":"2025-07-01T12:00:05+02:00","session":"","listen_ports":null,"connections":[]},"score":0,"reasons":null,"reason_count":0,"context":null,"issues":[{"affected":"/subject/sections","category":"truncated","description":"Removed some sections from process memory (originally 638)"}],"log_version":"v3.0.0"}` var findingObj Finding if err := json.Unmarshal([]byte(finding), &findingObj); err != nil { t.Fatalf("Failed to unmarshal finding: %v", err) diff --git a/thorlog/v3/matchstrings.go b/thorlog/v3/matchstrings.go index f90ee55..5b6553a 100644 --- a/thorlog/v3/matchstrings.go +++ b/thorlog/v3/matchstrings.go @@ -101,10 +101,18 @@ func (f MatchData) QuotedString() string { return matchingString } +// MatchString describes a sequence of bytes in an object +// that was matched on by a signature. type MatchString struct { - Match MatchData `json:"data"` - Context *MatchData `json:"context,omitempty"` - Offset *uint64 `json:"offset,omitempty"` + // Match contains the bytes that were matched. + Match MatchData `json:"data"` + // Context contains the bytes surrounding the matched bytes. + // This may be missing if no context is available. + Context *MatchData `json:"context,omitempty"` + // Offset contains the Match's offset within the Field + // where the data was matched. + Offset *uint64 `json:"offset,omitempty"` + // Field points to the field within the object that was matched on. Field *jsonlog.Reference `json:"field,omitempty"` HideOffset bool `json:"-"` } @@ -147,6 +155,8 @@ func (f MatchString) String() string { return matchString } +// MatchStrings is a list of matching byte sequences that explains +// why a specific signature matched on an object. type MatchStrings []MatchString const maxMatchStrings = 30 diff --git a/thorlog/v3/permissions.go b/thorlog/v3/permissions.go index d192a2e..4a06d60 100644 --- a/thorlog/v3/permissions.go +++ b/thorlog/v3/permissions.go @@ -28,7 +28,7 @@ type UnixPermissions struct { Owner string `json:"owner" textlog:"owner"` // FIXME: Could explicitly include name / UID Group string `json:"group" textlog:"group"` // FIXME: Could explicitly include name / GID - Mask PermissionMask `json:"permissions" textlog:"permissions"` + Mask PermissionMask `json:"mask" textlog:"permissions"` } func (p UnixPermissions) String() string { @@ -88,7 +88,7 @@ type WindowsPermissions struct { LogObjectHeader Owner string `json:"owner" textlog:"owner"` // FIXME: Could include information like the original SID - Permissions AclEntries `json:"permissions" textlog:"permissions" jsonschema:"nullable"` + Permissions AclEntries `json:"acl" textlog:"permissions" jsonschema:"nullable"` } func (p WindowsPermissions) String() string { diff --git a/thorlog/v3/reason.go b/thorlog/v3/reason.go index c8edc77..a0dbd64 100644 --- a/thorlog/v3/reason.go +++ b/thorlog/v3/reason.go @@ -7,12 +7,15 @@ import ( "github.com/NextronSystems/jsonlog" ) +// Reason describes a match of a single Signature on a ReportableObject. type Reason struct { jsonlog.ObjectHeader Summary string `json:"summary" textlog:"reason"` - Signature `json:"signature" textlog:",inline"` + // Signature contains details about the signature that matched. + Signature `json:"signature" textlog:",inline"` + // StringMatches contains the matches that explain why this signature matched. StringMatches MatchStrings `json:"matched" textlog:"matched" jsonschema:"nullable"` } @@ -30,18 +33,45 @@ func init() { AddLogObjectType(typeReason, &Reason{}) } +// Signature describes metadata about a signature that THOR uses to detect +// suspicious objects. type Signature struct { - Score int64 `json:"score" textlog:"subscore"` - Ref StringList `json:"ref" textlog:"ref" jsonschema:"nullable"` - Type Sigtype `json:"origin" textlog:"sigtype"` - Class Sigclass `json:"kind" textlog:"sigclass"` - Date string `json:"date,omitempty" textlog:"ruledate,omitempty"` - Tags StringList `json:"tags,omitempty" textlog:"tags,omitempty" jsonschema:"nullable"` - Rulename string `json:"rule_name,omitempty" textlog:"rulename,omitempty"` - LongDescription string `json:"description,omitempty" textlog:"description,omitempty"` - Author string `json:"author,omitempty" textlog:"author,omitempty"` - RuleId string `json:"id,omitempty" textlog:"id"` - FalsePositives StringList `json:"false_positives,omitempty" textlog:"falsepositives,omitempty" jsonschema:"nullable"` + // Score is a metric that combines severity and certainty for this signature. + // + // It is related to the Finding.Score, which is derived from the scores of all + // signatures that matched; however, signature scores are not limited to the + // 0 to 100 interval of finding scores, but may also be negative to indicate + // a likely false positive (which results in a score reduction on any related + // finding). + Score int64 `json:"score" textlog:"subscore"` + // Ref contains references (usually as links) for further information about + // the threat that is detected by this signature. + Ref StringList `json:"reference" textlog:"ref" jsonschema:"nullable"` + // Type indicates whether a signature was part of THOR's built in signature set + // or whether it was a custom signature provided by the user. + Type Sigtype `json:"origin" textlog:"sigtype"` + // Class is the sort of signature that this is (YARA Rule, Filename IOC, ...) + Class Sigclass `json:"kind" textlog:"sigclass"` + // Date is the date on which the signature was last modified. + Date string `json:"date,omitempty" textlog:"ruledate,omitempty"` + // Tags are short strings that help with grouping signatures. + // + // E.g. APT related signatures may be tagged "APT", or malware related signatures may be tagged "MAL". + Tags StringList `json:"tags,omitempty" textlog:"tags,omitempty" jsonschema:"nullable"` + // Rulename is the name of the signature (e.g. a YARA rule name). + Rulename string `json:"rule_name,omitempty" textlog:"rulename,omitempty"` + // LongDescription contains the description that the signature has about itself + // (e.g. "detects a webshell related to ...") + LongDescription string `json:"description,omitempty" textlog:"description,omitempty"` + // Author is the name of the person who wrote the signature. + Author string `json:"author,omitempty" textlog:"author,omitempty"` + // RuleId is a unique ID that identifies this signature. + // + // Not all classes of signatures may provide this field. + RuleId string `json:"id,omitempty" textlog:"id"` + // FalsePositives describes cases where this signature is known to produce matches + // even on benign data. + FalsePositives StringList `json:"false_positives,omitempty" textlog:"falsepositives,omitempty" jsonschema:"nullable"` } type Sigclass string