From a53ea0ea931f260d9e7ee6b9923a5dffefbdae89 Mon Sep 17 00:00:00 2001 From: ryandamour Date: Tue, 20 Oct 2020 22:06:41 -0700 Subject: [PATCH 1/9] Adding logic for MITRE enrichment --- README.md | 4 +++- setup.py | 2 +- sigmalint/modules/__init__.py | 1 + sigmalint/modules/mitre.py | 34 ++++++++++++++++++++++++++++++++++ sigmalint/sigmalint.py | 28 ++++++++++++++++++++++++++-- 5 files changed, 65 insertions(+), 4 deletions(-) create mode 100644 sigmalint/modules/__init__.py create mode 100644 sigmalint/modules/mitre.py diff --git a/README.md b/README.md index 5c67717..7aaffdc 100644 --- a/README.md +++ b/README.md @@ -4,12 +4,13 @@ A simple linter for Sigma rules ## Description -sigmalint is a command line interface for validating Sigma rules against the Sigma schema. +sigmalint is a command line interface for validating and enriching Sigma rules against the Sigma schema. The available arguments are: * `--sigmainput` - Path to a directory that comtains Sigma files or to a single Sigma file. * `--directory` - Flag for if sigmainput is a directory * `--method` - The schema validator that you wish to use (Default: rx) +* `--mitre` - Enrich Sigma file with MITRE content based off of any MITRE references in `tags` The available methods are: * `rx` - uses PyRx and the Rx schema from the Sigma repo @@ -32,6 +33,7 @@ Options: [required] --directory Flag for if sigmainput is a directory --method [rx|jsonschema|s2] Validation method. + --mitre Enrich Sigma file with MITRE content --help Show this message and exit. ``` diff --git a/setup.py b/setup.py index 1f4b3fd..bc7099c 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ author='Ryan Plas', author_email='ryan.plas@stage2sec.com', entry_points={"console_scripts": ["sigmalint = sigmalint.sigmalint:cli"]}, - packages=['sigmalint', 'sigmalint.schema'], + packages=['sigmalint', 'sigmalint.schema', 'sigmalint.modules'], package_dir={"": "."}, package_data={}, install_requires=[ diff --git a/sigmalint/modules/__init__.py b/sigmalint/modules/__init__.py new file mode 100644 index 0000000..d4747ea --- /dev/null +++ b/sigmalint/modules/__init__.py @@ -0,0 +1 @@ +from .mitre import mitre_pull diff --git a/sigmalint/modules/mitre.py b/sigmalint/modules/mitre.py new file mode 100644 index 0000000..a50703b --- /dev/null +++ b/sigmalint/modules/mitre.py @@ -0,0 +1,34 @@ +#!/usr/bin/python3 +from bs4 import BeautifulSoup +import requests +import re + +def mitre_pull(technique_id): + tactics = [] + + # Redirect + try: + url = "https://attack.mitre.org/techniques/{}/".format(str(technique_id)) + redirect = requests.get(url,allow_redirects=True) + redirect_url = re.search('url=/(.*)"',redirect.text) + + # Technique URL + mitre_url = "https://attack.mitre.org/{}/".format(str(redirect_url.group(1))) + mitre_request = requests.get(mitre_url,allow_redirects=False) + mitre_soup = BeautifulSoup(mitre_request.text,"html.parser") + + # Tactic ID's + tactic_id = mitre_soup.find('div', class_='card-data', id='card-tactics').get_text() + tactic_id = tactic_id.replace('Tactics:','').replace(' ','').replace('Tactic:\n','').strip("\n") + tactic_id = tactic_id.split(',') + tactics = tactic_id + + # Sub Techniques + sub_techniques_regex = re.search('Sub-technique of(.*?)',mitre_request.text,re.M | re.DOTALL) + sub_techniques = sub_techniques_regex.group(1).replace(': ','').replace(' ','').strip("\n") + sub_techniques = re.sub('<[^>]+>','',sub_techniques) + sub_techniques = sub_techniques.replace('\n','') + + return tactics, sub_techniques + except: + return None, None diff --git a/sigmalint/sigmalint.py b/sigmalint/sigmalint.py index 0dcdbd3..f42c21e 100644 --- a/sigmalint/sigmalint.py +++ b/sigmalint/sigmalint.py @@ -4,7 +4,9 @@ import yaml import pyrx import jsonschema +import re +from .modules.mitre import mitre_pull from .schema import rx_schema, json_schema, s2_schema rx = pyrx.Factory({'register_core_types': True}) @@ -15,7 +17,8 @@ @click.option('--sigmainput', type=click.Path(exists=True, file_okay=True, readable=True, resolve_path=True), help='Path to a directory that comtains Sigma files or to a single Sigma file.', required=True) @click.option('--directory', is_flag=True, help="Flag for if sigmainput is a directory") @click.option('--method', type=click.Choice(['rx', 'jsonschema', 's2'], case_sensitive=False), default='rx', help='Validation method.') -def cli(sigmainput, directory, method): +@click.option('--mitre', is_flag=True, help='Enrich Sigma file with MITRE content based off of any MITRE references in tags. This will append to the Sigma file.', required=False) +def cli(sigmainput, directory, method, mitre): results = [] filepaths = [] @@ -56,6 +59,27 @@ def cli(sigmainput, directory, method): result = False if len(errors) > 0 else True results.append({'result': result, 'reasons': errors, 'filename': filename}) + if mitre: + for tag in sigma_yaml_list[0]['tags']: + if len(re.findall(r'(?i)t\d{4}',tag)) > 0: + mitre_id = re.findall(r'(?i)t\d{4}',tag) + mitre_id = mitre_id[0].upper() + tactics, sub_techniques = mitre_pull(mitre_id) + ## Prevent multiple writes to file if `mitre` key already exists + if 'mitre' not in sigma_yaml_list[0]: + sigma_yaml_list[0]['mitre'] = {} + sigma_yaml_list[0]['mitre']['tactics'] = [] + sigma_yaml_list[0]['mitre']['subTechniques'] = [] + if tactics is not None and sub_techniques is not None: + for tactic in tactics: + ## Make sure we aren't duplicating + if tactic not in sigma_yaml_list[0]['mitre']['tactics']: + sigma_yaml_list[0]['mitre']['tactics'].append(tactic) + if sub_techniques not in sigma_yaml_list[0]['mitre']['subTechniques']: + sigma_yaml_list[0]['mitre']['subTechniques'].append(sub_techniques) + with open(os.path.join(sigmainput, filename), 'w') as f: + yaml.dump(sigma_yaml_list[0], f) + click.echo('Results:') for result in results: @@ -72,4 +96,4 @@ def cli(sigmainput, directory, method): click.echo('Total Valid Rule Files: {}'.format(str(len(results) - invalid_count) + "/" + str(len(results)))) click.echo('Total Invalid Rule Files: {}'.format(str(invalid_count) + "/" + str(len(results)))) - click.echo('Total Unsupported Rule Files (Multi-document): {}'.format(str(unsupported_count) + "/" + str(len(results)))) \ No newline at end of file + click.echo('Total Unsupported Rule Files (Multi-document): {}'.format(str(unsupported_count) + "/" + str(len(results)))) From e9d32816ffeeaa9e68597e2488e4167b1c8d38ee Mon Sep 17 00:00:00 2001 From: ryandamour Date: Thu, 22 Oct 2020 11:03:45 -0700 Subject: [PATCH 2/9] Added techniqueIds and reference links --- sigmalint/modules/mitre.py | 7 +++++-- sigmalint/sigmalint.py | 10 ++++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/sigmalint/modules/mitre.py b/sigmalint/modules/mitre.py index a50703b..3d72a4e 100644 --- a/sigmalint/modules/mitre.py +++ b/sigmalint/modules/mitre.py @@ -29,6 +29,9 @@ def mitre_pull(technique_id): sub_techniques = re.sub('<[^>]+>','',sub_techniques) sub_techniques = sub_techniques.replace('\n','') - return tactics, sub_techniques + # Reference + references = "https://attack.mitre.org/techniques/"+str(technique_id)+"/" + # Make sure we return technique_id if valid values were returned + return tactics, sub_techniques, technique_id, references except: - return None, None + return None, None, None, None diff --git a/sigmalint/sigmalint.py b/sigmalint/sigmalint.py index f42c21e..598cdea 100644 --- a/sigmalint/sigmalint.py +++ b/sigmalint/sigmalint.py @@ -64,19 +64,25 @@ def cli(sigmainput, directory, method, mitre): if len(re.findall(r'(?i)t\d{4}',tag)) > 0: mitre_id = re.findall(r'(?i)t\d{4}',tag) mitre_id = mitre_id[0].upper() - tactics, sub_techniques = mitre_pull(mitre_id) + tactics, sub_techniques, technique_id, references = mitre_pull(mitre_id) ## Prevent multiple writes to file if `mitre` key already exists if 'mitre' not in sigma_yaml_list[0]: sigma_yaml_list[0]['mitre'] = {} sigma_yaml_list[0]['mitre']['tactics'] = [] sigma_yaml_list[0]['mitre']['subTechniques'] = [] - if tactics is not None and sub_techniques is not None: + sigma_yaml_list[0]['mitre']['techniqueIds'] = [] + sigma_yaml_list[0]['mitre']['references'] = [] + if tactics is not None and sub_techniques is not None and technique_id is not None and references is not None: for tactic in tactics: ## Make sure we aren't duplicating if tactic not in sigma_yaml_list[0]['mitre']['tactics']: sigma_yaml_list[0]['mitre']['tactics'].append(tactic) if sub_techniques not in sigma_yaml_list[0]['mitre']['subTechniques']: sigma_yaml_list[0]['mitre']['subTechniques'].append(sub_techniques) + if technique_id not in sigma_yaml_list[0]['mitre']['techniqueIds']: + sigma_yaml_list[0]['mitre']['techniqueIds'].append(technique_id) + if technique_id not in sigma_yaml_list[0]['mitre']['references']: + sigma_yaml_list[0]['mitre']['references'].append(references) with open(os.path.join(sigmainput, filename), 'w') as f: yaml.dump(sigma_yaml_list[0], f) From 40f3b6b2adc86a69837458d733dcd961a9dc6898 Mon Sep 17 00:00:00 2001 From: ryandamour Date: Thu, 22 Oct 2020 11:07:41 -0700 Subject: [PATCH 3/9] Fix for duplicate references --- sigmalint/sigmalint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sigmalint/sigmalint.py b/sigmalint/sigmalint.py index 598cdea..73d695d 100644 --- a/sigmalint/sigmalint.py +++ b/sigmalint/sigmalint.py @@ -81,7 +81,7 @@ def cli(sigmainput, directory, method, mitre): sigma_yaml_list[0]['mitre']['subTechniques'].append(sub_techniques) if technique_id not in sigma_yaml_list[0]['mitre']['techniqueIds']: sigma_yaml_list[0]['mitre']['techniqueIds'].append(technique_id) - if technique_id not in sigma_yaml_list[0]['mitre']['references']: + if references not in sigma_yaml_list[0]['mitre']['references']: sigma_yaml_list[0]['mitre']['references'].append(references) with open(os.path.join(sigmainput, filename), 'w') as f: yaml.dump(sigma_yaml_list[0], f) From bfecb244a586d009d3f799645e918205332d746f Mon Sep 17 00:00:00 2001 From: ryandamour Date: Thu, 22 Oct 2020 13:40:26 -0700 Subject: [PATCH 4/9] Add bs4 to requirements --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 71c1210..814a489 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ pyrx==0.*,>=0.3.0 pytest==5.*,>=5.4.3 pytest-cov==2.*,>=2.10.0 pyyaml==5.*,>=5.3.1 +beautifulsoup4==4.*,>=4.9.3 From f4e31a900154957a123a3197acaac847a2b6c98a Mon Sep 17 00:00:00 2001 From: ryandamour Date: Thu, 22 Oct 2020 13:45:28 -0700 Subject: [PATCH 5/9] add bs4 to requirements --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bc7099c..7f02245 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ package_data={}, install_requires=[ 'click==7.*,>=7.1.2', 'jsonschema==3.*,>=3.2.0', 'pyrx==0.*,>=0.3.0', - 'pyyaml==5.*,>=5.3.1' + 'pyyaml==5.*,>=5.3.1', 'beautifulsoup4==4.*,>=4.9.3' ], extras_require={ "dev": ["pytest==5.*,>=5.4.3", "pytest-cov==2.*,>=2.10.0"]}, From aed2146923ea5319f8962be4763aa915372207d9 Mon Sep 17 00:00:00 2001 From: ryandamour Date: Thu, 22 Oct 2020 13:49:07 -0700 Subject: [PATCH 6/9] Add check for 'tags' key existence --- sigmalint/sigmalint.py | 52 ++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/sigmalint/sigmalint.py b/sigmalint/sigmalint.py index 73d695d..40683b8 100644 --- a/sigmalint/sigmalint.py +++ b/sigmalint/sigmalint.py @@ -60,31 +60,33 @@ def cli(sigmainput, directory, method, mitre): results.append({'result': result, 'reasons': errors, 'filename': filename}) if mitre: - for tag in sigma_yaml_list[0]['tags']: - if len(re.findall(r'(?i)t\d{4}',tag)) > 0: - mitre_id = re.findall(r'(?i)t\d{4}',tag) - mitre_id = mitre_id[0].upper() - tactics, sub_techniques, technique_id, references = mitre_pull(mitre_id) - ## Prevent multiple writes to file if `mitre` key already exists - if 'mitre' not in sigma_yaml_list[0]: - sigma_yaml_list[0]['mitre'] = {} - sigma_yaml_list[0]['mitre']['tactics'] = [] - sigma_yaml_list[0]['mitre']['subTechniques'] = [] - sigma_yaml_list[0]['mitre']['techniqueIds'] = [] - sigma_yaml_list[0]['mitre']['references'] = [] - if tactics is not None and sub_techniques is not None and technique_id is not None and references is not None: - for tactic in tactics: - ## Make sure we aren't duplicating - if tactic not in sigma_yaml_list[0]['mitre']['tactics']: - sigma_yaml_list[0]['mitre']['tactics'].append(tactic) - if sub_techniques not in sigma_yaml_list[0]['mitre']['subTechniques']: - sigma_yaml_list[0]['mitre']['subTechniques'].append(sub_techniques) - if technique_id not in sigma_yaml_list[0]['mitre']['techniqueIds']: - sigma_yaml_list[0]['mitre']['techniqueIds'].append(technique_id) - if references not in sigma_yaml_list[0]['mitre']['references']: - sigma_yaml_list[0]['mitre']['references'].append(references) - with open(os.path.join(sigmainput, filename), 'w') as f: - yaml.dump(sigma_yaml_list[0], f) + ## Most mentions of MITRE are currently found in the 'tags' field for Sigma rules + if 'tags' in sigma_yaml_list[0]: + for tag in sigma_yaml_list[0]['tags']: + if len(re.findall(r'(?i)t\d{4}',tag)) > 0: + mitre_id = re.findall(r'(?i)t\d{4}',tag) + mitre_id = mitre_id[0].upper() + tactics, sub_techniques, technique_id, references = mitre_pull(mitre_id) + ## Prevent multiple writes to file if `mitre` key already exists + if 'mitre' not in sigma_yaml_list[0]: + sigma_yaml_list[0]['mitre'] = {} + sigma_yaml_list[0]['mitre']['tactics'] = [] + sigma_yaml_list[0]['mitre']['subTechniques'] = [] + sigma_yaml_list[0]['mitre']['techniqueIds'] = [] + sigma_yaml_list[0]['mitre']['references'] = [] + if tactics is not None and sub_techniques is not None and technique_id is not None and references is not None: + for tactic in tactics: + ## Make sure we aren't duplicating + if tactic not in sigma_yaml_list[0]['mitre']['tactics']: + sigma_yaml_list[0]['mitre']['tactics'].append(tactic) + if sub_techniques not in sigma_yaml_list[0]['mitre']['subTechniques']: + sigma_yaml_list[0]['mitre']['subTechniques'].append(sub_techniques) + if technique_id not in sigma_yaml_list[0]['mitre']['techniqueIds']: + sigma_yaml_list[0]['mitre']['techniqueIds'].append(technique_id) + if references not in sigma_yaml_list[0]['mitre']['references']: + sigma_yaml_list[0]['mitre']['references'].append(references) + with open(os.path.join(sigmainput, filename), 'w') as f: + yaml.dump(sigma_yaml_list[0], f) click.echo('Results:') From 8e735a01d797fe8b08e6e6bf5699c1e0ff024a52 Mon Sep 17 00:00:00 2001 From: ryandamour Date: Thu, 22 Oct 2020 13:54:06 -0700 Subject: [PATCH 7/9] Add exception --- sigmalint/sigmalint.py | 54 +++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/sigmalint/sigmalint.py b/sigmalint/sigmalint.py index 40683b8..a3b16ee 100644 --- a/sigmalint/sigmalint.py +++ b/sigmalint/sigmalint.py @@ -62,31 +62,35 @@ def cli(sigmainput, directory, method, mitre): if mitre: ## Most mentions of MITRE are currently found in the 'tags' field for Sigma rules if 'tags' in sigma_yaml_list[0]: - for tag in sigma_yaml_list[0]['tags']: - if len(re.findall(r'(?i)t\d{4}',tag)) > 0: - mitre_id = re.findall(r'(?i)t\d{4}',tag) - mitre_id = mitre_id[0].upper() - tactics, sub_techniques, technique_id, references = mitre_pull(mitre_id) - ## Prevent multiple writes to file if `mitre` key already exists - if 'mitre' not in sigma_yaml_list[0]: - sigma_yaml_list[0]['mitre'] = {} - sigma_yaml_list[0]['mitre']['tactics'] = [] - sigma_yaml_list[0]['mitre']['subTechniques'] = [] - sigma_yaml_list[0]['mitre']['techniqueIds'] = [] - sigma_yaml_list[0]['mitre']['references'] = [] - if tactics is not None and sub_techniques is not None and technique_id is not None and references is not None: - for tactic in tactics: - ## Make sure we aren't duplicating - if tactic not in sigma_yaml_list[0]['mitre']['tactics']: - sigma_yaml_list[0]['mitre']['tactics'].append(tactic) - if sub_techniques not in sigma_yaml_list[0]['mitre']['subTechniques']: - sigma_yaml_list[0]['mitre']['subTechniques'].append(sub_techniques) - if technique_id not in sigma_yaml_list[0]['mitre']['techniqueIds']: - sigma_yaml_list[0]['mitre']['techniqueIds'].append(technique_id) - if references not in sigma_yaml_list[0]['mitre']['references']: - sigma_yaml_list[0]['mitre']['references'].append(references) - with open(os.path.join(sigmainput, filename), 'w') as f: - yaml.dump(sigma_yaml_list[0], f) + try: + for tag in sigma_yaml_list[0]['tags']: + if len(re.findall(r'(?i)t\d{4}',tag)) > 0: + mitre_id = re.findall(r'(?i)t\d{4}',tag) + mitre_id = mitre_id[0].upper() + tactics, sub_techniques, technique_id, references = mitre_pull(mitre_id) + ## Prevent multiple writes to file if `mitre` key already exists + if 'mitre' not in sigma_yaml_list[0]: + sigma_yaml_list[0]['mitre'] = {} + sigma_yaml_list[0]['mitre']['tactics'] = [] + sigma_yaml_list[0]['mitre']['subTechniques'] = [] + sigma_yaml_list[0]['mitre']['techniqueIds'] = [] + sigma_yaml_list[0]['mitre']['references'] = [] + if tactics is not None and sub_techniques is not None and technique_id is not None and references is not None: + for tactic in tactics: + ## Make sure we aren't duplicating + if tactic not in sigma_yaml_list[0]['mitre']['tactics']: + sigma_yaml_list[0]['mitre']['tactics'].append(tactic) + if sub_techniques not in sigma_yaml_list[0]['mitre']['subTechniques']: + sigma_yaml_list[0]['mitre']['subTechniques'].append(sub_techniques) + if technique_id not in sigma_yaml_list[0]['mitre']['techniqueIds']: + sigma_yaml_list[0]['mitre']['techniqueIds'].append(technique_id) + if references not in sigma_yaml_list[0]['mitre']['references']: + sigma_yaml_list[0]['mitre']['references'].append(references) + with open(os.path.join(sigmainput, filename), 'w') as f: + yaml.dump(sigma_yaml_list[0], f) + except: + click.secho('Unable to parse {}').format(os.path.join(sigmainput, result['filename'])), fg=color) + click.echo('Results:') From 2fc5bef1853bbd9caf6bb1e4ebbc2d48421feaad Mon Sep 17 00:00:00 2001 From: ryandamour Date: Thu, 22 Oct 2020 13:55:55 -0700 Subject: [PATCH 8/9] Fix secho syntax --- sigmalint/sigmalint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sigmalint/sigmalint.py b/sigmalint/sigmalint.py index a3b16ee..99f409c 100644 --- a/sigmalint/sigmalint.py +++ b/sigmalint/sigmalint.py @@ -89,7 +89,7 @@ def cli(sigmainput, directory, method, mitre): with open(os.path.join(sigmainput, filename), 'w') as f: yaml.dump(sigma_yaml_list[0], f) except: - click.secho('Unable to parse {}').format(os.path.join(sigmainput, result['filename'])), fg=color) + click.secho('Unable to parse {}'.format(os.path.join(sigmainput, result['filename'])), fg=color) click.echo('Results:') From dc7fd95a19f5604102bc0b82bc5af9d10f962b4a Mon Sep 17 00:00:00 2001 From: ryandamour Date: Thu, 22 Oct 2020 13:58:57 -0700 Subject: [PATCH 9/9] secho syntax fixes --- sigmalint/sigmalint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sigmalint/sigmalint.py b/sigmalint/sigmalint.py index 99f409c..2a9211e 100644 --- a/sigmalint/sigmalint.py +++ b/sigmalint/sigmalint.py @@ -89,7 +89,7 @@ def cli(sigmainput, directory, method, mitre): with open(os.path.join(sigmainput, filename), 'w') as f: yaml.dump(sigma_yaml_list[0], f) except: - click.secho('Unable to parse {}'.format(os.path.join(sigmainput, result['filename'])), fg=color) + click.secho('Unable to parse {}'.format(filename)) click.echo('Results:')