From 3b9f356530f90908ccafee6b3fee9f4870276a30 Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Sat, 2 Dec 2023 09:09:21 +0100 Subject: [PATCH 1/7] fix: [code] make panda happy --- CODE/generate_DISARM_pages.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/CODE/generate_DISARM_pages.py b/CODE/generate_DISARM_pages.py index 9481f70..7bac7ec 100644 --- a/CODE/generate_DISARM_pages.py +++ b/CODE/generate_DISARM_pages.py @@ -94,12 +94,12 @@ class Disarm: xlsx = pd.ExcelFile(frameworkfile) for sheetname in xlsx.sheet_names: metadata[sheetname] = xlsx.parse(sheetname) - metadata[sheetname].fillna('', inplace=True) + metadata[sheetname].replace(np.NaN, '', inplace=True) xlsx = pd.ExcelFile(datafile) for sheetname in xlsx.sheet_names: metadata[sheetname] = xlsx.parse(sheetname) - metadata[sheetname].fillna('', inplace=True) + metadata[sheetname].replace(np.NaN, '', inplace=True) # Create individual tables and dictionaries self.df_phases = metadata['phases'] @@ -111,11 +111,11 @@ class Disarm: self.df_tools = metadata['tools'] self.df_examples = metadata['examples'] self.df_counters = metadata['countermeasures'].sort_values('disarm_id') - self.df_counters[['tactic_id', 'tactic_name']] = self.df_counters['tactic'].str.split(' ', 1, expand=True) - self.df_counters[['metatechnique_id', 'metatechnique_name']] = self.df_counters['metatechnique'].str.split(' ', 1, expand=True) + self.df_counters[['tactic_id', 'tactic_name']] = self.df_counters['tactic'].str.split(' ', n=1, expand=True) + self.df_counters[['metatechnique_id', 'metatechnique_name']] = self.df_counters['metatechnique'].str.split(' ', n=1, expand=True) self.df_detections = metadata['detections'] - self.df_detections[['tactic_id', 'tactic_name']] = self.df_detections['tactic'].str.split(' ', 1, expand=True) -# self.df_detections[['metatechnique_id', 'metatechnique_name']] = self.df_detections['metatechnique'].str.split(' ', 1, expand=True) #FIXIT + self.df_detections[['tactic_id', 'tactic_name']] = self.df_detections['tactic'].str.split(' ', n=1, expand=True) +# self.df_detections[['metatechnique_id', 'metatechnique_name']] = self.df_detections['metatechnique'].str.split(' ', n=1, expand=True) #FIXIT self.df_actortypes = metadata['actortypes'] self.df_resources = metadata['resources'] self.df_responsetypes = metadata['responsetypes'] From cc38695dae3414f9303e1012c43042319f2cdd17 Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Sat, 2 Dec 2023 09:14:58 +0100 Subject: [PATCH 2/7] fix: fixes resource_counters_string wrong variable --- CODE/generate_DISARM_pages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODE/generate_DISARM_pages.py b/CODE/generate_DISARM_pages.py index 7bac7ec..18f2c41 100644 --- a/CODE/generate_DISARM_pages.py +++ b/CODE/generate_DISARM_pages.py @@ -301,7 +301,7 @@ class Disarm: resource_counters = self.cross_counterid_resourceid[self.cross_counterid_resourceid['resource_id']==resource_id] resource_counters = pd.merge(resource_counters, self.df_counters[['disarm_id', 'name', 'responsetype']]) row_string = '| [{0} {1}]({2}counters/{0}.md) | {3} |\n' - for index, row in actortype_counters.sort_values('disarm_id').iterrows(): + for index, row in resource_counters.sort_values('disarm_id').iterrows(): table_string += row_string.format(row['disarm_id'], row['name'], GENERATED_PAGES_FUDGE, row['responsetype']) return table_string From d2ca8fbcc7d4488730755fd41eef1c506e83d1ad Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Sat, 2 Dec 2023 18:08:26 +0100 Subject: [PATCH 3/7] new: [code] Generate DISARM as MISP Galaxy fixes #22 --- CODE/generate_DISARM_MISP_galaxy.py | 87 +++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100755 CODE/generate_DISARM_MISP_galaxy.py diff --git a/CODE/generate_DISARM_MISP_galaxy.py b/CODE/generate_DISARM_MISP_galaxy.py new file mode 100755 index 0000000..00645d4 --- /dev/null +++ b/CODE/generate_DISARM_MISP_galaxy.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +""" +Generate DISARM MISP galaxy + see also https://github.com/MISP/misp-galaxy + +Author: Christophe Vandeplas +License: AGPL-3 +""" + +from generate_DISARM_pages import Disarm +import json +import uuid +import os + + +class DisarmGalaxy: + def __init__(self, out_path=os.path.join('..', '..', 'misp-galaxy')): + self.disarm = Disarm() + self.out_path = out_path + + def generate_disarm_galaxy(self): + galaxy = {'name': 'DISARM Techniques', + 'type': 'disarm', + 'description': 'DISARM is a framework designed for describing and understanding disinformation incidents.', + 'uuid': str(uuid.uuid5(uuid.UUID("9319371e-2504-4128-8410-3741cebbcfd3"), 'disarm-galaxy-techniques')), + 'version': 1, + 'icon': 'map', + 'namespace': 'disarm', + 'kill_chain_order': { + 'disarm-tactics': [] + }} + + for k, v in self.disarm.tactics.items(): + galaxy['kill_chain_order']['disarm-tactics'].append(f'{v}') + + self.write_json_file(os.path.join(self.out_path, 'galaxies', 'disarm-techniques.json'), galaxy) + + def write_json_file(self, fname, file_data): + with open(fname, 'w') as f: + json.dump(file_data, f, indent=2, sort_keys=True, ensure_ascii=False) + f.write('\n') + + def generate_disarm_clusters(self): + cluster = {'authors': ['DISARM Project'], + 'category': 'disarm', + 'description': 'DISARM is a framework designed for describing and understanding disinformation incidents.', + 'name': 'DISARM Techniques', + 'source': 'https://github.com/misinfosecproject/amitt_framework', + 'type': 'disarm', + 'uuid': str(uuid.uuid5(uuid.UUID("9319371e-2504-4128-8410-3741cebbcfd3"), 'disarm-cluster-techniques')), + 'values': [], + 'version': 1} + + df = self.disarm.df_techniques + for i in range(len(df)): + t = { + 'uuid': str(uuid.uuid5(uuid.UUID("9319371e-2504-4128-8410-3741cebbcfd3"), df.values[i][0])), + 'value': f"{df.values[i][0]} - {df.values[i][1]}", + 'description': df.values[i][4], + 'meta': { + 'external_id': df.values[i][0], + 'kill_chain': [ + f'disarm-tactics:{self.disarm.tactics[df.values[i][3]]}' + ], + 'refs': [ + f'https://github.com/DISARMFoundation/DISARMframeworks/blob/main/generated_pages/techniques/{df.values[i][0]}.md' + ] + } + } + + cluster['values'].append(t) + + self.write_json_file(os.path.join(self.out_path, 'clusters', 'disarm-techniques.json'), cluster) + + pass + + +def main(): + disarm_galaxy = DisarmGalaxy() + disarm_galaxy.generate_disarm_galaxy() + disarm_galaxy.generate_disarm_clusters() + + +if __name__ == "__main__": + main() + print("All done, please look at the delta, and update the version number if needed.") + print("After that do ./jq_all_the_things.sh, commit, and then ./validate_all.sh.") From f12df578abe271670b43282d3cf249552debdb47 Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Tue, 19 Dec 2023 15:57:20 +0100 Subject: [PATCH 4/7] fix: [code] Fix MISP Galaxy value without ID --- CODE/generate_DISARM_MISP_galaxy.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CODE/generate_DISARM_MISP_galaxy.py b/CODE/generate_DISARM_MISP_galaxy.py index 00645d4..2906e54 100755 --- a/CODE/generate_DISARM_MISP_galaxy.py +++ b/CODE/generate_DISARM_MISP_galaxy.py @@ -50,12 +50,12 @@ class DisarmGalaxy: 'uuid': str(uuid.uuid5(uuid.UUID("9319371e-2504-4128-8410-3741cebbcfd3"), 'disarm-cluster-techniques')), 'values': [], 'version': 1} - + values = [] df = self.disarm.df_techniques for i in range(len(df)): t = { 'uuid': str(uuid.uuid5(uuid.UUID("9319371e-2504-4128-8410-3741cebbcfd3"), df.values[i][0])), - 'value': f"{df.values[i][0]} - {df.values[i][1]}", + 'value': f"{df.values[i][1]}", 'description': df.values[i][4], 'meta': { 'external_id': df.values[i][0], @@ -68,8 +68,8 @@ class DisarmGalaxy: } } - cluster['values'].append(t) - + values.append(t) + cluster['values'] = sorted(values, key=lambda x: x['meta']['external_id']) self.write_json_file(os.path.join(self.out_path, 'clusters', 'disarm-techniques.json'), cluster) pass From 8b8100611c94229accc2aad58c80ebb93f5524b2 Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Wed, 20 Dec 2023 11:29:33 +0100 Subject: [PATCH 5/7] new: [code] MISP Galaxy supports most models --- CODE/generate_DISARM_MISP_galaxy.py | 261 ++++++++++++++++++++++++---- CODE/generate_DISARM_pages.py | 3 + 2 files changed, 232 insertions(+), 32 deletions(-) diff --git a/CODE/generate_DISARM_MISP_galaxy.py b/CODE/generate_DISARM_MISP_galaxy.py index 2906e54..a1783f6 100755 --- a/CODE/generate_DISARM_MISP_galaxy.py +++ b/CODE/generate_DISARM_MISP_galaxy.py @@ -12,55 +12,60 @@ import json import uuid import os +DISARM_DESCRIPTION = 'DISARM is a framework designed for describing and understanding disinformation incidents.' +DISARM_CATEGORY = 'disarm' +DISARM_AUTHORS = ['DISARM Project'] +DISARM_SOURCE = 'https://github.com/DISARMFoundation/DISARMframeworks' +CORE_UUID = "9d6bd9d2-2cd3-4900-b61a-06cd64df3996" class DisarmGalaxy: def __init__(self, out_path=os.path.join('..', '..', 'misp-galaxy')): self.disarm = Disarm() self.out_path = out_path - def generate_disarm_galaxy(self): - galaxy = {'name': 'DISARM Techniques', - 'type': 'disarm', - 'description': 'DISARM is a framework designed for describing and understanding disinformation incidents.', - 'uuid': str(uuid.uuid5(uuid.UUID("9319371e-2504-4128-8410-3741cebbcfd3"), 'disarm-galaxy-techniques')), - 'version': 1, - 'icon': 'map', - 'namespace': 'disarm', - 'kill_chain_order': { - 'disarm-tactics': [] - }} - - for k, v in self.disarm.tactics.items(): - galaxy['kill_chain_order']['disarm-tactics'].append(f'{v}') - - self.write_json_file(os.path.join(self.out_path, 'galaxies', 'disarm-techniques.json'), galaxy) - def write_json_file(self, fname, file_data): with open(fname, 'w') as f: json.dump(file_data, f, indent=2, sort_keys=True, ensure_ascii=False) f.write('\n') - def generate_disarm_clusters(self): - cluster = {'authors': ['DISARM Project'], - 'category': 'disarm', - 'description': 'DISARM is a framework designed for describing and understanding disinformation incidents.', - 'name': 'DISARM Techniques', - 'source': 'https://github.com/misinfosecproject/amitt_framework', - 'type': 'disarm', - 'uuid': str(uuid.uuid5(uuid.UUID("9319371e-2504-4128-8410-3741cebbcfd3"), 'disarm-cluster-techniques')), + def generate_disarm_techniques_galaxy(self): + galaxy = {'name': 'Techniques', + 'type': 'disarm-techniques', + 'description': DISARM_DESCRIPTION, + 'uuid': str(uuid.uuid5(uuid.UUID(CORE_UUID), 'disarm-galaxy-techniques')), + 'version': 1, + 'icon': 'map', + 'namespace': 'disarm', + 'kill_chain_order': { + 'tactics': [] + }} + + for k, v in self.disarm.tactics.items(): + galaxy['kill_chain_order']['tactics'].append(f'{v}') + + self.write_json_file(os.path.join(self.out_path, 'galaxies', 'disarm-techniques.json'), galaxy) + + def generate_disarm_techniques_clusters(self): + cluster = {'authors': DISARM_AUTHORS, + 'category': DISARM_CATEGORY, + 'description': DISARM_DESCRIPTION, + 'name': 'Techniques', + 'source': DISARM_SOURCE, + 'type': 'disarm-techniques', + 'uuid': str(uuid.uuid5(uuid.UUID(CORE_UUID), 'disarm-cluster-techniques')), 'values': [], 'version': 1} values = [] df = self.disarm.df_techniques for i in range(len(df)): t = { - 'uuid': str(uuid.uuid5(uuid.UUID("9319371e-2504-4128-8410-3741cebbcfd3"), df.values[i][0])), - 'value': f"{df.values[i][1]}", + 'uuid': str(uuid.uuid5(uuid.UUID(CORE_UUID), df.values[i][0])), + 'value': df.values[i][1], 'description': df.values[i][4], 'meta': { 'external_id': df.values[i][0], 'kill_chain': [ - f'disarm-tactics:{self.disarm.tactics[df.values[i][3]]}' + f'tactics:{self.disarm.tactics[df.values[i][3]]}' ], 'refs': [ f'https://github.com/DISARMFoundation/DISARMframeworks/blob/main/generated_pages/techniques/{df.values[i][0]}.md' @@ -72,14 +77,206 @@ class DisarmGalaxy: cluster['values'] = sorted(values, key=lambda x: x['meta']['external_id']) self.write_json_file(os.path.join(self.out_path, 'clusters', 'disarm-techniques.json'), cluster) - pass + def generate_disarm_countermeasures_galaxy(self): + galaxy = {'name': 'Countermeasures', + 'type': 'disarm-countermeasures', + 'description': DISARM_DESCRIPTION, + 'uuid': str(uuid.uuid5(uuid.UUID(CORE_UUID), 'disarm-galaxy-counters')), + 'version': 1, + 'icon': 'shield-alt', + 'namespace': 'disarm', + 'kill_chain_order': { + 'tactics': [], + 'responsetypes': [], + 'metatechniques': [] + }} + + for k, v in self.disarm.tactics.items(): + galaxy['kill_chain_order']['tactics'].append(f'{v}') + for k, v in self.disarm.responsetypes.items(): + galaxy['kill_chain_order']['responsetypes'].append(f'{v}') + for k, v in self.disarm.metatechniques.items(): + galaxy['kill_chain_order']['metatechniques'].append(f'{v}') + + self.write_json_file(os.path.join(self.out_path, 'galaxies', 'disarm-countermeasures.json'), galaxy) + + def generate_disarm_countermeasures_clusters(self): + cluster = {'authors': DISARM_AUTHORS, + 'category': DISARM_CATEGORY, + 'description': DISARM_DESCRIPTION, + 'name': 'Countermeasures', + 'source': DISARM_SOURCE, + 'type': 'disarm-countermeasures', + 'uuid': str(uuid.uuid5(uuid.UUID(CORE_UUID), 'disarm-cluster-counters')), + 'values': [], + 'version': 1} + values = [] + df = self.disarm.df_counters + for i in range(len(df)): + kill_chain = [] + if self.disarm.tactics[df.values[i][15]]: + kill_chain.append(f'tactics:{self.disarm.tactics[df.values[i][15]]}') + if self.disarm.responsetypes[df.values[i][10]]: + kill_chain.append(f'responsetypes:{self.disarm.responsetypes[df.values[i][10]]}') + if self.disarm.metatechniques[df.values[i][17]]: + kill_chain.append(f'metatechniques:{self.disarm.metatechniques[df.values[i][17]]}') + + t = { + 'uuid': str(uuid.uuid5(uuid.UUID(CORE_UUID), df.values[i][0])), + 'value': df.values[i][1], + 'description': df.values[i][3], + 'meta': { + 'external_id': df.values[i][0], + 'kill_chain': kill_chain, + 'refs': [ + f'https://github.com/DISARMFoundation/DISARMframeworks/blob/main/generated_pages/counters/{df.values[i][0]}.md' + ] + } + } + + values.append(t) + cluster['values'] = sorted(values, key=lambda x: x['meta']['external_id']) + self.write_json_file(os.path.join(self.out_path, 'clusters', 'disarm-countermeasures.json'), cluster) + + def generate_disarm_detections_galaxy(self): + galaxy = {'name': 'Detections', + 'type': 'disarm-detections', + 'description': DISARM_DESCRIPTION, + 'uuid': str(uuid.uuid5(uuid.UUID(CORE_UUID), 'disarm-galaxy-detections')), + 'version': 1, + 'icon': 'bell', + 'namespace': 'disarm', + 'kill_chain_order': { + 'tactics': [], + 'responsetypes': [], + # 'metatechniques': [] + }} + + for k, v in self.disarm.tactics.items(): + galaxy['kill_chain_order']['tactics'].append(f'{v}') + for k, v in self.disarm.responsetypes.items(): + galaxy['kill_chain_order']['responsetypes'].append(f'{v}') + # for k, v in self.disarm.metatechniques.items(): + # galaxy['kill_chain_order']['metatechniques'].append(f'{v}') + + self.write_json_file(os.path.join(self.out_path, 'galaxies', 'disarm-detections.json'), galaxy) + + def generate_disarm_detections_clusters(self): + cluster = {'authors': DISARM_AUTHORS, + 'category': DISARM_CATEGORY, + 'description': DISARM_DESCRIPTION, + 'name': 'Detections', + 'source': DISARM_SOURCE, + 'type': 'disarm-detections', + 'uuid': str(uuid.uuid5(uuid.UUID(CORE_UUID), 'disarm-cluster-detections')), + 'values': [], + 'version': 1} + values = [] + df = self.disarm.df_detections + for i in range(len(df)): + + kill_chain = [] + try: + if self.disarm.tactics[df.values[i][14]]: + kill_chain.append(f'tactics:{self.disarm.tactics[df.values[i][14]]}') + except KeyError: + pass + try: + if self.disarm.responsetypes[df.values[i][10]]: + kill_chain.append(f'responsetypes:{self.disarm.responsetypes[df.values[i][10]]}') + except KeyError: + pass + # Metatechnique ID is not in the array + # if self.disarm.metatechniques[df.values[i][???]]: + # kill_chain.append(f'metatechniques:{self.disarm.metatechniques[df.values[i][???]]}') + + t = { + 'uuid': str(uuid.uuid5(uuid.UUID(CORE_UUID), df.values[i][0])), + 'value': df.values[i][1], + 'description': df.values[i][3], + 'meta': { + 'external_id': df.values[i][0], + 'kill_chain': kill_chain, + 'refs': [ + f'https://github.com/DISARMFoundation/DISARMframeworks/blob/main/generated_pages/detections/{df.values[i][0]}.md' + ] + } + } + + values.append(t) + cluster['values'] = sorted(values, key=lambda x: x['meta']['external_id']) + self.write_json_file(os.path.join(self.out_path, 'clusters', 'disarm-detections.json'), cluster) + + + def generate_disarm_actortypes_galaxy(self): + galaxy = {'name': 'Actor Types', + 'type': 'disarm-actortypes', + 'description': DISARM_DESCRIPTION, + 'uuid': str(uuid.uuid5(uuid.UUID(CORE_UUID), 'disarm-galaxy-actortypes')), + 'version': 1, + 'icon': 'user-secret', + 'namespace': 'disarm', + 'kill_chain_order': { + 'sectors': [] + }} + + for k, v in self.disarm.sectors.items(): + galaxy['kill_chain_order']['sectors'].append(f'{v}') + + self.write_json_file(os.path.join(self.out_path, 'galaxies', 'disarm-actortypes.json'), galaxy) + + def generate_disarm_actortypes_clusters(self): + cluster = {'authors': DISARM_AUTHORS, + 'category': DISARM_CATEGORY, + 'description': DISARM_DESCRIPTION, + 'name': 'Actor Types', + 'source': DISARM_SOURCE, + 'type': 'disarm-actortypes', + 'uuid': str(uuid.uuid5(uuid.UUID(CORE_UUID), 'disarm-cluster-actortypes')), + 'values': [], + 'version': 1} + values = [] + df = self.disarm.df_actortypes + for i in range(len(df)): + + kill_chain = [] + try: + sectors = df.values[i][3].split(',') + for sector in sectors: + sector = sector.strip() + if self.disarm.sectors[sector]: + kill_chain.append(f'sectors:{self.disarm.sectors[sector]}') + except KeyError: + pass + + t = { + 'uuid': str(uuid.uuid5(uuid.UUID(CORE_UUID), df.values[i][0])), + 'value': df.values[i][1], + 'description': df.values[i][2], + 'meta': { + 'external_id': df.values[i][0], + 'kill_chain': kill_chain, + 'refs': [ + f'https://github.com/DISARMFoundation/DISARMframeworks/blob/main/generated_pages/actortypes/{df.values[i][0]}.md' + ] + } + } + + values.append(t) + cluster['values'] = sorted(values, key=lambda x: x['meta']['external_id']) + self.write_json_file(os.path.join(self.out_path, 'clusters', 'disarm-actortypes.json'), cluster) def main(): disarm_galaxy = DisarmGalaxy() - disarm_galaxy.generate_disarm_galaxy() - disarm_galaxy.generate_disarm_clusters() - + disarm_galaxy.generate_disarm_techniques_galaxy() + disarm_galaxy.generate_disarm_techniques_clusters() + disarm_galaxy.generate_disarm_countermeasures_galaxy() + disarm_galaxy.generate_disarm_countermeasures_clusters() + disarm_galaxy.generate_disarm_detections_galaxy() + disarm_galaxy.generate_disarm_detections_clusters() + disarm_galaxy.generate_disarm_actortypes_galaxy() + disarm_galaxy.generate_disarm_actortypes_clusters() if __name__ == "__main__": main() diff --git a/CODE/generate_DISARM_pages.py b/CODE/generate_DISARM_pages.py index 18f2c41..93df35f 100644 --- a/CODE/generate_DISARM_pages.py +++ b/CODE/generate_DISARM_pages.py @@ -123,6 +123,7 @@ class Disarm: self.it = self.create_incident_technique_crosstable(metadata['incidenttechniques']) self.df_tactics = metadata['tactics'] self.df_playbooks = metadata['playbooks'] + self.df_sectors = metadata['sectors'] # Add columns containing lists of techniques and counters to the tactics dataframe self.df_techniques_per_tactic = self.df_techniques.groupby('tactic_id')['disarm_id'].apply(list).reset_index().rename({'disarm_id':'technique_ids'}, axis=1) @@ -136,8 +137,10 @@ class Disarm: self.techniques = self.make_object_dictionary(self.df_techniques) self.counters = self.make_object_dictionary(self.df_counters) self.metatechniques = self.make_object_dictionary(self.df_metatechniques) + self.responsetypes = self.make_object_dictionary(self.df_responsetypes) self.actortypes = self.make_object_dictionary(self.df_actortypes) self.resources = self.make_object_dictionary(self.df_resources) + self.sectors = self.make_object_dictionary(self.df_sectors) # Create the data table for each framework file self.num_tactics = len(self.df_tactics) From 197dcd3aea66dd3692fda74bcd204fd635458a78 Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Wed, 20 Dec 2023 16:18:39 +0100 Subject: [PATCH 6/7] new: [code] MISP Galaxy supports relations --- CODE/generate_DISARM_MISP_galaxy.py | 276 +++++++++++++++++++++------- 1 file changed, 206 insertions(+), 70 deletions(-) diff --git a/CODE/generate_DISARM_MISP_galaxy.py b/CODE/generate_DISARM_MISP_galaxy.py index a1783f6..cd00633 100755 --- a/CODE/generate_DISARM_MISP_galaxy.py +++ b/CODE/generate_DISARM_MISP_galaxy.py @@ -12,27 +12,47 @@ import json import uuid import os + DISARM_DESCRIPTION = 'DISARM is a framework designed for describing and understanding disinformation incidents.' DISARM_CATEGORY = 'disarm' DISARM_AUTHORS = ['DISARM Project'] DISARM_SOURCE = 'https://github.com/DISARMFoundation/DISARMframeworks' CORE_UUID = "9d6bd9d2-2cd3-4900-b61a-06cd64df3996" + class DisarmGalaxy: def __init__(self, out_path=os.path.join('..', '..', 'misp-galaxy')): self.disarm = Disarm() self.out_path = out_path + self.all_data = {} # variable that will contain everything + self.all_data_uuid = {} # used to compute references + + self.galaxy_types = ['techniques', 'countermeasures', 'detections', 'actortypes'] + + def generate_all_galaxies(self): + for galaxy_type in self.galaxy_types: + getattr(self, f'generate_{galaxy_type}_galaxy')() # also saves the files + + def generate_all_clusters(self): + # first build up the data + for galaxy_type in self.galaxy_types: + getattr(self, f'generate_{galaxy_type}_clusters')() + # write all to files + for galaxy_type in self.galaxy_types: + self.write_json_file(os.path.join(self.out_path, 'clusters', f'disarm-{galaxy_type}.json'), self.all_data[galaxy_type]) + def write_json_file(self, fname, file_data): with open(fname, 'w') as f: json.dump(file_data, f, indent=2, sort_keys=True, ensure_ascii=False) f.write('\n') - def generate_disarm_techniques_galaxy(self): + def generate_techniques_galaxy(self): + galaxy_type = 'techniques' galaxy = {'name': 'Techniques', - 'type': 'disarm-techniques', + 'type': f'disarm-{galaxy_type}', 'description': DISARM_DESCRIPTION, - 'uuid': str(uuid.uuid5(uuid.UUID(CORE_UUID), 'disarm-galaxy-techniques')), + 'uuid': str(uuid.uuid5(uuid.UUID(CORE_UUID), f'disarm-galaxy-{galaxy_type}')), 'version': 1, 'icon': 'map', 'namespace': 'disarm', @@ -43,45 +63,75 @@ class DisarmGalaxy: for k, v in self.disarm.tactics.items(): galaxy['kill_chain_order']['tactics'].append(f'{v}') - self.write_json_file(os.path.join(self.out_path, 'galaxies', 'disarm-techniques.json'), galaxy) + self.write_json_file(os.path.join(self.out_path, 'galaxies', f'disarm-{galaxy_type}.json'), galaxy) - def generate_disarm_techniques_clusters(self): + def generate_techniques_clusters(self): + galaxy_type = 'techniques' cluster = {'authors': DISARM_AUTHORS, 'category': DISARM_CATEGORY, 'description': DISARM_DESCRIPTION, 'name': 'Techniques', 'source': DISARM_SOURCE, - 'type': 'disarm-techniques', - 'uuid': str(uuid.uuid5(uuid.UUID(CORE_UUID), 'disarm-cluster-techniques')), + 'type': f'disarm-{galaxy_type}', + 'uuid': str(uuid.uuid5(uuid.UUID(CORE_UUID), f'disarm-cluster-{galaxy_type}')), 'values': [], 'version': 1} values = [] df = self.disarm.df_techniques for i in range(len(df)): - t = { - 'uuid': str(uuid.uuid5(uuid.UUID(CORE_UUID), df.values[i][0])), + entry_id = df.values[i][0] + kill_chain = [f'tactics:{self.disarm.tactics[df.values[i][3]]}'] + related = [] + # Countermeasures relations + mapping = self.disarm.cross_counterid_techniqueid[ + self.disarm.cross_counterid_techniqueid['technique_id'] == entry_id] + for index, row in mapping.sort_values('disarm_id').iterrows(): + related_id = row['disarm_id'] + related.append({ + "dest-uuid": str(uuid.uuid5(uuid.UUID(CORE_UUID), related_id)), + # "tags": [ + # "estimative-language:likelihood-probability=\"almost-certain\"" + # ], + "type": "blocked-by" # mitigated-by would be cleaner, but does not exist as relationship type + }) + # Detections relations + mapping = self.disarm.cross_detectionid_techniqueid[ + self.disarm.cross_detectionid_techniqueid['technique_id'] == entry_id] + for index, row in mapping.sort_values('disarm_id').iterrows(): + related_id = row['disarm_id'] + related.append({ + "dest-uuid": str(uuid.uuid5(uuid.UUID(CORE_UUID), related_id)), + # "tags": [ + # "estimative-language:likelihood-probability=\"almost-certain\"" + # ], + "type": "detected-by" + }) + + value = { + 'uuid': str(uuid.uuid5(uuid.UUID(CORE_UUID), entry_id)), 'value': df.values[i][1], 'description': df.values[i][4], 'meta': { - 'external_id': df.values[i][0], - 'kill_chain': [ - f'tactics:{self.disarm.tactics[df.values[i][3]]}' - ], + 'external_id': entry_id, + 'kill_chain': kill_chain, 'refs': [ - f'https://github.com/DISARMFoundation/DISARMframeworks/blob/main/generated_pages/techniques/{df.values[i][0]}.md' + f'https://github.com/DISARMFoundation/DISARMframeworks/blob/main/generated_pages/{galaxy_type}/{entry_id}.md' ] - } + }, + 'related': related } + values.append(value) + self.all_data_uuid[value['uuid']] = value - values.append(t) cluster['values'] = sorted(values, key=lambda x: x['meta']['external_id']) - self.write_json_file(os.path.join(self.out_path, 'clusters', 'disarm-techniques.json'), cluster) + self.all_data[galaxy_type] = cluster - def generate_disarm_countermeasures_galaxy(self): + def generate_countermeasures_galaxy(self): + galaxy_type = 'countermeasures' galaxy = {'name': 'Countermeasures', - 'type': 'disarm-countermeasures', + 'type': f'disarm-{galaxy_type}', 'description': DISARM_DESCRIPTION, - 'uuid': str(uuid.uuid5(uuid.UUID(CORE_UUID), 'disarm-galaxy-counters')), + 'uuid': str(uuid.uuid5(uuid.UUID(CORE_UUID), f'disarm-galaxy-{galaxy_type}')), 'version': 1, 'icon': 'shield-alt', 'namespace': 'disarm', @@ -98,21 +148,23 @@ class DisarmGalaxy: for k, v in self.disarm.metatechniques.items(): galaxy['kill_chain_order']['metatechniques'].append(f'{v}') - self.write_json_file(os.path.join(self.out_path, 'galaxies', 'disarm-countermeasures.json'), galaxy) + self.write_json_file(os.path.join(self.out_path, 'galaxies', f'disarm-{galaxy_type}.json'), galaxy) - def generate_disarm_countermeasures_clusters(self): + def generate_countermeasures_clusters(self): + galaxy_type = 'countermeasures' cluster = {'authors': DISARM_AUTHORS, 'category': DISARM_CATEGORY, 'description': DISARM_DESCRIPTION, 'name': 'Countermeasures', 'source': DISARM_SOURCE, - 'type': 'disarm-countermeasures', - 'uuid': str(uuid.uuid5(uuid.UUID(CORE_UUID), 'disarm-cluster-counters')), + 'type': f'disarm-{galaxy_type}', + 'uuid': str(uuid.uuid5(uuid.UUID(CORE_UUID), f'disarm-cluster-{galaxy_type}')), 'values': [], 'version': 1} values = [] df = self.disarm.df_counters for i in range(len(df)): + entry_id = df.values[i][0] kill_chain = [] if self.disarm.tactics[df.values[i][15]]: kill_chain.append(f'tactics:{self.disarm.tactics[df.values[i][15]]}') @@ -121,28 +173,58 @@ class DisarmGalaxy: if self.disarm.metatechniques[df.values[i][17]]: kill_chain.append(f'metatechniques:{self.disarm.metatechniques[df.values[i][17]]}') - t = { - 'uuid': str(uuid.uuid5(uuid.UUID(CORE_UUID), df.values[i][0])), + related = [] + # Techniques relations + mapping = self.disarm.cross_counterid_techniqueid[ + self.disarm.cross_counterid_techniqueid['disarm_id'] == entry_id] + for index, row in mapping.sort_values('technique_id').iterrows(): + related_id = row['technique_id'] + related.append({ + "dest-uuid": str(uuid.uuid5(uuid.UUID(CORE_UUID), related_id)), + # "tags": [ + # "estimative-language:likelihood-probability=\"almost-certain\"" + # ], + "type": "blocks" # mitigated would be cleaner, but mitigated-by does not exist as relationship type + }) + # Actortype relations + mapping = self.disarm.cross_counterid_actortypeid[ + self.disarm.cross_counterid_actortypeid['disarm_id'] == entry_id] + for index, row in mapping.sort_values('actortype_id').iterrows(): + related_id = row['actortype_id'] + related.append({ + "dest-uuid": str(uuid.uuid5(uuid.UUID(CORE_UUID), related_id)), + # "tags": [ + # "estimative-language:likelihood-probability=\"almost-certain\"" + # ], + "type": "affected-by" + # mitigated-by would be cleaner, but mitigated-by does not exist as relationship type + }) + + value = { + 'uuid': str(uuid.uuid5(uuid.UUID(CORE_UUID), entry_id)), 'value': df.values[i][1], 'description': df.values[i][3], 'meta': { - 'external_id': df.values[i][0], + 'external_id': entry_id, 'kill_chain': kill_chain, 'refs': [ - f'https://github.com/DISARMFoundation/DISARMframeworks/blob/main/generated_pages/counters/{df.values[i][0]}.md' + f'https://github.com/DISARMFoundation/DISARMframeworks/blob/main/generated_pages/counters/{entry_id}.md' ] - } + }, + 'related': related } + values.append(value) + self.all_data_uuid[value['uuid']] = value - values.append(t) cluster['values'] = sorted(values, key=lambda x: x['meta']['external_id']) - self.write_json_file(os.path.join(self.out_path, 'clusters', 'disarm-countermeasures.json'), cluster) + self.all_data[galaxy_type] = cluster - def generate_disarm_detections_galaxy(self): + def generate_detections_galaxy(self): + galaxy_type = 'detections' galaxy = {'name': 'Detections', - 'type': 'disarm-detections', + 'type': f'disarm-{galaxy_type}', 'description': DISARM_DESCRIPTION, - 'uuid': str(uuid.uuid5(uuid.UUID(CORE_UUID), 'disarm-galaxy-detections')), + 'uuid': str(uuid.uuid5(uuid.UUID(CORE_UUID), f'disarm-galaxy-{galaxy_type}')), 'version': 1, 'icon': 'bell', 'namespace': 'disarm', @@ -159,22 +241,23 @@ class DisarmGalaxy: # for k, v in self.disarm.metatechniques.items(): # galaxy['kill_chain_order']['metatechniques'].append(f'{v}') - self.write_json_file(os.path.join(self.out_path, 'galaxies', 'disarm-detections.json'), galaxy) + self.write_json_file(os.path.join(self.out_path, 'galaxies', f'disarm-{galaxy_type}.json'), galaxy) - def generate_disarm_detections_clusters(self): + def generate_detections_clusters(self): + galaxy_type = 'detections' cluster = {'authors': DISARM_AUTHORS, 'category': DISARM_CATEGORY, 'description': DISARM_DESCRIPTION, 'name': 'Detections', 'source': DISARM_SOURCE, - 'type': 'disarm-detections', - 'uuid': str(uuid.uuid5(uuid.UUID(CORE_UUID), 'disarm-cluster-detections')), + 'type': f'disarm-{galaxy_type}', + 'uuid': str(uuid.uuid5(uuid.UUID(CORE_UUID), f'disarm-cluster-{galaxy_type}')), 'values': [], 'version': 1} values = [] df = self.disarm.df_detections for i in range(len(df)): - + entry_id = df.values[i][0] kill_chain = [] try: if self.disarm.tactics[df.values[i][14]]: @@ -190,29 +273,58 @@ class DisarmGalaxy: # if self.disarm.metatechniques[df.values[i][???]]: # kill_chain.append(f'metatechniques:{self.disarm.metatechniques[df.values[i][???]]}') - t = { - 'uuid': str(uuid.uuid5(uuid.UUID(CORE_UUID), df.values[i][0])), + related = [] + # Techniques relations + mapping = self.disarm.cross_detectionid_techniqueid[ + self.disarm.cross_detectionid_techniqueid['disarm_id'] == entry_id] + for index, row in mapping.sort_values('technique_id').iterrows(): + related_id = row['technique_id'] + related.append({ + "dest-uuid": str(uuid.uuid5(uuid.UUID(CORE_UUID), related_id)), + # "tags": [ + # "estimative-language:likelihood-probability=\"almost-certain\"" + # ], + "type": "detects" + }) + # Actortypes relations + mapping = self.disarm.cross_detectionid_actortypeid[ + self.disarm.cross_detectionid_actortypeid['disarm_id'] == entry_id] + for index, row in mapping.sort_values('actortype_id').iterrows(): + related_id = row['actortype_id'] + related.append({ + "dest-uuid": str(uuid.uuid5(uuid.UUID(CORE_UUID), related_id)), + # "tags": [ + # "estimative-language:likelihood-probability=\"almost-certain\"" + # ], + "type": "detected-by" + # mitigated-by would be cleaner, but mitigated-by does not exist as relationship type + }) + + value = { + 'uuid': str(uuid.uuid5(uuid.UUID(CORE_UUID), entry_id)), 'value': df.values[i][1], 'description': df.values[i][3], 'meta': { - 'external_id': df.values[i][0], + 'external_id': entry_id, 'kill_chain': kill_chain, 'refs': [ - f'https://github.com/DISARMFoundation/DISARMframeworks/blob/main/generated_pages/detections/{df.values[i][0]}.md' + f'https://github.com/DISARMFoundation/DISARMframeworks/blob/main/generated_pages/{galaxy_type}/{entry_id}.md' ] - } + }, + 'related': related } + values.append(value) + self.all_data_uuid[value['uuid']] = value - values.append(t) cluster['values'] = sorted(values, key=lambda x: x['meta']['external_id']) - self.write_json_file(os.path.join(self.out_path, 'clusters', 'disarm-detections.json'), cluster) + self.all_data[galaxy_type] = cluster - - def generate_disarm_actortypes_galaxy(self): + def generate_actortypes_galaxy(self): + galaxy_type = 'actortypes' galaxy = {'name': 'Actor Types', - 'type': 'disarm-actortypes', + 'type': f'disarm-{galaxy_type}', 'description': DISARM_DESCRIPTION, - 'uuid': str(uuid.uuid5(uuid.UUID(CORE_UUID), 'disarm-galaxy-actortypes')), + 'uuid': str(uuid.uuid5(uuid.UUID(CORE_UUID), f'disarm-galaxy-{galaxy_type}')), 'version': 1, 'icon': 'user-secret', 'namespace': 'disarm', @@ -223,22 +335,23 @@ class DisarmGalaxy: for k, v in self.disarm.sectors.items(): galaxy['kill_chain_order']['sectors'].append(f'{v}') - self.write_json_file(os.path.join(self.out_path, 'galaxies', 'disarm-actortypes.json'), galaxy) + self.write_json_file(os.path.join(self.out_path, 'galaxies', f'disarm-{galaxy_type}.json'), galaxy) - def generate_disarm_actortypes_clusters(self): + def generate_actortypes_clusters(self): + galaxy_type = 'actortypes' cluster = {'authors': DISARM_AUTHORS, 'category': DISARM_CATEGORY, 'description': DISARM_DESCRIPTION, 'name': 'Actor Types', 'source': DISARM_SOURCE, - 'type': 'disarm-actortypes', - 'uuid': str(uuid.uuid5(uuid.UUID(CORE_UUID), 'disarm-cluster-actortypes')), + 'type': f'disarm-{galaxy_type}', + 'uuid': str(uuid.uuid5(uuid.UUID(CORE_UUID), f'disarm-cluster-{galaxy_type}')), 'values': [], 'version': 1} values = [] df = self.disarm.df_actortypes for i in range(len(df)): - + entry_id = df.values[i][0] kill_chain = [] try: sectors = df.values[i][3].split(',') @@ -249,34 +362,57 @@ class DisarmGalaxy: except KeyError: pass - t = { - 'uuid': str(uuid.uuid5(uuid.UUID(CORE_UUID), df.values[i][0])), + related = [] + # Countermeasures relations + mapping = self.disarm.cross_counterid_actortypeid[ + self.disarm.cross_counterid_actortypeid['actortype_id'] == entry_id] + for index, row in mapping.sort_values('disarm_id').iterrows(): + related_id = row['disarm_id'] + related.append({ + "dest-uuid": str(uuid.uuid5(uuid.UUID(CORE_UUID), related_id)), + # "tags": [ + # "estimative-language:likelihood-probability=\"almost-certain\"" + # ], + "type": "affects" + }) + # Detections relations + mapping = self.disarm.cross_detectionid_actortypeid[ + self.disarm.cross_detectionid_actortypeid['actortype_id'] == entry_id] + for index, row in mapping.sort_values('disarm_id').iterrows(): + related_id = row['disarm_id'] + related.append({ + "dest-uuid": str(uuid.uuid5(uuid.UUID(CORE_UUID), related_id)), + # "tags": [ + # "estimative-language:likelihood-probability=\"almost-certain\"" + # ], + "type": "detects" + }) + + value = { + 'uuid': str(uuid.uuid5(uuid.UUID(CORE_UUID), entry_id)), 'value': df.values[i][1], 'description': df.values[i][2], 'meta': { - 'external_id': df.values[i][0], + 'external_id': entry_id, 'kill_chain': kill_chain, 'refs': [ - f'https://github.com/DISARMFoundation/DISARMframeworks/blob/main/generated_pages/actortypes/{df.values[i][0]}.md' + f'https://github.com/DISARMFoundation/DISARMframeworks/blob/main/generated_pages/{galaxy_type}/{entry_id}.md' ] - } + }, + 'related': related } - values.append(t) + values.append(value) + self.all_data_uuid[value['uuid']] = value cluster['values'] = sorted(values, key=lambda x: x['meta']['external_id']) - self.write_json_file(os.path.join(self.out_path, 'clusters', 'disarm-actortypes.json'), cluster) + self.all_data[galaxy_type] = cluster def main(): disarm_galaxy = DisarmGalaxy() - disarm_galaxy.generate_disarm_techniques_galaxy() - disarm_galaxy.generate_disarm_techniques_clusters() - disarm_galaxy.generate_disarm_countermeasures_galaxy() - disarm_galaxy.generate_disarm_countermeasures_clusters() - disarm_galaxy.generate_disarm_detections_galaxy() - disarm_galaxy.generate_disarm_detections_clusters() - disarm_galaxy.generate_disarm_actortypes_galaxy() - disarm_galaxy.generate_disarm_actortypes_clusters() + disarm_galaxy.generate_all_galaxies() + disarm_galaxy.generate_all_clusters() + if __name__ == "__main__": main() From 352e5411ec20a003241b30bd9a9d5df53ce304d4 Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Thu, 21 Dec 2023 09:16:38 +0100 Subject: [PATCH 7/7] fix: [code] MISP Galaxy skips duplicate values --- CODE/generate_DISARM_MISP_galaxy.py | 63 +++++++++++------------------ 1 file changed, 24 insertions(+), 39 deletions(-) diff --git a/CODE/generate_DISARM_MISP_galaxy.py b/CODE/generate_DISARM_MISP_galaxy.py index cd00633..21e2912 100755 --- a/CODE/generate_DISARM_MISP_galaxy.py +++ b/CODE/generate_DISARM_MISP_galaxy.py @@ -25,9 +25,6 @@ class DisarmGalaxy: self.disarm = Disarm() self.out_path = out_path - self.all_data = {} # variable that will contain everything - self.all_data_uuid = {} # used to compute references - self.galaxy_types = ['techniques', 'countermeasures', 'detections', 'actortypes'] def generate_all_galaxies(self): @@ -35,12 +32,8 @@ class DisarmGalaxy: getattr(self, f'generate_{galaxy_type}_galaxy')() # also saves the files def generate_all_clusters(self): - # first build up the data for galaxy_type in self.galaxy_types: getattr(self, f'generate_{galaxy_type}_clusters')() - # write all to files - for galaxy_type in self.galaxy_types: - self.write_json_file(os.path.join(self.out_path, 'clusters', f'disarm-{galaxy_type}.json'), self.all_data[galaxy_type]) def write_json_file(self, fname, file_data): with open(fname, 'w') as f: @@ -77,8 +70,13 @@ class DisarmGalaxy: 'values': [], 'version': 1} values = [] + seen_values = [] df = self.disarm.df_techniques for i in range(len(df)): + if df.values[i][1] in seen_values: # remove duplicates + continue + seen_values.append(df.values[i][1]) + entry_id = df.values[i][0] kill_chain = [f'tactics:{self.disarm.tactics[df.values[i][3]]}'] related = [] @@ -89,9 +87,6 @@ class DisarmGalaxy: related_id = row['disarm_id'] related.append({ "dest-uuid": str(uuid.uuid5(uuid.UUID(CORE_UUID), related_id)), - # "tags": [ - # "estimative-language:likelihood-probability=\"almost-certain\"" - # ], "type": "blocked-by" # mitigated-by would be cleaner, but does not exist as relationship type }) # Detections relations @@ -101,9 +96,6 @@ class DisarmGalaxy: related_id = row['disarm_id'] related.append({ "dest-uuid": str(uuid.uuid5(uuid.UUID(CORE_UUID), related_id)), - # "tags": [ - # "estimative-language:likelihood-probability=\"almost-certain\"" - # ], "type": "detected-by" }) @@ -121,10 +113,9 @@ class DisarmGalaxy: 'related': related } values.append(value) - self.all_data_uuid[value['uuid']] = value cluster['values'] = sorted(values, key=lambda x: x['meta']['external_id']) - self.all_data[galaxy_type] = cluster + self.write_json_file(os.path.join(self.out_path, 'clusters', f'disarm-{galaxy_type}.json'), cluster) def generate_countermeasures_galaxy(self): galaxy_type = 'countermeasures' @@ -162,8 +153,13 @@ class DisarmGalaxy: 'values': [], 'version': 1} values = [] + seen_values = [] df = self.disarm.df_counters for i in range(len(df)): + if df.values[i][1] in seen_values: # remove duplicates + continue + seen_values.append(df.values[i][1]) + entry_id = df.values[i][0] kill_chain = [] if self.disarm.tactics[df.values[i][15]]: @@ -181,9 +177,6 @@ class DisarmGalaxy: related_id = row['technique_id'] related.append({ "dest-uuid": str(uuid.uuid5(uuid.UUID(CORE_UUID), related_id)), - # "tags": [ - # "estimative-language:likelihood-probability=\"almost-certain\"" - # ], "type": "blocks" # mitigated would be cleaner, but mitigated-by does not exist as relationship type }) # Actortype relations @@ -193,9 +186,6 @@ class DisarmGalaxy: related_id = row['actortype_id'] related.append({ "dest-uuid": str(uuid.uuid5(uuid.UUID(CORE_UUID), related_id)), - # "tags": [ - # "estimative-language:likelihood-probability=\"almost-certain\"" - # ], "type": "affected-by" # mitigated-by would be cleaner, but mitigated-by does not exist as relationship type }) @@ -214,10 +204,9 @@ class DisarmGalaxy: 'related': related } values.append(value) - self.all_data_uuid[value['uuid']] = value cluster['values'] = sorted(values, key=lambda x: x['meta']['external_id']) - self.all_data[galaxy_type] = cluster + self.write_json_file(os.path.join(self.out_path, 'clusters', f'disarm-{galaxy_type}.json'), cluster) def generate_detections_galaxy(self): galaxy_type = 'detections' @@ -255,8 +244,13 @@ class DisarmGalaxy: 'values': [], 'version': 1} values = [] + seen_values = [] df = self.disarm.df_detections for i in range(len(df)): + if df.values[i][1] in seen_values: # remove duplicates + continue + seen_values.append(df.values[i][1]) + entry_id = df.values[i][0] kill_chain = [] try: @@ -281,9 +275,6 @@ class DisarmGalaxy: related_id = row['technique_id'] related.append({ "dest-uuid": str(uuid.uuid5(uuid.UUID(CORE_UUID), related_id)), - # "tags": [ - # "estimative-language:likelihood-probability=\"almost-certain\"" - # ], "type": "detects" }) # Actortypes relations @@ -293,9 +284,6 @@ class DisarmGalaxy: related_id = row['actortype_id'] related.append({ "dest-uuid": str(uuid.uuid5(uuid.UUID(CORE_UUID), related_id)), - # "tags": [ - # "estimative-language:likelihood-probability=\"almost-certain\"" - # ], "type": "detected-by" # mitigated-by would be cleaner, but mitigated-by does not exist as relationship type }) @@ -314,10 +302,9 @@ class DisarmGalaxy: 'related': related } values.append(value) - self.all_data_uuid[value['uuid']] = value cluster['values'] = sorted(values, key=lambda x: x['meta']['external_id']) - self.all_data[galaxy_type] = cluster + self.write_json_file(os.path.join(self.out_path, 'clusters', f'disarm-{galaxy_type}.json'), cluster) def generate_actortypes_galaxy(self): galaxy_type = 'actortypes' @@ -349,8 +336,13 @@ class DisarmGalaxy: 'values': [], 'version': 1} values = [] + seen_values = [] df = self.disarm.df_actortypes for i in range(len(df)): + if df.values[i][1] in seen_values: # remove duplicates + continue + seen_values.append(df.values[i][1]) + entry_id = df.values[i][0] kill_chain = [] try: @@ -370,9 +362,6 @@ class DisarmGalaxy: related_id = row['disarm_id'] related.append({ "dest-uuid": str(uuid.uuid5(uuid.UUID(CORE_UUID), related_id)), - # "tags": [ - # "estimative-language:likelihood-probability=\"almost-certain\"" - # ], "type": "affects" }) # Detections relations @@ -382,9 +371,6 @@ class DisarmGalaxy: related_id = row['disarm_id'] related.append({ "dest-uuid": str(uuid.uuid5(uuid.UUID(CORE_UUID), related_id)), - # "tags": [ - # "estimative-language:likelihood-probability=\"almost-certain\"" - # ], "type": "detects" }) @@ -403,9 +389,8 @@ class DisarmGalaxy: } values.append(value) - self.all_data_uuid[value['uuid']] = value cluster['values'] = sorted(values, key=lambda x: x['meta']['external_id']) - self.all_data[galaxy_type] = cluster + self.write_json_file(os.path.join(self.out_path, 'clusters', f'disarm-{galaxy_type}.json'), cluster) def main():