Added EEAS framework objects and STIX generator

Added framework objects:
- Added technique T0066 "Degrade adversary" to TA02
- Added technique T0067 "Plan to discredit credible sources" to TA02
- Added technique T0068 "respond to breaking news event" to TA02
- Added technique T0069  "respond to active crisis" to TA02
- Added technique T0070 "Analyze existing communities" to TA02
- Added technique T0071 "Find echo chambers" to TA13
- Added technique T0072  "Segment audiences" to TA13

Added STIX generator from repo DISARM-stix2, and added code to generate github files, databases, and STIX from the same Jupyter notebook.

CODE/.ipynb_checkpoints/generate_DISARM_pages-checkpoint.ipynb

@@ -4,9 +4,16 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "# Generate DISARM github files\n",
+    "# Generate DISARM files\n",
     "\n",
-    "Generate all the DISARM github files from the DISARM master spreadsheets, being careful to retain any comments people have made below the \"don't write above this\" line in them. "
+    "Generate DISARM files and database objects from the DISARM master spreadsheets."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Step 1. Generate DISARM github pages"
    ]
   },
   {
@@ -54,9 +61,64 @@
     "disarm.generate_and_write_datafiles()"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Step 2. Generate DISARM STIX objects"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "'conduct-center-of-gravity-analysis' is not a recognized DISARM Tactic.\n",
+      "'drive-offline-activity' is not a recognized DISARM Tactic.\n"
+     ]
+    }
+   ],
+   "source": [
+    "import sys\n",
+    "sys.path.insert(0, \"DISARM-STIX2\")\n",
+    "from main import generate_disarm_stix\n",
+    "\n",
+    "generate_disarm_stix()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Step 3. Generate DISARM database objects"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from generate_disarm_sql import generate_disarm_sql\n",
+    "\n",
+    "generate_disarm_sql('sqlite')\n",
+    "generate_disarm_sql('postgresql_local')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Step 4 (optional). Look at datasets"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
    "metadata": {
     "scrolled": true
    },
@@ -88,6 +150,7 @@
        "      <th>sector_ids</th>\n",
        "      <th>framework_ids</th>\n",
        "      <th>longname</th>\n",
+       "      <th>Jon's comments/questions</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
@@ -99,6 +162,7 @@
        "      <td>S001, S002, S003, S004, S005, S006, S007, S008...</td>\n",
        "      <td>FW01, FW02</td>\n",
        "      <td>A001 - data scientist</td>\n",
+       "      <td>What actual actions do data scientists execute...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
@@ -108,6 +172,7 @@
        "      <td>S001, S002, S003, S004, S005, S006, S007, S008...</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A002 - target</td>\n",
+       "      <td>Both red and blue framework users will refer t...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
@@ -117,6 +182,7 @@
        "      <td>S001, S002, S003, S004, S005, S006, S007, S008...</td>\n",
        "      <td>FW01, FW02</td>\n",
        "      <td>A003 - trusted authority</td>\n",
+       "      <td>Is A003 best summarised by the word \"Inluencer...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
@@ -126,6 +192,7 @@
        "      <td>S002</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A004 - activist</td>\n",
+       "      <td>What does this actortype do?</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
@@ -135,6 +202,7 @@
        "      <td>S002</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A005 - community group</td>\n",
+       "      <td>What does this actortype do?</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>5</th>\n",
@@ -144,6 +212,7 @@
        "      <td>S002</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A006 - educator</td>\n",
+       "      <td></td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>6</th>\n",
@@ -153,6 +222,7 @@
        "      <td>S002</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A007 - factchecker</td>\n",
+       "      <td></td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>7</th>\n",
@@ -162,6 +232,7 @@
        "      <td>S002</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A008 - library</td>\n",
+       "      <td>Is this an actortype? A sub-sector, yes. But w...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>8</th>\n",
@@ -171,6 +242,7 @@
        "      <td>S002</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A009 - NGO</td>\n",
+       "      <td>This is a sector, not an actortype</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>9</th>\n",
@@ -180,6 +252,7 @@
        "      <td>S002</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A010 - religious organisation</td>\n",
+       "      <td>\"Faith communities\" are a sub-sector of the Ci...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>10</th>\n",
@@ -189,6 +262,7 @@
        "      <td>S002</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A011 - school</td>\n",
+       "      <td>This is a subsector of S004 - not an actortype...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>11</th>\n",
@@ -198,6 +272,7 @@
        "      <td>S006</td>\n",
        "      <td>FW01\\nFW02</td>\n",
        "      <td>A012 - account owner</td>\n",
+       "      <td></td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>12</th>\n",
@@ -207,6 +282,7 @@
        "      <td>S006</td>\n",
        "      <td>FW01\\nFW02</td>\n",
        "      <td>A013 - content creator</td>\n",
+       "      <td></td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>13</th>\n",
@@ -216,6 +292,7 @@
        "      <td>S006</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A014 - elves</td>\n",
+       "      <td>??? SJ, you have told me, but I've forgotten</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>14</th>\n",
@@ -225,6 +302,7 @@
        "      <td>S006</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A015 - general public</td>\n",
+       "      <td>Duplication of a field in the sector object. B...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>15</th>\n",
@@ -234,6 +312,7 @@
        "      <td>S006</td>\n",
        "      <td>FW01\\nFW02</td>\n",
        "      <td>A016 - influencer</td>\n",
+       "      <td></td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>16</th>\n",
@@ -243,6 +322,7 @@
        "      <td>S003</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A017 - coordinating body</td>\n",
+       "      <td>\"Response coordinator\"? Or just 'Coordinator\"</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>17</th>\n",
@@ -252,6 +332,7 @@
        "      <td>S003</td>\n",
        "      <td>FW01\\nFW02</td>\n",
        "      <td>A018 - government</td>\n",
+       "      <td>DHS (A017) is \"government\". Again, this is a f...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>18</th>\n",
@@ -261,6 +342,7 @@
        "      <td>S003</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A019 - military</td>\n",
+       "      <td>A019 is a sub-sector of \"government\". Again, t...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>19</th>\n",
@@ -270,6 +352,7 @@
        "      <td>S003</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A020 - policy maker</td>\n",
+       "      <td></td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>20</th>\n",
@@ -279,6 +362,7 @@
        "      <td>S010</td>\n",
        "      <td>FW01\\nFW02</td>\n",
        "      <td>A021 - media organisation</td>\n",
+       "      <td>Not an actortype. The actor is perhaps as the ...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>21</th>\n",
@@ -288,6 +372,7 @@
        "      <td>S009</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A022 - company</td>\n",
+       "      <td></td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>22</th>\n",
@@ -297,6 +382,7 @@
        "      <td>S008</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A023 - adtech provider</td>\n",
+       "      <td>Should Adtech be a called-out subsector of S00...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>23</th>\n",
@@ -306,6 +392,7 @@
        "      <td>S008</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A024 - developer</td>\n",
+       "      <td></td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>24</th>\n",
@@ -315,6 +402,7 @@
        "      <td>S008</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A025 - funding_site_admin</td>\n",
+       "      <td>What is this?</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>25</th>\n",
@@ -324,6 +412,7 @@
        "      <td>S008</td>\n",
        "      <td>FW01, FW02</td>\n",
        "      <td>A026 - games designer</td>\n",
+       "      <td>Just \"Designer\"? Would the TTP context not mak...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>26</th>\n",
@@ -333,6 +422,7 @@
        "      <td>S008</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A027 - information security</td>\n",
+       "      <td></td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>27</th>\n",
@@ -342,6 +432,7 @@
        "      <td>S008</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A028 - platform administrator</td>\n",
+       "      <td>Just \"Administrator\"?</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>28</th>\n",
@@ -351,6 +442,7 @@
        "      <td>S008</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A029 - server admininistrator</td>\n",
+       "      <td>Just \"Administrator\"? (de-duplicating A028)</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>29</th>\n",
@@ -360,6 +452,7 @@
        "      <td>S007</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A030 - platforms</td>\n",
+       "      <td>The same as S007? If there is an action here, ...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>30</th>\n",
@@ -369,6 +462,7 @@
        "      <td>S007</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A031 - social media platform adminstrator</td>\n",
+       "      <td>Not needed, given A028. The sector selection d...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>31</th>\n",
@@ -378,6 +472,7 @@
        "      <td>S007</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A032 - social media platform outreach</td>\n",
+       "      <td>What does this do?</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>32</th>\n",
@@ -387,6 +482,7 @@
        "      <td>S007</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A033 - social media platform owner</td>\n",
+       "      <td></td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
@@ -498,48 +594,84 @@
        "31                                               S007          FW02   \n",
        "32                                               S007          FW02   \n",
        "\n",
-       "                                     longname  \n",
-       "0                      A001 - data scientist   \n",
-       "1                               A002 - target  \n",
-       "2                   A003 - trusted authority   \n",
-       "3                             A004 - activist  \n",
-       "4                      A005 - community group  \n",
-       "5                             A006 - educator  \n",
-       "6                          A007 - factchecker  \n",
-       "7                              A008 - library  \n",
-       "8                                  A009 - NGO  \n",
-       "9              A010 - religious organisation   \n",
-       "10                             A011 - school   \n",
-       "11                       A012 - account owner  \n",
-       "12                    A013 - content creator   \n",
-       "13                               A014 - elves  \n",
-       "14                      A015 - general public  \n",
-       "15                          A016 - influencer  \n",
-       "16                   A017 - coordinating body  \n",
-       "17                         A018 - government   \n",
-       "18                           A019 - military   \n",
-       "19                        A020 - policy maker  \n",
-       "20                  A021 - media organisation  \n",
-       "21                             A022 - company  \n",
-       "22                     A023 - adtech provider  \n",
-       "23                           A024 - developer  \n",
-       "24                  A025 - funding_site_admin  \n",
-       "25                      A026 - games designer  \n",
-       "26                A027 - information security  \n",
-       "27              A028 - platform administrator  \n",
-       "28             A029 - server admininistrator   \n",
-       "29                          A030 - platforms   \n",
-       "30  A031 - social media platform adminstrator  \n",
-       "31     A032 - social media platform outreach   \n",
-       "32         A033 - social media platform owner  "
+       "                                     longname  \\\n",
+       "0                      A001 - data scientist    \n",
+       "1                               A002 - target   \n",
+       "2                   A003 - trusted authority    \n",
+       "3                             A004 - activist   \n",
+       "4                      A005 - community group   \n",
+       "5                             A006 - educator   \n",
+       "6                          A007 - factchecker   \n",
+       "7                              A008 - library   \n",
+       "8                                  A009 - NGO   \n",
+       "9              A010 - religious organisation    \n",
+       "10                             A011 - school    \n",
+       "11                       A012 - account owner   \n",
+       "12                    A013 - content creator    \n",
+       "13                               A014 - elves   \n",
+       "14                      A015 - general public   \n",
+       "15                          A016 - influencer   \n",
+       "16                   A017 - coordinating body   \n",
+       "17                         A018 - government    \n",
+       "18                           A019 - military    \n",
+       "19                        A020 - policy maker   \n",
+       "20                  A021 - media organisation   \n",
+       "21                             A022 - company   \n",
+       "22                     A023 - adtech provider   \n",
+       "23                           A024 - developer   \n",
+       "24                  A025 - funding_site_admin   \n",
+       "25                      A026 - games designer   \n",
+       "26                A027 - information security   \n",
+       "27              A028 - platform administrator   \n",
+       "28             A029 - server admininistrator    \n",
+       "29                          A030 - platforms    \n",
+       "30  A031 - social media platform adminstrator   \n",
+       "31     A032 - social media platform outreach    \n",
+       "32         A033 - social media platform owner   \n",
+       "\n",
+       "                             Jon's comments/questions  \n",
+       "0   What actual actions do data scientists execute...  \n",
+       "1   Both red and blue framework users will refer t...  \n",
+       "2   Is A003 best summarised by the word \"Inluencer...  \n",
+       "3                        What does this actortype do?  \n",
+       "4                        What does this actortype do?  \n",
+       "5                                                      \n",
+       "6                                                      \n",
+       "7   Is this an actortype? A sub-sector, yes. But w...  \n",
+       "8                  This is a sector, not an actortype  \n",
+       "9   \"Faith communities\" are a sub-sector of the Ci...  \n",
+       "10  This is a subsector of S004 - not an actortype...  \n",
+       "11                                                     \n",
+       "12                                                     \n",
+       "13       ??? SJ, you have told me, but I've forgotten  \n",
+       "14  Duplication of a field in the sector object. B...  \n",
+       "15                                                     \n",
+       "16      \"Response coordinator\"? Or just 'Coordinator\"  \n",
+       "17  DHS (A017) is \"government\". Again, this is a f...  \n",
+       "18  A019 is a sub-sector of \"government\". Again, t...  \n",
+       "19                                                     \n",
+       "20  Not an actortype. The actor is perhaps as the ...  \n",
+       "21                                                     \n",
+       "22  Should Adtech be a called-out subsector of S00...  \n",
+       "23                                                     \n",
+       "24                                      What is this?  \n",
+       "25  Just \"Designer\"? Would the TTP context not mak...  \n",
+       "26                                                     \n",
+       "27                              Just \"Administrator\"?  \n",
+       "28        Just \"Administrator\"? (de-duplicating A028)  \n",
+       "29  The same as S007? If there is an action here, ...  \n",
+       "30  Not needed, given A028. The sector selection d...  \n",
+       "31                                 What does this do?  \n",
+       "32                                                     "
       ]
      },
-     "execution_count": 2,
+     "execution_count": 4,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
+    "pd.set_option('display.max_rows', 1000)\n",
     "disarm.df_actortypes"
    ]
   },

CODE/DISARM-STIX2/README.md

@@ -0,0 +1,33 @@
+# DISARM STIX2 Generator
+
+## Usage
+
+1. Clone this repository.
+2. Download the latest version of the DISARM Framework xlsx [here](https://github.com/DISARMFoundation/DISARMframeworks).
+3. Copy the xlsx to the root directory of this repository.
+4. Run `python3 main.py` to generate STIX objects in the `output/` folder.
+5. `output/DISARM.json` contains the complete STIX bundle.  The folders in `output/` contain individual objects for reference.
+
+## DISARM STIX2
+
+The DISARM STIX2 Generator encodes the DISARM object into the corresponding STIX2 object shown in the following table.
+
+| DISARM    | STIX2                 |
+|-----------|-----------------------|
+| Matrix    | Matrix (MITRE custom) |
+| Tactic    | Tactic (MITRE custom) |
+| Technique | AttackPattern         |
+
+## MITRE ATT&CK Navigator
+
+DISARM STIX is compatible with the MITRE ATT&CK Navigator.
+
+DISARM object types, such as `Matrix`, `Tatic` are prefixed with `x-mitre--` for compatibility reasons.
+
+DISARM `AttackPattern` objects also contain `x_mitre_is_subtechnique` and `x_mitre_platforms` properties for compatability.  These properties cannot be removed without upstream changes to the ATT&CK Navigator.
+
+## OpenCTI
+
+DISARM STIX can be imported into OpenCTI via the OpenCTI STIX Importer plugin which is installed in OpenCTI by default.
+Alternatively, use the OpenCTI DISARM plugin to continuously pull the latest DISARM STIX.
+

CODE/DISARM-STIX2/helpers/file.py

@@ -0,0 +1,82 @@
+import json
+import os
+from stix2 import Bundle
+import shutil
+
+outdir = '../generated_files/DISARM_STIX/'
+
+def write_disarm_dir(dir, outdir=outdir):
+    """
+
+     Args:
+        dir (str): a directory name
+
+    Returns:
+
+    """
+    try:
+        os.mkdir(outdir)
+    except FileExistsError:
+        pass
+
+    try:
+        os.mkdir(outdir + dir)
+    except FileExistsError:
+        pass
+
+
+def clean_output_dir(outdir=outdir):
+    """Recursively delete the output folder.
+
+    Returns:
+
+    """
+    try:
+        os.mkdir(outdir)
+    except FileExistsError:
+        pass
+
+    shutil.rmtree(outdir)
+
+
+def write_file(file_name, file_data):
+    """Write a JSON file to outdir
+
+    Args:
+        file_name (str): a file name
+        file_data (str): the file json data
+
+    Returns:
+
+    """
+    with open(file_name, 'w') as f:
+        # f.write(json.dumps(file_data, sort_keys=True, indent=4))
+        f.write(file_data.serialize(pretty=True))
+        f.write('\n')
+
+
+def write_files(stix_objects, outdir=outdir):
+    """
+
+    Args:
+        stix_objects:
+
+    Returns:
+
+    """
+    for i in stix_objects:
+        write_disarm_dir(i.type)
+        write_file(outdir+f"{i.type}/{i.id}.json", Bundle(i, allow_custom=True))
+
+
+def write_bundle(bundle, bundle_name, outdir=outdir):
+    """
+
+    Args:
+        bundle:
+        bundle_name:
+
+    Returns:
+
+    """
+    write_file(outdir+f"{bundle_name}.json", bundle)

CODE/DISARM-STIX2/helpers/xlsx.py

@@ -0,0 +1,19 @@
+import pandas as pd
+
+
+def load_excel_data(infile):
+    """Load an xlsx document.
+
+    Args:
+        infile (str): Path to an xlsx file.
+
+    Returns:
+        dict: xlsx sheets
+
+    """
+    sheets = {}
+    xlsx = pd.ExcelFile(infile)
+    for sheetname in xlsx.sheet_names:
+        sheets[sheetname] = xlsx.parse(sheetname)
+    return sheets
+

CODE/DISARM-STIX2/main.py

@@ -0,0 +1,49 @@
+# A STIX bundle generator for the DISARM Framework.
+#
+# Author: Roger Johnston, Twitter: @VV_X_7
+# License: GPL-3
+
+import pandas as pd
+import openpyxl
+from stix2 import (Bundle, AttackPattern, ThreatActor, IntrusionSet, Relationship, CustomObject, properties,
+                   Malware, Tool, Campaign, Identity, MarkingDefinition, ExternalReference, StatementMarking,
+                   GranularMarking, Location, MemoryStore, Filter)
+from stix2.properties import (ReferenceProperty, ListProperty, StringProperty, TimestampProperty, BooleanProperty, IntegerProperty)
+
+import helpers
+from objects import tactic, technique, matrix, bundle
+from helpers import xlsx, file
+
+
+def generate_disarm_stix():
+    """Generates a DISARM STIX bundle.
+
+    Returns:
+
+    """
+    data = helpers.xlsx.load_excel_data("../DISARM_MASTER_DATA/DISARM_FRAMEWORKS_MASTER.xlsx")
+
+    tactics = tactic.make_disarm_tactics(data)
+    techniques = technique.make_disarm_techniques(data)
+    navigator_matrix = matrix.make_disarm_matrix(tactics)
+
+    stix_objects = []
+    stix_objects.append(tactics)
+    stix_objects.append(techniques)
+    stix_objects.append(navigator_matrix)
+
+    stix_objects = [item for sublist in stix_objects for item in sublist]
+
+    # for i in stix_objects:
+    #     print(i)
+
+    disarm_bundle = bundle.make_stix_bundle(stix_objects)
+
+    helpers.file.clean_output_dir()
+    helpers.file.write_files(stix_objects)
+    helpers.file.write_bundle(disarm_bundle, "DISARM")
+
+
+
+if __name__ == "__main__":
+    generate_disarm_stix()

CODE/DISARM-STIX2/objects/bundle.py

@@ -0,0 +1,15 @@
+from stix2 import Bundle
+
+
+def make_stix_bundle(stix_objects):
+    """Makes a STIX Bundle object.
+
+    Args:
+        stix_objects (list): A list of STIX objects.
+
+    Returns:
+        Bundle: A STIX Bundle object.
+
+    """
+    bundle = Bundle(stix_objects, allow_custom=True)
+    return bundle

CODE/DISARM-STIX2/objects/identity.py

@@ -0,0 +1,16 @@
+from stix2 import Identity
+
+
+def make_disarm_identity():
+    """Creates the default DISARM identity used for indicating authorship of various components in the bundle.
+
+    Returns:
+        identity: a STIX Identity object
+
+    """
+    identity = Identity(
+        name="DISARM Foundation",
+        identity_class="organization",
+        description="DISARM is a framework designed for describing and understanding disinformation incidents.",
+    )
+    return identity

CODE/DISARM-STIX2/objects/marking_definition.py

@@ -0,0 +1,11 @@
+from stix2 import MarkingDefinition, StatementMarking
+from objects import identity
+
+
+def make_disarm_marking_definition():
+    marking_definition = MarkingDefinition(
+        definition_type="statement",
+        created_by_ref=identity.make_disarm_identity(),
+        definition=StatementMarking(statement="CC-BY-SA-4.0 DISARM Foundation")
+    )
+    return marking_definition

CODE/DISARM-STIX2/objects/matrix.py

@@ -0,0 +1,51 @@
+from stix2 import CustomObject, properties, ExternalReference
+
+import objects.marking_definition
+from objects import identity, marking_definition
+
+
+@CustomObject('x-mitre-matrix', [
+    ('name', properties.StringProperty(required=True)),
+    ('description', properties.StringProperty(required=True)),
+    ('tactic_refs', properties.ListProperty(properties.ReferenceProperty(valid_types="SDO"), required=True))
+])
+class Matrix(object):
+    def __init__(self, **kwargs):
+        if True:
+            pass
+
+
+def make_disarm_matrix(tactics):
+    """Creates a Matrix object.
+
+    Args:
+        tactics: A list of Tactic objects.
+
+    Returns:
+
+    """
+    description = 'DISARM is a framework designed for describing and understanding disinformation incidents.'
+    external_references = [
+        {
+            "external_id": "DISARM",
+            "source_name": "DISARM",
+            "url": "https://github.com/DISARMFoundation"
+        }
+    ]
+    name = 'DISARM Framework'
+
+    # print(tactics)
+    # p =[i.id for i in tactics]
+    # r = properties.ReferenceProperty()
+    # f = properties.ListProperty(r)
+
+    tactic_refs = [i.id for i in tactics]
+
+    matrix = Matrix(
+        name=name,
+        description=description,
+        external_references=external_references,
+        tactic_refs=tactic_refs,
+        allow_custom=True
+    )
+    return [matrix]

CODE/DISARM-STIX2/objects/relationship.py

@@ -0,0 +1,21 @@
+from stix2 import Relationship, properties, ExternalReference
+
+
+def make_disarm_subtechnique_relationship(source, target):
+    """Creates a relationship between the parent technique and sub-technique.
+
+    Args:
+        source (str): Subtechnique ID
+        target (str): Parent technique ID
+
+    Returns:
+        A Relationship object.
+
+    """
+    relationship = Relationship(
+        source_ref=source,
+        target_ref=target,
+        relationship_type="subtechnique-of"
+    )
+
+    return relationship

CODE/DISARM-STIX2/objects/tactic.py

@@ -0,0 +1,57 @@
+from stix2 import CustomObject, properties, ExternalReference
+
+import objects.marking_definition
+from objects import identity, marking_definition
+
+
+@CustomObject('x-mitre-tactic', [
+    ('name', properties.StringProperty(required=True)),
+    ('description', properties.StringProperty(required=True)),
+    ('x_mitre_shortname', properties.StringProperty(required=True)),
+    ('external_references', properties.ListProperty(ExternalReference))
+])
+class Tactic(object):
+    def __init__(self, x_mitre_shortname=None, **kwargs):
+        if x_mitre_shortname and x_mitre_shortname not in ["strategic-planning", "objective-planning",
+                                                           "develop-people", "develop-persona",
+                                                           "develop-networks", "microtargeting", "develop-content",
+                                                           "channel-selection", "pump-priming", "exposure",
+                                                           "go-physical",
+                                                           "persistence", "measure-effectiveness"]:
+            # raise ValueError("'%s' is not a recognized DISARM Tactic." % x_mitre_shortname)
+            print("'%s' is not a recognized DISARM Tactic." % x_mitre_shortname)
+
+
+def make_disarm_tactics(data):
+    """Create all DISARM tactic objects.
+
+    Args:
+        data: The xlsx tactic sheet.
+
+    Returns:
+        A list of Tactics.
+
+    """
+    tactics = []
+    for t in data["tactics"].values.tolist():
+        external_references = [
+            {
+                'external_id': f'{t[0]}',
+                'source_name': 'DISARM',
+                'url': f'https://github.com/DISARMFoundation/DISARM_framework/blob/master/tactics/{t[0]}.md'
+            }
+        ]
+
+        tactic = Tactic(
+            name=f"{t[1]}",
+            description=f"{t[5]}",
+            x_mitre_shortname=f'{t[1].lower().replace(" ", "-")}',
+            external_references=external_references,
+            object_marking_refs=objects.marking_definition.make_disarm_marking_definition(),
+            created_by_ref=objects.identity.make_disarm_identity()
+        )
+
+        tactics.append(tactic)
+
+    return tactics
+

CODE/DISARM-STIX2/objects/technique.py

@@ -0,0 +1,71 @@
+from stix2 import AttackPattern, properties, ExternalReference
+import objects.marking_definition
+import pandas as pd
+from objects import identity, marking_definition
+
+
+def make_disarm_techniques(data):
+    """Create all DISARM Techniques objects.
+
+    Args:
+        data: The xlsx technique sheet.
+
+    Returns:
+        A list of Techniques.
+
+    """
+    tacdict = pd.Series(data["tactics"].name.values, index=data["tactics"].disarm_id).to_dict()
+    techniques = []
+    for t in data["techniques"].values.tolist():
+        external_references = [
+            {
+                'external_id': f'{t[0]}',
+                'source_name': 'DISARM',
+                'url': f'https://github.com/DISARMFoundation/DISARM_framework/blob/master/techniques/{t[0]}.md'
+            }
+        ]
+
+        kill_chain_phases = [
+            {
+                'phase_name': tacdict[t[3]].replace(' ', '-').lower(),
+                'kill_chain_name': 'mitre-attack'
+            }
+        ]
+
+        subtechnique = t[0].split(".")
+        x_mitre_is_subtechnique = False
+        if len(subtechnique) > 1:
+            x_mitre_is_subtechnique = True
+
+        # MITRE ATT&CK Navigator expect techniques to have at least one of these platforms.
+        # Without one, the technique will not render in the Navigator.
+        x_mitre_platforms = 'Windows', 'Linux', 'Mac'
+
+        technique = AttackPattern(
+            name=f"{t[1]}",
+            description=f"{t[3]}",
+            external_references=external_references,
+            object_marking_refs=objects.marking_definition.make_disarm_marking_definition(),
+            created_by_ref=objects.identity.make_disarm_identity(),
+            kill_chain_phases=kill_chain_phases,
+            custom_properties={
+                'x_mitre_platforms': x_mitre_platforms,
+                'x_mitre_version': "1,0",
+                'x_mitre_is_subtechnique': x_mitre_is_subtechnique
+            }
+        )
+
+        techniques.append(technique)
+    return techniques
+
+
+def make_subtechnique_map(techinques):
+    """
+
+    Args:
+        techinques:
+
+    Returns:
+
+    """
+    pass

CODE/DISARM-STIX2/requirements.txt

@@ -0,0 +1,3 @@
+pandas
+openpyxl
+stix2

CODE/generate_DISARM_pages.ipynb

@@ -4,9 +4,16 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "# Generate DISARM github files\n",
+    "# Generate DISARM files\n",
     "\n",
-    "Generate all the DISARM github files from the DISARM master spreadsheets, being careful to retain any comments people have made below the \"don't write above this\" line in them. "
+    "Generate DISARM files and database objects from the DISARM master spreadsheets."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Step 1. Generate DISARM github pages"
    ]
   },
   {
@@ -28,96 +35,14 @@
       "updated ../generated_pages/phases_index.md\n",
       "Temp: objecttype tactic\n",
       "updated ../generated_pages/tactics_index.md\n",
-      "Updating ../generated_pages/tactics/TA01.md\n",
-      "Updating ../generated_pages/tactics/TA02.md\n",
-      "Updating ../generated_pages/tactics/TA13.md\n",
-      "Updating ../generated_pages/tactics/TA04.md\n",
-      "Updating ../generated_pages/tactics/TA05.md\n",
-      "Updating ../generated_pages/tactics/TA06.md\n",
-      "Updating ../generated_pages/tactics/TA08.md\n",
-      "Updating ../generated_pages/tactics/TA10.md\n",
       "Temp: objecttype technique\n",
       "updated ../generated_pages/techniques_index.md\n",
-      "Updating ../generated_pages/techniques/T0004.md\n",
-      "Updating ../generated_pages/techniques/T0006.md\n",
-      "Updating ../generated_pages/techniques/T0011.md\n",
-      "Updating ../generated_pages/techniques/T0014.md\n",
-      "Updating ../generated_pages/techniques/T0015.md\n",
-      "Updating ../generated_pages/techniques/T0017.md\n",
-      "Updating ../generated_pages/techniques/T0018.md\n",
-      "Updating ../generated_pages/techniques/T0026.md\n",
-      "Updating ../generated_pages/techniques/T0065.md\n",
       "Temp: objecttype task\n",
       "updated ../generated_pages/tasks_index.md\n",
       "Temp: objecttype incident\n",
       "updated ../generated_pages/incidents_index.md\n",
-      "Updating ../generated_pages/incidents/I00002.md\n",
-      "Updating ../generated_pages/incidents/I00005.md\n",
-      "Updating ../generated_pages/incidents/I00006.md\n",
-      "Updating ../generated_pages/incidents/I00017.md\n",
-      "Updating ../generated_pages/incidents/I00042.md\n",
       "Temp: objecttype counter\n",
       "updated ../generated_pages/counters_index.md\n",
-      "Updating ../generated_pages/counters/C00006.md\n",
-      "Updating ../generated_pages/counters/C00008.md\n",
-      "Updating ../generated_pages/counters/C00010.md\n",
-      "Updating ../generated_pages/counters/C00011.md\n",
-      "Updating ../generated_pages/counters/C00012.md\n",
-      "Updating ../generated_pages/counters/C00014.md\n",
-      "Updating ../generated_pages/counters/C00016.md\n",
-      "Updating ../generated_pages/counters/C00021.md\n",
-      "Updating ../generated_pages/counters/C00022.md\n",
-      "Updating ../generated_pages/counters/C00024.md\n",
-      "Updating ../generated_pages/counters/C00027.md\n",
-      "Updating ../generated_pages/counters/C00029.md\n",
-      "Updating ../generated_pages/counters/C00030.md\n",
-      "Updating ../generated_pages/counters/C00031.md\n",
-      "Updating ../generated_pages/counters/C00036.md\n",
-      "Updating ../generated_pages/counters/C00042.md\n",
-      "Updating ../generated_pages/counters/C00053.md\n",
-      "Updating ../generated_pages/counters/C00058.md\n",
-      "Updating ../generated_pages/counters/C00059.md\n",
-      "Updating ../generated_pages/counters/C00065.md\n",
-      "Updating ../generated_pages/counters/C00066.md\n",
-      "Updating ../generated_pages/counters/C00067.md\n",
-      "Updating ../generated_pages/counters/C00070.md\n",
-      "Updating ../generated_pages/counters/C00073.md\n",
-      "Updating ../generated_pages/counters/C00074.md\n",
-      "Updating ../generated_pages/counters/C00076.md\n",
-      "Updating ../generated_pages/counters/C00080.md\n",
-      "Updating ../generated_pages/counters/C00081.md\n",
-      "Updating ../generated_pages/counters/C00082.md\n",
-      "Updating ../generated_pages/counters/C00084.md\n",
-      "Updating ../generated_pages/counters/C00085.md\n",
-      "Updating ../generated_pages/counters/C00093.md\n",
-      "Updating ../generated_pages/counters/C00094.md\n",
-      "Updating ../generated_pages/counters/C00096.md\n",
-      "Updating ../generated_pages/counters/C00098.md\n",
-      "Updating ../generated_pages/counters/C00105.md\n",
-      "Updating ../generated_pages/counters/C00107.md\n",
-      "Updating ../generated_pages/counters/C00117.md\n",
-      "Updating ../generated_pages/counters/C00119.md\n",
-      "Updating ../generated_pages/counters/C00122.md\n",
-      "Updating ../generated_pages/counters/C00125.md\n",
-      "Updating ../generated_pages/counters/C00126.md\n",
-      "Updating ../generated_pages/counters/C00129.md\n",
-      "Updating ../generated_pages/counters/C00133.md\n",
-      "Updating ../generated_pages/counters/C00153.md\n",
-      "Updating ../generated_pages/counters/C00155.md\n",
-      "Updating ../generated_pages/counters/C00156.md\n",
-      "Updating ../generated_pages/counters/C00161.md\n",
-      "Updating ../generated_pages/counters/C00162.md\n",
-      "Updating ../generated_pages/counters/C00164.md\n",
-      "Updating ../generated_pages/counters/C00169.md\n",
-      "Updating ../generated_pages/counters/C00176.md\n",
-      "Updating ../generated_pages/counters/C00178.md\n",
-      "Updating ../generated_pages/counters/C00182.md\n",
-      "Updating ../generated_pages/counters/C00189.md\n",
-      "Updating ../generated_pages/counters/C00195.md\n",
-      "Updating ../generated_pages/counters/C00197.md\n",
-      "Updating ../generated_pages/counters/C00211.md\n",
-      "Updating ../generated_pages/counters/C00216.md\n",
-      "Updating ../generated_pages/counters/C00219.md\n",
       "Temp: objecttype metatechnique\n",
       "updated ../generated_pages/metatechniques_index.md\n",
       "Temp: objecttype actortype\n",
@@ -136,9 +61,64 @@
     "disarm.generate_and_write_datafiles()"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Step 2. Generate DISARM STIX objects"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "'conduct-center-of-gravity-analysis' is not a recognized DISARM Tactic.\n",
+      "'drive-offline-activity' is not a recognized DISARM Tactic.\n"
+     ]
+    }
+   ],
+   "source": [
+    "import sys\n",
+    "sys.path.insert(0, \"DISARM-STIX2\")\n",
+    "from main import generate_disarm_stix\n",
+    "\n",
+    "generate_disarm_stix()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Step 3. Generate DISARM database objects"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from generate_disarm_sql import generate_disarm_sql\n",
+    "\n",
+    "generate_disarm_sql('sqlite')\n",
+    "generate_disarm_sql('postgresql_local')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Step 4 (optional). Look at datasets"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
    "metadata": {
     "scrolled": true
    },
@@ -170,6 +150,7 @@
        "      <th>sector_ids</th>\n",
        "      <th>framework_ids</th>\n",
        "      <th>longname</th>\n",
+       "      <th>Jon's comments/questions</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
@@ -181,6 +162,7 @@
        "      <td>S001, S002, S003, S004, S005, S006, S007, S008...</td>\n",
        "      <td>FW01, FW02</td>\n",
        "      <td>A001 - data scientist</td>\n",
+       "      <td>What actual actions do data scientists execute...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
@@ -190,6 +172,7 @@
        "      <td>S001, S002, S003, S004, S005, S006, S007, S008...</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A002 - target</td>\n",
+       "      <td>Both red and blue framework users will refer t...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
@@ -199,6 +182,7 @@
        "      <td>S001, S002, S003, S004, S005, S006, S007, S008...</td>\n",
        "      <td>FW01, FW02</td>\n",
        "      <td>A003 - trusted authority</td>\n",
+       "      <td>Is A003 best summarised by the word \"Inluencer...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
@@ -208,6 +192,7 @@
        "      <td>S002</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A004 - activist</td>\n",
+       "      <td>What does this actortype do?</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
@@ -217,6 +202,7 @@
        "      <td>S002</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A005 - community group</td>\n",
+       "      <td>What does this actortype do?</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>5</th>\n",
@@ -226,6 +212,7 @@
        "      <td>S002</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A006 - educator</td>\n",
+       "      <td></td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>6</th>\n",
@@ -235,6 +222,7 @@
        "      <td>S002</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A007 - factchecker</td>\n",
+       "      <td></td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>7</th>\n",
@@ -244,6 +232,7 @@
        "      <td>S002</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A008 - library</td>\n",
+       "      <td>Is this an actortype? A sub-sector, yes. But w...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>8</th>\n",
@@ -253,6 +242,7 @@
        "      <td>S002</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A009 - NGO</td>\n",
+       "      <td>This is a sector, not an actortype</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>9</th>\n",
@@ -262,6 +252,7 @@
        "      <td>S002</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A010 - religious organisation</td>\n",
+       "      <td>\"Faith communities\" are a sub-sector of the Ci...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>10</th>\n",
@@ -271,6 +262,7 @@
        "      <td>S002</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A011 - school</td>\n",
+       "      <td>This is a subsector of S004 - not an actortype...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>11</th>\n",
@@ -280,6 +272,7 @@
        "      <td>S006</td>\n",
        "      <td>FW01\\nFW02</td>\n",
        "      <td>A012 - account owner</td>\n",
+       "      <td></td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>12</th>\n",
@@ -289,6 +282,7 @@
        "      <td>S006</td>\n",
        "      <td>FW01\\nFW02</td>\n",
        "      <td>A013 - content creator</td>\n",
+       "      <td></td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>13</th>\n",
@@ -298,6 +292,7 @@
        "      <td>S006</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A014 - elves</td>\n",
+       "      <td>??? SJ, you have told me, but I've forgotten</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>14</th>\n",
@@ -307,6 +302,7 @@
        "      <td>S006</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A015 - general public</td>\n",
+       "      <td>Duplication of a field in the sector object. B...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>15</th>\n",
@@ -316,6 +312,7 @@
        "      <td>S006</td>\n",
        "      <td>FW01\\nFW02</td>\n",
        "      <td>A016 - influencer</td>\n",
+       "      <td></td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>16</th>\n",
@@ -325,6 +322,7 @@
        "      <td>S003</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A017 - coordinating body</td>\n",
+       "      <td>\"Response coordinator\"? Or just 'Coordinator\"</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>17</th>\n",
@@ -334,6 +332,7 @@
        "      <td>S003</td>\n",
        "      <td>FW01\\nFW02</td>\n",
        "      <td>A018 - government</td>\n",
+       "      <td>DHS (A017) is \"government\". Again, this is a f...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>18</th>\n",
@@ -343,6 +342,7 @@
        "      <td>S003</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A019 - military</td>\n",
+       "      <td>A019 is a sub-sector of \"government\". Again, t...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>19</th>\n",
@@ -352,6 +352,7 @@
        "      <td>S003</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A020 - policy maker</td>\n",
+       "      <td></td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>20</th>\n",
@@ -361,6 +362,7 @@
        "      <td>S010</td>\n",
        "      <td>FW01\\nFW02</td>\n",
        "      <td>A021 - media organisation</td>\n",
+       "      <td>Not an actortype. The actor is perhaps as the ...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>21</th>\n",
@@ -370,6 +372,7 @@
        "      <td>S009</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A022 - company</td>\n",
+       "      <td></td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>22</th>\n",
@@ -379,6 +382,7 @@
        "      <td>S008</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A023 - adtech provider</td>\n",
+       "      <td>Should Adtech be a called-out subsector of S00...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>23</th>\n",
@@ -388,6 +392,7 @@
        "      <td>S008</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A024 - developer</td>\n",
+       "      <td></td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>24</th>\n",
@@ -397,6 +402,7 @@
        "      <td>S008</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A025 - funding_site_admin</td>\n",
+       "      <td>What is this?</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>25</th>\n",
@@ -406,6 +412,7 @@
        "      <td>S008</td>\n",
        "      <td>FW01, FW02</td>\n",
        "      <td>A026 - games designer</td>\n",
+       "      <td>Just \"Designer\"? Would the TTP context not mak...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>26</th>\n",
@@ -415,6 +422,7 @@
        "      <td>S008</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A027 - information security</td>\n",
+       "      <td></td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>27</th>\n",
@@ -424,6 +432,7 @@
        "      <td>S008</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A028 - platform administrator</td>\n",
+       "      <td>Just \"Administrator\"?</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>28</th>\n",
@@ -433,6 +442,7 @@
        "      <td>S008</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A029 - server admininistrator</td>\n",
+       "      <td>Just \"Administrator\"? (de-duplicating A028)</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>29</th>\n",
@@ -442,6 +452,7 @@
        "      <td>S007</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A030 - platforms</td>\n",
+       "      <td>The same as S007? If there is an action here, ...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>30</th>\n",
@@ -451,6 +462,7 @@
        "      <td>S007</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A031 - social media platform adminstrator</td>\n",
+       "      <td>Not needed, given A028. The sector selection d...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>31</th>\n",
@@ -460,6 +472,7 @@
        "      <td>S007</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A032 - social media platform outreach</td>\n",
+       "      <td>What does this do?</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>32</th>\n",
@@ -469,6 +482,7 @@
        "      <td>S007</td>\n",
        "      <td>FW02</td>\n",
        "      <td>A033 - social media platform owner</td>\n",
+       "      <td></td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
@@ -580,48 +594,84 @@
        "31                                               S007          FW02   \n",
        "32                                               S007          FW02   \n",
        "\n",
-       "                                     longname  \n",
-       "0                      A001 - data scientist   \n",
-       "1                               A002 - target  \n",
-       "2                   A003 - trusted authority   \n",
-       "3                             A004 - activist  \n",
-       "4                      A005 - community group  \n",
-       "5                             A006 - educator  \n",
-       "6                          A007 - factchecker  \n",
-       "7                              A008 - library  \n",
-       "8                                  A009 - NGO  \n",
-       "9              A010 - religious organisation   \n",
-       "10                             A011 - school   \n",
-       "11                       A012 - account owner  \n",
-       "12                    A013 - content creator   \n",
-       "13                               A014 - elves  \n",
-       "14                      A015 - general public  \n",
-       "15                          A016 - influencer  \n",
-       "16                   A017 - coordinating body  \n",
-       "17                         A018 - government   \n",
-       "18                           A019 - military   \n",
-       "19                        A020 - policy maker  \n",
-       "20                  A021 - media organisation  \n",
-       "21                             A022 - company  \n",
-       "22                     A023 - adtech provider  \n",
-       "23                           A024 - developer  \n",
-       "24                  A025 - funding_site_admin  \n",
-       "25                      A026 - games designer  \n",
-       "26                A027 - information security  \n",
-       "27              A028 - platform administrator  \n",
-       "28             A029 - server admininistrator   \n",
-       "29                          A030 - platforms   \n",
-       "30  A031 - social media platform adminstrator  \n",
-       "31     A032 - social media platform outreach   \n",
-       "32         A033 - social media platform owner  "
+       "                                     longname  \\\n",
+       "0                      A001 - data scientist    \n",
+       "1                               A002 - target   \n",
+       "2                   A003 - trusted authority    \n",
+       "3                             A004 - activist   \n",
+       "4                      A005 - community group   \n",
+       "5                             A006 - educator   \n",
+       "6                          A007 - factchecker   \n",
+       "7                              A008 - library   \n",
+       "8                                  A009 - NGO   \n",
+       "9              A010 - religious organisation    \n",
+       "10                             A011 - school    \n",
+       "11                       A012 - account owner   \n",
+       "12                    A013 - content creator    \n",
+       "13                               A014 - elves   \n",
+       "14                      A015 - general public   \n",
+       "15                          A016 - influencer   \n",
+       "16                   A017 - coordinating body   \n",
+       "17                         A018 - government    \n",
+       "18                           A019 - military    \n",
+       "19                        A020 - policy maker   \n",
+       "20                  A021 - media organisation   \n",
+       "21                             A022 - company   \n",
+       "22                     A023 - adtech provider   \n",
+       "23                           A024 - developer   \n",
+       "24                  A025 - funding_site_admin   \n",
+       "25                      A026 - games designer   \n",
+       "26                A027 - information security   \n",
+       "27              A028 - platform administrator   \n",
+       "28             A029 - server admininistrator    \n",
+       "29                          A030 - platforms    \n",
+       "30  A031 - social media platform adminstrator   \n",
+       "31     A032 - social media platform outreach    \n",
+       "32         A033 - social media platform owner   \n",
+       "\n",
+       "                             Jon's comments/questions  \n",
+       "0   What actual actions do data scientists execute...  \n",
+       "1   Both red and blue framework users will refer t...  \n",
+       "2   Is A003 best summarised by the word \"Inluencer...  \n",
+       "3                        What does this actortype do?  \n",
+       "4                        What does this actortype do?  \n",
+       "5                                                      \n",
+       "6                                                      \n",
+       "7   Is this an actortype? A sub-sector, yes. But w...  \n",
+       "8                  This is a sector, not an actortype  \n",
+       "9   \"Faith communities\" are a sub-sector of the Ci...  \n",
+       "10  This is a subsector of S004 - not an actortype...  \n",
+       "11                                                     \n",
+       "12                                                     \n",
+       "13       ??? SJ, you have told me, but I've forgotten  \n",
+       "14  Duplication of a field in the sector object. B...  \n",
+       "15                                                     \n",
+       "16      \"Response coordinator\"? Or just 'Coordinator\"  \n",
+       "17  DHS (A017) is \"government\". Again, this is a f...  \n",
+       "18  A019 is a sub-sector of \"government\". Again, t...  \n",
+       "19                                                     \n",
+       "20  Not an actortype. The actor is perhaps as the ...  \n",
+       "21                                                     \n",
+       "22  Should Adtech be a called-out subsector of S00...  \n",
+       "23                                                     \n",
+       "24                                      What is this?  \n",
+       "25  Just \"Designer\"? Would the TTP context not mak...  \n",
+       "26                                                     \n",
+       "27                              Just \"Administrator\"?  \n",
+       "28        Just \"Administrator\"? (de-duplicating A028)  \n",
+       "29  The same as S007? If there is an action here, ...  \n",
+       "30  Not needed, given A028. The sector selection d...  \n",
+       "31                                 What does this do?  \n",
+       "32                                                     "
       ]
      },
-     "execution_count": 2,
+     "execution_count": 4,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
+    "pd.set_option('display.max_rows', 1000)\n",
     "disarm.df_actortypes"
    ]
   },

CODE/generate_disarm_sql.py

@@ -0,0 +1,147 @@
+''' Generate DISARM sql databases
+
+# Author: SJ Terp, Twitter @bodaceacat
+# License: GPL-3
+
+'''
+
+import pandas as pd
+import sqlite3 as sql
+import os
+from sqlalchemy import create_engine
+from generate_DISARM_pages import Disarm
+
+
+
+def generate_disarm_sql(dbasetype='sqlite'):
+	''' Generate SQL
+	Expected database types include 
+		'sqlite': used to export database to other apps
+		'postgresql_local': used in local instances of the DISARM explorer
+		'heroku': used in Heroku-based instances of the DISARM explorer, although current 
+		practice at DISARM is to post a dump of the postgresql database to it instead. 
+	'''
+
+	# Generate DISARM datasets
+	disarm = Disarm()
+
+
+	# Create connection to database
+	if dbasetype == 'sqlite':
+	    # Generate minimal sqlite database from the  variables
+	    conn = sql.connect('../generated_files/DISARM_database.sqlite')
+	elif dbasetype == 'postgresql_local':
+	    conn = create_engine('postgresql://sara:@localhost:5432/disarmsite')
+
+	    
+	# Add table to database    
+	def add_table(dataframe, tablename, columns):    
+	    # Create sql-appropriate table from dataframe
+	    colnames = ', '.join(['{} TEXT NOT NULL'.format(col) for col in columns])
+	    newtable = dataframe[columns].copy().applymap(str)
+	    newtable['id'] = range(1,len(newtable)+1)
+	    
+	    # send to database
+	    if dbasetype == 'sqlite':
+	        conn.execute("DROP TABLE IF EXISTS {}".format(tablename))
+	        conn.execute('''CREATE TABLE {} (id INTEGER PRIMARY KEY AUTOINCREMENT, {});'''.format(tablename, colnames))
+	        newtable.to_sql(tablename, conn, index=False, if_exists='append')
+	        conn.commit()
+	    elif dbasetype == 'postgresql_local':
+	        newtable.to_sql(tablename, conn, index=False, if_exists='replace')
+	    return newtable
+
+
+	# Build a cross-table
+	def object_tactics_techniques(objectcol, objecttable, crosstable):
+	    # objects to techniques
+	    ctech = crosstable.copy()
+	    ctech = ctech[(ctech['technique_id'] != '') & (~ctech['technique_id'].str.startswith('TA'))]
+	    ctech.rename(columns={'disarm_id':objectcol}, inplace=True)
+	    ctech['summary'] = 'N/A'
+
+	    # objects to tactics
+	    ctact = crosstable[crosstable['technique_id'].str.startswith('TA')].copy()
+	    ctact.rename(columns={'disarm_id':objectcol, 'technique_id': 'tactic_id'}, inplace=True)
+	    ctact['main_tactic'] = 'N'
+	    ctactmain = objecttable[['disarm_id', 'tactic_id']].copy()
+	    ctactmain.rename(columns={'disarm_id':objectcol}, inplace=True)
+	    ctactmain['main_tactic'] = 'Y'
+	    ctact = pd.concat([ctact, ctactmain], ignore_index=True, sort=False)
+	    ctact['summary'] = 'N/A'
+	    return(ctech, ctact)
+
+	#Load all the tables - Heroku needs them in correct order... 
+
+	# -- frameworks --
+
+	#newtable = add_table(disarm.df_actortypes, 'actor_type', ['disarm_id', 'sector_id', 'framework_id', 'name', 'summary'])
+	newtable = add_table(disarm.df_counters, 'counter', ['disarm_id', 'tactic_id', 'metatechnique_id', 'name', 'summary'])
+	newtable = add_table(disarm.df_detections, 'detection', ['disarm_id', 'tactic_id', 'name', 'summary'])
+	newtable = add_table(disarm.df_frameworks, 'framework', ['disarm_id', 'name', 'summary'])
+	newtable = add_table(disarm.df_metatechniques, 'metatechnique', ['disarm_id', 'name', 'summary'])
+	newtable = add_table(disarm.df_phases, 'phase', ['disarm_id', 'name', 'rank', 'summary'])
+	newtable = add_table(disarm.df_playbooks, 'playbook', ['disarm_id', 'object_id', 'name', 'summary'])
+	newtable = add_table(disarm.df_resources, 'resource', ['disarm_id', 'name', 'summary', 'resource_type'])
+	newtable = add_table(disarm.df_responsetypes, 'responsetype', ['disarm_id', 'name', 'summary'])
+	#newtable = add_table(disarm.df_sector, 'sector', ['disarm_id', 'name', 'summary'])
+	newtable = add_table(disarm.df_tactics, 'tactic', ['disarm_id', 'phase_id', 'name', 'rank', 'summary'])
+	newtable = add_table(disarm.df_tasks, 'task', ['disarm_id', 'tactic_id', 'framework_id', 'name', 'summary'])
+	newtable = add_table(disarm.df_techniques, 'technique', ['disarm_id', 'tactic_id', 'name', 'summary'])
+
+	(ctech, ctact) = object_tactics_techniques('counter_id', disarm.df_counters, disarm.cross_counterid_techniqueid)
+	newtable = add_table(ctech, 'counter_technique', ['counter_id', 'technique_id', 'summary'])
+	newtable = add_table(ctact, 'counter_tactic', ['counter_id', 'tactic_id', 'main_tactic', 'summary'])
+
+	(dtech, dtact) = object_tactics_techniques('detection_id', disarm.df_detections, disarm.cross_detectionid_techniqueid)
+	newtable = add_table(dtech, 'detection_technique', ['detection_id', 'technique_id', 'summary'])
+	newtable = add_table(dtact, 'detection_tactic', ['detection_id', 'tactic_id', 'main_tactic', 'summary'])
+
+	# -- datasets --
+
+	newtable = add_table(disarm.df_examples, 'example', ['disarm_id', 'object_id', 'name', 'summary'])
+
+	# dataset
+	# reference - create this from other tables. 
+	# Also need incidentcounter etc - create from data
+	newtable = add_table(disarm.df_externalgroups, 'externalgroup', ['disarm_id', 'name', 'url', 'summary', 
+	                                                'sector', 'primary_role', 'secondary_role', 
+	                                                'primary_subject', 'secondary_subject', 
+	                                                'volunteers', 'region', 'country', 
+	                                                'twitter_handle'])
+	newtable = add_table(disarm.df_incidents, 'incident', ['disarm_id', 'name', 'summary', 
+	                                                      'year_started', 'attributions_seen', 
+	                                                      'found_in_country', 'objecttype'])
+	newtable = add_table(disarm.df_tools, 'tool', ['disarm_id', 'name', 'summary',
+	                                              'externalgroup', 'url', 'category', 
+	                                              'disinformation_use', 'cogseccollab_use', 
+	                                              'function', 'code_url', 'artifacts', 
+	                                              'automation', 'platform', 'accessibility'])
+	# incidenttechnique crosstable
+	it = vars(disarm)['it'][['disarm_id', 'name', 'summary', 'disarm_id_incident', 'disarm_id_technique']].copy()
+	it.rename(columns={'disarm_id_incident':'incident_id', 'disarm_id_technique': 'technique_id'}, inplace=True)
+	newtable = add_table(it, 'incident_technique', ['disarm_id', 'name', 'summary', 
+	                                                'incident_id', 'technique_id'])
+
+	# Load in users table, and close connection
+	if dbasetype == 'sqlite':
+	    conn.execute("DROP TABLE IF EXISTS {}".format('users'))
+	    conn.execute('''CREATE TABLE users (id INTEGER PRIMARY KEY AUTOINCREMENT, username TEXT NOT NULL UNIQUE, password TEXT NOT NULL);''')
+	    conn.close()
+	else:
+	    dfusers = pd.DataFrame([['test','testing']], columns=['username', 'password'])
+	    usertable = add_table(dfusers, 'users', ['username', 'password'])
+
+	    return
+
+
+
+''' main, if we need it
+'''
+def main():
+    generate_disarm_sql()
+
+
+if __name__ == "__main__":
+    main()
+