chg: [misp_modules] parse modules result

OVERVIEW.md

@@ -82,6 +82,16 @@ Redis and ARDB overview
 | ------ | ------ | ------ |
 | ail:all_role | **role** | **int, role priority (1=admin)** |
 
+##### MISP Modules:
+
+| Set Key | Value |
+| ------ | ------ |
+| enabled_misp_modules | **module name** |
+
+| Key | Value |
+| ------ | ------ |
+| misp_module:**module name** | **module dict** |
+
 ## DB2 - TermFreq:
 
 ##### Set:

bin/packages/MispModules.py

@@ -2,6 +2,7 @@
 
 import os
 import json
+import redis
 import requests
 import configparser
 
@@ -9,6 +10,19 @@ misp_module_url = 'http://localhost:6666'
 
 default_config_path = os.path.join(os.environ['AIL_HOME'], 'configs', 'misp_modules.cfg')
 
+configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
+if not os.path.exists(configfile):
+    raise Exception('Unable to find the configuration file. \
+                    Did you set environment variables? \
+                    Or activate the virtualenv.')
+cfg = configparser.ConfigParser()
+cfg.read(configfile)
+r_serv = redis.StrictRedis(
+    host=cfg.get("ARDB_DB", "host"),
+    port=cfg.getint("ARDB_DB", "port"),
+    db=cfg.getint("ARDB_DB", "db"),
+    decode_responses=True)
+
 def init_config(config_path=default_config_path):
     config = configparser.ConfigParser()
     if os.path.isfile(config_path):
@@ -27,49 +41,89 @@ def init_module_config(module_json, config, config_path=default_config_path):
             for config_var in module_json['meta']['config']:
                 if config_var not in config[module_json['name']]:
                     config.set(module_json['name'], config_var, '')
-        else:
-            print(module_json['name'])
     return config
 
-def build_enrichment_request_json(module_name, var_name, var_value):
+def load_modules_list():
-    request_dict = {'module': module_name, var_name: var_value}
-    # # TODO: add error handler
-    return json.dumps(request_dict)
-
-def misp_module_enrichement_request(misp_module_url, misp_module_port, request_content):
-    endpoint_url = '{}:{}/query'.format(misp_module_url, misp_module_port)
-    req = requests.post(endpoint_url, headers={'Content-Type': 'application/json'}, data=request_content)
-    if req.status_code == 200:
-        print(req.json())
-
-if __name__ == "__main__":
     req = requests.get('{}/modules'.format(misp_module_url))
-
     if req.status_code == 200:
         all_misp_modules = req.json()
         all_modules = []
         for module_json in all_misp_modules:
 
             #filter module-types
-            if 'hover' in module_json['meta']['module-type']:
+            if 'hover' in module_json['meta']['module-type'] or 'expansion' in module_json['meta']['module-type']:
                 all_modules.append(module_json)
 
-            # if 'expansion' in module_json['meta']['module-type']:
+            # # TODO: handle import/export modules
-            #     all_expansion.append(module_json['name'])
 
         config = init_config()
+        r_serv.delete('misp_modules')
         for module_json in all_modules:
             config = init_module_config(module_json, config, config_path=default_config_path)
+            r_serv.hset('misp_modules', module_json['name'], json.dumps(module_json))
 
         with open(default_config_path, 'w') as f:
             config.write(f)
 
-        misp_module_url = 'http://localhost'
-        misp_module_port = 6666
-        test_content = build_enrichment_request_json('btc_steroids', 'btc', 'btc address')
-        misp_module_enrichement_request(misp_module_url, misp_module_port, test_content)
-
-
     else:
         print('Error: Module service not reachable.')
-        print(req)
+
+
+def build_config_json(module_name):
+    misp_module_config = configparser.ConfigParser()
+    misp_module_config.read(default_config_path)
+    dict_config = {}
+    if module_name in misp_module_config:
+        for config_key in misp_module_config[module_name]:
+            config_value = misp_module_config[module_name][config_key]
+            if config_value:
+                dict_config[config_key] = config_value
+    return dict_config
+
+def build_enrichment_request_json(module_name, var_name, var_value):
+    # # TODO: add error handler
+    request_dict = {'module': module_name, var_name: var_value}
+    # add config
+    config_json = build_config_json(module_name)
+    if config_json:
+        request_dict['config'] = config_json
+    return json.dumps(request_dict)
+
+def misp_module_enrichment_request(misp_module_url, misp_module_port, request_content):
+    # # TODO: check if module is enabled
+    endpoint_url = '{}:{}/query'.format(misp_module_url, misp_module_port)
+    req = requests.post(endpoint_url, headers={'Content-Type': 'application/json'}, data=request_content)
+    if req.status_code == 200:
+        response = req.json()
+        if response:
+            return parse_module_enrichment_response(response)
+    else:
+        print('error: {} Enrichment service not reachable.'.format(req.status_code,))
+        return ''
+
+def parse_module_enrichment_response(misp_module_response):
+    print(misp_module_response)
+    response_values = []
+    if 'results' in misp_module_response:
+        # # TODO: handle misp_format (Attribute, Object, Tags)
+        response_types = []
+        for result in misp_module_response['results']:
+            # get all types
+            for resp_type in result['types']:
+                response_types.append(resp_type)
+            # get all values
+            for resp_value in result['values']:
+                response_values.append(resp_value)
+        # TODO: handle / verify / use response types
+        #print(response_types)
+    return response_values
+
+if __name__ == "__main__":
+
+    load_modules_list()
+
+    misp_module_url = 'http://localhost'
+    misp_module_port = 6666
+    test_content = build_enrichment_request_json('btc_steroids', 'btc', 'btc_add')
+    print(test_content)
+    misp_module_enrichment_request(misp_module_url, misp_module_port, test_content)