chg: [merge] master

bin/Crawler.py

@@ -341,13 +341,27 @@ if __name__ == '__main__':
 
     faup = Faup()
 
+    # get HAR files
+    default_crawler_har = p.config.getboolean("Crawler", "default_crawler_har")
+    if default_crawler_har:
+        default_crawler_har = 1
+    else:
+        default_crawler_har = 0
+
+    # get PNG files
+    default_crawler_png = p.config.getboolean("Crawler", "default_crawler_png")
+    if default_crawler_png:
+        default_crawler_png = 1
+    else:
+        default_crawler_png = 0
+
     # Default crawler options
     default_crawler_config = {'html': 1,
-                              'har': 1,
-                              'png': 1,
+                              'har': default_crawler_har,
+                              'png': default_crawler_png,
                               'depth_limit': p.config.getint("Crawler", "crawler_depth_limit"),
-                              'closespider_pagecount': 50,
-                              'user_agent': 'Mozilla/5.0 (Windows NT 6.1; rv:24.0) Gecko/20100101 Firefox/24.0'}
+                              'closespider_pagecount': p.config.getint("Crawler", "default_crawler_closespider_pagecount"),
+                              'user_agent': p.config.get("Crawler", "default_crawler_user_agent")}
 
     # Track launched crawler
     r_cache.sadd('all_crawler', splash_port)

bin/packages/Correlation.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import redis
+
+import Flask_config
+
+r_serv_metadata = Flask_config.r_serv_metadata
+
+
+class Correlation(object):
+
+    def __init__(self, correlation_name):
+        self.correlation_name = correlation_name
+
+    def _exist_corelation_field(self, correlation_type, field_name):
+        return r_serv_metadata.exists('set_{}_{}:{}'.format(self.correlation_name, correlation_type, field_name))
+
+
+    def _get_items(self, correlation_type, field_name):
+        res =  r_serv_metadata.smembers('set_{}_{}:{}'.format(self.correlation_name, correlation_type, field_name))
+        if res:
+            return list(res)
+        else:
+            return []
+
+
+    def _get_metadata(self, correlation_type, field_name):
+        meta_dict = {}
+        meta_dict['first_seen'] = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, correlation_type, field_name), 'first_seen')
+        meta_dict['last_seen'] = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, correlation_type, field_name), 'last_seen')
+        return meta_dict
+
+    def _get_correlation_by_date(self, correlation_type, date):
+        return r_serv_metadata.hkeys('{}:{}:{}'.format(self.correlation_name, correlation_type, date))
+
+    def verify_correlation_field_request(self, request_dict, correlation_type):
+        if not request_dict:
+            return Response({'status': 'error', 'reason': 'Malformed JSON'}, 400)
+
+        field_name = request_dict.get(correlation_type, None)
+        if not field_name:
+            return ( {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400 )
+        if not self._exist_corelation_field(correlation_type, field_name):
+            return ( {'status': 'error', 'reason': 'Item not found'}, 404 )
+
+    def get_correlation(self, request_dict, correlation_type, field_name):
+        dict_resp = {}
+
+        if request_dict.get('items'):
+            dict_resp['items'] = self._get_items(correlation_type, field_name)
+
+        if request_dict.get('metadata'):
+            dict_resp['metadata'] = self._get_metadata(correlation_type, field_name)
+
+        dict_resp[correlation_type] = field_name
+
+        return (dict_resp, 200)
+
+
+
+
+#cryptocurrency_all:cryptocurrency name	cryptocurrency address	nb seen

bin/packages/Cryptocurrency.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import redis
+
+from hashlib import sha256
+
+import Flask_config
+from Correlation import Correlation
+
+r_serv_metadata = Flask_config.r_serv_metadata
+
+digits58 = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
+
+cryptocurrency = Correlation('cryptocurrency')
+
+def decode_base58(bc, length):
+    n = 0
+    for char in bc:
+        n = n * 58 + digits58.index(char)
+    return n.to_bytes(length, 'big')
+
+def check_bitcoin_address(bc):
+    try:
+        bcbytes = decode_base58(bc, 25)
+        return bcbytes[-4:] == sha256(sha256(bcbytes[:-4]).digest()).digest()[:4]
+    except Exception:
+        return False
+
+def verify_cryptocurrency_address(cryptocurrency_type, cryptocurrency_address):
+    if cryptocurrency_type == 'bitcoin':
+        return check_bitcoin_address(cryptocurrency_address)
+    else:
+        return True
+
+
+def get_cryptocurrency(request_dict, cryptocurrency_type):
+    # basic verification
+    res = cryptocurrency.verify_correlation_field_request(request_dict, cryptocurrency_type)
+    if res:
+        return res
+    # cerify address
+    field_name = request_dict.get(cryptocurrency_type)
+    if not verify_cryptocurrency_address(cryptocurrency_type, field_name):
+        return ( {'status': 'error', 'reason': 'Invalid Cryptocurrency address'}, 400 )
+
+    return cryptocurrency.get_correlation(request_dict, cryptocurrency_type, field_name)

bin/packages/Item.py

@@ -14,6 +14,7 @@ import Tag
 
 PASTES_FOLDER = Flask_config.PASTES_FOLDER
 r_cache = Flask_config.r_cache
+r_serv_metadata = Flask_config.r_serv_metadata
 
 def exist_item(item_id):
     if os.path.isfile(os.path.join(PASTES_FOLDER, item_id)):
@@ -93,4 +94,43 @@ def get_item(request_dict):
     if lines_info:
         dict_item['lines'] = get_lines_info(item_id, dict_item.get('content', 'None'))
 
+    if request_dict.get('pgp'):
+        dict_item['pgp'] = {}
+        if request_dict['pgp'].get('key'):
+            dict_item['pgp']['key'] = get_item_pgp_key(item_id)
+        if request_dict['pgp'].get('mail'):
+            dict_item['pgp']['mail'] = get_item_pgp_mail(item_id)
+        if request_dict['pgp'].get('name'):
+            dict_item['pgp']['name'] = get_item_pgp_name(item_id)
+
+    if request_dict.get('cryptocurrency'):
+        dict_item['cryptocurrency'] = {}
+        if request_dict['cryptocurrency'].get('bitcoin'):
+            dict_item['cryptocurrency']['bitcoin'] = get_item_bitcoin(item_id)
+
     return (dict_item, 200)
+
+
+###
+### correlation
+###
+
+def _get_item_correlation(correlation_name, correlation_type, item_id):
+    print('item_{}_{}:{}'.format(correlation_name, correlation_type, item_id))
+    res = r_serv_metadata.smembers('item_{}_{}:{}'.format(correlation_name, correlation_type, item_id))
+    if res:
+        return list(res)
+    else:
+        return []
+
+def get_item_bitcoin(item_id):
+    return _get_item_correlation('cryptocurrency', 'bitcoin', item_id)
+
+def get_item_pgp_key(item_id):
+    return _get_item_correlation('pgpdump', 'key', item_id)
+
+def get_item_pgp_name(item_id):
+    return _get_item_correlation('pgpdump', 'name', item_id)
+
+def get_item_pgp_mail(item_id):
+    return _get_item_correlation('pgpdump', 'mail', item_id)

bin/packages/Pgp.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import redis
+
+from hashlib import sha256
+
+import Flask_config
+from Correlation import Correlation
+
+r_serv_metadata = Flask_config.r_serv_metadata
+
+pgpdump = Correlation('pgpdump')
+
+
+def get_pgp(request_dict, pgp_type):
+    # basic verification
+    res = pgpdump.verify_correlation_field_request(request_dict, pgp_type)
+    if res:
+        return res
+    # cerify address
+    field_name = request_dict.get(pgp_type)
+
+    return pgpdump.get_correlation(request_dict, pgp_type, field_name)

bin/packages/config.cfg.sample

@@ -255,5 +255,9 @@ db = 0
 [Crawler]
 activate_crawler = False
 crawler_depth_limit = 1
+default_crawler_har = True
+default_crawler_png = True
+default_crawler_closespider_pagecount = 50
+default_crawler_user_agent = Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0
 splash_url = http://127.0.0.1
 splash_port = 8050-8052

doc/README.md

@@ -250,6 +250,25 @@ Get item. Filter requested field.
   - get item lines info
   - *boolean*
   - default: `false`
+- `cryptocurrency`
+  - `bitcoin`
+    - get item bitcoin adress
+    - *boolean*
+    - default: `false`
+- `pgp`
+  - `key`
+    - get item pgp key
+    - *boolean*
+    - default: `false`
+  - `mail`
+    - get item pgp mail
+    - *boolean*
+    - default: `false`
+  - `name`
+    - get item pgp name
+    - *boolean*
+    - default: `false`
+
 
 #### JSON response
 - `content`
@@ -276,6 +295,20 @@ Get item. Filter requested field.
       - `nb`
         - nb lines item
         - *int*
+- `cryptocurrency`
+  - `bitcoin`
+    - item bitcoin adress
+    - *list*
+- `pgp`
+  - `key`
+    - item pgp keys
+    - *list*
+  - `mail`
+    - item pgp mails
+    - *list*
+  - `name`
+    - item pgp name
+    - *list*
 
 
 #### Example
@@ -299,12 +332,28 @@ curl https://127.0.0.1:7000/api/v1/get/item --header "Authorization: iHc1_ChZxj1
 ```json
   {
     "content": "dsvcdsvcdsc vvvv",
+    "cryptocurrency": {
+      "bitcoin": [
+        "132M1aGTGodHkQNh1augLeMjEXH51wgoCc"
+      ]
+    },
     "date": "20190726",
     "id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz",
     "lines": {
       "max_length": 19,
       "nb": 1
     },
+    "pgp": {
+      "key": [
+        "0x5180D21F4C20F975"
+      ],
+      "mail": [
+        "mail@test.test"
+      ],
+      "name": [
+        "user_test"
+      ]
+    },
     "size": 0.03,
     "tags": [
       "misp-galaxy:stealer=\"Vidar\"",
@@ -546,10 +595,10 @@ Get tag metadata.
   - *str*
 - `first_seen`
   - date: first seen
-  - *str - YYMMDD*
+  - *str - YYYYMMDD*
 - `last_seen`
-  - date: first seen
-  - *str - YYMMDD*
+  - date: last seen
+  - *str - YYYYMMDD*
 #### Example
 ```
 curl https://127.0.0.1:7000/api/v1/get/tag/metadata --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST
@@ -581,6 +630,116 @@ curl https://127.0.0.1:7000/api/v1/get/tag/metadata --header "Authorization: iHc
 
 
 
+## Cryptocurrency
+
+
+
+### Get bitcoin metadata: `api/v1/get/cryptocurrency/bitcoin/metadata`<a name="get_cryptocurrency_bitcoin_metadata"></a>
+
+#### Description
+Get all metdata from a bitcoin address.
+
+**Method** : `POST`
+
+#### Parameters
+- `bitcoin`
+  - bitcoin address
+  - *str*
+  - mandatory
+
+#### JSON response
+- `bitcoin`
+  - bitcoin address
+  - *str*
+- `first_seen`
+  - date: first seen
+  - *str - YYYYMMDD*
+- `last_seen`
+  - date: last seen
+  - *str - YYYYMMDD*
+#### Example
+```
+curl https://127.0.0.1:7000/api/v1/get/cryptocurrency/bitcoin/metadata --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST
+```
+
+#### input.json Example
+```json
+  {
+    "bitcoin": "3DZfm5TQaJKcJm9PsuaWmSz9XmHMLxVv3y"
+  }
+```
+
+#### Expected Success Response
+**HTTP Status Code** : `200`
+```json
+  {
+    "bitcoin": "3DZfm5TQaJKcJm9PsuaWmSz9XmHMLxVv3y",
+    "first_seen": "20190605",
+    "last_seen": "20190726"
+  }
+```
+
+#### Expected Fail Response
+**HTTP Status Code** : `404`
+```json
+  {"status": "error", "reason": "Item not found"}
+```
+
+
+
+### Get bitcoin metadata: `api/v1/get/cryptocurrency/bitcoin/item`<a name="get_cryptocurrency_bitcoin_item"></a>
+
+#### Description
+Get all items related to a bitcoin address.
+
+**Method** : `POST`
+
+#### Parameters
+- `bitcoin`
+  - bitcoin address
+  - *str*
+  - mandatory
+
+#### JSON response
+- `bitcoin`
+  - bitcoin address
+  - *str*
+- `items`
+  - list of item id
+  - *list*
+#### Example
+```
+curl https://127.0.0.1:7000/api/v1/get/cryptocurrency/bitcoin/item --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST
+```
+
+#### input.json Example
+```json
+  {
+    "bitcoin": "3DZfm5TQaJKcJm9PsuaWmSz9XmHMLxVv3y"
+  }
+```
+
+#### Expected Success Response
+**HTTP Status Code** : `200`
+```json
+  {
+    "bitcoin": "3DZfm5TQaJKcJm9PsuaWmSz9XmHMLxVv3y",
+    "items": [
+      "archive/2019/08/26/test_bitcoin001",
+      "archive/2019/08/26/test_bitcoin002",
+      "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz"
+    ]
+  }
+```
+
+#### Expected Fail Response
+**HTTP Status Code** : `404`
+```json
+  {"status": "error", "reason": "Item not found"}
+```
+
+
+
 
 
 
@@ -777,9 +936,6 @@ curl https://127.0.0.1:7000/api/v1/add/tracker/term --header "Authorization: iHc
 
 
 
-
-
-
 ## Import management