chg: [merge] master

This commit is contained in:
Terrtia 2019-09-04 10:24:46 +02:00
commit bd8735a6e5
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
116 changed files with 639 additions and 59 deletions

View file

@ -341,13 +341,27 @@ if __name__ == '__main__':
faup = Faup()
# get HAR files
default_crawler_har = p.config.getboolean("Crawler", "default_crawler_har")
if default_crawler_har:
default_crawler_har = 1
else:
default_crawler_har = 0
# get PNG files
default_crawler_png = p.config.getboolean("Crawler", "default_crawler_png")
if default_crawler_png:
default_crawler_png = 1
else:
default_crawler_png = 0
# Default crawler options
default_crawler_config = {'html': 1,
'har': 1,
'png': 1,
'har': default_crawler_har,
'png': default_crawler_png,
'depth_limit': p.config.getint("Crawler", "crawler_depth_limit"),
'closespider_pagecount': 50,
'user_agent': 'Mozilla/5.0 (Windows NT 6.1; rv:24.0) Gecko/20100101 Firefox/24.0'}
'closespider_pagecount': p.config.getint("Crawler", "default_crawler_closespider_pagecount"),
'user_agent': p.config.get("Crawler", "default_crawler_user_agent")}
# Track launched crawler
r_cache.sadd('all_crawler', splash_port)

64
bin/packages/Correlation.py Executable file
View file

@ -0,0 +1,64 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import redis
import Flask_config
r_serv_metadata = Flask_config.r_serv_metadata
class Correlation(object):
def __init__(self, correlation_name):
self.correlation_name = correlation_name
def _exist_corelation_field(self, correlation_type, field_name):
return r_serv_metadata.exists('set_{}_{}:{}'.format(self.correlation_name, correlation_type, field_name))
def _get_items(self, correlation_type, field_name):
res = r_serv_metadata.smembers('set_{}_{}:{}'.format(self.correlation_name, correlation_type, field_name))
if res:
return list(res)
else:
return []
def _get_metadata(self, correlation_type, field_name):
meta_dict = {}
meta_dict['first_seen'] = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, correlation_type, field_name), 'first_seen')
meta_dict['last_seen'] = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, correlation_type, field_name), 'last_seen')
return meta_dict
def _get_correlation_by_date(self, correlation_type, date):
return r_serv_metadata.hkeys('{}:{}:{}'.format(self.correlation_name, correlation_type, date))
def verify_correlation_field_request(self, request_dict, correlation_type):
if not request_dict:
return Response({'status': 'error', 'reason': 'Malformed JSON'}, 400)
field_name = request_dict.get(correlation_type, None)
if not field_name:
return ( {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400 )
if not self._exist_corelation_field(correlation_type, field_name):
return ( {'status': 'error', 'reason': 'Item not found'}, 404 )
def get_correlation(self, request_dict, correlation_type, field_name):
dict_resp = {}
if request_dict.get('items'):
dict_resp['items'] = self._get_items(correlation_type, field_name)
if request_dict.get('metadata'):
dict_resp['metadata'] = self._get_metadata(correlation_type, field_name)
dict_resp[correlation_type] = field_name
return (dict_resp, 200)
#cryptocurrency_all:cryptocurrency name cryptocurrency address nb seen

48
bin/packages/Cryptocurrency.py Executable file
View file

@ -0,0 +1,48 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import redis
from hashlib import sha256
import Flask_config
from Correlation import Correlation
r_serv_metadata = Flask_config.r_serv_metadata
digits58 = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
cryptocurrency = Correlation('cryptocurrency')
def decode_base58(bc, length):
n = 0
for char in bc:
n = n * 58 + digits58.index(char)
return n.to_bytes(length, 'big')
def check_bitcoin_address(bc):
try:
bcbytes = decode_base58(bc, 25)
return bcbytes[-4:] == sha256(sha256(bcbytes[:-4]).digest()).digest()[:4]
except Exception:
return False
def verify_cryptocurrency_address(cryptocurrency_type, cryptocurrency_address):
if cryptocurrency_type == 'bitcoin':
return check_bitcoin_address(cryptocurrency_address)
else:
return True
def get_cryptocurrency(request_dict, cryptocurrency_type):
# basic verification
res = cryptocurrency.verify_correlation_field_request(request_dict, cryptocurrency_type)
if res:
return res
# cerify address
field_name = request_dict.get(cryptocurrency_type)
if not verify_cryptocurrency_address(cryptocurrency_type, field_name):
return ( {'status': 'error', 'reason': 'Invalid Cryptocurrency address'}, 400 )
return cryptocurrency.get_correlation(request_dict, cryptocurrency_type, field_name)

View file

@ -14,6 +14,7 @@ import Tag
PASTES_FOLDER = Flask_config.PASTES_FOLDER
r_cache = Flask_config.r_cache
r_serv_metadata = Flask_config.r_serv_metadata
def exist_item(item_id):
if os.path.isfile(os.path.join(PASTES_FOLDER, item_id)):
@ -93,4 +94,43 @@ def get_item(request_dict):
if lines_info:
dict_item['lines'] = get_lines_info(item_id, dict_item.get('content', 'None'))
if request_dict.get('pgp'):
dict_item['pgp'] = {}
if request_dict['pgp'].get('key'):
dict_item['pgp']['key'] = get_item_pgp_key(item_id)
if request_dict['pgp'].get('mail'):
dict_item['pgp']['mail'] = get_item_pgp_mail(item_id)
if request_dict['pgp'].get('name'):
dict_item['pgp']['name'] = get_item_pgp_name(item_id)
if request_dict.get('cryptocurrency'):
dict_item['cryptocurrency'] = {}
if request_dict['cryptocurrency'].get('bitcoin'):
dict_item['cryptocurrency']['bitcoin'] = get_item_bitcoin(item_id)
return (dict_item, 200)
###
### correlation
###
def _get_item_correlation(correlation_name, correlation_type, item_id):
print('item_{}_{}:{}'.format(correlation_name, correlation_type, item_id))
res = r_serv_metadata.smembers('item_{}_{}:{}'.format(correlation_name, correlation_type, item_id))
if res:
return list(res)
else:
return []
def get_item_bitcoin(item_id):
return _get_item_correlation('cryptocurrency', 'bitcoin', item_id)
def get_item_pgp_key(item_id):
return _get_item_correlation('pgpdump', 'key', item_id)
def get_item_pgp_name(item_id):
return _get_item_correlation('pgpdump', 'name', item_id)
def get_item_pgp_mail(item_id):
return _get_item_correlation('pgpdump', 'mail', item_id)

25
bin/packages/Pgp.py Executable file
View file

@ -0,0 +1,25 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import redis
from hashlib import sha256
import Flask_config
from Correlation import Correlation
r_serv_metadata = Flask_config.r_serv_metadata
pgpdump = Correlation('pgpdump')
def get_pgp(request_dict, pgp_type):
# basic verification
res = pgpdump.verify_correlation_field_request(request_dict, pgp_type)
if res:
return res
# cerify address
field_name = request_dict.get(pgp_type)
return pgpdump.get_correlation(request_dict, pgp_type, field_name)

View file

@ -255,5 +255,9 @@ db = 0
[Crawler]
activate_crawler = False
crawler_depth_limit = 1
default_crawler_har = True
default_crawler_png = True
default_crawler_closespider_pagecount = 50
default_crawler_user_agent = Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0
splash_url = http://127.0.0.1
splash_port = 8050-8052

View file

@ -250,6 +250,25 @@ Get item. Filter requested field.
- get item lines info
- *boolean*
- default: `false`
- `cryptocurrency`
- `bitcoin`
- get item bitcoin adress
- *boolean*
- default: `false`
- `pgp`
- `key`
- get item pgp key
- *boolean*
- default: `false`
- `mail`
- get item pgp mail
- *boolean*
- default: `false`
- `name`
- get item pgp name
- *boolean*
- default: `false`
#### JSON response
- `content`
@ -276,6 +295,20 @@ Get item. Filter requested field.
- `nb`
- nb lines item
- *int*
- `cryptocurrency`
- `bitcoin`
- item bitcoin adress
- *list*
- `pgp`
- `key`
- item pgp keys
- *list*
- `mail`
- item pgp mails
- *list*
- `name`
- item pgp name
- *list*
#### Example
@ -299,12 +332,28 @@ curl https://127.0.0.1:7000/api/v1/get/item --header "Authorization: iHc1_ChZxj1
```json
{
"content": "dsvcdsvcdsc vvvv",
"cryptocurrency": {
"bitcoin": [
"132M1aGTGodHkQNh1augLeMjEXH51wgoCc"
]
},
"date": "20190726",
"id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz",
"lines": {
"max_length": 19,
"nb": 1
},
"pgp": {
"key": [
"0x5180D21F4C20F975"
],
"mail": [
"mail@test.test"
],
"name": [
"user_test"
]
},
"size": 0.03,
"tags": [
"misp-galaxy:stealer=\"Vidar\"",
@ -546,10 +595,10 @@ Get tag metadata.
- *str*
- `first_seen`
- date: first seen
- *str - YYMMDD*
- *str - YYYYMMDD*
- `last_seen`
- date: first seen
- *str - YYMMDD*
- date: last seen
- *str - YYYYMMDD*
#### Example
```
curl https://127.0.0.1:7000/api/v1/get/tag/metadata --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST
@ -581,6 +630,116 @@ curl https://127.0.0.1:7000/api/v1/get/tag/metadata --header "Authorization: iHc
## Cryptocurrency
### Get bitcoin metadata: `api/v1/get/cryptocurrency/bitcoin/metadata`<a name="get_cryptocurrency_bitcoin_metadata"></a>
#### Description
Get all metdata from a bitcoin address.
**Method** : `POST`
#### Parameters
- `bitcoin`
- bitcoin address
- *str*
- mandatory
#### JSON response
- `bitcoin`
- bitcoin address
- *str*
- `first_seen`
- date: first seen
- *str - YYYYMMDD*
- `last_seen`
- date: last seen
- *str - YYYYMMDD*
#### Example
```
curl https://127.0.0.1:7000/api/v1/get/cryptocurrency/bitcoin/metadata --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST
```
#### input.json Example
```json
{
"bitcoin": "3DZfm5TQaJKcJm9PsuaWmSz9XmHMLxVv3y"
}
```
#### Expected Success Response
**HTTP Status Code** : `200`
```json
{
"bitcoin": "3DZfm5TQaJKcJm9PsuaWmSz9XmHMLxVv3y",
"first_seen": "20190605",
"last_seen": "20190726"
}
```
#### Expected Fail Response
**HTTP Status Code** : `404`
```json
{"status": "error", "reason": "Item not found"}
```
### Get bitcoin metadata: `api/v1/get/cryptocurrency/bitcoin/item`<a name="get_cryptocurrency_bitcoin_item"></a>
#### Description
Get all items related to a bitcoin address.
**Method** : `POST`
#### Parameters
- `bitcoin`
- bitcoin address
- *str*
- mandatory
#### JSON response
- `bitcoin`
- bitcoin address
- *str*
- `items`
- list of item id
- *list*
#### Example
```
curl https://127.0.0.1:7000/api/v1/get/cryptocurrency/bitcoin/item --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST
```
#### input.json Example
```json
{
"bitcoin": "3DZfm5TQaJKcJm9PsuaWmSz9XmHMLxVv3y"
}
```
#### Expected Success Response
**HTTP Status Code** : `200`
```json
{
"bitcoin": "3DZfm5TQaJKcJm9PsuaWmSz9XmHMLxVv3y",
"items": [
"archive/2019/08/26/test_bitcoin001",
"archive/2019/08/26/test_bitcoin002",
"submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz"
]
}
```
#### Expected Fail Response
**HTTP Status Code** : `404`
```json
{"status": "error", "reason": "Item not found"}
```
@ -777,9 +936,6 @@ curl https://127.0.0.1:7000/api/v1/add/tracker/term --header "Authorization: iHc
## Import management

View file

Before

Width:  |  Height:  |  Size: 5.6 KiB

After

Width:  |  Height:  |  Size: 5.6 KiB

Some files were not shown because too many files have changed in this diff Show more