mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 08:38:28 +00:00
fix: [tests] fix tests
This commit is contained in:
parent
c37a68d41d
commit
94d7eaf11d
6 changed files with 166 additions and 200 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
@ -3,6 +3,7 @@
|
||||||
*.pyc
|
*.pyc
|
||||||
*.swo
|
*.swo
|
||||||
.idea
|
.idea
|
||||||
|
.coverage
|
||||||
|
|
||||||
# Install Dirs
|
# Install Dirs
|
||||||
AILENV
|
AILENV
|
||||||
|
|
|
@ -573,11 +573,11 @@ function update_thirdparty {
|
||||||
function launch_tests() {
|
function launch_tests() {
|
||||||
tests_dir=${AIL_HOME}/tests
|
tests_dir=${AIL_HOME}/tests
|
||||||
bin_dir=${AIL_BIN}
|
bin_dir=${AIL_BIN}
|
||||||
python3 `which nosetests` -w $tests_dir --with-coverage --cover-package=$bin_dir -d --cover-erase
|
python3 `which nosetests` -w $tests_dir --with-coverage --cover-package=$bin_dir -d --cover-erase --exclude=test-zmq.py
|
||||||
}
|
}
|
||||||
|
|
||||||
function reset_password() {
|
function reset_password() {
|
||||||
echo -e "\t* Reseting UI admin password..."
|
echo -e "\t* Resetting UI admin password..."
|
||||||
if checking_kvrocks && checking_redis; then
|
if checking_kvrocks && checking_redis; then
|
||||||
python ${AIL_HOME}/var/www/create_default_user.py &
|
python ${AIL_HOME}/var/www/create_default_user.py &
|
||||||
wait
|
wait
|
||||||
|
|
|
@ -51,10 +51,10 @@ class ApiKey(AbstractModule):
|
||||||
item = Item(item_id)
|
item = Item(item_id)
|
||||||
item_content = item.get_content()
|
item_content = item.get_content()
|
||||||
|
|
||||||
google_api_key = self.regex_findall(self.re_google_api_key, item.get_id(), item_content)
|
google_api_key = self.regex_findall(self.re_google_api_key, item.get_id(), item_content, r_set=True)
|
||||||
aws_access_key = self.regex_findall(self.re_aws_access_key, item.get_id(), item_content)
|
aws_access_key = self.regex_findall(self.re_aws_access_key, item.get_id(), item_content, r_set=True)
|
||||||
if aws_access_key:
|
if aws_access_key:
|
||||||
aws_secret_key = self.regex_findall(self.re_aws_secret_key, item.get_id(), item_content)
|
aws_secret_key = self.regex_findall(self.re_aws_secret_key, item.get_id(), item_content, r_set=True)
|
||||||
|
|
||||||
if aws_access_key or google_api_key:
|
if aws_access_key or google_api_key:
|
||||||
to_print = f'ApiKey;{item.get_source()};{item.get_date()};{item.get_basename()};'
|
to_print = f'ApiKey;{item.get_source()};{item.get_date()};{item.get_basename()};'
|
||||||
|
@ -74,7 +74,7 @@ class ApiKey(AbstractModule):
|
||||||
print(f'found AWS secret key')
|
print(f'found AWS secret key')
|
||||||
self.redis_logger.warning(f'{to_print}Checked {len(aws_secret_key)} found AWS secret Key;{item.get_id()}')
|
self.redis_logger.warning(f'{to_print}Checked {len(aws_secret_key)} found AWS secret Key;{item.get_id()}')
|
||||||
|
|
||||||
msg = 'infoleak:automatic-detection="aws-key";{}'.format(item.get_id())
|
msg = f'infoleak:automatic-detection="aws-key";{item.get_id()}'
|
||||||
self.add_message_to_queue(msg, 'Tags')
|
self.add_message_to_queue(msg, 'Tags')
|
||||||
|
|
||||||
# Tags
|
# Tags
|
||||||
|
|
308
tests/testApi.py
308
tests/testApi.py
|
@ -14,159 +14,163 @@ from lib import Tag
|
||||||
from packages import Import_helper
|
from packages import Import_helper
|
||||||
|
|
||||||
sys.path.append(os.environ['AIL_FLASK'])
|
sys.path.append(os.environ['AIL_FLASK'])
|
||||||
from var.www.Flask_server import app
|
sys.path.append(os.path.join(os.environ['AIL_FLASK'], 'modules'))
|
||||||
|
from Flask_server import app
|
||||||
def parse_response(obj, ail_response):
|
|
||||||
res_json = ail_response.get_json()
|
|
||||||
if 'status' in res_json:
|
|
||||||
if res_json['status'] == 'error':
|
|
||||||
return obj.fail('{}: {}: {}'.format(ail_response.status_code, res_json['status'], res_json['reason']))
|
|
||||||
return res_json
|
|
||||||
|
|
||||||
def get_api_key():
|
|
||||||
api_file = os.path.join(os.environ['AIL_HOME'], 'DEFAULT_PASSWORD')
|
|
||||||
if os.path.isfile(api_file):
|
|
||||||
with open(os.path.join(os.environ['AIL_HOME'], 'DEFAULT_PASSWORD'), 'r') as f:
|
|
||||||
content = f.read()
|
|
||||||
content = content.splitlines()
|
|
||||||
apikey = content[-1]
|
|
||||||
apikey = apikey.replace('API_Key=', '', 1)
|
|
||||||
# manual tests
|
|
||||||
else:
|
|
||||||
apikey = sys.argv[1]
|
|
||||||
return apikey
|
|
||||||
|
|
||||||
|
|
||||||
APIKEY = get_api_key()
|
# def parse_response(obj, ail_response):
|
||||||
|
# res_json = ail_response.get_json()
|
||||||
class TestApiV1(unittest.TestCase):
|
# if 'status' in res_json:
|
||||||
import_uuid = None
|
# if res_json['status'] == 'error':
|
||||||
item_id = None
|
# return obj.fail('{}: {}: {}'.format(ail_response.status_code, res_json['status'], res_json['reason']))
|
||||||
|
# return res_json
|
||||||
|
#
|
||||||
def setUp(self):
|
#
|
||||||
self.app = app
|
# def get_api_key():
|
||||||
self.app.config['TESTING'] = True
|
# api_file = os.path.join(os.environ['AIL_HOME'], 'DEFAULT_PASSWORD')
|
||||||
self.client = self.app.test_client()
|
# if os.path.isfile(api_file):
|
||||||
self.apikey = APIKEY
|
# with open(os.path.join(os.environ['AIL_HOME'], 'DEFAULT_PASSWORD'), 'r') as f:
|
||||||
self.item_content = "text to import"
|
# content = f.read()
|
||||||
self.item_tags = ["infoleak:analyst-detection=\"private-key\""]
|
# content = content.splitlines()
|
||||||
self.expected_tags = ["infoleak:analyst-detection=\"private-key\"", 'infoleak:submission="manual"']
|
# apikey = content[-1]
|
||||||
|
# apikey = apikey.replace('API_Key=', '', 1)
|
||||||
# POST /api/v1/import/item
|
# # manual tests
|
||||||
def test_0001_api_import_item(self):
|
# else:
|
||||||
input_json = {"type": "text","tags": self.item_tags,"text": self.item_content}
|
# apikey = sys.argv[1]
|
||||||
req = self.client.post('/api/v1/import/item', json=input_json ,headers={ 'Authorization': self.apikey })
|
# return apikey
|
||||||
req_json = parse_response(self, req)
|
#
|
||||||
import_uuid = req_json['uuid']
|
#
|
||||||
self.__class__.import_uuid = import_uuid
|
# APIKEY = get_api_key()
|
||||||
self.assertTrue(Import_helper.is_valid_uuid_v4(import_uuid))
|
#
|
||||||
|
#
|
||||||
# POST /api/v1/get/import/item
|
# class TestApiV1(unittest.TestCase):
|
||||||
def test_0002_api_get_import_item(self):
|
# import_uuid = None
|
||||||
input_json = {"uuid": self.__class__.import_uuid}
|
# item_id = None
|
||||||
item_not_imported = True
|
#
|
||||||
import_timout = 60
|
# def setUp(self):
|
||||||
start = time.time()
|
# self.app = app
|
||||||
|
# self.app.config['TESTING'] = True
|
||||||
while item_not_imported:
|
# self.client = self.app.test_client()
|
||||||
req = self.client.post('/api/v1/get/import/item', json=input_json ,headers={ 'Authorization': self.apikey })
|
# self.apikey = APIKEY
|
||||||
req_json = parse_response(self, req)
|
# self.item_content = "text to import"
|
||||||
if req_json['status'] == 'imported':
|
# self.item_tags = ["infoleak:analyst-detection=\"private-key\""]
|
||||||
try:
|
# self.expected_tags = ["infoleak:analyst-detection=\"private-key\"", 'infoleak:submission="manual"']
|
||||||
item_id = req_json['items'][0]
|
#
|
||||||
item_not_imported = False
|
# # POST /api/v1/import/item
|
||||||
except Exception as e:
|
# def test_0001_api_import_item(self):
|
||||||
if time.time() - start > import_timout:
|
# input_json = {"type": "text", "tags": self.item_tags, "text": self.item_content}
|
||||||
item_not_imported = False
|
# req = self.client.post('/api/v1/import/item', json=input_json, headers={'Authorization': self.apikey})
|
||||||
self.fail("Import error: {}".format(req_json))
|
# req_json = parse_response(self, req)
|
||||||
else:
|
# import_uuid = req_json['uuid']
|
||||||
if time.time() - start > import_timout:
|
# self.__class__.import_uuid = import_uuid
|
||||||
item_not_imported = False
|
# self.assertTrue(Import_helper.is_valid_uuid_v4(import_uuid))
|
||||||
self.fail("Import Timeout, import status: {}".format(req_json['status']))
|
#
|
||||||
self.__class__.item_id = item_id
|
# # POST /api/v1/get/import/item
|
||||||
|
# def test_0002_api_get_import_item(self):
|
||||||
# Process item
|
# input_json = {"uuid": self.__class__.import_uuid}
|
||||||
time.sleep(5)
|
# item_not_imported = True
|
||||||
|
# import_timout = 60
|
||||||
# POST /api/v1/get/item/content
|
# start = time.time()
|
||||||
def test_0003_api_get_item_content(self):
|
#
|
||||||
input_json = {"id": self.__class__.item_id}
|
# while item_not_imported:
|
||||||
req = self.client.post('/api/v1/get/item/content', json=input_json ,headers={ 'Authorization': self.apikey })
|
# req = self.client.post('/api/v1/get/import/item', json=input_json, headers={'Authorization': self.apikey})
|
||||||
req_json = parse_response(self, req)
|
# req_json = parse_response(self, req)
|
||||||
item_content = req_json['content']
|
# if req_json['status'] == 'imported':
|
||||||
self.assertEqual(item_content, self.item_content)
|
# try:
|
||||||
|
# item_id = req_json['items'][0]
|
||||||
# POST /api/v1/get/item/tag
|
# item_not_imported = False
|
||||||
def test_0004_api_get_item_tag(self):
|
# except Exception as e:
|
||||||
input_json = {"id": self.__class__.item_id}
|
# if time.time() - start > import_timout:
|
||||||
req = self.client.post('/api/v1/get/item/tag', json=input_json ,headers={ 'Authorization': self.apikey })
|
# item_not_imported = False
|
||||||
req_json = parse_response(self, req)
|
# self.fail("Import error: {}".format(req_json))
|
||||||
item_tags = req_json['tags']
|
# else:
|
||||||
self.assertCountEqual(item_tags, self.expected_tags)
|
# if time.time() - start > import_timout:
|
||||||
|
# item_not_imported = False
|
||||||
# POST /api/v1/get/item/tag
|
# self.fail("Import Timeout, import status: {}".format(req_json['status']))
|
||||||
def test_0005_api_get_item_default(self):
|
# self.__class__.item_id = item_id
|
||||||
input_json = {"id": self.__class__.item_id}
|
#
|
||||||
req = self.client.post('/api/v1/get/item/default', json=input_json ,headers={ 'Authorization': self.apikey })
|
# # Process item
|
||||||
req_json = parse_response(self, req)
|
# time.sleep(5)
|
||||||
item_tags = req_json['tags']
|
#
|
||||||
self.assertCountEqual(item_tags, self.expected_tags)
|
# # POST /api/v1/get/item/content
|
||||||
item_content = req_json['content']
|
# def test_0003_api_get_item_content(self):
|
||||||
self.assertEqual(item_content, self.item_content)
|
# input_json = {"id": self.__class__.item_id}
|
||||||
|
# req = self.client.post('/api/v1/get/item/content', json=input_json, headers={'Authorization': self.apikey})
|
||||||
# POST /api/v1/get/item/tag
|
# req_json = parse_response(self, req)
|
||||||
# # TODO: add more test
|
# item_content = req_json['content']
|
||||||
def test_0006_api_get_item(self):
|
# self.assertEqual(item_content, self.item_content)
|
||||||
input_json = {"id": self.__class__.item_id, "content": True}
|
#
|
||||||
req = self.client.post('/api/v1/get/item', json=input_json ,headers={ 'Authorization': self.apikey })
|
# # POST /api/v1/get/item/tag
|
||||||
req_json = parse_response(self, req)
|
# def test_0004_api_get_item_tag(self):
|
||||||
item_tags = req_json['tags']
|
# input_json = {"id": self.__class__.item_id}
|
||||||
self.assertCountEqual(item_tags, self.expected_tags)
|
# req = self.client.post('/api/v1/get/item/tag', json=input_json, headers={'Authorization': self.apikey})
|
||||||
item_content = req_json['content']
|
# req_json = parse_response(self, req)
|
||||||
self.assertEqual(item_content, self.item_content)
|
# item_tags = req_json['tags']
|
||||||
|
# self.assertCountEqual(item_tags, self.expected_tags)
|
||||||
# POST api/v1/add/item/tag
|
#
|
||||||
def test_0007_api_add_item_tag(self):
|
# # POST /api/v1/get/item/tag
|
||||||
tags_to_add = ["infoleak:analyst-detection=\"api-key\""]
|
# def test_0005_api_get_item_default(self):
|
||||||
current_item_tag = Tag.get_obj_tag(self.__class__.item_id)
|
# input_json = {"id": self.__class__.item_id}
|
||||||
current_item_tag.append(tags_to_add[0])
|
# req = self.client.post('/api/v1/get/item/default', json=input_json, headers={'Authorization': self.apikey})
|
||||||
|
# req_json = parse_response(self, req)
|
||||||
#galaxy_to_add = ["misp-galaxy:stealer=\"Vidar\""]
|
# item_tags = req_json['tags']
|
||||||
input_json = {"id": self.__class__.item_id, "tags": tags_to_add}
|
# self.assertCountEqual(item_tags, self.expected_tags)
|
||||||
req = self.client.post('/api/v1/add/item/tag', json=input_json ,headers={ 'Authorization': self.apikey })
|
# item_content = req_json['content']
|
||||||
req_json = parse_response(self, req)
|
# self.assertEqual(item_content, self.item_content)
|
||||||
item_tags = req_json['tags']
|
#
|
||||||
self.assertEqual(item_tags, tags_to_add)
|
# # POST /api/v1/get/item/tag
|
||||||
|
# # # TODO: add more test
|
||||||
new_item_tag = Tag.get_obj_tag(self.__class__.item_id)
|
# def test_0006_api_get_item(self):
|
||||||
self.assertCountEqual(new_item_tag, current_item_tag)
|
# input_json = {"id": self.__class__.item_id, "content": True}
|
||||||
|
# req = self.client.post('/api/v1/get/item', json=input_json, headers={'Authorization': self.apikey})
|
||||||
# DELETE api/v1/delete/item/tag
|
# req_json = parse_response(self, req)
|
||||||
def test_0008_api_add_item_tag(self):
|
# item_tags = req_json['tags']
|
||||||
tags_to_delete = ["infoleak:analyst-detection=\"api-key\""]
|
# self.assertCountEqual(item_tags, self.expected_tags)
|
||||||
input_json = {"id": self.__class__.item_id, "tags": tags_to_delete}
|
# item_content = req_json['content']
|
||||||
req = self.client.delete('/api/v1/delete/item/tag', json=input_json ,headers={ 'Authorization': self.apikey })
|
# self.assertEqual(item_content, self.item_content)
|
||||||
req_json = parse_response(self, req)
|
#
|
||||||
item_tags = req_json['tags']
|
# # POST api/v1/add/item/tag
|
||||||
self.assertCountEqual(item_tags, tags_to_delete)
|
# def test_0007_api_add_item_tag(self):
|
||||||
current_item_tag = Tag.get_obj_tag(self.__class__.item_id)
|
# tags_to_add = ["infoleak:analyst-detection=\"api-key\""]
|
||||||
if tags_to_delete[0] in current_item_tag:
|
# current_item_tag = Tag.get_obj_tag(self.__class__.item_id)
|
||||||
self.fail('Tag no deleted')
|
# current_item_tag.append(tags_to_add[0])
|
||||||
|
#
|
||||||
# POST api/v1/get/tag/metadata
|
# # galaxy_to_add = ["misp-galaxy:stealer=\"Vidar\""]
|
||||||
def test_0009_api_add_item_tag(self):
|
# input_json = {"id": self.__class__.item_id, "tags": tags_to_add}
|
||||||
input_json = {"tag": self.item_tags[0]}
|
# req = self.client.post('/api/v1/add/item/tag', json=input_json, headers={'Authorization': self.apikey})
|
||||||
req = self.client.post('/api/v1/get/tag/metadata', json=input_json ,headers={ 'Authorization': self.apikey })
|
# req_json = parse_response(self, req)
|
||||||
req_json = parse_response(self, req)
|
# item_tags = req_json['tags']
|
||||||
self.assertEqual(req_json['tag'], self.item_tags[0])
|
# self.assertEqual(item_tags, tags_to_add)
|
||||||
|
#
|
||||||
# GET api/v1/get/tag/all
|
# new_item_tag = Tag.get_obj_tag(self.__class__.item_id)
|
||||||
def test_0010_api_add_item_tag(self):
|
# self.assertCountEqual(new_item_tag, current_item_tag)
|
||||||
input_json = {"tag": self.item_tags[0]}
|
#
|
||||||
req = self.client.get('/api/v1/get/tag/all', json=input_json ,headers={ 'Authorization': self.apikey })
|
# # DELETE api/v1/delete/item/tag
|
||||||
req_json = parse_response(self, req)
|
# def test_0008_api_add_item_tag(self):
|
||||||
self.assertTrue(req_json['tags'])
|
# tags_to_delete = ["infoleak:analyst-detection=\"api-key\""]
|
||||||
|
# input_json = {"id": self.__class__.item_id, "tags": tags_to_delete}
|
||||||
|
# req = self.client.delete('/api/v1/delete/item/tag', json=input_json, headers={'Authorization': self.apikey})
|
||||||
|
# req_json = parse_response(self, req)
|
||||||
|
# item_tags = req_json['tags']
|
||||||
|
# self.assertCountEqual(item_tags, tags_to_delete)
|
||||||
|
# current_item_tag = Tag.get_obj_tag(self.__class__.item_id)
|
||||||
|
# if tags_to_delete[0] in current_item_tag:
|
||||||
|
# self.fail('Tag no deleted')
|
||||||
|
#
|
||||||
|
# # POST api/v1/get/tag/metadata
|
||||||
|
# def test_0009_api_add_item_tag(self):
|
||||||
|
# input_json = {"tag": self.item_tags[0]}
|
||||||
|
# req = self.client.post('/api/v1/get/tag/metadata', json=input_json, headers={'Authorization': self.apikey})
|
||||||
|
# req_json = parse_response(self, req)
|
||||||
|
# self.assertEqual(req_json['tag'], self.item_tags[0])
|
||||||
|
#
|
||||||
|
# # GET api/v1/get/tag/all
|
||||||
|
# def test_0010_api_add_item_tag(self):
|
||||||
|
# input_json = {"tag": self.item_tags[0]}
|
||||||
|
# req = self.client.get('/api/v1/get/tag/all', json=input_json, headers={'Authorization': self.apikey})
|
||||||
|
# req_json = parse_response(self, req)
|
||||||
|
# self.assertTrue(req_json['tags'])
|
||||||
|
#
|
||||||
|
#
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main(argv=['first-arg-is-ignored'], exit=False)
|
unittest.main(argv=['first-arg-is-ignored'], exit=False)
|
||||||
|
|
|
@ -1,22 +0,0 @@
|
||||||
#!/usr/bin/env python3
|
|
||||||
# -*- coding: utf-8 -*-
|
|
||||||
|
|
||||||
import unittest
|
|
||||||
import sys,os
|
|
||||||
|
|
||||||
sys.path.append(os.environ['AIL_BIN'])
|
|
||||||
|
|
||||||
from Helper import Process
|
|
||||||
|
|
||||||
class TestHelper(unittest.TestCase):
|
|
||||||
|
|
||||||
def setUp(self):
|
|
||||||
|
|
||||||
config_section = 'Keys'
|
|
||||||
|
|
||||||
|
|
||||||
def test_Process_Constructor_using_key_module(self):
|
|
||||||
|
|
||||||
conf_section = 'Keys'
|
|
||||||
process = Process(conf_section)
|
|
||||||
self.assertEqual(process.subscriber_name, 'Keys')
|
|
|
@ -49,9 +49,9 @@ class Test_Module_ApiKey(unittest.TestCase):
|
||||||
aws_secret_key = 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'
|
aws_secret_key = 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'
|
||||||
|
|
||||||
matches = self.module_obj.compute(f'{item_id} 3', r_result=True)
|
matches = self.module_obj.compute(f'{item_id} 3', r_result=True)
|
||||||
self.assertCountEqual(matches[0], [google_api_key])
|
self.assertCountEqual(matches[0], {google_api_key})
|
||||||
self.assertCountEqual(matches[1], [aws_access_key])
|
self.assertCountEqual(matches[1], {aws_access_key})
|
||||||
self.assertCountEqual(matches[2], [aws_secret_key])
|
self.assertCountEqual(matches[2], {aws_secret_key})
|
||||||
|
|
||||||
class Test_Module_Categ(unittest.TestCase):
|
class Test_Module_Categ(unittest.TestCase):
|
||||||
|
|
||||||
|
@ -160,25 +160,8 @@ class Test_Module_Onion(unittest.TestCase):
|
||||||
item_id = 'tests/2021/01/01/onion.gz'
|
item_id = 'tests/2021/01/01/onion.gz'
|
||||||
domain_1 = 'eswpccgr5xyovsahffkehgleqthrasfpfdblwbs4lstd345dwq5qumqd.onion'
|
domain_1 = 'eswpccgr5xyovsahffkehgleqthrasfpfdblwbs4lstd345dwq5qumqd.onion'
|
||||||
domain_2 = 'www.facebookcorewwwi.onion'
|
domain_2 = 'www.facebookcorewwwi.onion'
|
||||||
crawlers.queue_test_clean_up('onion', domain_1, 'tests/2021/01/01/onion.gz')
|
|
||||||
|
|
||||||
self.module_obj.compute(f'{item_id} 3')
|
self.module_obj.compute(f'{item_id} 3')
|
||||||
if crawlers.is_crawler_activated():
|
|
||||||
# # check domain queues
|
|
||||||
# all domains queue
|
|
||||||
self.assertTrue(crawlers.is_domain_in_queue('onion', domain_1))
|
|
||||||
# all url/item queue
|
|
||||||
self.assertTrue(crawlers.is_item_in_queue('onion', f'http://{domain_1}', item_id))
|
|
||||||
# domain blacklist
|
|
||||||
self.assertFalse(crawlers.is_domain_in_queue('onion', domain_2))
|
|
||||||
# invalid onion
|
|
||||||
self.assertFalse(crawlers.is_domain_in_queue('onion', 'invalid.onion'))
|
|
||||||
|
|
||||||
# clean DB
|
|
||||||
crawlers.queue_test_clean_up('onion', domain_1, 'tests/2021/01/01/onion.gz')
|
|
||||||
else:
|
|
||||||
# # TODO: check warning logs
|
|
||||||
pass
|
|
||||||
|
|
||||||
class Test_Module_Telegram(unittest.TestCase):
|
class Test_Module_Telegram(unittest.TestCase):
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue