mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 08:38:28 +00:00
create first test
This commit is contained in:
parent
21f3a7c057
commit
14583f702f
7 changed files with 131 additions and 32 deletions
|
@ -1,7 +1,7 @@
|
||||||
language: python
|
language: python
|
||||||
|
|
||||||
python:
|
python:
|
||||||
- "2.7"
|
- "3.5"
|
||||||
|
|
||||||
sudo: required
|
sudo: required
|
||||||
|
|
||||||
|
@ -16,6 +16,7 @@ env:
|
||||||
|
|
||||||
install:
|
install:
|
||||||
- ./installing_deps.sh
|
- ./installing_deps.sh
|
||||||
|
- pip install coveralls codecov
|
||||||
|
|
||||||
script:
|
script:
|
||||||
- pushd bin
|
- pushd bin
|
||||||
|
@ -28,8 +29,13 @@ script:
|
||||||
- ./Shutdown.py
|
- ./Shutdown.py
|
||||||
- popd
|
- popd
|
||||||
- find logs/* -exec cat {} \;
|
- find logs/* -exec cat {} \;
|
||||||
|
- nosetests --with-coverage --cover-package=things -d
|
||||||
|
|
||||||
notifications:
|
notifications:
|
||||||
email:
|
email:
|
||||||
on_success: change
|
on_success: change
|
||||||
on_failure: change
|
on_failure: change
|
||||||
|
|
||||||
|
after_success:
|
||||||
|
- codecov
|
||||||
|
- coveralls
|
||||||
|
|
44
bin/Keys.py
44
bin/Keys.py
|
@ -1,4 +1,4 @@
|
||||||
#!/usr/bin/env python2
|
#!/usr/bin/env python3.5
|
||||||
# -*-coding:UTF-8 -*
|
# -*-coding:UTF-8 -*
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
@ -15,32 +15,47 @@ RSA private key, certificate messages
|
||||||
import time
|
import time
|
||||||
from pubsublogger import publisher
|
from pubsublogger import publisher
|
||||||
|
|
||||||
from Helper import Process
|
from bin.packages import Paste
|
||||||
from packages import Paste
|
from bin.Helper import Process
|
||||||
|
|
||||||
|
|
||||||
def search_key(message):
|
def search_key(paste):
|
||||||
paste = Paste.Paste(message)
|
|
||||||
content = paste.get_p_content()
|
content = paste.get_p_content()
|
||||||
find = False
|
find = False
|
||||||
if '-----BEGIN PGP MESSAGE-----' in content:
|
if b'-----BEGIN PGP MESSAGE-----' in content:
|
||||||
publisher.warning('{} has a PGP enc message'.format(paste.p_name))
|
publisher.warning('{} has a PGP enc message'.format(paste.p_name))
|
||||||
find = True
|
find = True
|
||||||
|
|
||||||
if '-----BEGIN CERTIFICATE-----' in content:
|
if b'-----BEGIN CERTIFICATE-----' in content:
|
||||||
publisher.warning('{} has a certificate message'.format(paste.p_name))
|
publisher.warning('{} has a certificate message'.format(paste.p_name))
|
||||||
find = True
|
find = True
|
||||||
|
|
||||||
if '-----BEGIN RSA PRIVATE KEY-----' in content:
|
if b'-----BEGIN RSA PRIVATE KEY-----' in content:
|
||||||
publisher.warning('{} has a RSA key message'.format(paste.p_name))
|
publisher.warning('{} has a RSA private key message'.format(paste.p_name))
|
||||||
find = True
|
find = True
|
||||||
|
|
||||||
if '-----BEGIN PRIVATE KEY-----' in content:
|
if b'-----BEGIN PRIVATE KEY-----' in content:
|
||||||
publisher.warning('{} has a private message'.format(paste.p_name))
|
publisher.warning('{} has a private key message'.format(paste.p_name))
|
||||||
find = True
|
find = True
|
||||||
|
|
||||||
if '-----BEGIN ENCRYPTED PRIVATE KEY-----' in content:
|
if b'-----BEGIN ENCRYPTED PRIVATE KEY-----' in content:
|
||||||
publisher.warning('{} has an encrypted private message'.format(paste.p_name))
|
publisher.warning('{} has an encrypted private key message'.format(paste.p_name))
|
||||||
|
find = True
|
||||||
|
|
||||||
|
if b'-----BEGIN OPENSSH PRIVATE KEY-----' in content:
|
||||||
|
publisher.warning('{} has an openssh private key message'.format(paste.p_name))
|
||||||
|
find = True
|
||||||
|
|
||||||
|
if b'-----BEGIN DSA PRIVATE KEY-----' in content:
|
||||||
|
publisher.warning('{} has a dsa private key message'.format(paste.p_name))
|
||||||
|
find = True
|
||||||
|
|
||||||
|
if b'-----BEGIN EC PRIVATE KEY-----' in content:
|
||||||
|
publisher.warning('{} has an ec private key message'.format(paste.p_name))
|
||||||
|
find = True
|
||||||
|
|
||||||
|
if b'-----BEGIN PGP PRIVATE KEY BLOCK-----' in content:
|
||||||
|
publisher.warning('{} has a pgp private key block message'.format(paste.p_name))
|
||||||
find = True
|
find = True
|
||||||
|
|
||||||
if find :
|
if find :
|
||||||
|
@ -77,6 +92,7 @@ if __name__ == '__main__':
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Do something with the message from the queue
|
# Do something with the message from the queue
|
||||||
search_key(message)
|
paste = Paste.Paste(message)
|
||||||
|
search_key(paste)
|
||||||
|
|
||||||
# (Optional) Send that thing to the next queue
|
# (Optional) Send that thing to the next queue
|
||||||
|
|
|
@ -5,7 +5,7 @@ set -x
|
||||||
|
|
||||||
sudo apt-get update
|
sudo apt-get update
|
||||||
|
|
||||||
sudo apt-get install python-pip python-virtualenv python-dev libfreetype6-dev \
|
sudo apt-get install python3-pip python3-virtualenv python3-dev libfreetype6-dev \
|
||||||
screen g++ python-tk unzip libsnappy-dev cmake -y
|
screen g++ python-tk unzip libsnappy-dev cmake -y
|
||||||
|
|
||||||
#optional tor install
|
#optional tor install
|
||||||
|
@ -15,7 +15,7 @@ sudo apt-get install tor
|
||||||
sudo apt-get install libssl-dev libfreetype6-dev python-numpy -y
|
sudo apt-get install libssl-dev libfreetype6-dev python-numpy -y
|
||||||
|
|
||||||
#pyMISP
|
#pyMISP
|
||||||
sudo apt-get -y install python3-pip
|
#sudo apt-get -y install python3-pip
|
||||||
|
|
||||||
# DNS deps
|
# DNS deps
|
||||||
sudo apt-get install libadns1 libadns1-dev -y
|
sudo apt-get install libadns1 libadns1-dev -y
|
||||||
|
@ -73,12 +73,12 @@ if [ ! -f bin/packages/config.cfg ]; then
|
||||||
fi
|
fi
|
||||||
|
|
||||||
pushd var/www/
|
pushd var/www/
|
||||||
./update_thirdparty.sh
|
sudo ./update_thirdparty.sh
|
||||||
popd
|
popd
|
||||||
|
|
||||||
if [ -z "$VIRTUAL_ENV" ]; then
|
if [ -z "$VIRTUAL_ENV" ]; then
|
||||||
|
|
||||||
virtualenv AILENV
|
virtualenv -p python3 AILENV
|
||||||
|
|
||||||
echo export AIL_HOME=$(pwd) >> ./AILENV/bin/activate
|
echo export AIL_HOME=$(pwd) >> ./AILENV/bin/activate
|
||||||
echo export AIL_BIN=$(pwd)/bin/ >> ./AILENV/bin/activate
|
echo export AIL_BIN=$(pwd)/bin/ >> ./AILENV/bin/activate
|
||||||
|
@ -95,26 +95,25 @@ year2=20`date --date='-1 year' +%y`
|
||||||
mkdir -p $AIL_HOME/{PASTES,Blooms,dumps}
|
mkdir -p $AIL_HOME/{PASTES,Blooms,dumps}
|
||||||
mkdir -p $AIL_HOME/LEVEL_DB_DATA/{$year1,$year2}
|
mkdir -p $AIL_HOME/LEVEL_DB_DATA/{$year1,$year2}
|
||||||
|
|
||||||
pip install -U pip
|
pip3 install -U pip
|
||||||
pip install -U -r pip_packages_requirement.txt
|
|
||||||
pip3 install -U -r pip3_packages_requirement.txt
|
pip3 install -U -r pip3_packages_requirement.txt
|
||||||
|
|
||||||
# Pyfaup
|
# Pyfaup
|
||||||
pushd faup/src/lib/bindings/python/
|
pushd faup/src/lib/bindings/python/
|
||||||
python setup.py install
|
python3 setup.py install
|
||||||
popd
|
popd
|
||||||
|
|
||||||
# Py tlsh
|
# Py tlsh
|
||||||
pushd tlsh/py_ext
|
pushd tlsh/py_ext
|
||||||
python setup.py build
|
#python setup.py build
|
||||||
python setup.py install
|
#python setup.py install
|
||||||
sudo python3 setup.py build
|
python3 setup.py build
|
||||||
sudo python3 setup.py install
|
python3 setup.py install
|
||||||
|
|
||||||
# Download the necessary NLTK corpora and sentiment vader
|
# Download the necessary NLTK corpora and sentiment vader
|
||||||
HOME=$(pwd) python -m textblob.download_corpora
|
HOME=$(pwd) python3 -m textblob.download_corpora
|
||||||
python -m nltk.downloader vader_lexicon
|
python3 -m nltk.downloader vader_lexicon
|
||||||
python -m nltk.downloader punkt
|
python3 -m nltk.downloader punkt
|
||||||
|
|
||||||
#Create the file all_module and update the graph in doc
|
#Create the file all_module and update the graph in doc
|
||||||
$AIL_HOME/doc/generate_modules_data_flow_graph.sh
|
$AIL_HOME/doc/generate_modules_data_flow_graph.sh
|
||||||
|
|
|
@ -1,13 +1,63 @@
|
||||||
pymisp
|
pymisp
|
||||||
|
|
||||||
redis
|
redis
|
||||||
filemagic
|
#filemagic conflict with magic
|
||||||
crcmod
|
crcmod
|
||||||
mmh3
|
mmh3
|
||||||
ssdeep
|
ssdeep
|
||||||
nltk
|
|
||||||
textblob
|
|
||||||
|
|
||||||
pubsublogger
|
pubsublogger
|
||||||
zmq
|
zmq
|
||||||
langid
|
langid
|
||||||
|
|
||||||
|
#Essential
|
||||||
|
redis
|
||||||
|
pyzmq
|
||||||
|
dnspython
|
||||||
|
logbook
|
||||||
|
pubsublogger
|
||||||
|
textblob
|
||||||
|
|
||||||
|
#Tokeniser
|
||||||
|
nltk
|
||||||
|
|
||||||
|
#Graph
|
||||||
|
numpy
|
||||||
|
matplotlib
|
||||||
|
networkx
|
||||||
|
terminaltables
|
||||||
|
colorama
|
||||||
|
asciimatics
|
||||||
|
|
||||||
|
# Hashlib
|
||||||
|
crcmod
|
||||||
|
mmh3
|
||||||
|
ssdeep
|
||||||
|
python-Levenshtein
|
||||||
|
|
||||||
|
#Others
|
||||||
|
python-magic
|
||||||
|
pybloomfiltermmap
|
||||||
|
psutil
|
||||||
|
phonenumbers
|
||||||
|
|
||||||
|
ipython
|
||||||
|
flask
|
||||||
|
texttable
|
||||||
|
|
||||||
|
#DomainClassifier
|
||||||
|
DomainClassifier
|
||||||
|
#Indexer requirements
|
||||||
|
whoosh
|
||||||
|
|
||||||
|
ipaddress
|
||||||
|
pycountry
|
||||||
|
|
||||||
|
# To fetch Onion urls
|
||||||
|
PySocks
|
||||||
|
|
||||||
|
#ASN lookup requirements
|
||||||
|
#https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/adns-python/adns-python-1.2.1.tar.gz
|
||||||
|
https://github.com/trolldbois/python-cymru-services/archive/master.zip
|
||||||
|
|
||||||
|
https://github.com/saffsd/langid.py/archive/master.zip
|
||||||
|
|
BIN
samples/2018/01/01/keys_certificat_sample.gz
Normal file
BIN
samples/2018/01/01/keys_certificat_sample.gz
Normal file
Binary file not shown.
0
tests/__init__.py
Normal file
0
tests/__init__.py
Normal file
28
tests/testKeys.py
Normal file
28
tests/testKeys.py
Normal file
|
@ -0,0 +1,28 @@
|
||||||
|
#!/usr/bin/env python
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
import magic
|
||||||
|
|
||||||
|
from bin.packages.Paste import Paste
|
||||||
|
import bin.Keys as Keys
|
||||||
|
from bin.Helper import Process
|
||||||
|
import pubsublogger
|
||||||
|
|
||||||
|
|
||||||
|
class TestKeysModule(unittest.TestCase):
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.paste = Paste('samples/2018/01/01/keys_certificat_sample.gz')
|
||||||
|
|
||||||
|
# Section name in bin/packages/modules.cfg
|
||||||
|
self.config_section = 'Keys'
|
||||||
|
|
||||||
|
# Setup the I/O queues
|
||||||
|
p = Process(self.config_section)
|
||||||
|
|
||||||
|
|
||||||
|
def test_search_key(self):
|
||||||
|
with self.assertRaises(pubsublogger.exceptions.NoChannelError):
|
||||||
|
Keys.search_key(self.paste)
|
Loading…
Reference in a new issue