create first test

This commit is contained in:
Terrtia 2018-04-11 10:14:33 +02:00
parent 21f3a7c057
commit 14583f702f
7 changed files with 131 additions and 32 deletions

View file

@ -1,7 +1,7 @@
language: python language: python
python: python:
- "2.7" - "3.5"
sudo: required sudo: required
@ -16,6 +16,7 @@ env:
install: install:
- ./installing_deps.sh - ./installing_deps.sh
- pip install coveralls codecov
script: script:
- pushd bin - pushd bin
@ -28,8 +29,13 @@ script:
- ./Shutdown.py - ./Shutdown.py
- popd - popd
- find logs/* -exec cat {} \; - find logs/* -exec cat {} \;
- nosetests --with-coverage --cover-package=things -d
notifications: notifications:
email: email:
on_success: change on_success: change
on_failure: change on_failure: change
after_success:
- codecov
- coveralls

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python2 #!/usr/bin/env python3.5
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
""" """
@ -15,32 +15,47 @@ RSA private key, certificate messages
import time import time
from pubsublogger import publisher from pubsublogger import publisher
from Helper import Process from bin.packages import Paste
from packages import Paste from bin.Helper import Process
def search_key(message): def search_key(paste):
paste = Paste.Paste(message)
content = paste.get_p_content() content = paste.get_p_content()
find = False find = False
if '-----BEGIN PGP MESSAGE-----' in content: if b'-----BEGIN PGP MESSAGE-----' in content:
publisher.warning('{} has a PGP enc message'.format(paste.p_name)) publisher.warning('{} has a PGP enc message'.format(paste.p_name))
find = True find = True
if '-----BEGIN CERTIFICATE-----' in content: if b'-----BEGIN CERTIFICATE-----' in content:
publisher.warning('{} has a certificate message'.format(paste.p_name)) publisher.warning('{} has a certificate message'.format(paste.p_name))
find = True find = True
if '-----BEGIN RSA PRIVATE KEY-----' in content: if b'-----BEGIN RSA PRIVATE KEY-----' in content:
publisher.warning('{} has a RSA key message'.format(paste.p_name)) publisher.warning('{} has a RSA private key message'.format(paste.p_name))
find = True find = True
if '-----BEGIN PRIVATE KEY-----' in content: if b'-----BEGIN PRIVATE KEY-----' in content:
publisher.warning('{} has a private message'.format(paste.p_name)) publisher.warning('{} has a private key message'.format(paste.p_name))
find = True find = True
if '-----BEGIN ENCRYPTED PRIVATE KEY-----' in content: if b'-----BEGIN ENCRYPTED PRIVATE KEY-----' in content:
publisher.warning('{} has an encrypted private message'.format(paste.p_name)) publisher.warning('{} has an encrypted private key message'.format(paste.p_name))
find = True
if b'-----BEGIN OPENSSH PRIVATE KEY-----' in content:
publisher.warning('{} has an openssh private key message'.format(paste.p_name))
find = True
if b'-----BEGIN DSA PRIVATE KEY-----' in content:
publisher.warning('{} has a dsa private key message'.format(paste.p_name))
find = True
if b'-----BEGIN EC PRIVATE KEY-----' in content:
publisher.warning('{} has an ec private key message'.format(paste.p_name))
find = True
if b'-----BEGIN PGP PRIVATE KEY BLOCK-----' in content:
publisher.warning('{} has a pgp private key block message'.format(paste.p_name))
find = True find = True
if find : if find :
@ -77,6 +92,7 @@ if __name__ == '__main__':
continue continue
# Do something with the message from the queue # Do something with the message from the queue
search_key(message) paste = Paste.Paste(message)
search_key(paste)
# (Optional) Send that thing to the next queue # (Optional) Send that thing to the next queue

View file

@ -5,7 +5,7 @@ set -x
sudo apt-get update sudo apt-get update
sudo apt-get install python-pip python-virtualenv python-dev libfreetype6-dev \ sudo apt-get install python3-pip python3-virtualenv python3-dev libfreetype6-dev \
screen g++ python-tk unzip libsnappy-dev cmake -y screen g++ python-tk unzip libsnappy-dev cmake -y
#optional tor install #optional tor install
@ -15,7 +15,7 @@ sudo apt-get install tor
sudo apt-get install libssl-dev libfreetype6-dev python-numpy -y sudo apt-get install libssl-dev libfreetype6-dev python-numpy -y
#pyMISP #pyMISP
sudo apt-get -y install python3-pip #sudo apt-get -y install python3-pip
# DNS deps # DNS deps
sudo apt-get install libadns1 libadns1-dev -y sudo apt-get install libadns1 libadns1-dev -y
@ -73,12 +73,12 @@ if [ ! -f bin/packages/config.cfg ]; then
fi fi
pushd var/www/ pushd var/www/
./update_thirdparty.sh sudo ./update_thirdparty.sh
popd popd
if [ -z "$VIRTUAL_ENV" ]; then if [ -z "$VIRTUAL_ENV" ]; then
virtualenv AILENV virtualenv -p python3 AILENV
echo export AIL_HOME=$(pwd) >> ./AILENV/bin/activate echo export AIL_HOME=$(pwd) >> ./AILENV/bin/activate
echo export AIL_BIN=$(pwd)/bin/ >> ./AILENV/bin/activate echo export AIL_BIN=$(pwd)/bin/ >> ./AILENV/bin/activate
@ -95,26 +95,25 @@ year2=20`date --date='-1 year' +%y`
mkdir -p $AIL_HOME/{PASTES,Blooms,dumps} mkdir -p $AIL_HOME/{PASTES,Blooms,dumps}
mkdir -p $AIL_HOME/LEVEL_DB_DATA/{$year1,$year2} mkdir -p $AIL_HOME/LEVEL_DB_DATA/{$year1,$year2}
pip install -U pip pip3 install -U pip
pip install -U -r pip_packages_requirement.txt
pip3 install -U -r pip3_packages_requirement.txt pip3 install -U -r pip3_packages_requirement.txt
# Pyfaup # Pyfaup
pushd faup/src/lib/bindings/python/ pushd faup/src/lib/bindings/python/
python setup.py install python3 setup.py install
popd popd
# Py tlsh # Py tlsh
pushd tlsh/py_ext pushd tlsh/py_ext
python setup.py build #python setup.py build
python setup.py install #python setup.py install
sudo python3 setup.py build python3 setup.py build
sudo python3 setup.py install python3 setup.py install
# Download the necessary NLTK corpora and sentiment vader # Download the necessary NLTK corpora and sentiment vader
HOME=$(pwd) python -m textblob.download_corpora HOME=$(pwd) python3 -m textblob.download_corpora
python -m nltk.downloader vader_lexicon python3 -m nltk.downloader vader_lexicon
python -m nltk.downloader punkt python3 -m nltk.downloader punkt
#Create the file all_module and update the graph in doc #Create the file all_module and update the graph in doc
$AIL_HOME/doc/generate_modules_data_flow_graph.sh $AIL_HOME/doc/generate_modules_data_flow_graph.sh

View file

@ -1,13 +1,63 @@
pymisp pymisp
redis redis
filemagic #filemagic conflict with magic
crcmod crcmod
mmh3 mmh3
ssdeep ssdeep
nltk
textblob
pubsublogger pubsublogger
zmq zmq
langid langid
#Essential
redis
pyzmq
dnspython
logbook
pubsublogger
textblob
#Tokeniser
nltk
#Graph
numpy
matplotlib
networkx
terminaltables
colorama
asciimatics
# Hashlib
crcmod
mmh3
ssdeep
python-Levenshtein
#Others
python-magic
pybloomfiltermmap
psutil
phonenumbers
ipython
flask
texttable
#DomainClassifier
DomainClassifier
#Indexer requirements
whoosh
ipaddress
pycountry
# To fetch Onion urls
PySocks
#ASN lookup requirements
#https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/adns-python/adns-python-1.2.1.tar.gz
https://github.com/trolldbois/python-cymru-services/archive/master.zip
https://github.com/saffsd/langid.py/archive/master.zip

Binary file not shown.

0
tests/__init__.py Normal file
View file

28
tests/testKeys.py Normal file
View file

@ -0,0 +1,28 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import unittest
import magic
from bin.packages.Paste import Paste
import bin.Keys as Keys
from bin.Helper import Process
import pubsublogger
class TestKeysModule(unittest.TestCase):
def setUp(self):
self.paste = Paste('samples/2018/01/01/keys_certificat_sample.gz')
# Section name in bin/packages/modules.cfg
self.config_section = 'Keys'
# Setup the I/O queues
p = Process(self.config_section)
def test_search_key(self):
with self.assertRaises(pubsublogger.exceptions.NoChannelError):
Keys.search_key(self.paste)