From 14583f702f56ffa32cb36e494743da2854b063e1 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 11 Apr 2018 10:14:33 +0200 Subject: [PATCH] create first test --- .travis.yml | 8 ++- bin/Keys.py | 44 ++++++++++----- installing_deps.sh | 27 +++++---- pip3_packages_requirement.txt | 56 ++++++++++++++++++- samples/2018/01/01/keys_certificat_sample.gz | Bin 0 -> 78 bytes tests/__init__.py | 0 tests/testKeys.py | 28 ++++++++++ 7 files changed, 131 insertions(+), 32 deletions(-) create mode 100644 samples/2018/01/01/keys_certificat_sample.gz create mode 100644 tests/__init__.py create mode 100644 tests/testKeys.py diff --git a/.travis.yml b/.travis.yml index 554d0967..cbef2841 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: python python: - - "2.7" + - "3.5" sudo: required @@ -16,6 +16,7 @@ env: install: - ./installing_deps.sh + - pip install coveralls codecov script: - pushd bin @@ -28,8 +29,13 @@ script: - ./Shutdown.py - popd - find logs/* -exec cat {} \; + - nosetests --with-coverage --cover-package=things -d notifications: email: on_success: change on_failure: change + +after_success: + - codecov + - coveralls diff --git a/bin/Keys.py b/bin/Keys.py index 0e474b40..71c04602 100755 --- a/bin/Keys.py +++ b/bin/Keys.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3.5 # -*-coding:UTF-8 -* """ @@ -15,32 +15,47 @@ RSA private key, certificate messages import time from pubsublogger import publisher -from Helper import Process -from packages import Paste +from bin.packages import Paste +from bin.Helper import Process -def search_key(message): - paste = Paste.Paste(message) +def search_key(paste): content = paste.get_p_content() find = False - if '-----BEGIN PGP MESSAGE-----' in content: + if b'-----BEGIN PGP MESSAGE-----' in content: publisher.warning('{} has a PGP enc message'.format(paste.p_name)) find = True - if '-----BEGIN CERTIFICATE-----' in content: + if b'-----BEGIN CERTIFICATE-----' in content: publisher.warning('{} has a certificate message'.format(paste.p_name)) find = True - if '-----BEGIN RSA PRIVATE KEY-----' in content: - publisher.warning('{} has a RSA key message'.format(paste.p_name)) + if b'-----BEGIN RSA PRIVATE KEY-----' in content: + publisher.warning('{} has a RSA private key message'.format(paste.p_name)) find = True - if '-----BEGIN PRIVATE KEY-----' in content: - publisher.warning('{} has a private message'.format(paste.p_name)) + if b'-----BEGIN PRIVATE KEY-----' in content: + publisher.warning('{} has a private key message'.format(paste.p_name)) find = True - if '-----BEGIN ENCRYPTED PRIVATE KEY-----' in content: - publisher.warning('{} has an encrypted private message'.format(paste.p_name)) + if b'-----BEGIN ENCRYPTED PRIVATE KEY-----' in content: + publisher.warning('{} has an encrypted private key message'.format(paste.p_name)) + find = True + + if b'-----BEGIN OPENSSH PRIVATE KEY-----' in content: + publisher.warning('{} has an openssh private key message'.format(paste.p_name)) + find = True + + if b'-----BEGIN DSA PRIVATE KEY-----' in content: + publisher.warning('{} has a dsa private key message'.format(paste.p_name)) + find = True + + if b'-----BEGIN EC PRIVATE KEY-----' in content: + publisher.warning('{} has an ec private key message'.format(paste.p_name)) + find = True + + if b'-----BEGIN PGP PRIVATE KEY BLOCK-----' in content: + publisher.warning('{} has a pgp private key block message'.format(paste.p_name)) find = True if find : @@ -77,6 +92,7 @@ if __name__ == '__main__': continue # Do something with the message from the queue - search_key(message) + paste = Paste.Paste(message) + search_key(paste) # (Optional) Send that thing to the next queue diff --git a/installing_deps.sh b/installing_deps.sh index c600f604..3f8b7ee4 100755 --- a/installing_deps.sh +++ b/installing_deps.sh @@ -5,7 +5,7 @@ set -x sudo apt-get update -sudo apt-get install python-pip python-virtualenv python-dev libfreetype6-dev \ +sudo apt-get install python3-pip python3-virtualenv python3-dev libfreetype6-dev \ screen g++ python-tk unzip libsnappy-dev cmake -y #optional tor install @@ -15,7 +15,7 @@ sudo apt-get install tor sudo apt-get install libssl-dev libfreetype6-dev python-numpy -y #pyMISP -sudo apt-get -y install python3-pip +#sudo apt-get -y install python3-pip # DNS deps sudo apt-get install libadns1 libadns1-dev -y @@ -73,12 +73,12 @@ if [ ! -f bin/packages/config.cfg ]; then fi pushd var/www/ -./update_thirdparty.sh +sudo ./update_thirdparty.sh popd if [ -z "$VIRTUAL_ENV" ]; then - virtualenv AILENV + virtualenv -p python3 AILENV echo export AIL_HOME=$(pwd) >> ./AILENV/bin/activate echo export AIL_BIN=$(pwd)/bin/ >> ./AILENV/bin/activate @@ -95,26 +95,25 @@ year2=20`date --date='-1 year' +%y` mkdir -p $AIL_HOME/{PASTES,Blooms,dumps} mkdir -p $AIL_HOME/LEVEL_DB_DATA/{$year1,$year2} -pip install -U pip -pip install -U -r pip_packages_requirement.txt +pip3 install -U pip pip3 install -U -r pip3_packages_requirement.txt # Pyfaup pushd faup/src/lib/bindings/python/ -python setup.py install +python3 setup.py install popd # Py tlsh pushd tlsh/py_ext -python setup.py build -python setup.py install -sudo python3 setup.py build -sudo python3 setup.py install +#python setup.py build +#python setup.py install +python3 setup.py build +python3 setup.py install # Download the necessary NLTK corpora and sentiment vader -HOME=$(pwd) python -m textblob.download_corpora -python -m nltk.downloader vader_lexicon -python -m nltk.downloader punkt +HOME=$(pwd) python3 -m textblob.download_corpora +python3 -m nltk.downloader vader_lexicon +python3 -m nltk.downloader punkt #Create the file all_module and update the graph in doc $AIL_HOME/doc/generate_modules_data_flow_graph.sh diff --git a/pip3_packages_requirement.txt b/pip3_packages_requirement.txt index 57cf60eb..e30b5826 100644 --- a/pip3_packages_requirement.txt +++ b/pip3_packages_requirement.txt @@ -1,13 +1,63 @@ pymisp redis -filemagic +#filemagic conflict with magic crcmod mmh3 ssdeep -nltk -textblob pubsublogger zmq langid + +#Essential +redis +pyzmq +dnspython +logbook +pubsublogger +textblob + +#Tokeniser +nltk + +#Graph +numpy +matplotlib +networkx +terminaltables +colorama +asciimatics + +# Hashlib +crcmod +mmh3 +ssdeep +python-Levenshtein + +#Others +python-magic +pybloomfiltermmap +psutil +phonenumbers + +ipython +flask +texttable + +#DomainClassifier +DomainClassifier +#Indexer requirements +whoosh + +ipaddress +pycountry + +# To fetch Onion urls +PySocks + +#ASN lookup requirements +#https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/adns-python/adns-python-1.2.1.tar.gz +https://github.com/trolldbois/python-cymru-services/archive/master.zip + +https://github.com/saffsd/langid.py/archive/master.zip diff --git a/samples/2018/01/01/keys_certificat_sample.gz b/samples/2018/01/01/keys_certificat_sample.gz new file mode 100644 index 0000000000000000000000000000000000000000..d3427e1039a5240f2e5bb59f190caf9f212eb4c8 GIT binary patch literal 78 zcmV-U0I~lciwFqXs?1sd18Ze@b6;a+a&&2CX=7n@Uvpt?aBO7&)71q6Cs%h*KLuyk kpb$?tPiMyvSCF7CS0YZp1yEkD0GzxYYXATM literal 0 HcmV?d00001 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/testKeys.py b/tests/testKeys.py new file mode 100644 index 00000000..3559a205 --- /dev/null +++ b/tests/testKeys.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import unittest + +import magic + +from bin.packages.Paste import Paste +import bin.Keys as Keys +from bin.Helper import Process +import pubsublogger + + +class TestKeysModule(unittest.TestCase): + + def setUp(self): + self.paste = Paste('samples/2018/01/01/keys_certificat_sample.gz') + + # Section name in bin/packages/modules.cfg + self.config_section = 'Keys' + + # Setup the I/O queues + p = Process(self.config_section) + + + def test_search_key(self): + with self.assertRaises(pubsublogger.exceptions.NoChannelError): + Keys.search_key(self.paste)