create first test

.travis.yml

@@ -1,7 +1,7 @@
 language: python
 
 python:
-    - "2.7"
+    - "3.5"
 
 sudo: required
 
@@ -16,6 +16,7 @@ env:
 
 install:
     - ./installing_deps.sh
+    - pip install coveralls codecov
 
 script:
     - pushd bin
@@ -28,8 +29,13 @@ script:
     - ./Shutdown.py
     - popd
     - find logs/* -exec cat {} \;
+    - nosetests --with-coverage --cover-package=things -d
 
 notifications:
     email:
         on_success: change
         on_failure: change
+
+after_success:
+    - codecov
+    - coveralls

bin/Keys.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 
 """
@@ -15,32 +15,47 @@ RSA private key, certificate messages
 import time
 from pubsublogger import publisher
 
-from Helper import Process
-from packages import Paste
+from bin.packages import Paste
+from bin.Helper import Process
 
 
-def search_key(message):
-    paste = Paste.Paste(message)
+def search_key(paste):
     content = paste.get_p_content()
     find = False
-    if '-----BEGIN PGP MESSAGE-----' in content:
+    if b'-----BEGIN PGP MESSAGE-----' in content:
         publisher.warning('{} has a PGP enc message'.format(paste.p_name))
         find = True
 
-    if '-----BEGIN CERTIFICATE-----' in content:
+    if b'-----BEGIN CERTIFICATE-----' in content:
         publisher.warning('{} has a certificate message'.format(paste.p_name))
         find = True
 
-    if '-----BEGIN RSA PRIVATE KEY-----' in content:
-        publisher.warning('{} has a RSA key message'.format(paste.p_name))
+    if b'-----BEGIN RSA PRIVATE KEY-----' in content:
+        publisher.warning('{} has a RSA private key message'.format(paste.p_name))
         find = True
 
-    if '-----BEGIN PRIVATE KEY-----' in content:
-        publisher.warning('{} has a private message'.format(paste.p_name))
+    if b'-----BEGIN PRIVATE KEY-----' in content:
+        publisher.warning('{} has a private key message'.format(paste.p_name))
         find = True
 
-    if '-----BEGIN ENCRYPTED PRIVATE KEY-----' in content:
-        publisher.warning('{} has an encrypted private message'.format(paste.p_name))
+    if b'-----BEGIN ENCRYPTED PRIVATE KEY-----' in content:
+        publisher.warning('{} has an encrypted private key message'.format(paste.p_name))
+        find = True
+
+    if b'-----BEGIN OPENSSH PRIVATE KEY-----' in content:
+        publisher.warning('{} has an openssh private key message'.format(paste.p_name))
+        find = True
+
+    if b'-----BEGIN DSA PRIVATE KEY-----' in content:
+        publisher.warning('{} has a dsa private key message'.format(paste.p_name))
+        find = True
+
+    if b'-----BEGIN EC PRIVATE KEY-----' in content:
+        publisher.warning('{} has an ec private key message'.format(paste.p_name))
+        find = True
+
+    if b'-----BEGIN PGP PRIVATE KEY BLOCK-----' in content:
+        publisher.warning('{} has a pgp private key block message'.format(paste.p_name))
         find = True
 
     if find :
@@ -77,6 +92,7 @@ if __name__ == '__main__':
             continue
 
         # Do something with the message from the queue
-        search_key(message)
+        paste = Paste.Paste(message)
+        search_key(paste)
 
         # (Optional) Send that thing to the next queue

installing_deps.sh

@@ -5,7 +5,7 @@ set -x
 
 sudo apt-get update
 
-sudo apt-get install python-pip python-virtualenv python-dev libfreetype6-dev \
+sudo apt-get install python3-pip python3-virtualenv python3-dev libfreetype6-dev \
     screen g++ python-tk unzip libsnappy-dev cmake -y
 
 #optional tor install
@@ -15,7 +15,7 @@ sudo apt-get install tor
 sudo apt-get install libssl-dev libfreetype6-dev python-numpy -y
 
 #pyMISP
-sudo apt-get -y install python3-pip
+#sudo apt-get -y install python3-pip
 
 # DNS deps
 sudo apt-get install libadns1 libadns1-dev -y
@@ -73,12 +73,12 @@ if [ ! -f bin/packages/config.cfg ]; then
 fi
 
 pushd var/www/
-./update_thirdparty.sh
+sudo ./update_thirdparty.sh
 popd
 
 if [ -z "$VIRTUAL_ENV" ]; then
 
-    virtualenv AILENV
+    virtualenv -p python3 AILENV
 
     echo export AIL_HOME=$(pwd) >> ./AILENV/bin/activate
     echo export AIL_BIN=$(pwd)/bin/ >> ./AILENV/bin/activate
@@ -95,26 +95,25 @@ year2=20`date --date='-1 year' +%y`
 mkdir -p $AIL_HOME/{PASTES,Blooms,dumps}
 mkdir -p $AIL_HOME/LEVEL_DB_DATA/{$year1,$year2}
 
-pip install -U pip
-pip install -U -r pip_packages_requirement.txt
+pip3 install -U pip
 pip3 install -U -r pip3_packages_requirement.txt
 
 # Pyfaup
 pushd faup/src/lib/bindings/python/
-python setup.py install
+python3 setup.py install
 popd
 
 # Py tlsh
 pushd tlsh/py_ext
-python setup.py build
-python setup.py install
-sudo python3 setup.py build
-sudo python3 setup.py install
+#python setup.py build
+#python setup.py install
+python3 setup.py build
+python3 setup.py install
 
 # Download the necessary NLTK corpora and sentiment vader
-HOME=$(pwd) python -m textblob.download_corpora
-python -m nltk.downloader vader_lexicon
-python -m nltk.downloader punkt
+HOME=$(pwd) python3 -m textblob.download_corpora
+python3 -m nltk.downloader vader_lexicon
+python3 -m nltk.downloader punkt
 
 #Create the file all_module and update the graph in doc
 $AIL_HOME/doc/generate_modules_data_flow_graph.sh

pip3_packages_requirement.txt

@@ -1,13 +1,63 @@
 pymisp
 
 redis
-filemagic
+#filemagic conflict with magic
 crcmod
 mmh3
 ssdeep
-nltk
-textblob
 
 pubsublogger
 zmq
 langid
+
+#Essential
+redis
+pyzmq
+dnspython
+logbook
+pubsublogger
+textblob
+
+#Tokeniser
+nltk
+
+#Graph
+numpy
+matplotlib
+networkx
+terminaltables
+colorama
+asciimatics
+
+# Hashlib
+crcmod
+mmh3
+ssdeep
+python-Levenshtein
+
+#Others
+python-magic
+pybloomfiltermmap
+psutil
+phonenumbers
+
+ipython
+flask
+texttable
+
+#DomainClassifier
+DomainClassifier
+#Indexer requirements
+whoosh
+
+ipaddress
+pycountry
+
+# To fetch Onion urls
+PySocks
+
+#ASN lookup requirements
+#https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/adns-python/adns-python-1.2.1.tar.gz
+https://github.com/trolldbois/python-cymru-services/archive/master.zip
+
+https://github.com/saffsd/langid.py/archive/master.zip

tests/testKeys.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import unittest
+
+import magic
+
+from bin.packages.Paste import Paste
+import bin.Keys as Keys
+from bin.Helper import Process
+import pubsublogger
+
+
+class TestKeysModule(unittest.TestCase):
+
+    def setUp(self):
+        self.paste = Paste('samples/2018/01/01/keys_certificat_sample.gz')
+
+        # Section name in bin/packages/modules.cfg
+        self.config_section = 'Keys'
+
+        # Setup the I/O queues
+        p = Process(self.config_section)
+
+
+    def test_search_key(self):
+        with self.assertRaises(pubsublogger.exceptions.NoChannelError):
+            Keys.search_key(self.paste)