From fed83aba889c1f2699952460e1b1fd8bfc4b9d1d Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Mon, 3 Sep 2018 15:52:42 +0200
Subject: [PATCH] Added more documentation/descriptions/...

---
 bin/Bitcoin.py              |  5 -----
 bin/Categ.py                | 10 +---------
 bin/Credential.py           | 10 ++++++++++
 bin/Curve.py                |  5 -----
 bin/Decoder.py              |  2 +-
 bin/DomClassifier.py        |  2 +-
 bin/Duplicates.py           |  2 --
 bin/Global.py               | 11 +----------
 bin/Helper.py               |  3 ---
 bin/Indexer.py              |  4 ++--
 bin/Lines.py                | 14 ++------------
 bin/Mail.py                 |  2 +-
 bin/Mixer.py                |  3 ++-
 bin/ModulesInformationV2.py |  4 ++++
 bin/Onion.py                | 13 +------------
 bin/Tags.py                 |  2 +-
 bin/Tokenize.py             |  6 +-----
 bin/Update-conf.py          |  4 ++++
 bin/import_dir.py           |  4 ++++
 bin/submit_paste.py         |  4 ++++
 20 files changed, 40 insertions(+), 70 deletions(-)

diff --git a/bin/Bitcoin.py b/bin/Bitcoin.py
index 5ec2199f..7958787d 100755
--- a/bin/Bitcoin.py
+++ b/bin/Bitcoin.py
@@ -8,11 +8,6 @@ It trying to extract Bitcoin address and secret key from paste
 
     ..seealso:: Paste method (get_regex)
 
-Requirements
-------------
-
-*Need running Redis instances. (Redis).
-
 """
 
 from packages import Paste
diff --git a/bin/Categ.py b/bin/Categ.py
index cf78f90f..175c3d76 100755
--- a/bin/Categ.py
+++ b/bin/Categ.py
@@ -4,12 +4,9 @@
 The ZMQ_PubSub_Categ Module
 ============================
 
-This module is consuming the Redis-list created by the ZMQ_PubSub_Tokenize_Q
-Module.
-
 Each words files created under /files/ are representing categories.
 This modules take these files and compare them to
-the stream of data given by the ZMQ_PubSub_Tokenize_Q  Module.
+the stream of data.
 
 When a word from a paste match one or more of these words file, the filename of
 the paste is published/forwarded to the next modules.
@@ -25,15 +22,10 @@ Implementing modules can start here, create your own category file,
 and then create your own module to treat the specific paste matching this
 category.
 
-..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
-the same Subscriber name in both of them.
-
 Requirements
 ------------
 
-*Need running Redis instances. (Redis)
 *Categories files of words in /files/ need to be created
-*Need the ZMQ_PubSub_Tokenize_Q Module running to be able to work properly.
 
 """
 import os
diff --git a/bin/Credential.py b/bin/Credential.py
index d1016586..bddb88ce 100755
--- a/bin/Credential.py
+++ b/bin/Credential.py
@@ -162,3 +162,13 @@ if __name__ == "__main__":
             for partCred in splitedCred:
                 if len(partCred) > minimumLengthThreshold:
                     server_cred.sadd(partCred, uniq_num_cred)
+
+            build_top_username(uniq_num_path, paste._get_p_date(), server_cred)
+
+# hset:
+# mostPostedUser:DATE -> uniq_cred_id -> count
+def build_top_username(uniq, date, serv):
+    pass
+
+
+
diff --git a/bin/Curve.py b/bin/Curve.py
index 8e228039..045128c8 100755
--- a/bin/Curve.py
+++ b/bin/Curve.py
@@ -1,16 +1,11 @@
 #!/usr/bin/env python3
 # -*-coding:UTF-8 -*
 """
-This module is consuming the Redis-list created by the ZMQ_Sub_Curve_Q Module.
-
 This modules update a .csv file used to draw curves representing selected
 words and their occurency per day.
 
 ..note:: The channel will have the name of the file created.
 
-..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
-the same Subscriber name in both of them.
-
 
 This Module is also used for term frequency.
 
diff --git a/bin/Decoder.py b/bin/Decoder.py
index af385fed..534ede3b 100755
--- a/bin/Decoder.py
+++ b/bin/Decoder.py
@@ -3,7 +3,7 @@
 """
     Decoder module
 
-    Dectect Binary and decode it
+    Dectect and decode content based on the provided decoder function.
 """
 import time
 import os
diff --git a/bin/DomClassifier.py b/bin/DomClassifier.py
index aed87a55..37d041ea 100755
--- a/bin/DomClassifier.py
+++ b/bin/DomClassifier.py
@@ -6,7 +6,7 @@ The DomClassifier Module
 ============================
 
 The DomClassifier modules extract and classify Internet domains/hostnames/IP addresses from
-the out output of the Global module.
+the output of the Global module. Also performs DNS lookup.
 
 """
 import time
diff --git a/bin/Duplicates.py b/bin/Duplicates.py
index 0c24bec1..4d8fffa4 100755
--- a/bin/Duplicates.py
+++ b/bin/Duplicates.py
@@ -9,8 +9,6 @@ This huge module is, in short term, checking duplicates.
 Its input comes from other modules, namely:
     Credential, CreditCard, Keys, Mails, SQLinjectionDetection, CVE and Phone
 
-This one differ from v1 by only using redis and not json file stored on disk
-
 Perform comparisions with ssdeep and tlsh
 
 Requirements:
diff --git a/bin/Global.py b/bin/Global.py
index 32a3656b..a4863ab0 100755
--- a/bin/Global.py
+++ b/bin/Global.py
@@ -4,21 +4,12 @@
 The ZMQ_Feed_Q Module
 =====================
 
-This module is consuming the Redis-list created by the ZMQ_Feed_Q Module,
+This module is consuming the Redis-list created by the Mixer Module,
 And save the paste on disk to allow others modules to work on them.
 
 ..todo:: Be able to choose to delete or not the saved paste after processing.
 ..todo:: Store the empty paste (unprocessed) somewhere in Redis.
 
-..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
-the same Subscriber name in both of them.
-
-Requirements
-------------
-
-*Need running Redis instances.
-*Need the ZMQ_Feed_Q Module running to be able to work properly.
-
 """
 import base64
 import os
diff --git a/bin/Helper.py b/bin/Helper.py
index d90388f5..4f6f3d7a 100755
--- a/bin/Helper.py
+++ b/bin/Helper.py
@@ -7,9 +7,6 @@ Queue helper module
 This module subscribe to a Publisher stream and put the received messages
 into a Redis-list waiting to be popped later by others scripts.
 
-..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
-the same Subscriber name in both of them.
-
 """
 import redis
 import configparser
diff --git a/bin/Indexer.py b/bin/Indexer.py
index 1d1ece4b..87eb9344 100755
--- a/bin/Indexer.py
+++ b/bin/Indexer.py
@@ -2,10 +2,10 @@
 # -*-coding:UTF-8 -*
 
 """
-The ZMQ_Sub_Indexer Module
+The Indexer Module
 ============================
 
-The ZMQ_Sub_Indexer modules is fetching the list of files to be processed
+This module is fetching the list of files to be processed
 and index each file with a full-text indexer (Whoosh until now).
 
 """
diff --git a/bin/Lines.py b/bin/Lines.py
index 8c9f6827..f357a0b2 100755
--- a/bin/Lines.py
+++ b/bin/Lines.py
@@ -2,11 +2,10 @@
 # -*-coding:UTF-8 -*
 
 """
-The ZMQ_PubSub_Lines Module
+The Lines Module
 ============================
 
-This module is consuming the Redis-list created by the ZMQ_PubSub_Line_Q
-Module.
+This module is consuming the Redis-list created by Global Module.
 
 It perform a sorting on the line's length and publish/forward them to
 differents channels:
@@ -17,15 +16,6 @@ differents channels:
 The collected informations about the processed pastes
 (number of lines and maximum length line) are stored in Redis.
 
-..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
-the same Subscriber name in both of them.
-
-Requirements
-------------
-
-*Need running Redis instances. (LevelDB & Redis)
-*Need the ZMQ_PubSub_Line_Q Module running to be able to work properly.
-
 """
 import argparse
 import time
diff --git a/bin/Mail.py b/bin/Mail.py
index c1d8cf70..139e8245 100755
--- a/bin/Mail.py
+++ b/bin/Mail.py
@@ -2,7 +2,7 @@
 # -*-coding:UTF-8 -*
 
 """
-The CreditCards Module
+The Mail Module
 ======================
 
 This module is consuming the Redis-list created by the Categ module.
diff --git a/bin/Mixer.py b/bin/Mixer.py
index 96f20815..7423e99b 100755
--- a/bin/Mixer.py
+++ b/bin/Mixer.py
@@ -4,7 +4,8 @@
 The Mixer Module
 ================
 
-This module is consuming the Redis-list created by the ZMQ_Feed_Q Module.
+This module is consuming the Redis-list created by the ZMQ_Feed Module.
+It is responsible to feeding data to the all plateform.
 
 This module take all the feeds provided in the config.
 Depending on the configuration, this module will process the feed as follow:
diff --git a/bin/ModulesInformationV2.py b/bin/ModulesInformationV2.py
index 36b397ca..cfdd8548 100755
--- a/bin/ModulesInformationV2.py
+++ b/bin/ModulesInformationV2.py
@@ -1,5 +1,9 @@
 #!/usr/bin/env python3
 # -*-coding:UTF-8 -*
+'''
+This module can be used to see the health of modules in the system, and also
+perform basics statistics about performances.
+'''
 
 from asciimatics.widgets import Frame, ListBox, Layout, Divider, Text, \
     Button, Label
diff --git a/bin/Onion.py b/bin/Onion.py
index 277f1c71..f1435b10 100755
--- a/bin/Onion.py
+++ b/bin/Onion.py
@@ -1,25 +1,14 @@
 #!/usr/bin/env python3
 # -*-coding:UTF-8 -*
 """
-The ZMQ_Sub_Onion Module
+The Onion Module
 ============================
 
-This module is consuming the Redis-list created by the ZMQ_Sub_Onion_Q Module.
-
 It trying to extract url from paste and returning only ones which are tor
 related (.onion)
 
     ..seealso:: Paste method (get_regex)
 
-..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
-the same Subscriber name in both of them.
-
-Requirements
-------------
-
-*Need running Redis instances. (Redis)
-*Need the ZMQ_Sub_Onion_Q Module running to be able to work properly.
-
 """
 import pprint
 import time
diff --git a/bin/Tags.py b/bin/Tags.py
index 0a178fef..d16da6fa 100755
--- a/bin/Tags.py
+++ b/bin/Tags.py
@@ -5,7 +5,7 @@
 The Tags Module
 ================================
 
-This module create tags.
+This module create tags on pastes.
 
 """
 import redis
diff --git a/bin/Tokenize.py b/bin/Tokenize.py
index 698b4fbc..8b6f2812 100755
--- a/bin/Tokenize.py
+++ b/bin/Tokenize.py
@@ -4,8 +4,7 @@
 The Tokenize Module
 ===================
 
-This module is consuming the Redis-list created by the ZMQ_PubSub_Tokenize_Q
-Module.
+This module is consuming the Redis-list created by the Global Module.
 
 It tokenize the content of the paste and publish the result in the following
 format:
@@ -13,9 +12,6 @@ format:
 
     ..seealso:: Paste method (_get_top_words)
 
-..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
-the same Subscriber name in both of them.
-
 Requirements
 ------------
 
diff --git a/bin/Update-conf.py b/bin/Update-conf.py
index 0d04fb88..4ca80837 100755
--- a/bin/Update-conf.py
+++ b/bin/Update-conf.py
@@ -8,6 +8,10 @@ from collections import OrderedDict
 import sys
 import shutil
 
+'''
+Helper module to update the configuration file, looking if the config is up-to-date.
+'''
+
 
 #return true if the configuration is up-to-date
 def main():
diff --git a/bin/import_dir.py b/bin/import_dir.py
index d8360631..c91bcd96 100755
--- a/bin/import_dir.py
+++ b/bin/import_dir.py
@@ -1,6 +1,10 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
+'''
+This module can be used to import a directory and its subsequent folder in AIL.
+''
+
 import zmq
 import base64
 from io import StringIO
diff --git a/bin/submit_paste.py b/bin/submit_paste.py
index a999ec39..3193db70 100755
--- a/bin/submit_paste.py
+++ b/bin/submit_paste.py
@@ -1,6 +1,10 @@
 #!/usr/bin/env python3
 # -*-coding:UTF-8 -*
 
+'''
+This module can be used to manually submit a paste to the system.
+'''
+
 import configparser
 import os
 import sys