Merge pull request #242 from mokaddem/moreComments

Added more documentation/descriptions/...

bin/Bitcoin.py

@@ -8,11 +8,6 @@ It trying to extract Bitcoin address and secret key from paste
 
     ..seealso:: Paste method (get_regex)
 
-Requirements
-------------
-
-*Need running Redis instances. (Redis).
-
 """
 
 from packages import Paste

bin/Categ.py

@@ -4,12 +4,9 @@
 The ZMQ_PubSub_Categ Module
 ============================
 
-This module is consuming the Redis-list created by the ZMQ_PubSub_Tokenize_Q
-Module.
-
 Each words files created under /files/ are representing categories.
 This modules take these files and compare them to
-the stream of data given by the ZMQ_PubSub_Tokenize_Q  Module.
+the stream of data.
 
 When a word from a paste match one or more of these words file, the filename of
 the paste is published/forwarded to the next modules.
@@ -25,15 +22,10 @@ Implementing modules can start here, create your own category file,
 and then create your own module to treat the specific paste matching this
 category.
 
-..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
-the same Subscriber name in both of them.
-
 Requirements
 ------------
 
-*Need running Redis instances. (Redis)
 *Categories files of words in /files/ need to be created
-*Need the ZMQ_PubSub_Tokenize_Q Module running to be able to work properly.
 
 """
 import os

bin/Credential.py

@@ -162,3 +162,13 @@ if __name__ == "__main__":
             for partCred in splitedCred:
                 if len(partCred) > minimumLengthThreshold:
                     server_cred.sadd(partCred, uniq_num_cred)
+
+            build_top_username(uniq_num_path, paste._get_p_date(), server_cred)
+
+# hset:
+# mostPostedUser:DATE -> uniq_cred_id -> count
+def build_top_username(uniq, date, serv):
+    pass
+
+
+

bin/Curve.py

@@ -1,16 +1,11 @@
 #!/usr/bin/env python3
 # -*-coding:UTF-8 -*
 """
-This module is consuming the Redis-list created by the ZMQ_Sub_Curve_Q Module.
-
 This modules update a .csv file used to draw curves representing selected
 words and their occurency per day.
 
 ..note:: The channel will have the name of the file created.
 
-..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
-the same Subscriber name in both of them.
-
 
 This Module is also used for term frequency.
 

bin/Decoder.py

@@ -3,7 +3,7 @@
 """
     Decoder module
 
-    Dectect Binary and decode it
+    Dectect and decode content based on the provided decoder function.
 """
 import time
 import os

bin/DomClassifier.py

@@ -6,7 +6,7 @@ The DomClassifier Module
 ============================
 
 The DomClassifier modules extract and classify Internet domains/hostnames/IP addresses from
-the out output of the Global module.
+the output of the Global module. Also performs DNS lookup.
 
 """
 import time

bin/Duplicates.py

@@ -9,8 +9,6 @@ This huge module is, in short term, checking duplicates.
 Its input comes from other modules, namely:
     Credential, CreditCard, Keys, Mails, SQLinjectionDetection, CVE and Phone
 
-This one differ from v1 by only using redis and not json file stored on disk
-
 Perform comparisions with ssdeep and tlsh
 
 Requirements:

bin/Global.py

@@ -4,21 +4,12 @@
 The ZMQ_Feed_Q Module
 =====================
 
-This module is consuming the Redis-list created by the ZMQ_Feed_Q Module,
+This module is consuming the Redis-list created by the Mixer Module,
 And save the paste on disk to allow others modules to work on them.
 
 ..todo:: Be able to choose to delete or not the saved paste after processing.
 ..todo:: Store the empty paste (unprocessed) somewhere in Redis.
 
-..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
-the same Subscriber name in both of them.
-
-Requirements
-------------
-
-*Need running Redis instances.
-*Need the ZMQ_Feed_Q Module running to be able to work properly.
-
 """
 import base64
 import os

bin/Helper.py

@@ -7,9 +7,6 @@ Queue helper module
 This module subscribe to a Publisher stream and put the received messages
 into a Redis-list waiting to be popped later by others scripts.
 
-..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
-the same Subscriber name in both of them.
-
 """
 import redis
 import configparser

bin/Indexer.py

@@ -2,10 +2,10 @@
 # -*-coding:UTF-8 -*
 
 """
-The ZMQ_Sub_Indexer Module
+The Indexer Module
 ============================
 
-The ZMQ_Sub_Indexer modules is fetching the list of files to be processed
+This module is fetching the list of files to be processed
 and index each file with a full-text indexer (Whoosh until now).
 
 """

bin/Lines.py

@@ -2,11 +2,10 @@
 # -*-coding:UTF-8 -*
 
 """
-The ZMQ_PubSub_Lines Module
+The Lines Module
 ============================
 
-This module is consuming the Redis-list created by the ZMQ_PubSub_Line_Q
-Module.
+This module is consuming the Redis-list created by Global Module.
 
 It perform a sorting on the line's length and publish/forward them to
 differents channels:
@@ -17,15 +16,6 @@ differents channels:
 The collected informations about the processed pastes
 (number of lines and maximum length line) are stored in Redis.
 
-..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
-the same Subscriber name in both of them.
-
-Requirements
-------------
-
-*Need running Redis instances. (LevelDB & Redis)
-*Need the ZMQ_PubSub_Line_Q Module running to be able to work properly.
-
 """
 import argparse
 import time

bin/Mail.py

@@ -2,7 +2,7 @@
 # -*-coding:UTF-8 -*
 
 """
-The CreditCards Module
+The Mail Module
 ======================
 
 This module is consuming the Redis-list created by the Categ module.

bin/Mixer.py

@@ -4,7 +4,8 @@
 The Mixer Module
 ================
 
-This module is consuming the Redis-list created by the ZMQ_Feed_Q Module.
+This module is consuming the Redis-list created by the ZMQ_Feed Module.
+It is responsible to feeding data to the all plateform.
 
 This module take all the feeds provided in the config.
 Depending on the configuration, this module will process the feed as follow:

bin/ModulesInformationV2.py

@@ -1,5 +1,9 @@
 #!/usr/bin/env python3
 # -*-coding:UTF-8 -*
+'''
+This module can be used to see the health of modules in the system, and also
+perform basics statistics about performances.
+'''
 
 from asciimatics.widgets import Frame, ListBox, Layout, Divider, Text, \
     Button, Label

bin/Onion.py

@@ -1,25 +1,14 @@
 #!/usr/bin/env python3
 # -*-coding:UTF-8 -*
 """
-The ZMQ_Sub_Onion Module
+The Onion Module
 ============================
 
-This module is consuming the Redis-list created by the ZMQ_Sub_Onion_Q Module.
-
 It trying to extract url from paste and returning only ones which are tor
 related (.onion)
 
     ..seealso:: Paste method (get_regex)
 
-..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
-the same Subscriber name in both of them.
-
-Requirements
-------------
-
-*Need running Redis instances. (Redis)
-*Need the ZMQ_Sub_Onion_Q Module running to be able to work properly.
-
 """
 import pprint
 import time

bin/Tags.py

@@ -5,7 +5,7 @@
 The Tags Module
 ================================
 
-This module create tags.
+This module create tags on pastes.
 
 """
 import redis

bin/Tokenize.py

@@ -4,8 +4,7 @@
 The Tokenize Module
 ===================
 
-This module is consuming the Redis-list created by the ZMQ_PubSub_Tokenize_Q
-Module.
+This module is consuming the Redis-list created by the Global Module.
 
 It tokenize the content of the paste and publish the result in the following
 format:
@@ -13,9 +12,6 @@ format:
 
     ..seealso:: Paste method (_get_top_words)
 
-..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
-the same Subscriber name in both of them.
-
 Requirements
 ------------
 

bin/Update-conf.py

@@ -8,6 +8,10 @@ from collections import OrderedDict
 import sys
 import shutil
 
+'''
+Helper module to update the configuration file, looking if the config is up-to-date.
+'''
+
 
 #return true if the configuration is up-to-date
 def main():

bin/import_dir.py

@@ -1,6 +1,10 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
+'''
+This module can be used to import a directory and its subsequent folder in AIL.
+''
+
 import zmq
 import base64
 from io import StringIO

bin/submit_paste.py

@@ -1,6 +1,10 @@
 #!/usr/bin/env python3
 # -*-coding:UTF-8 -*
 
+'''
+This module can be used to manually submit a paste to the system.
+'''
+
 import configparser
 import os
 import sys