Merge pull request #242 from mokaddem/moreComments

Added more documentation/descriptions/...
This commit is contained in:
Thirion Aurélien 2018-09-11 13:49:53 +02:00 committed by GitHub
commit 544561acd5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
20 changed files with 40 additions and 70 deletions

View file

@ -8,11 +8,6 @@ It trying to extract Bitcoin address and secret key from paste
..seealso:: Paste method (get_regex) ..seealso:: Paste method (get_regex)
Requirements
------------
*Need running Redis instances. (Redis).
""" """
from packages import Paste from packages import Paste

View file

@ -4,12 +4,9 @@
The ZMQ_PubSub_Categ Module The ZMQ_PubSub_Categ Module
============================ ============================
This module is consuming the Redis-list created by the ZMQ_PubSub_Tokenize_Q
Module.
Each words files created under /files/ are representing categories. Each words files created under /files/ are representing categories.
This modules take these files and compare them to This modules take these files and compare them to
the stream of data given by the ZMQ_PubSub_Tokenize_Q Module. the stream of data.
When a word from a paste match one or more of these words file, the filename of When a word from a paste match one or more of these words file, the filename of
the paste is published/forwarded to the next modules. the paste is published/forwarded to the next modules.
@ -25,15 +22,10 @@ Implementing modules can start here, create your own category file,
and then create your own module to treat the specific paste matching this and then create your own module to treat the specific paste matching this
category. category.
..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
the same Subscriber name in both of them.
Requirements Requirements
------------ ------------
*Need running Redis instances. (Redis)
*Categories files of words in /files/ need to be created *Categories files of words in /files/ need to be created
*Need the ZMQ_PubSub_Tokenize_Q Module running to be able to work properly.
""" """
import os import os

View file

@ -162,3 +162,13 @@ if __name__ == "__main__":
for partCred in splitedCred: for partCred in splitedCred:
if len(partCred) > minimumLengthThreshold: if len(partCred) > minimumLengthThreshold:
server_cred.sadd(partCred, uniq_num_cred) server_cred.sadd(partCred, uniq_num_cred)
build_top_username(uniq_num_path, paste._get_p_date(), server_cred)
# hset:
# mostPostedUser:DATE -> uniq_cred_id -> count
def build_top_username(uniq, date, serv):
pass

View file

@ -1,16 +1,11 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
""" """
This module is consuming the Redis-list created by the ZMQ_Sub_Curve_Q Module.
This modules update a .csv file used to draw curves representing selected This modules update a .csv file used to draw curves representing selected
words and their occurency per day. words and their occurency per day.
..note:: The channel will have the name of the file created. ..note:: The channel will have the name of the file created.
..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
the same Subscriber name in both of them.
This Module is also used for term frequency. This Module is also used for term frequency.

View file

@ -3,7 +3,7 @@
""" """
Decoder module Decoder module
Dectect Binary and decode it Dectect and decode content based on the provided decoder function.
""" """
import time import time
import os import os

View file

@ -6,7 +6,7 @@ The DomClassifier Module
============================ ============================
The DomClassifier modules extract and classify Internet domains/hostnames/IP addresses from The DomClassifier modules extract and classify Internet domains/hostnames/IP addresses from
the out output of the Global module. the output of the Global module. Also performs DNS lookup.
""" """
import time import time

View file

@ -9,8 +9,6 @@ This huge module is, in short term, checking duplicates.
Its input comes from other modules, namely: Its input comes from other modules, namely:
Credential, CreditCard, Keys, Mails, SQLinjectionDetection, CVE and Phone Credential, CreditCard, Keys, Mails, SQLinjectionDetection, CVE and Phone
This one differ from v1 by only using redis and not json file stored on disk
Perform comparisions with ssdeep and tlsh Perform comparisions with ssdeep and tlsh
Requirements: Requirements:

View file

@ -4,21 +4,12 @@
The ZMQ_Feed_Q Module The ZMQ_Feed_Q Module
===================== =====================
This module is consuming the Redis-list created by the ZMQ_Feed_Q Module, This module is consuming the Redis-list created by the Mixer Module,
And save the paste on disk to allow others modules to work on them. And save the paste on disk to allow others modules to work on them.
..todo:: Be able to choose to delete or not the saved paste after processing. ..todo:: Be able to choose to delete or not the saved paste after processing.
..todo:: Store the empty paste (unprocessed) somewhere in Redis. ..todo:: Store the empty paste (unprocessed) somewhere in Redis.
..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
the same Subscriber name in both of them.
Requirements
------------
*Need running Redis instances.
*Need the ZMQ_Feed_Q Module running to be able to work properly.
""" """
import base64 import base64
import os import os

View file

@ -7,9 +7,6 @@ Queue helper module
This module subscribe to a Publisher stream and put the received messages This module subscribe to a Publisher stream and put the received messages
into a Redis-list waiting to be popped later by others scripts. into a Redis-list waiting to be popped later by others scripts.
..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
the same Subscriber name in both of them.
""" """
import redis import redis
import configparser import configparser

View file

@ -2,10 +2,10 @@
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
""" """
The ZMQ_Sub_Indexer Module The Indexer Module
============================ ============================
The ZMQ_Sub_Indexer modules is fetching the list of files to be processed This module is fetching the list of files to be processed
and index each file with a full-text indexer (Whoosh until now). and index each file with a full-text indexer (Whoosh until now).
""" """

View file

@ -2,11 +2,10 @@
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
""" """
The ZMQ_PubSub_Lines Module The Lines Module
============================ ============================
This module is consuming the Redis-list created by the ZMQ_PubSub_Line_Q This module is consuming the Redis-list created by Global Module.
Module.
It perform a sorting on the line's length and publish/forward them to It perform a sorting on the line's length and publish/forward them to
differents channels: differents channels:
@ -17,15 +16,6 @@ differents channels:
The collected informations about the processed pastes The collected informations about the processed pastes
(number of lines and maximum length line) are stored in Redis. (number of lines and maximum length line) are stored in Redis.
..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
the same Subscriber name in both of them.
Requirements
------------
*Need running Redis instances. (LevelDB & Redis)
*Need the ZMQ_PubSub_Line_Q Module running to be able to work properly.
""" """
import argparse import argparse
import time import time

View file

@ -2,7 +2,7 @@
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
""" """
The CreditCards Module The Mail Module
====================== ======================
This module is consuming the Redis-list created by the Categ module. This module is consuming the Redis-list created by the Categ module.

View file

@ -4,7 +4,8 @@
The Mixer Module The Mixer Module
================ ================
This module is consuming the Redis-list created by the ZMQ_Feed_Q Module. This module is consuming the Redis-list created by the ZMQ_Feed Module.
It is responsible to feeding data to the all plateform.
This module take all the feeds provided in the config. This module take all the feeds provided in the config.
Depending on the configuration, this module will process the feed as follow: Depending on the configuration, this module will process the feed as follow:

View file

@ -1,5 +1,9 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
'''
This module can be used to see the health of modules in the system, and also
perform basics statistics about performances.
'''
from asciimatics.widgets import Frame, ListBox, Layout, Divider, Text, \ from asciimatics.widgets import Frame, ListBox, Layout, Divider, Text, \
Button, Label Button, Label

View file

@ -1,25 +1,14 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
""" """
The ZMQ_Sub_Onion Module The Onion Module
============================ ============================
This module is consuming the Redis-list created by the ZMQ_Sub_Onion_Q Module.
It trying to extract url from paste and returning only ones which are tor It trying to extract url from paste and returning only ones which are tor
related (.onion) related (.onion)
..seealso:: Paste method (get_regex) ..seealso:: Paste method (get_regex)
..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
the same Subscriber name in both of them.
Requirements
------------
*Need running Redis instances. (Redis)
*Need the ZMQ_Sub_Onion_Q Module running to be able to work properly.
""" """
import pprint import pprint
import time import time

View file

@ -5,7 +5,7 @@
The Tags Module The Tags Module
================================ ================================
This module create tags. This module create tags on pastes.
""" """
import redis import redis

View file

@ -4,8 +4,7 @@
The Tokenize Module The Tokenize Module
=================== ===================
This module is consuming the Redis-list created by the ZMQ_PubSub_Tokenize_Q This module is consuming the Redis-list created by the Global Module.
Module.
It tokenize the content of the paste and publish the result in the following It tokenize the content of the paste and publish the result in the following
format: format:
@ -13,9 +12,6 @@ format:
..seealso:: Paste method (_get_top_words) ..seealso:: Paste method (_get_top_words)
..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
the same Subscriber name in both of them.
Requirements Requirements
------------ ------------

View file

@ -8,6 +8,10 @@ from collections import OrderedDict
import sys import sys
import shutil import shutil
'''
Helper module to update the configuration file, looking if the config is up-to-date.
'''
#return true if the configuration is up-to-date #return true if the configuration is up-to-date
def main(): def main():

View file

@ -1,6 +1,10 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
'''
This module can be used to import a directory and its subsequent folder in AIL.
''
import zmq import zmq
import base64 import base64
from io import StringIO from io import StringIO

View file

@ -1,6 +1,10 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
'''
This module can be used to manually submit a paste to the system.
'''
import configparser import configparser
import os import os
import sys import sys