Merge pull request #242 from mokaddem/moreComments

Added more documentation/descriptions/...
This commit is contained in:
Thirion Aurélien 2018-09-11 13:49:53 +02:00 committed by GitHub
commit 544561acd5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
20 changed files with 40 additions and 70 deletions

View file

@ -8,11 +8,6 @@ It trying to extract Bitcoin address and secret key from paste
..seealso:: Paste method (get_regex)
Requirements
------------
*Need running Redis instances. (Redis).
"""
from packages import Paste

View file

@ -4,12 +4,9 @@
The ZMQ_PubSub_Categ Module
============================
This module is consuming the Redis-list created by the ZMQ_PubSub_Tokenize_Q
Module.
Each words files created under /files/ are representing categories.
This modules take these files and compare them to
the stream of data given by the ZMQ_PubSub_Tokenize_Q Module.
the stream of data.
When a word from a paste match one or more of these words file, the filename of
the paste is published/forwarded to the next modules.
@ -25,15 +22,10 @@ Implementing modules can start here, create your own category file,
and then create your own module to treat the specific paste matching this
category.
..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
the same Subscriber name in both of them.
Requirements
------------
*Need running Redis instances. (Redis)
*Categories files of words in /files/ need to be created
*Need the ZMQ_PubSub_Tokenize_Q Module running to be able to work properly.
"""
import os

View file

@ -162,3 +162,13 @@ if __name__ == "__main__":
for partCred in splitedCred:
if len(partCred) > minimumLengthThreshold:
server_cred.sadd(partCred, uniq_num_cred)
build_top_username(uniq_num_path, paste._get_p_date(), server_cred)
# hset:
# mostPostedUser:DATE -> uniq_cred_id -> count
def build_top_username(uniq, date, serv):
pass

View file

@ -1,16 +1,11 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
"""
This module is consuming the Redis-list created by the ZMQ_Sub_Curve_Q Module.
This modules update a .csv file used to draw curves representing selected
words and their occurency per day.
..note:: The channel will have the name of the file created.
..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
the same Subscriber name in both of them.
This Module is also used for term frequency.

View file

@ -3,7 +3,7 @@
"""
Decoder module
Dectect Binary and decode it
Dectect and decode content based on the provided decoder function.
"""
import time
import os

View file

@ -6,7 +6,7 @@ The DomClassifier Module
============================
The DomClassifier modules extract and classify Internet domains/hostnames/IP addresses from
the out output of the Global module.
the output of the Global module. Also performs DNS lookup.
"""
import time

View file

@ -9,8 +9,6 @@ This huge module is, in short term, checking duplicates.
Its input comes from other modules, namely:
Credential, CreditCard, Keys, Mails, SQLinjectionDetection, CVE and Phone
This one differ from v1 by only using redis and not json file stored on disk
Perform comparisions with ssdeep and tlsh
Requirements:

View file

@ -4,21 +4,12 @@
The ZMQ_Feed_Q Module
=====================
This module is consuming the Redis-list created by the ZMQ_Feed_Q Module,
This module is consuming the Redis-list created by the Mixer Module,
And save the paste on disk to allow others modules to work on them.
..todo:: Be able to choose to delete or not the saved paste after processing.
..todo:: Store the empty paste (unprocessed) somewhere in Redis.
..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
the same Subscriber name in both of them.
Requirements
------------
*Need running Redis instances.
*Need the ZMQ_Feed_Q Module running to be able to work properly.
"""
import base64
import os

View file

@ -7,9 +7,6 @@ Queue helper module
This module subscribe to a Publisher stream and put the received messages
into a Redis-list waiting to be popped later by others scripts.
..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
the same Subscriber name in both of them.
"""
import redis
import configparser

View file

@ -2,10 +2,10 @@
# -*-coding:UTF-8 -*
"""
The ZMQ_Sub_Indexer Module
The Indexer Module
============================
The ZMQ_Sub_Indexer modules is fetching the list of files to be processed
This module is fetching the list of files to be processed
and index each file with a full-text indexer (Whoosh until now).
"""

View file

@ -2,11 +2,10 @@
# -*-coding:UTF-8 -*
"""
The ZMQ_PubSub_Lines Module
The Lines Module
============================
This module is consuming the Redis-list created by the ZMQ_PubSub_Line_Q
Module.
This module is consuming the Redis-list created by Global Module.
It perform a sorting on the line's length and publish/forward them to
differents channels:
@ -17,15 +16,6 @@ differents channels:
The collected informations about the processed pastes
(number of lines and maximum length line) are stored in Redis.
..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
the same Subscriber name in both of them.
Requirements
------------
*Need running Redis instances. (LevelDB & Redis)
*Need the ZMQ_PubSub_Line_Q Module running to be able to work properly.
"""
import argparse
import time

View file

@ -2,7 +2,7 @@
# -*-coding:UTF-8 -*
"""
The CreditCards Module
The Mail Module
======================
This module is consuming the Redis-list created by the Categ module.

View file

@ -4,7 +4,8 @@
The Mixer Module
================
This module is consuming the Redis-list created by the ZMQ_Feed_Q Module.
This module is consuming the Redis-list created by the ZMQ_Feed Module.
It is responsible to feeding data to the all plateform.
This module take all the feeds provided in the config.
Depending on the configuration, this module will process the feed as follow:

View file

@ -1,5 +1,9 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
'''
This module can be used to see the health of modules in the system, and also
perform basics statistics about performances.
'''
from asciimatics.widgets import Frame, ListBox, Layout, Divider, Text, \
Button, Label

View file

@ -1,25 +1,14 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
"""
The ZMQ_Sub_Onion Module
The Onion Module
============================
This module is consuming the Redis-list created by the ZMQ_Sub_Onion_Q Module.
It trying to extract url from paste and returning only ones which are tor
related (.onion)
..seealso:: Paste method (get_regex)
..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
the same Subscriber name in both of them.
Requirements
------------
*Need running Redis instances. (Redis)
*Need the ZMQ_Sub_Onion_Q Module running to be able to work properly.
"""
import pprint
import time

View file

@ -5,7 +5,7 @@
The Tags Module
================================
This module create tags.
This module create tags on pastes.
"""
import redis

View file

@ -4,8 +4,7 @@
The Tokenize Module
===================
This module is consuming the Redis-list created by the ZMQ_PubSub_Tokenize_Q
Module.
This module is consuming the Redis-list created by the Global Module.
It tokenize the content of the paste and publish the result in the following
format:
@ -13,9 +12,6 @@ format:
..seealso:: Paste method (_get_top_words)
..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
the same Subscriber name in both of them.
Requirements
------------

View file

@ -8,6 +8,10 @@ from collections import OrderedDict
import sys
import shutil
'''
Helper module to update the configuration file, looking if the config is up-to-date.
'''
#return true if the configuration is up-to-date
def main():

View file

@ -1,6 +1,10 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
'''
This module can be used to import a directory and its subsequent folder in AIL.
''
import zmq
import base64
from io import StringIO

View file

@ -1,6 +1,10 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
'''
This module can be used to manually submit a paste to the system.
'''
import configparser
import os
import sys