mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 08:38:28 +00:00
Merge pull request #242 from mokaddem/moreComments
Added more documentation/descriptions/...
This commit is contained in:
commit
544561acd5
20 changed files with 40 additions and 70 deletions
|
@ -8,11 +8,6 @@ It trying to extract Bitcoin address and secret key from paste
|
||||||
|
|
||||||
..seealso:: Paste method (get_regex)
|
..seealso:: Paste method (get_regex)
|
||||||
|
|
||||||
Requirements
|
|
||||||
------------
|
|
||||||
|
|
||||||
*Need running Redis instances. (Redis).
|
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from packages import Paste
|
from packages import Paste
|
||||||
|
|
10
bin/Categ.py
10
bin/Categ.py
|
@ -4,12 +4,9 @@
|
||||||
The ZMQ_PubSub_Categ Module
|
The ZMQ_PubSub_Categ Module
|
||||||
============================
|
============================
|
||||||
|
|
||||||
This module is consuming the Redis-list created by the ZMQ_PubSub_Tokenize_Q
|
|
||||||
Module.
|
|
||||||
|
|
||||||
Each words files created under /files/ are representing categories.
|
Each words files created under /files/ are representing categories.
|
||||||
This modules take these files and compare them to
|
This modules take these files and compare them to
|
||||||
the stream of data given by the ZMQ_PubSub_Tokenize_Q Module.
|
the stream of data.
|
||||||
|
|
||||||
When a word from a paste match one or more of these words file, the filename of
|
When a word from a paste match one or more of these words file, the filename of
|
||||||
the paste is published/forwarded to the next modules.
|
the paste is published/forwarded to the next modules.
|
||||||
|
@ -25,15 +22,10 @@ Implementing modules can start here, create your own category file,
|
||||||
and then create your own module to treat the specific paste matching this
|
and then create your own module to treat the specific paste matching this
|
||||||
category.
|
category.
|
||||||
|
|
||||||
..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
|
|
||||||
the same Subscriber name in both of them.
|
|
||||||
|
|
||||||
Requirements
|
Requirements
|
||||||
------------
|
------------
|
||||||
|
|
||||||
*Need running Redis instances. (Redis)
|
|
||||||
*Categories files of words in /files/ need to be created
|
*Categories files of words in /files/ need to be created
|
||||||
*Need the ZMQ_PubSub_Tokenize_Q Module running to be able to work properly.
|
|
||||||
|
|
||||||
"""
|
"""
|
||||||
import os
|
import os
|
||||||
|
|
|
@ -162,3 +162,13 @@ if __name__ == "__main__":
|
||||||
for partCred in splitedCred:
|
for partCred in splitedCred:
|
||||||
if len(partCred) > minimumLengthThreshold:
|
if len(partCred) > minimumLengthThreshold:
|
||||||
server_cred.sadd(partCred, uniq_num_cred)
|
server_cred.sadd(partCred, uniq_num_cred)
|
||||||
|
|
||||||
|
build_top_username(uniq_num_path, paste._get_p_date(), server_cred)
|
||||||
|
|
||||||
|
# hset:
|
||||||
|
# mostPostedUser:DATE -> uniq_cred_id -> count
|
||||||
|
def build_top_username(uniq, date, serv):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -1,16 +1,11 @@
|
||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
# -*-coding:UTF-8 -*
|
# -*-coding:UTF-8 -*
|
||||||
"""
|
"""
|
||||||
This module is consuming the Redis-list created by the ZMQ_Sub_Curve_Q Module.
|
|
||||||
|
|
||||||
This modules update a .csv file used to draw curves representing selected
|
This modules update a .csv file used to draw curves representing selected
|
||||||
words and their occurency per day.
|
words and their occurency per day.
|
||||||
|
|
||||||
..note:: The channel will have the name of the file created.
|
..note:: The channel will have the name of the file created.
|
||||||
|
|
||||||
..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
|
|
||||||
the same Subscriber name in both of them.
|
|
||||||
|
|
||||||
|
|
||||||
This Module is also used for term frequency.
|
This Module is also used for term frequency.
|
||||||
|
|
||||||
|
|
|
@ -3,7 +3,7 @@
|
||||||
"""
|
"""
|
||||||
Decoder module
|
Decoder module
|
||||||
|
|
||||||
Dectect Binary and decode it
|
Dectect and decode content based on the provided decoder function.
|
||||||
"""
|
"""
|
||||||
import time
|
import time
|
||||||
import os
|
import os
|
||||||
|
|
|
@ -6,7 +6,7 @@ The DomClassifier Module
|
||||||
============================
|
============================
|
||||||
|
|
||||||
The DomClassifier modules extract and classify Internet domains/hostnames/IP addresses from
|
The DomClassifier modules extract and classify Internet domains/hostnames/IP addresses from
|
||||||
the out output of the Global module.
|
the output of the Global module. Also performs DNS lookup.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
import time
|
import time
|
||||||
|
|
|
@ -9,8 +9,6 @@ This huge module is, in short term, checking duplicates.
|
||||||
Its input comes from other modules, namely:
|
Its input comes from other modules, namely:
|
||||||
Credential, CreditCard, Keys, Mails, SQLinjectionDetection, CVE and Phone
|
Credential, CreditCard, Keys, Mails, SQLinjectionDetection, CVE and Phone
|
||||||
|
|
||||||
This one differ from v1 by only using redis and not json file stored on disk
|
|
||||||
|
|
||||||
Perform comparisions with ssdeep and tlsh
|
Perform comparisions with ssdeep and tlsh
|
||||||
|
|
||||||
Requirements:
|
Requirements:
|
||||||
|
|
|
@ -4,21 +4,12 @@
|
||||||
The ZMQ_Feed_Q Module
|
The ZMQ_Feed_Q Module
|
||||||
=====================
|
=====================
|
||||||
|
|
||||||
This module is consuming the Redis-list created by the ZMQ_Feed_Q Module,
|
This module is consuming the Redis-list created by the Mixer Module,
|
||||||
And save the paste on disk to allow others modules to work on them.
|
And save the paste on disk to allow others modules to work on them.
|
||||||
|
|
||||||
..todo:: Be able to choose to delete or not the saved paste after processing.
|
..todo:: Be able to choose to delete or not the saved paste after processing.
|
||||||
..todo:: Store the empty paste (unprocessed) somewhere in Redis.
|
..todo:: Store the empty paste (unprocessed) somewhere in Redis.
|
||||||
|
|
||||||
..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
|
|
||||||
the same Subscriber name in both of them.
|
|
||||||
|
|
||||||
Requirements
|
|
||||||
------------
|
|
||||||
|
|
||||||
*Need running Redis instances.
|
|
||||||
*Need the ZMQ_Feed_Q Module running to be able to work properly.
|
|
||||||
|
|
||||||
"""
|
"""
|
||||||
import base64
|
import base64
|
||||||
import os
|
import os
|
||||||
|
|
|
@ -7,9 +7,6 @@ Queue helper module
|
||||||
This module subscribe to a Publisher stream and put the received messages
|
This module subscribe to a Publisher stream and put the received messages
|
||||||
into a Redis-list waiting to be popped later by others scripts.
|
into a Redis-list waiting to be popped later by others scripts.
|
||||||
|
|
||||||
..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
|
|
||||||
the same Subscriber name in both of them.
|
|
||||||
|
|
||||||
"""
|
"""
|
||||||
import redis
|
import redis
|
||||||
import configparser
|
import configparser
|
||||||
|
|
|
@ -2,10 +2,10 @@
|
||||||
# -*-coding:UTF-8 -*
|
# -*-coding:UTF-8 -*
|
||||||
|
|
||||||
"""
|
"""
|
||||||
The ZMQ_Sub_Indexer Module
|
The Indexer Module
|
||||||
============================
|
============================
|
||||||
|
|
||||||
The ZMQ_Sub_Indexer modules is fetching the list of files to be processed
|
This module is fetching the list of files to be processed
|
||||||
and index each file with a full-text indexer (Whoosh until now).
|
and index each file with a full-text indexer (Whoosh until now).
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
14
bin/Lines.py
14
bin/Lines.py
|
@ -2,11 +2,10 @@
|
||||||
# -*-coding:UTF-8 -*
|
# -*-coding:UTF-8 -*
|
||||||
|
|
||||||
"""
|
"""
|
||||||
The ZMQ_PubSub_Lines Module
|
The Lines Module
|
||||||
============================
|
============================
|
||||||
|
|
||||||
This module is consuming the Redis-list created by the ZMQ_PubSub_Line_Q
|
This module is consuming the Redis-list created by Global Module.
|
||||||
Module.
|
|
||||||
|
|
||||||
It perform a sorting on the line's length and publish/forward them to
|
It perform a sorting on the line's length and publish/forward them to
|
||||||
differents channels:
|
differents channels:
|
||||||
|
@ -17,15 +16,6 @@ differents channels:
|
||||||
The collected informations about the processed pastes
|
The collected informations about the processed pastes
|
||||||
(number of lines and maximum length line) are stored in Redis.
|
(number of lines and maximum length line) are stored in Redis.
|
||||||
|
|
||||||
..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
|
|
||||||
the same Subscriber name in both of them.
|
|
||||||
|
|
||||||
Requirements
|
|
||||||
------------
|
|
||||||
|
|
||||||
*Need running Redis instances. (LevelDB & Redis)
|
|
||||||
*Need the ZMQ_PubSub_Line_Q Module running to be able to work properly.
|
|
||||||
|
|
||||||
"""
|
"""
|
||||||
import argparse
|
import argparse
|
||||||
import time
|
import time
|
||||||
|
|
|
@ -2,7 +2,7 @@
|
||||||
# -*-coding:UTF-8 -*
|
# -*-coding:UTF-8 -*
|
||||||
|
|
||||||
"""
|
"""
|
||||||
The CreditCards Module
|
The Mail Module
|
||||||
======================
|
======================
|
||||||
|
|
||||||
This module is consuming the Redis-list created by the Categ module.
|
This module is consuming the Redis-list created by the Categ module.
|
||||||
|
|
|
@ -4,7 +4,8 @@
|
||||||
The Mixer Module
|
The Mixer Module
|
||||||
================
|
================
|
||||||
|
|
||||||
This module is consuming the Redis-list created by the ZMQ_Feed_Q Module.
|
This module is consuming the Redis-list created by the ZMQ_Feed Module.
|
||||||
|
It is responsible to feeding data to the all plateform.
|
||||||
|
|
||||||
This module take all the feeds provided in the config.
|
This module take all the feeds provided in the config.
|
||||||
Depending on the configuration, this module will process the feed as follow:
|
Depending on the configuration, this module will process the feed as follow:
|
||||||
|
|
|
@ -1,5 +1,9 @@
|
||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
# -*-coding:UTF-8 -*
|
# -*-coding:UTF-8 -*
|
||||||
|
'''
|
||||||
|
This module can be used to see the health of modules in the system, and also
|
||||||
|
perform basics statistics about performances.
|
||||||
|
'''
|
||||||
|
|
||||||
from asciimatics.widgets import Frame, ListBox, Layout, Divider, Text, \
|
from asciimatics.widgets import Frame, ListBox, Layout, Divider, Text, \
|
||||||
Button, Label
|
Button, Label
|
||||||
|
|
13
bin/Onion.py
13
bin/Onion.py
|
@ -1,25 +1,14 @@
|
||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
# -*-coding:UTF-8 -*
|
# -*-coding:UTF-8 -*
|
||||||
"""
|
"""
|
||||||
The ZMQ_Sub_Onion Module
|
The Onion Module
|
||||||
============================
|
============================
|
||||||
|
|
||||||
This module is consuming the Redis-list created by the ZMQ_Sub_Onion_Q Module.
|
|
||||||
|
|
||||||
It trying to extract url from paste and returning only ones which are tor
|
It trying to extract url from paste and returning only ones which are tor
|
||||||
related (.onion)
|
related (.onion)
|
||||||
|
|
||||||
..seealso:: Paste method (get_regex)
|
..seealso:: Paste method (get_regex)
|
||||||
|
|
||||||
..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
|
|
||||||
the same Subscriber name in both of them.
|
|
||||||
|
|
||||||
Requirements
|
|
||||||
------------
|
|
||||||
|
|
||||||
*Need running Redis instances. (Redis)
|
|
||||||
*Need the ZMQ_Sub_Onion_Q Module running to be able to work properly.
|
|
||||||
|
|
||||||
"""
|
"""
|
||||||
import pprint
|
import pprint
|
||||||
import time
|
import time
|
||||||
|
|
|
@ -5,7 +5,7 @@
|
||||||
The Tags Module
|
The Tags Module
|
||||||
================================
|
================================
|
||||||
|
|
||||||
This module create tags.
|
This module create tags on pastes.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
import redis
|
import redis
|
||||||
|
|
|
@ -4,8 +4,7 @@
|
||||||
The Tokenize Module
|
The Tokenize Module
|
||||||
===================
|
===================
|
||||||
|
|
||||||
This module is consuming the Redis-list created by the ZMQ_PubSub_Tokenize_Q
|
This module is consuming the Redis-list created by the Global Module.
|
||||||
Module.
|
|
||||||
|
|
||||||
It tokenize the content of the paste and publish the result in the following
|
It tokenize the content of the paste and publish the result in the following
|
||||||
format:
|
format:
|
||||||
|
@ -13,9 +12,6 @@ format:
|
||||||
|
|
||||||
..seealso:: Paste method (_get_top_words)
|
..seealso:: Paste method (_get_top_words)
|
||||||
|
|
||||||
..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
|
|
||||||
the same Subscriber name in both of them.
|
|
||||||
|
|
||||||
Requirements
|
Requirements
|
||||||
------------
|
------------
|
||||||
|
|
||||||
|
|
|
@ -8,6 +8,10 @@ from collections import OrderedDict
|
||||||
import sys
|
import sys
|
||||||
import shutil
|
import shutil
|
||||||
|
|
||||||
|
'''
|
||||||
|
Helper module to update the configuration file, looking if the config is up-to-date.
|
||||||
|
'''
|
||||||
|
|
||||||
|
|
||||||
#return true if the configuration is up-to-date
|
#return true if the configuration is up-to-date
|
||||||
def main():
|
def main():
|
||||||
|
|
|
@ -1,6 +1,10 @@
|
||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
'''
|
||||||
|
This module can be used to import a directory and its subsequent folder in AIL.
|
||||||
|
''
|
||||||
|
|
||||||
import zmq
|
import zmq
|
||||||
import base64
|
import base64
|
||||||
from io import StringIO
|
from io import StringIO
|
||||||
|
|
|
@ -1,6 +1,10 @@
|
||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
# -*-coding:UTF-8 -*
|
# -*-coding:UTF-8 -*
|
||||||
|
|
||||||
|
'''
|
||||||
|
This module can be used to manually submit a paste to the system.
|
||||||
|
'''
|
||||||
|
|
||||||
import configparser
|
import configparser
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
|
|
Loading…
Reference in a new issue