munin-host.py

#!/usr/bin/env python3
"""Checks IPs and Hosts read from an input file on Virustotal"""

__AUTHOR__ = 'Florian Roth'
__VERSION__ = "0.3 February 2019"

"""
Install dependencies with:
pip install simplejson colorama IPy pickle pycurl
"""

import configparser
import json
import signal
import urllib
from urllib.parse import urlparse
import urllib.request
import pycurl
from io import BytesIO
import platform
import time
import re
import os
import sys
import traceback
import subprocess
import argparse
import socket
import ssl
import dns.resolver
from IPy import IP
from colorama import init, Fore, Back, Style
from lib.helper import generateResultFilename

URLS = {'ip': r'https://www.virustotal.com/vtapi/v2/ip-address/report',
        'domain': r'https://www.virustotal.com/vtapi/v2/domain/report'}
WAIT_TIME = 15  # Public API allows 4 request per minute, so we wait 15 secs by default
IP_WHITE_LIST = ['1.0.0.127', '127.0.0.1']
OWNER_WHITE_LIST = ['Google Inc.', 'Facebook, Inc.', 'CloudFlare, Inc.', 'Microsoft Corporation',
                    'Akamai Technologies, Inc.']  # not yet used
DOMAIN_WHITE_LIST = ['sourceforge.net']
RES_TARGETS = {'ip': 'hostname', 'domain': 'ip_address'}


def fetch_ip_and_domains(line):
    """
    Extracts IPs and Domains from a log line
    """
    domains = []
    # Modify line to easily extract IPs and Domains from reports
    # get 183.200.23[.]213
    mod_line = line.replace("[", "").replace("]", "")
    ip_pattern = r'\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b'
    ips = re.findall(ip_pattern, mod_line)
    # Exact
    #domain_pattern = r'\b(?=.{4,253}$)(((?!-)[a-zA-Z0-9-]{1,63}(?<!-)\.)+[a-zA-Z0-9-]{3,40}\.[a-zA-Z]{2,6})\b'
    # Allows for domain names only
    domain_pattern = r'([\s\n]|^)(([a-zA-Z0-9-]{3,40}\.)*[a-zA-Z]{2,6})\b'
    domains_raw = re.findall(domain_pattern, mod_line)
    for domain in domains_raw:
        tld = domain[1].split(".")[-1]
        if not is_valid_tld(tld):
            continue
        domains.append(domain[1])
    return ips, domains


def is_valid_tld(tld):
    try:
        dns.resolver.query(tld + '.', 'SOA')
        return True
    except dns.resolver.NXDOMAIN:
        return False


def is_ip(value):
    ip_pattern = r'^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b$'
    if re.match(ip_pattern, value):
        return True
    return False


def is_private(ip):
    ip = IP(ip)
    if ip.iptype() == "PRIVATE":
        return True
    return False


def is_resolvable(domain):
    try:
        socket.gethostbyname(domain)
        return True
    except Exception as e:
        # traceback.print_exc()
        return False


def is_pingable(ip):
    """
    Ping the target IP
    :param ip:
    :return:
    """
    try:
        # Ping parameters as function of OS
        ping_str = "-n 1 -w 500" if platform.system().lower() == "windows" else "-c 1 -W 500"
        # Ping
        subprocess.check_output("ping {0} {1}".format(ping_str, ip),
                                stderr=subprocess.STDOUT,
                                shell=True)
        return True
    except Exception as e:
        # traceback.print_exc()
        return False


def saveCache(cache, fileName):
    """
    Saves the cache database as pickle dump to a file
    :param cache:
    :param fileName:
    :return:
    """
    with open(fileName, 'w') as fh:
        fh.write(json.dumps(cache))


def loadCache(fileName):
    """
    Load cache database as json dump from file
    :param fileName:
    :return:
    """
    try:
        with open(fileName, 'r') as fh:
            return json.loads(fh.read()), True
    except Exception as e:
        traceback.print_exc()
        return [], False


def print_highlighted(line, hl_color=Back.WHITE):
    """
    Print a highlighted line
    """
    try:
        # Highlight positives
        colorer = re.compile(r'([^\s]+) POSITIVES: ([1-9]) ')
        line = colorer.sub(Fore.YELLOW + r'\1 ' + 'POSITIVES: ' + Fore.YELLOW + r'\2 ' + Style.RESET_ALL, line)
        colorer = re.compile(r'([^\s]+) POSITIVES: ([0-9]+) ')
        line = colorer.sub(Fore.RED + r'\1 ' + 'POSITIVES: ' + Fore.RED + r'\2 ' + Style.RESET_ALL, line)
        # Keyword highlight
        colorer = re.compile(r'([A-Z_]{2,}:)\s', re.VERBOSE)
        line = colorer.sub(Fore.BLACK + hl_color + r'\1' + Style.RESET_ALL + ' ', line)
        print(line)
    except Exception as e:
        pass


def process_lines(lines, debug=False):
    """
    Process the input file line by line
    """
    # Counter
    linenr = 0
    # Elements
    elements = []

    # Loop through lines -----------------------------------------------------------------------------------------------
    for line in lines:
        # Skip comments
        if line.startswith("#"):
            continue

        ips, domains = fetch_ip_and_domains(line)
        if debug:
            if len(ips):
                print("[D] IPs: {0}".format(', '.join(ips)))
            if len(domains):
                print("[D] Domains: {0}".format(', '.join(domains)))

        # Line number
        linenr += 1

        # If no IP or Domain found in line
        if len(ips) < 1 and len(domains) < 1:
            continue

        # Elements
        for i in ips:
            elements.append({"value": i, "type": "ip"})
        for h in domains:
            elements.append({"value": h, "type": "domain"})

    return elements


def process_elements(elements, result_file, max_items, nocsv=False, dups=False, noresolve=False, ping=False,
                     debug=False):
    # Counter
    c = 0

    while c < len(elements):

        # Get the current element
        element = elements[c]
        value = element["value"]
        cat = element["type"]
        status = "processed"

        if debug:
            print("i: %d - Elements in queue: %d Element in check: %s" % (c, len(elements), element))

        # Count
        c += 1

        # Cache ------------------------------------------------------------------------------------------------
        #if value in cache:
        #    if dups:
        #        # Colorized head of each hash check
        #        print_highlighted("\n{0}: {1}".format(str.upper(cat), value), Back.CYAN)
        #        print_highlighted("RESULT: %s (from cache)" % cache[value])
        #    continue

        # Skips ------------------------------------------------------------------------------------------------
        # Is private
        if cat == 'ip':
            # Skip private IPs
            if is_private(value):
                if debug:
                    # Add to cache
                    status = 'skipped'
                    print("[D] IP {0} is a private IP - skipping".format(value))
                continue
            # Skip unreachable systems
            if ping:
                if not is_pingable(value):
                    # Add to cache
                    status = 'skipped'
                    if debug:
                        print("[D] IP {0} ping failed - skipping".format(value))
                    continue

        # White lists
        for dom in DOMAIN_WHITE_LIST:
            if dom in value:
                print_highlighted("Domain white-listed - skipping this host SYSTEM: %s" % value)
                status = 'skipped'

        for iwl in IP_WHITE_LIST:
            if iwl == value:
                print_highlighted("IP white-listed - skipping this host SYSTEM: %s" % value)
                status = 'skipped'

        if status == 'skipped':
            continue

        # Is resolvable
        if not noresolve:
            if cat == 'domain':
                if not is_resolvable(value):
                    # Add to cache
                    status = 'skipped'
                    continue

        # Head -------------------------------------------------------------------------------------------------
        # Colorized head of each hash check
        print_highlighted("\n{0}: {1}".format(str.upper(cat), value), Back.CYAN)

        # VT API Request ---------------------------------------------------------------------------------------
        # Prepare VT API request
        parameters = {cat: value, "apikey": VT_PUBLIC_API_KEY}
        success = False

        while not success:
            try:
                parameters = {cat: value, 'apikey': VT_PUBLIC_API_KEY}
                if debug:
                    print("URL: %s" % URLS[cat])
                    print("PARAMS: %s" % parameters)

                # remove in new version
                ctx = ssl.create_default_context()
                ctx.check_hostname = False
                ctx.verify_mode = ssl.CERT_NONE

                response = urllib.request.urlopen('%s?%s' % (URLS[cat], urllib.parse.urlencode(parameters)), context=ctx).read()

                response_dict = json.loads(response)
                success = True
            except Exception as e:
                if debug:
                    print("RESPONSE: %s" % response)
                    traceback.print_exc()
                    # print "Error requesting VT results"
                pass
        if debug:
            print(json.dumps(response_dict, indent=4, sort_keys=True))

        # Process results --------------------------------------------------------------------------------------
        result = "- / -"
        rating = "unknown"
        owner = "-"
        country = "-"
        positives = 0
        total = 0
        sample_positives = 0
        sample_total = 0
        resolutions = []
        urls = []
        samples = []
        res_color = Back.CYAN
        shown_messages = {}

        # print simplejson.dumps(response_dict, sort_keys=True, indent=4)
        if response_dict.get("response_code") > 0:

            # Predefine Rating
            rating = "clean"

            # Other Info
            owner = response_dict.get("as_owner")
            country = response_dict.get("country")

            # WHITE LIST CHECKS
            if owner:
                for owl in OWNER_WHITE_LIST:
                    if owl in owner:
                        print_highlighted("Owner white-listed - skipping this host OWNER: %s" % owner)

            # Resolutions
            if 'resolutions' in response_dict:
                resolution_list = response_dict['resolutions']
                for i, res in enumerate(resolution_list):
                    resolutions.append({'target': res[RES_TARGETS[cat]], 'last_resolved': res['last_resolved']})
                    if i < max_items:
                        print_highlighted("HOST: {0} LAST_RESOLVED: {1}".format(res[RES_TARGETS[cat]],
                                                                                res['last_resolved']))
                    else:
                        if "hosts" not in shown_messages:
                            sys.stdout.write("Others found: ")
                            shown_messages["hosts"] = True
                        sys.stdout.write(".")
                    # Add the IP to the elements
                    if args.recursive:
                        new_value = res[RES_TARGETS[cat]]
                        if is_ip(new_value):
                            elements.append({'value': new_value, 'type': 'ip'})
                        else:
                            elements.append({'value': new_value, 'type': 'domain'})
                if "hosts" in shown_messages:
                    sys.stdout.write("\n")

            # URL matches
            if 'detected_urls' in response_dict:
                detected_urls = response_dict['detected_urls']
                for i, url in enumerate(detected_urls):
                    positives_url = url['positives']
                    total_url = url['total']
                    urls.append({'url': url['url'], 'positives': positives_url, 'total': total_url})
                    if i < max_items and args.download:
                        print_highlighted("URL: {0} POSITIVES: {1} TOTAL: {2}".format(url['url'],
                                                                                      positives_url,
                                                                                      total_url))
                    else:
                        if "urls" not in shown_messages:
                            sys.stdout.write("Others found: ")
                            shown_messages["urls"] = True
                        sys.stdout.write(".")
                    positives += positives_url
                    total += total_url
                    # Download URL
                    if args.download:
                        download_url(value, url['url'])
                if "urls" in shown_messages:
                    sys.stdout.write("\n")

            # Samples
            if 'detected_communicating_samples' in response_dict:
                samples_list = response_dict['detected_communicating_samples']
                for i, sample in enumerate(samples_list):
                    positives_sample = sample['positives']
                    total_sample = sample['total']
                    date = sample['date']
                    sha256 = sample['sha256']
                    samples.append({'sample': sha256, 'positives': positives_sample, 'total': total_sample,
                                    'date': date})
                    if i < max_items:
                        print_highlighted("SAMPLE: {0} POSITIVES: {1} TOTAL: {2} "
                                          "DATE: {3}".format(sha256, positives_sample, total_sample, date))
                    else:
                        if "samples" not in shown_messages:
                            sys.stdout.write("Others found: ")
                            shown_messages["samples"] = True
                        sys.stdout.write(".")
                    sample_positives += positives_sample
                    sample_total += total_sample
                if "samples" in shown_messages:
                    sys.stdout.write("\n")

            # Calculations -------------------------------------------------------------------------------------
            # Rating
            # Calculate ratio
            if positives > 0 and total > 0:
                ratio = (float(positives) / float(total)) * 100
                # Set rating
                if ratio > 3 and rating == "clean":
                    rating = "suspicious"
                if ratio > 10 and (rating == "clean" or rating == "suspicious"):
                    rating = "malicious"

            # Type
            res_color = Back.GREEN
            if rating == "suspicious":
                res_color = Back.YELLOW
            if rating == "malicious":
                res_color = Back.RED

            # Result -------------------------------------------------------------------------------------------
            result = "%s / %s" % (positives, total)
            print_highlighted("COUNTRY: {0} OWNER: {1}".format(country, owner))
            print_highlighted("POSITIVES: %s RATING: %s" % (result, rating), hl_color=res_color)

        else:
            # Print the highlighted result line
            print_highlighted("POSITIVES: %s RATING: %s" % (result, rating), hl_color=res_color)

        # CSV OUTPUT -------------------------------------------------------------------------------------------
        # Add to log file
        if not nocsv:
            # Hosts string
            targets = []
            for r in resolutions:
                targets.append(r['target'])
            targets_value = ', '.join(targets)
            # Malicious samples
            mal_samples = []
            for s in samples:
                if s['positives'] > 3:
                    mal_samples.append(s['sample'])
            samples_value = ', '.join(mal_samples)
            # urls = ', '.join("%s=%r" % (key,val) for (key,val) in urls.iteritems())
            # samples = ', '.join("%s=%r" % (key,val) for (key,val) in samples.iteritems())
            result_line = "{0};{1};{2};{3};{4};{5};{6};{7}\n".format(value, rating, owner, country,
                                                                     positives, total,
                                                                     samples_value, targets_value)
            with open(result_file, "a") as fh_results:
                fh_results.write(result_line)

        # Add to cache -----------------------------------------------------------------------------------------
        cache.append({'value': value,
                      'rating': rating,
                      'owner': owner,
                      'country': country,
                      'positives': positives,
                      'total': total,
                      'status': status,
                      })

        # Wait -------------------------------------------------------------------------------------------------
        # Wait some time for the next request
        time.sleep(WAIT_TIME)


def download_url(host_id, url):
    """
    Downloads an URL and stores the response to a directory with named as the host/IP
    :param host_id:
    :param url:
    :return:
    """
    output = BytesIO()
    header = BytesIO()

    print("[>] Trying to download URL: %s" % url)
    # Download file
    try:
        # 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)')
        c = pycurl.Curl()
        c.setopt(c.URL, url)
        c.setopt(pycurl.CONNECTTIMEOUT, 10)
        c.setopt(pycurl.TIMEOUT, 180)
        c.setopt(pycurl.FOLLOWLOCATION, 1)
        c.setopt(pycurl.USERAGENT, 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0)')
        c.setopt(c.WRITEFUNCTION, output.write)
        c.setopt(c.HEADERFUNCTION, header.write)
        c.setopt(pycurl.SSL_VERIFYHOST, 0)
        c.setopt(pycurl.SSL_VERIFYPEER, 0)
        c.perform()
        # Header parsing
        header_info = header_function(header.getvalue())
    except Exception as e:
        if args.debug:
            traceback.print_exc()
        print_highlighted("[-] Error MESSAGE: %s" % str(e))

    # Write File
    if c.getinfo(c.RESPONSE_CODE) == 200:
        # Check folder
        out_path = os.path.join(os.path.abspath(args.d), host_id)
        if not os.path.exists(out_path):
            os.makedirs(out_path)
        # Output file name
        r = urlparse(url)
        filename = r.path.split("/")[-1]
        if filename == "":
            if r.path != "/":
                out_path = os.path.join(out_path, r.path.lstrip("/"))
            filename = "index.dat"
        out_filename = os.path.join(os.path.abspath(out_path), filename)
        # Write file
        try:
            with open(out_filename, 'wb') as f:
                f.write(output.getvalue())
                print_highlighted("[+] Successfully saved to FILE: %s" % out_filename)
        except Exception as e:
            if args.debug:
                traceback.print_exc()
            print("[-] Failed to write file %s (use --debug for more info)" % out_filename)
    else:
        try:
            print_highlighted("[i] Response CODE: %s MIME_TYPE: %s SIZE: %s" % (
                                  str(c.getinfo(c.RESPONSE_CODE)),
                                  header_info['content-type'],
                                  header_info['content-length'])
                              )
        except Exception as e:
            print_highlighted("[-] Response CODE: %s" % str(c.getinfo(c.RESPONSE_CODE)))

    output.close()


def header_function(header_raw):
    """
    Process header info
    Example from pycurl quick start guide http://pycurl.io/docs/latest/quickstart.html
    :param header_line:
    :return:
    """
    headers = {}
    header_lines = header_raw.splitlines()

    for header_line in header_lines:

        # HTTP standard specifies that headers are encoded in iso-8859-1.
        # On Python 2, decoding step can be skipped.
        # On Python 3, decoding step is required.
        header_line = header_line.decode('iso-8859-1')

        # Header lines include the first status line (HTTP/1.x ...).
        # We are going to ignore all lines that don't have a colon in them.
        # This will botch headers that are split on multiple lines...
        if ':' not in header_line:
            continue

        # Break the header line into header name and value.
        name, value = header_line.split(':', 1)

        # Remove whitespace that may be present.
        # Header lines include the trailing newline, and there may be whitespace
        # around the colon.
        name = name.strip()
        value = value.strip()

        # Header names are case insensitive.
        # Lowercase name here.
        name = name.lower()

        # Now we can actually record the header name and value.
        headers[name] = value

    return headers


def signal_handler(signal, frame):
    print("\n[+] Saving {0} cache entries to file {1}".format(len(cache), args.c))
    saveCache(cache, args.c)
    sys.exit(0)


if __name__ == '__main__':

    signal.signal(signal.SIGINT, signal_handler)
    init(autoreset=False)

    print(Style.RESET_ALL)
    print(Fore.BLACK + Back.WHITE)
    print(" ".ljust(80))
    print("   _________   _    _   ______  _____  ______          ".ljust(80))
    print("  | | | | | \\ | |  | | | |  \\ \\  | |  | |  \\ \\     /.) ".ljust(80))
    print("  | | | | | | | |  | | | |  | |  | |  | |  | |    /)\\| ".ljust(80))
    print("  |_| |_| |_| \\_|__|_| |_|  |_| _|_|_ |_|  |_|   // /  ".ljust(80))
    print("  > IP AND DOMAIN CHECKER                       /'\" \"  ".ljust(80))
    print(" ".ljust(80))
    print(("  " + __AUTHOR__ + " - " + __VERSION__ + "").ljust(80))
    print(" ".ljust(80) + Style.RESET_ALL)
    print(Style.RESET_ALL + " ")

    parser = argparse.ArgumentParser(description='Virustotal Online Checker (IP/Domain)')
    parser.add_argument('-f', help='File to process (hash line by line OR csv with hash in each line - auto-detects '
                                   'position and comment)', metavar='path', default='')
    parser.add_argument('-o', help='Output file for results (CSV)', metavar='output', default='')
    parser.add_argument('-m', help='Maximum number of items (urls, hosts, samples) to show', metavar='max-items',
                        default=10)
    parser.add_argument('-c', help='Name of the cache database file (default: vt-hosts-db.json)', metavar='cache-db',
                        default='vt-hosts-db.json')
    parser.add_argument('-i', help='Name of the ini file that holds the API keys', metavar='ini-file',
                        default='munin.ini')
    parser.add_argument('--nocache', action='store_true', help='Do not use the load the cache db (vt-check-cache.pkl)',
                        default=False)
    parser.add_argument('--nocsv', action='store_true', help='Do not write a CSV with the results', default=False)
    parser.add_argument('--recursive', action='store_true', help='Process the resolved IPs as well', default=False)
    parser.add_argument('--download', action='store_true',
                        help='Try to download the URLs (directories with host/ip names)', default=False)
    parser.add_argument('-d', help='Store the downloads to the given directory', metavar='download_path',
                        default='./')
    parser.add_argument('--dups', action='store_true', help='Do not skip duplicate hashes', default=False)
    parser.add_argument('--noresolve', action='store_true', help='Do not perform DNS resolve test on found domain '
                                                                 'names', default=False)
    parser.add_argument('--ping', action='store_true', help='Perform ping check on IPs (speeds up process if many '
                                                            'public but internally routed IPs appear in text file)',
                        default=False)
    parser.add_argument('--debug', action='store_true', default=False, help='Debug output')

    args = parser.parse_args()

    # Read the config file
    config = configparser.ConfigParser()
    try:
        config.read(args.i)
        VT_PUBLIC_API_KEY = config['DEFAULT']['VT_PUBLIC_API_KEY']
        MAL_SHARE_API_KEY = config['DEFAULT']['MAL_SHARE_API_KEY']
        PAYLOAD_SEC_API_KEY = config['DEFAULT']['PAYLOAD_SEC_API_KEY']
    except Exception as e:
        traceback.print_exc()
        print("[E] Config file '%s' not found" % args.i)

    # Check input file
    if args.f == '':
        print("[E] Please provide an input file with '-f inputfile'\n")
        parser.print_help()
        sys.exit(1)
    if not os.path.exists(args.f):
        print("[E] Cannot find input file {0}".format(args.f))
        sys.exit(1)

    # Caches
    cache = []
    # Trying to load cache from json dump
    if not args.nocache:
        cache, success = loadCache(args.c)
        if success:
            print("[+] {0} cache entries read from cache database: {1}".format(len(cache), args.c))
        else:
            print("[-] No cache database found")
            print("[+] Analyzed IPs/domains will be written to cache database: {0}".format(args.c))
        print("[+] You can always interrupt the scan by pressing CTRL+C without loosing the scan state")

    # Open input file
    try:
        with open(args.f, 'r') as fh_input:
            lines = fh_input.readlines()
    except Exception as e:
        print("[E] Cannot read input file")
        sys.exit(1)

    # Result file
    if not args.nocsv:
        alreadyExists, result_file = generateResultFilename(args.f, args.o)
        if alreadyExists:
            print("[+] Found results CSV from previous run: {0}".format(result_file))
            print("[+] Appending results to file: {0}".format(result_file))
        else:
            print("[+] Writing results to new file: {0}".format(result_file))
            try:
                with open(result_file, 'w') as fh_results:
                    fh_results.write(
                        "IP;Rating;Owner;Country Code;Positives;Total;Malicious Samples;Hosts\n")
            except Exception as e:
                print("[E] Cannot write CSV export file: {0}".format(result_file))

    # Process the input lines
    elements = process_lines(lines, args.debug)

    # Process elements
    process_elements(elements, result_file, int(args.m), args.nocsv, args.dups, args.noresolve, args.ping,
                     args.debug)

    # Write Cache
    print("\n[+] Saving {0} cache entries to file {1}".format(len(cache), args.c))
    saveCache(cache, args.c)

    print(Style.RESET_ALL)