Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Issue 14 check local branch #47

Closed
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,4 @@
/dist/
/truffleHog.egg-info/
*/__pycache__/
files.txt
33 changes: 33 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,44 @@ truffleHog file:///user/dxa4481/codeprojects/truffleHog/

![Example](https://i.imgur.com/YAXndLD.png)

## Usage Details
```
usage: truffleHog [-h] [--json] git_repo

Find secrets hidden in the depths of git.

positional arguments:
git_repo Git Repository: Remote URL or local filesystem for secret searching

optional arguments:
-h, --help show this help message and exit
--json Output in JSON
```

## Install
Automatically install via the [Python Package Indexer, PIP](https://pypi.python.org/pypi/pip)
```
pip install truffleHog
```

or manually install by via clone/download the repository and install with setup.py

```
git clone https://github.com/dxa4481/truffleHog.git
cd truffleHog.git
python setup.py install --record files.txt
```

## Uninstall
```
pip uninstall truffleHog
```

or manually uninstall if installed locally via repo
```
cat files.txt | xargs rm -rf
```

## How it works
This module will go through the entire commit history of each branch, and check each diff from each commit, and evaluate the shannon entropy for both the base64 char set and hexidecimal char set for every blob of text greater than 20 characters comprised of those character sets in each diff. If at any point a high entropy string >20 characters is detected, it will print to the screen.

Expand Down
65 changes: 57 additions & 8 deletions truffleHog/truffleHog.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,22 +8,68 @@
import argparse
import tempfile
import os
import subprocess
import json
import stat
import urllib
from git import Repo

# Global Vars
BASE64_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="
HEX_CHARS = "1234567890abcdefABCDEF"

def main():
parser = argparse.ArgumentParser(description='Find secrets hidden in the depths of git.')
parser.add_argument('--json', dest="output_json", action="store_true", help="Output in JSON")
parser.add_argument('git_url', type=str, help='URL for secret searching')
parser.add_argument('git_repo', type=str, help='Git Repository: Remote URL or local filesystem for secret searching')
args = parser.parse_args()
output = find_strings(args.git_url, args.output_json)
output = find_strings(args.git_repo, args.output_json)
project_path = output["project_path"]
shutil.rmtree(project_path, onerror=del_rw)

def is_abs_url(url=None):
"""Checks if string is absolute URL or not and returns boolean denoting True or False if valid URL.

BASE64_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="
HEX_CHARS = "1234567890abcdefABCDEF"
:param url: A string, absolute URL.
"""
if url is None or url.strip() == '':
return False
if not url.strip().lower()[:4] == 'http':
return False
try:
rc = urllib.urlopen(url).getcode()
if rc == 200:
return True
else:
sys.stderr.write("Unable to reach remote Git repo over HTTP due to response code : " + str(rc))
return False
except Exception as e:
sys.stderr.write("Invalid URL " + e.message)
return False

def is_git_dir(filepath=None):
"""Checks if filepath is directory and if directory itself is a git repository.
Returns boolean denoting True or False if valid directory and git repository.

:param filepath: A string, filepath of the directory.
"""
if filepath is None or filepath.strip() == '':
return False

if filepath[-4:] != '.git':
filepath += '/.git'

# verify actual git directory
if not os.path.isdir(filepath) and os.path.exists(filepath):
try:
rc = subprocess.call(['git', 'rev-parse'], cwd=filepath)
return (rc == 0)
except OSError as e:
sys.stderr.write("Unable to invoke cmd 'git rev-parse' due to following: " + e.message)
return False
else:
sys.stderr.write("Invalid filepath - directory does not exist " + filepath)
return False

def del_rw(action, name, exc):
os.chmod(name, stat.S_IWRITE)
Expand All @@ -42,7 +88,6 @@ def shannon_entropy(data, iterator):
entropy += - p_x*math.log(p_x, 2)
return entropy


def get_strings_of_set(word, char_set, threshold=20):
count = 0
letters = ""
Expand Down Expand Up @@ -70,9 +115,13 @@ class bcolors:
BOLD = '\033[1m'
UNDERLINE = '\033[4m'

def find_strings(git_url, printJson=False):
project_path = tempfile.mkdtemp()
Repo.clone_from(git_url, project_path)
def find_strings(git_repo, printJson=False):
if is_abs_url(git_repo) or is_git_dir(git_repo):
project_path = tempfile.mkdtemp()
Repo.clone_from(git_repo, project_path)
else:
sys.exit("\nUnable to scan for secrets due to invalid Git repository " + git_repo)

output = {"entropicDiffs": []}
repo = Repo(project_path)
already_searched = set()
Expand Down