http analysis; fixes in script; formatting; score-calc tweaking

This commit is contained in:
anon 2023-01-15 19:13:59 +01:00
parent c83cfc9e74
commit c12f1d3e51
9 changed files with 329 additions and 113 deletions

274
main.py
View File

@ -4,117 +4,195 @@ import re
from flagging import scoring
onionReport = os.getenv("ONIONSCAN_REPORT")
#http_headers = os.getenv("HTTP_HEADERS")
httpHeaders = os.getenv("HTTP_HEADERS")
onionFlag = 1
httpFlag = 1
securityHeaders = {
"X-Frame-Options":"DENY",
"X-XSS-Protection":0,
"X-Content-Type-Options":"nosniff",
"Referrer-Policy":"strict-origin-when-cross-origin",
"Content-Type":"text/html; charset=UTF-8",
"Set-Cookie":"HttpOnly; Secure; SameSite=Strict",
"Strict-Transport-Security":"max-age=63072000; includeSubDomains; preload",
"Content-Security-Policy":"default-src 'none'; script-src 'self'; connect-src 'self'; img-src 'self'; style-src 'self'; frame-ancestors 'self'; form-action 'self'",
"Cross-Origin-Opener-Policy":"same-origin",
"Cross-Origin-Embedder-Policy":"require-corp",
"Cross-Origin-Resource-Policy":"same-site",
"Permissions-Policy":"microphone=(); geolocation=(); interest-cohort=()",
"X-DNS-Prefetch-Control":"off",
}
badHeaders = [
"Access-Control-Allow-Origin",
"Expect-CT",
"X-Powered-By",
"X-AspNet-Version",
"X-AspNetMvc-Version",
"Public-Key-Pins",
"Server",
"ETag"
]
print("Analysis started with base score at 100")
print("")
if len(onionReport) == 0:
print("OnionScan report not found, exiting...")
exit()
print("OnionScan report not found, skipping...")
onionFlag = 0
onionReport = json.loads(onionReport)
#http_headers = json.loads(http_headers)
if onionFlag == 1:
onionReport = json.loads(onionReport)
print("Starting analysis...")
print("Starting at the base score 100")
baseScore = 100
print("OnionScan analysis:")
baseScore = 100
hiddenService = onionReport['hiddenService']
print("Hidden service address:", hiddenService)
if hiddenService == " http://ciadotgov4sjwlzihbbgxnqg3xiyrg7so2r2o3lt5wz5ypk4sxyjstad.onion":
baseScore = 0
print("Score goes down, now:", baseScore)
print("This hidden service is likely owned by CIA.")
scoring(baseScore)
exit()
hiddenService = onionReport['hiddenService']
print("\t Hidden service address:", hiddenService)
if hiddenService == " http://ciadotgov4sjwlzihbbgxnqg3xiyrg7so2r2o3lt5wz5ypk4sxyjstad.onion":
baseScore = 0
print("\t Score goes down, now:", baseScore)
print("\t This hidden service is likely owned by CIA.")
scoring(baseScore)
exit()
ssh = onionReport['sshDetected']
print("SSH?", ssh)
if ssh:
baseScore = baseScore * 0.67
print("Score goes down, now:", baseScore)
print("SSH key:", onionReport['sshKey'])
ssh = onionReport['sshDetected']
print("\t SSH?", ssh)
if ssh:
baseScore = baseScore * 0.67
print("\t Score goes down, now:", baseScore)
print("\t SSH key:", onionReport['sshKey'])
ftp = onionReport['ftpDetected']
print("FTP?", ftp)
if ftp:
baseScore = baseScore * 0.67
print("Score goes down, now:", baseScore)
print("FTP fingerprint:", onionReport['ftpFingerprint'])
print("FTP banner:", onionReport['ftpBanner'])
ftp = onionReport['ftpDetected']
print("\t FTP?", ftp)
if ftp:
baseScore = baseScore * 0.67
print("\t Score goes down, now:", baseScore)
print("\t FTP fingerprint:", onionReport['ftpFingerprint'])
print("\t FTP banner:", onionReport['ftpBanner'])
ftp = onionReport['ftpDetected']
smtp = onionReport['smtpDetected']
print("SMTP?", smtp)
if smtp:
baseScore = baseScore * 0.67
print("Score goes down, now:", baseScore)
print("SMTP fingerprint:", onionReport['smtpFingerprint'])
print("SMTP banner:", onionReport['smtpBanner'])
smtp = onionReport['smtpDetected']
print("\t SMTP?", smtp)
if smtp:
baseScore = baseScore * 0.67
print("\t Score goes down, now:", baseScore)
print("\t SMTP fingerprint:", onionReport['smtpFingerprint'])
print("\t SMTP banner:", onionReport['smtpBanner'])
bitcoin = onionReport['bitcoinDetected']
print("Bitcoin?", bitcoin)
if bitcoin:
baseScore = baseScore * 0.81
print("Score goes down, now:", baseScore)
bitcoinInfo = onionReport['bitcoinServices']['bitcoin']
print("Bitcoin user agent:", bitcoinInfo['userAgent'])
print("Bitcoin version:", bitcoinInfo['protocolVersion'])
print("Bitcoin onion peers:", bitcoinInfo['onionPeers'])
bitcoin = onionReport['bitcoinDetected']
print("\t Bitcoin?", bitcoin)
if bitcoin:
baseScore = baseScore * 0.81
print("\t Score goes down, now:", baseScore)
bitcoinInfo = onionReport['bitcoinServices']['bitcoin']
print("\t Bitcoin user agent:", bitcoinInfo['userAgent'])
print("\t Bitcoin version:", bitcoinInfo['protocolVersion'])
print("\t Bitcoin onion peers:", bitcoinInfo['onionPeers'])
idReport = onionReport['identifierReport']
idReport = onionReport['identifierReport']
privateKey = idReport['privateKeyDetected']
print("Private key found?", privateKey)
if privateKey:
baseScore = baseScore * 0.63
print("Score goes down, now:", baseScore)
privateKey = idReport['privateKeyDetected']
print("\t Private key found?", privateKey)
if privateKey:
baseScore = baseScore * 0.63
print("\t Score goes down, now:", baseScore)
apacheStatus = idReport['foundApacheModStatus']
print("Apache status found?", apacheStatus)
if apacheStatus:
baseScore = baseScore * 0.87
print("Score goes down, now:", baseScore)
ipAddress = idReport['ipAddresses']
print("IP address leakage?", ipAddress)
if ipAddress:
baseScore = baseScore * 0.55
print("Score goes down, now:", baseScore)
emailAddress = idReport['emailAddresses']
print("Email address found?", emailAddress)
if emailAddress:
baseScore = baseScore * 0.959
print("Score goes down, now:", baseScore)
analyticsId = idReport['analyticsIDs']
print("Analytics tags?", analyticsId)
if analyticsId:
baseScore = baseScore * 0.6
print("Score goes down, now:", baseScore)
risks = onionReport['simpleReport']['risks']
print("OnionScan detected risks:\n")
for r in risks:
t = r['title']
print("\tName:", t)
s = r['severity']
print("\tSeverity:", s)
if s == "info":
baseScore = baseScore * 0.999
print("\tScore goes down, now:", baseScore)
if s == "low":
baseScore = baseScore * 0.959
print("\tScore goes down, now:", baseScore)
if s == "medium":
baseScore = baseScore * 0.939
print("\tScore goes down, now:", baseScore)
if s == "high":
apacheStatus = idReport['foundApacheModStatus']
print("\t Apache status found?", apacheStatus)
if apacheStatus:
baseScore = baseScore * 0.87
print("\tScore goes down, now:", baseScore)
if s == "critical":
baseScore = baseScore * 0.77
print("\tScore goes down, now:", baseScore)
print("")
print("\t Score goes down, now:", baseScore)
ipAddress = idReport['ipAddresses']
print("\t IP address leakage?", ipAddress)
if ipAddress:
baseScore = baseScore * 0.55
print("\t Score goes down, now:", baseScore)
emailAddress = idReport['emailAddresses']
print("\t Email address found?", emailAddress)
if emailAddress:
baseScore = baseScore * 0.959
print("\t Score goes down, now:", baseScore)
analyticsId = idReport['analyticsIDs']
print("\t Analytics tags?", analyticsId)
if analyticsId:
baseScore = baseScore * 0.6
print("\t Score goes down, now:", baseScore)
risks = onionReport['simpleReport']['risks']
if not risks:
print("\t No risk detected.")
print("")
else:
print("\t OnionScan detected risks:\n")
for r in risks:
t = r['title']
print("\t Name:", t)
s = r['severity']
print("\t Severity:", s)
if s == "info":
baseScore = baseScore * 0.999
print("\t Score goes down, now:", baseScore)
if s == "low":
baseScore = baseScore * 0.959
print("\t Score goes down, now:", baseScore)
if s == "medium":
baseScore = baseScore * 0.939
print("\t Score goes down, now:", baseScore)
if s == "high":
baseScore = baseScore * 0.87
print("\t Score goes down, now:", baseScore)
if s == "critical":
baseScore = baseScore * 0.77
print("\t Score goes down, now:", baseScore)
print("")
if len(httpHeaders) == 0:
print("HTTP Headers not found, skipping...")
httpFlag = 0
if httpFlag == 1:
httpHeaders = json.loads(httpHeaders)
print("HTTP headers analysis:")
for badHeader in badHeaders:
if badHeader in httpHeaders:
baseScore = baseScore * 0.993
print("\t Found", badHeader, "in HTTP headers.")
print("\t Score goes down, now:", baseScore)
for secureHeader in securityHeaders:
if secureHeader in httpHeaders:
if securityHeaders[secureHeader] != httpHeaders[secureHeader]:
baseScore = baseScore * 0.987
print("\t", secureHeader, "is present, but have diffrent value than expected.")
print("\t Present value:", httpHeaders[secureHeader])
print("\t Expected value:", securityHeaders[secureHeader])
print("\t Score goes down, now:", baseScore)
else:
print("\t", secureHeader, "is present and set correctly.")
else:
baseScore = baseScore * 0.983
print("\t",secureHeader, "not found.")
print("\t Score goes down, now:", baseScore)
if "Expect-CT" in httpHeaders:
baseScore = baseScore * 0.983
print("\t This site is using Expect-CT header, it is recommended to not use it.")
print("\t Check https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Expect-CT for details.")
print("\t Score goes down, now:", baseScore)
if "Access-Control-Allow-Origin" in httpHeaders:
baseScore = baseScore * 0.989
print("\t This site is using Access-Control-Allow-Origin header, which allows to relax SOP.")
print("\t Score goes down, now:", baseScore)
if onionFlag or httpFlag:
scoring(baseScore)
scoring(baseScore)
print("Analysis ended.")

1
node_modules/.bin/curl-headers-to-json generated vendored Symbolic link
View File

@ -0,0 +1 @@
../curl-headers-to-json/index.js

15
node_modules/.package-lock.json generated vendored Normal file
View File

@ -0,0 +1,15 @@
{
"name": "shallot",
"lockfileVersion": 3,
"requires": true,
"packages": {
"node_modules/curl-headers-to-json": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/curl-headers-to-json/-/curl-headers-to-json-1.0.1.tgz",
"integrity": "sha512-PNbfCuu4UdsTmSAHwN4R0m34KzvOvDjP3mJ6XNfHzOdeTskUb/eaVV110y/SnT6h6HDA7fisCNgQvRP/vH9obQ==",
"bin": {
"curl-headers-to-json": "index.js"
}
}
}
}

12
node_modules/curl-headers-to-json/README.md generated vendored Normal file
View File

@ -0,0 +1,12 @@
# curl-headers-to-json
A tool to format the output of cURL headers into JSON.
Example:
```
curl -sSL -D - https://api.github.com -o /dev/null | npx curl-headers-to-json | jq
```
or if you grabbed the output already:
```
npx curl-headers-to-json headers.txt | jq
```

56
node_modules/curl-headers-to-json/index.js generated vendored Executable file
View File

@ -0,0 +1,56 @@
#!/usr/bin/env node
const fs = require('fs');
const util = require('util');
const readFile = util.promisify(fs.readFile);
const args = process.argv.slice(2);
const shellParam = args[0];
const isTTY = process.stdin.isTTY;
const stdin = process.stdin;
const stdout = process.stdout;
if (isTTY && args.length === 0) {
console.log('Usage: ');
} else if (isTTY && args.length !== 0) {
handleShellArguments();
} else {
handleStdin();
}
function handleStdin() {
let headersData = '';
stdin.setEncoding('utf8');
stdin.on('readable', () => {
const chuck = stdin.read();
if(chuck !== null){
headersData += chuck;
}
});
stdin.on('end', () => process.stdout.write(JSON.stringify(format(headersData))));
}
function format(data) {
const lines = data.split(/\r?\n|\r/g);
const result = {};
lines.shift();
for(let line of lines) {
if(line.trim() === '') {
continue;
}
const [key, value] = line.split(': ');
result[key] = value.replace(/\\r/g, '');
}
return result;
}
async function handleShellArguments(){
try {
const input = await readFile(shellParam, { encoding: 'utf8' });
process.stdout.write(JSON.stringify(format(input)))
} catch (e) {
console.log(`An error ocurred while trying to open the file: ${e.message}`);
}
}

18
node_modules/curl-headers-to-json/package.json generated vendored Normal file
View File

@ -0,0 +1,18 @@
{
"name": "curl-headers-to-json",
"bin": {
"curl-headers-to-json": "./index.js"
},
"version": "1.0.1",
"description": "Formats cURL headers into JSON format",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 0"
},
"keywords": [
"curl",
"json"
],
"author": "Alejandro Oviedo <alejandro.oviedo.g@gmail.com>",
"license": "MIT"
}

20
package-lock.json generated Normal file
View File

@ -0,0 +1,20 @@
{
"name": "shallot",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"dependencies": {
"curl-headers-to-json": "^1.0.1"
}
},
"node_modules/curl-headers-to-json": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/curl-headers-to-json/-/curl-headers-to-json-1.0.1.tgz",
"integrity": "sha512-PNbfCuu4UdsTmSAHwN4R0m34KzvOvDjP3mJ6XNfHzOdeTskUb/eaVV110y/SnT6h6HDA7fisCNgQvRP/vH9obQ==",
"bin": {
"curl-headers-to-json": "index.js"
}
}
}
}

5
package.json Normal file
View File

@ -0,0 +1,5 @@
{
"dependencies": {
"curl-headers-to-json": "^1.0.1"
}
}

View File

@ -1,16 +1,16 @@
#!/bin/bash
set -xe
set -e
# Setting env
SCAN_DATE=`date "+%F-%H-%M"`
export SHALLOT_DIR="/tmp/shallot-$SCAN_DATE"
mkdir -p $SHALLOT_DIR
echo "Shallot scritp v0.0.2"
echo "Shallot scritp v0.1.1"
if [[ $# -eq 0 ]] ; then
echo "[ERROR] No arguments was passed, exiting..."
echo "[ERRO] No arguments was passed, exiting..."
exit 1
fi
@ -18,49 +18,60 @@ if [[ $# -ge 2 ]] ; then
echo "[WARRNING] Too much argument was passed, this script uses only first one."
fi
[ $(type -P "npx") ] || echo "[ERRO] npx is not in the path, install npm first!"
export ONIONSITE=$1
echo "[INFO] Checking if Tor Browser proxy is running..."
echo "Checking if Tor Browser proxy is running..."
NETSTAT_OUTPUT=`netstat -tlnp 2> /dev/null`
#NETSTAT_OUTPUT=`netstat -tlnp 2> /dev/null`
#IF_TOR_RUNNING=`echo $NETSTAT_OUTPUT | grep -Ezqv "/tor" && echo 0 || echo 1`
IF_TOR_RUNNING=`ps -eaf | grep -i tor |sed '/^$/d' | wc -l`
if [[ "$IF_TOR_RUNNING" > 1 ]] ; then
echo "[INFO] Tor is running!"
else
echo "[ERROR] Tor is not running, start Tor Browser and connect to Tor, then restart this scritp"
echo "[ERRO] Tor is not running, start Tor Browser and connect to Tor, then restart this scritp"
exit 1;
fi
echo "Checking Onion Service, address: $1"
IS_ADDRESS_ONION=`echo $1 | grep -Ei ".onion$" | wc -c`
if [[ $IS_ADDRESS_ONION > 0 ]] ; then
echo "Checking Onion Service, address: $1"
else
echo "[ERRO] Looks like $1 is not an onion site, exiting..."
exit 1
fi
echo ""
# OnionScan
echo "Runnning OnionScan aginst address, this will take a while..."
echo "[INFO] Runnning OnionScan aginst address, this will take a while..."
export ONIONSCAN_REPORT=$(onionscan --jsonReport --torProxyAddress "127.0.0.1:9150" $1 2>$SHALLOT_DIR/onionscan_error.log | jq)
echo $ONIONSCAN_REPORT > $SHALLOT_DIR/onionscan_result.txt
if [ $? ] ; then
echo "OnionScan done! Saved in $SHALLOT_DIR/onionscan_result.txt"
echo "[INFO] OnionScan done! Saved in $SHALLOT_DIR/onionscan_result.txt"
else
echo "[ERROR] Error occured, exiting, check $SHALLOT_DIR/onionscan_error.log for details."
exit 1
echo "[ERRO] Error occured, exiting, check $SHALLOT_DIR/onionscan_error.log for details."
fi
# HTTP Headers
echo "Scanning HTTP headers, wait..."
export HTTP_HEADERS=$(proxychains -q -f /etc/proxychains4.conf /usr/bin/curl -I -s $1 | tail -n +3 | sed 's/\r//g' | head -n -1 | jq -R 'split(":")|{(.[0]) : .[1]}' 2>$SHALLOT_DIR/http_headers_error.log)
#export HTTP_HEADERS=$(proxychains -q -f /etc/proxychains4.conf /usr/bin/curl -I -s $1 | tail -n +3 | sed 's/\r//g' | head -n -1 | jq -R 'split(":")|{(.[0]) : .[1]}' | sed 's/\\"//g' 2>$SHALLOT_DIR/http_headers_error.log)
export HTTP_HEADERS=$(proxychains -q -f /etc/proxychains4.conf /usr/bin/curl -LIs -D - $1 -o /dev/null | npx curl-headers-to-json | sed 's/\\"//g' | jq 2>$SHALLOT_DIR/http_headers_error.log)
echo $HTTP_HEADERS > $SHALLOT_DIR/http_headers.txt
if [ $? ] ; then
echo "HTTP headers done! Saved in $SHALLOT_DIR/http_headers.txt"
echo "[INFO] HTTP headers done! Saved in $SHALLOT_DIR/http_headers.txt"
else
echo "[ERROR] Error occured, check $SHALLOT_DIR/http_headers_error.log"
exit 1
echo "[ERRO] Error occured, check $SHALLOT_DIR/http_headers_error.log"
fi
# Report analysis
python3 main.py
echo "Works done, exiting."
exit 0