From c12f1d3e5199af841877b13e96d1756f601fe06a Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 15 Jan 2023 19:13:59 +0100 Subject: [PATCH] http analysis; fixes in script; formatting; score-calc tweaking --- main.py | 274 +++++++++++------- node_modules/.bin/curl-headers-to-json | 1 + node_modules/.package-lock.json | 15 + node_modules/curl-headers-to-json/README.md | 12 + node_modules/curl-headers-to-json/index.js | 56 ++++ .../curl-headers-to-json/package.json | 18 ++ package-lock.json | 20 ++ package.json | 5 + shallot.sh | 41 ++- 9 files changed, 329 insertions(+), 113 deletions(-) create mode 120000 node_modules/.bin/curl-headers-to-json create mode 100644 node_modules/.package-lock.json create mode 100644 node_modules/curl-headers-to-json/README.md create mode 100755 node_modules/curl-headers-to-json/index.js create mode 100644 node_modules/curl-headers-to-json/package.json create mode 100644 package-lock.json create mode 100644 package.json diff --git a/main.py b/main.py index 2f539fd..123fe5b 100644 --- a/main.py +++ b/main.py @@ -4,117 +4,195 @@ import re from flagging import scoring onionReport = os.getenv("ONIONSCAN_REPORT") -#http_headers = os.getenv("HTTP_HEADERS") +httpHeaders = os.getenv("HTTP_HEADERS") + +onionFlag = 1 +httpFlag = 1 + +securityHeaders = { + "X-Frame-Options":"DENY", + "X-XSS-Protection":0, + "X-Content-Type-Options":"nosniff", + "Referrer-Policy":"strict-origin-when-cross-origin", + "Content-Type":"text/html; charset=UTF-8", + "Set-Cookie":"HttpOnly; Secure; SameSite=Strict", + "Strict-Transport-Security":"max-age=63072000; includeSubDomains; preload", + "Content-Security-Policy":"default-src 'none'; script-src 'self'; connect-src 'self'; img-src 'self'; style-src 'self'; frame-ancestors 'self'; form-action 'self'", + "Cross-Origin-Opener-Policy":"same-origin", + "Cross-Origin-Embedder-Policy":"require-corp", + "Cross-Origin-Resource-Policy":"same-site", + "Permissions-Policy":"microphone=(); geolocation=(); interest-cohort=()", + "X-DNS-Prefetch-Control":"off", +} + +badHeaders = [ + "Access-Control-Allow-Origin", + "Expect-CT", + "X-Powered-By", + "X-AspNet-Version", + "X-AspNetMvc-Version", + "Public-Key-Pins", + "Server", + "ETag" +] + +print("Analysis started with base score at 100") +print("") if len(onionReport) == 0: - print("OnionScan report not found, exiting...") - exit() + print("OnionScan report not found, skipping...") + onionFlag = 0 -onionReport = json.loads(onionReport) -#http_headers = json.loads(http_headers) +if onionFlag == 1: + onionReport = json.loads(onionReport) -print("Starting analysis...") -print("Starting at the base score 100") -baseScore = 100 + print("OnionScan analysis:") + baseScore = 100 -hiddenService = onionReport['hiddenService'] -print("Hidden service address:", hiddenService) -if hiddenService == " http://ciadotgov4sjwlzihbbgxnqg3xiyrg7so2r2o3lt5wz5ypk4sxyjstad.onion": - baseScore = 0 - print("Score goes down, now:", baseScore) - print("This hidden service is likely owned by CIA.") - scoring(baseScore) - exit() + hiddenService = onionReport['hiddenService'] + print("\t Hidden service address:", hiddenService) + if hiddenService == " http://ciadotgov4sjwlzihbbgxnqg3xiyrg7so2r2o3lt5wz5ypk4sxyjstad.onion": + baseScore = 0 + print("\t Score goes down, now:", baseScore) + print("\t This hidden service is likely owned by CIA.") + scoring(baseScore) + exit() -ssh = onionReport['sshDetected'] -print("SSH?", ssh) -if ssh: - baseScore = baseScore * 0.67 - print("Score goes down, now:", baseScore) - print("SSH key:", onionReport['sshKey']) + ssh = onionReport['sshDetected'] + print("\t SSH?", ssh) + if ssh: + baseScore = baseScore * 0.67 + print("\t Score goes down, now:", baseScore) + print("\t SSH key:", onionReport['sshKey']) -ftp = onionReport['ftpDetected'] -print("FTP?", ftp) -if ftp: - baseScore = baseScore * 0.67 - print("Score goes down, now:", baseScore) - print("FTP fingerprint:", onionReport['ftpFingerprint']) - print("FTP banner:", onionReport['ftpBanner']) ftp = onionReport['ftpDetected'] + print("\t FTP?", ftp) + if ftp: + baseScore = baseScore * 0.67 + print("\t Score goes down, now:", baseScore) + print("\t FTP fingerprint:", onionReport['ftpFingerprint']) + print("\t FTP banner:", onionReport['ftpBanner']) + ftp = onionReport['ftpDetected'] -smtp = onionReport['smtpDetected'] -print("SMTP?", smtp) -if smtp: - baseScore = baseScore * 0.67 - print("Score goes down, now:", baseScore) - print("SMTP fingerprint:", onionReport['smtpFingerprint']) - print("SMTP banner:", onionReport['smtpBanner']) + smtp = onionReport['smtpDetected'] + print("\t SMTP?", smtp) + if smtp: + baseScore = baseScore * 0.67 + print("\t Score goes down, now:", baseScore) + print("\t SMTP fingerprint:", onionReport['smtpFingerprint']) + print("\t SMTP banner:", onionReport['smtpBanner']) -bitcoin = onionReport['bitcoinDetected'] -print("Bitcoin?", bitcoin) -if bitcoin: - baseScore = baseScore * 0.81 - print("Score goes down, now:", baseScore) - bitcoinInfo = onionReport['bitcoinServices']['bitcoin'] - print("Bitcoin user agent:", bitcoinInfo['userAgent']) - print("Bitcoin version:", bitcoinInfo['protocolVersion']) - print("Bitcoin onion peers:", bitcoinInfo['onionPeers']) + bitcoin = onionReport['bitcoinDetected'] + print("\t Bitcoin?", bitcoin) + if bitcoin: + baseScore = baseScore * 0.81 + print("\t Score goes down, now:", baseScore) + bitcoinInfo = onionReport['bitcoinServices']['bitcoin'] + print("\t Bitcoin user agent:", bitcoinInfo['userAgent']) + print("\t Bitcoin version:", bitcoinInfo['protocolVersion']) + print("\t Bitcoin onion peers:", bitcoinInfo['onionPeers']) -idReport = onionReport['identifierReport'] + idReport = onionReport['identifierReport'] -privateKey = idReport['privateKeyDetected'] -print("Private key found?", privateKey) -if privateKey: - baseScore = baseScore * 0.63 - print("Score goes down, now:", baseScore) + privateKey = idReport['privateKeyDetected'] + print("\t Private key found?", privateKey) + if privateKey: + baseScore = baseScore * 0.63 + print("\t Score goes down, now:", baseScore) -apacheStatus = idReport['foundApacheModStatus'] -print("Apache status found?", apacheStatus) -if apacheStatus: - baseScore = baseScore * 0.87 - print("Score goes down, now:", baseScore) - -ipAddress = idReport['ipAddresses'] -print("IP address leakage?", ipAddress) -if ipAddress: - baseScore = baseScore * 0.55 - print("Score goes down, now:", baseScore) - -emailAddress = idReport['emailAddresses'] -print("Email address found?", emailAddress) -if emailAddress: - baseScore = baseScore * 0.959 - print("Score goes down, now:", baseScore) - -analyticsId = idReport['analyticsIDs'] -print("Analytics tags?", analyticsId) -if analyticsId: - baseScore = baseScore * 0.6 - print("Score goes down, now:", baseScore) - -risks = onionReport['simpleReport']['risks'] -print("OnionScan detected risks:\n") -for r in risks: - t = r['title'] - print("\tName:", t) - s = r['severity'] - print("\tSeverity:", s) - if s == "info": - baseScore = baseScore * 0.999 - print("\tScore goes down, now:", baseScore) - if s == "low": - baseScore = baseScore * 0.959 - print("\tScore goes down, now:", baseScore) - if s == "medium": - baseScore = baseScore * 0.939 - print("\tScore goes down, now:", baseScore) - if s == "high": + apacheStatus = idReport['foundApacheModStatus'] + print("\t Apache status found?", apacheStatus) + if apacheStatus: baseScore = baseScore * 0.87 - print("\tScore goes down, now:", baseScore) - if s == "critical": - baseScore = baseScore * 0.77 - print("\tScore goes down, now:", baseScore) - print("") + print("\t Score goes down, now:", baseScore) + + ipAddress = idReport['ipAddresses'] + print("\t IP address leakage?", ipAddress) + if ipAddress: + baseScore = baseScore * 0.55 + print("\t Score goes down, now:", baseScore) + + emailAddress = idReport['emailAddresses'] + print("\t Email address found?", emailAddress) + if emailAddress: + baseScore = baseScore * 0.959 + print("\t Score goes down, now:", baseScore) + + analyticsId = idReport['analyticsIDs'] + print("\t Analytics tags?", analyticsId) + if analyticsId: + baseScore = baseScore * 0.6 + print("\t Score goes down, now:", baseScore) + + risks = onionReport['simpleReport']['risks'] + if not risks: + print("\t No risk detected.") + print("") + else: + print("\t OnionScan detected risks:\n") + for r in risks: + t = r['title'] + print("\t Name:", t) + s = r['severity'] + print("\t Severity:", s) + if s == "info": + baseScore = baseScore * 0.999 + print("\t Score goes down, now:", baseScore) + if s == "low": + baseScore = baseScore * 0.959 + print("\t Score goes down, now:", baseScore) + if s == "medium": + baseScore = baseScore * 0.939 + print("\t Score goes down, now:", baseScore) + if s == "high": + baseScore = baseScore * 0.87 + print("\t Score goes down, now:", baseScore) + if s == "critical": + baseScore = baseScore * 0.77 + print("\t Score goes down, now:", baseScore) + print("") + +if len(httpHeaders) == 0: + print("HTTP Headers not found, skipping...") + httpFlag = 0 + +if httpFlag == 1: + httpHeaders = json.loads(httpHeaders) + print("HTTP headers analysis:") + for badHeader in badHeaders: + if badHeader in httpHeaders: + baseScore = baseScore * 0.993 + print("\t Found", badHeader, "in HTTP headers.") + print("\t Score goes down, now:", baseScore) + + for secureHeader in securityHeaders: + if secureHeader in httpHeaders: + if securityHeaders[secureHeader] != httpHeaders[secureHeader]: + baseScore = baseScore * 0.987 + print("\t", secureHeader, "is present, but have diffrent value than expected.") + print("\t Present value:", httpHeaders[secureHeader]) + print("\t Expected value:", securityHeaders[secureHeader]) + print("\t Score goes down, now:", baseScore) + else: + print("\t", secureHeader, "is present and set correctly.") + else: + baseScore = baseScore * 0.983 + print("\t",secureHeader, "not found.") + print("\t Score goes down, now:", baseScore) + + if "Expect-CT" in httpHeaders: + baseScore = baseScore * 0.983 + print("\t This site is using Expect-CT header, it is recommended to not use it.") + print("\t Check https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Expect-CT for details.") + print("\t Score goes down, now:", baseScore) + + if "Access-Control-Allow-Origin" in httpHeaders: + baseScore = baseScore * 0.989 + print("\t This site is using Access-Control-Allow-Origin header, which allows to relax SOP.") + print("\t Score goes down, now:", baseScore) + + +if onionFlag or httpFlag: + scoring(baseScore) -scoring(baseScore) print("Analysis ended.") \ No newline at end of file diff --git a/node_modules/.bin/curl-headers-to-json b/node_modules/.bin/curl-headers-to-json new file mode 120000 index 0000000..a825016 --- /dev/null +++ b/node_modules/.bin/curl-headers-to-json @@ -0,0 +1 @@ +../curl-headers-to-json/index.js \ No newline at end of file diff --git a/node_modules/.package-lock.json b/node_modules/.package-lock.json new file mode 100644 index 0000000..38e7954 --- /dev/null +++ b/node_modules/.package-lock.json @@ -0,0 +1,15 @@ +{ + "name": "shallot", + "lockfileVersion": 3, + "requires": true, + "packages": { + "node_modules/curl-headers-to-json": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/curl-headers-to-json/-/curl-headers-to-json-1.0.1.tgz", + "integrity": "sha512-PNbfCuu4UdsTmSAHwN4R0m34KzvOvDjP3mJ6XNfHzOdeTskUb/eaVV110y/SnT6h6HDA7fisCNgQvRP/vH9obQ==", + "bin": { + "curl-headers-to-json": "index.js" + } + } + } +} diff --git a/node_modules/curl-headers-to-json/README.md b/node_modules/curl-headers-to-json/README.md new file mode 100644 index 0000000..c9a3844 --- /dev/null +++ b/node_modules/curl-headers-to-json/README.md @@ -0,0 +1,12 @@ +# curl-headers-to-json + +A tool to format the output of cURL headers into JSON. + +Example: +``` +curl -sSL -D - https://api.github.com -o /dev/null | npx curl-headers-to-json | jq +``` +or if you grabbed the output already: +``` +npx curl-headers-to-json headers.txt | jq +``` \ No newline at end of file diff --git a/node_modules/curl-headers-to-json/index.js b/node_modules/curl-headers-to-json/index.js new file mode 100755 index 0000000..a40f52d --- /dev/null +++ b/node_modules/curl-headers-to-json/index.js @@ -0,0 +1,56 @@ +#!/usr/bin/env node + +const fs = require('fs'); +const util = require('util'); +const readFile = util.promisify(fs.readFile); +const args = process.argv.slice(2); +const shellParam = args[0]; + +const isTTY = process.stdin.isTTY; +const stdin = process.stdin; +const stdout = process.stdout; + +if (isTTY && args.length === 0) { + console.log('Usage: '); +} else if (isTTY && args.length !== 0) { + handleShellArguments(); +} else { + handleStdin(); +} + +function handleStdin() { + let headersData = ''; + + stdin.setEncoding('utf8'); + stdin.on('readable', () => { + const chuck = stdin.read(); + if(chuck !== null){ + headersData += chuck; + } + }); + stdin.on('end', () => process.stdout.write(JSON.stringify(format(headersData)))); +} + + +function format(data) { + const lines = data.split(/\r?\n|\r/g); + const result = {}; + lines.shift(); + for(let line of lines) { + if(line.trim() === '') { + continue; + } + const [key, value] = line.split(': '); + result[key] = value.replace(/\\r/g, ''); + } + return result; +} + +async function handleShellArguments(){ + try { + const input = await readFile(shellParam, { encoding: 'utf8' }); + process.stdout.write(JSON.stringify(format(input))) + } catch (e) { + console.log(`An error ocurred while trying to open the file: ${e.message}`); + } +} \ No newline at end of file diff --git a/node_modules/curl-headers-to-json/package.json b/node_modules/curl-headers-to-json/package.json new file mode 100644 index 0000000..107c206 --- /dev/null +++ b/node_modules/curl-headers-to-json/package.json @@ -0,0 +1,18 @@ +{ + "name": "curl-headers-to-json", + "bin": { + "curl-headers-to-json": "./index.js" + }, + "version": "1.0.1", + "description": "Formats cURL headers into JSON format", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 0" + }, + "keywords": [ + "curl", + "json" + ], + "author": "Alejandro Oviedo ", + "license": "MIT" +} diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..6d7c8eb --- /dev/null +++ b/package-lock.json @@ -0,0 +1,20 @@ +{ + "name": "shallot", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "dependencies": { + "curl-headers-to-json": "^1.0.1" + } + }, + "node_modules/curl-headers-to-json": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/curl-headers-to-json/-/curl-headers-to-json-1.0.1.tgz", + "integrity": "sha512-PNbfCuu4UdsTmSAHwN4R0m34KzvOvDjP3mJ6XNfHzOdeTskUb/eaVV110y/SnT6h6HDA7fisCNgQvRP/vH9obQ==", + "bin": { + "curl-headers-to-json": "index.js" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..2ab03e8 --- /dev/null +++ b/package.json @@ -0,0 +1,5 @@ +{ + "dependencies": { + "curl-headers-to-json": "^1.0.1" + } +} diff --git a/shallot.sh b/shallot.sh index f876a81..e6861cb 100755 --- a/shallot.sh +++ b/shallot.sh @@ -1,16 +1,16 @@ #!/bin/bash -set -xe +set -e # Setting env SCAN_DATE=`date "+%F-%H-%M"` export SHALLOT_DIR="/tmp/shallot-$SCAN_DATE" mkdir -p $SHALLOT_DIR -echo "Shallot scritp v0.0.2" +echo "Shallot scritp v0.1.1" if [[ $# -eq 0 ]] ; then - echo "[ERROR] No arguments was passed, exiting..." + echo "[ERRO] No arguments was passed, exiting..." exit 1 fi @@ -18,49 +18,60 @@ if [[ $# -ge 2 ]] ; then echo "[WARRNING] Too much argument was passed, this script uses only first one." fi +[ $(type -P "npx") ] || echo "[ERRO] npx is not in the path, install npm first!" + + export ONIONSITE=$1 -echo "[INFO] Checking if Tor Browser proxy is running..." +echo "Checking if Tor Browser proxy is running..." -NETSTAT_OUTPUT=`netstat -tlnp 2> /dev/null` +#NETSTAT_OUTPUT=`netstat -tlnp 2> /dev/null` #IF_TOR_RUNNING=`echo $NETSTAT_OUTPUT | grep -Ezqv "/tor" && echo 0 || echo 1` IF_TOR_RUNNING=`ps -eaf | grep -i tor |sed '/^$/d' | wc -l` if [[ "$IF_TOR_RUNNING" > 1 ]] ; then echo "[INFO] Tor is running!" else - echo "[ERROR] Tor is not running, start Tor Browser and connect to Tor, then restart this scritp" + echo "[ERRO] Tor is not running, start Tor Browser and connect to Tor, then restart this scritp" exit 1; fi -echo "Checking Onion Service, address: $1" +IS_ADDRESS_ONION=`echo $1 | grep -Ei ".onion$" | wc -c` +if [[ $IS_ADDRESS_ONION > 0 ]] ; then + echo "Checking Onion Service, address: $1" +else + echo "[ERRO] Looks like $1 is not an onion site, exiting..." + exit 1 +fi echo "" # OnionScan -echo "Runnning OnionScan aginst address, this will take a while..." +echo "[INFO] Runnning OnionScan aginst address, this will take a while..." export ONIONSCAN_REPORT=$(onionscan --jsonReport --torProxyAddress "127.0.0.1:9150" $1 2>$SHALLOT_DIR/onionscan_error.log | jq) echo $ONIONSCAN_REPORT > $SHALLOT_DIR/onionscan_result.txt if [ $? ] ; then - echo "OnionScan done! Saved in $SHALLOT_DIR/onionscan_result.txt" + echo "[INFO] OnionScan done! Saved in $SHALLOT_DIR/onionscan_result.txt" else - echo "[ERROR] Error occured, exiting, check $SHALLOT_DIR/onionscan_error.log for details." - exit 1 + echo "[ERRO] Error occured, exiting, check $SHALLOT_DIR/onionscan_error.log for details." fi # HTTP Headers echo "Scanning HTTP headers, wait..." -export HTTP_HEADERS=$(proxychains -q -f /etc/proxychains4.conf /usr/bin/curl -I -s $1 | tail -n +3 | sed 's/\r//g' | head -n -1 | jq -R 'split(":")|{(.[0]) : .[1]}' 2>$SHALLOT_DIR/http_headers_error.log) +#export HTTP_HEADERS=$(proxychains -q -f /etc/proxychains4.conf /usr/bin/curl -I -s $1 | tail -n +3 | sed 's/\r//g' | head -n -1 | jq -R 'split(":")|{(.[0]) : .[1]}' | sed 's/\\"//g' 2>$SHALLOT_DIR/http_headers_error.log) +export HTTP_HEADERS=$(proxychains -q -f /etc/proxychains4.conf /usr/bin/curl -LIs -D - $1 -o /dev/null | npx curl-headers-to-json | sed 's/\\"//g' | jq 2>$SHALLOT_DIR/http_headers_error.log) echo $HTTP_HEADERS > $SHALLOT_DIR/http_headers.txt if [ $? ] ; then - echo "HTTP headers done! Saved in $SHALLOT_DIR/http_headers.txt" + echo "[INFO] HTTP headers done! Saved in $SHALLOT_DIR/http_headers.txt" else - echo "[ERROR] Error occured, check $SHALLOT_DIR/http_headers_error.log" - exit 1 + echo "[ERRO] Error occured, check $SHALLOT_DIR/http_headers_error.log" fi # Report analysis python3 main.py + +echo "Works done, exiting." +exit 0 \ No newline at end of file