Ability to scan screen content for malicious links

Signed-off-by: Tavi <tavi@divested.dev>
This commit is contained in:
Tavi 2024-05-22 19:36:10 -04:00
parent cde1d8ff69
commit 02dd4f624e
No known key found for this signature in database
GPG key ID: E599F62ECBAEAF2E
14 changed files with 271 additions and 45 deletions

View file

@ -6,8 +6,8 @@ android {
applicationId "us.spotco.malwarescanner" applicationId "us.spotco.malwarescanner"
minSdkVersion 16 minSdkVersion 16
targetSdkVersion 32 targetSdkVersion 32
versionCode 310 versionCode 311
versionName "3.10" versionName "3.11"
resConfigs 'en', 'af', 'cs', 'de', 'el', 'es', 'fi', 'fr', 'gl', 'it', 'pl', 'pt', 'pt-rBR', 'ru', 'tr', 'zh-rCN' resConfigs 'en', 'af', 'cs', 'de', 'el', 'es', 'fi', 'fr', 'gl', 'it', 'pl', 'pt', 'pt-rBR', 'ru', 'tr', 'zh-rCN'
} }
buildTypes { buildTypes {

View file

@ -58,6 +58,19 @@
android:exported="false" android:exported="false"
android:label="Realtime Malware Scanner" /> android:label="Realtime Malware Scanner" />
<service
android:name=".LinkScannerService"
android:exported="false"
android:permission="android.permission.BIND_ACCESSIBILITY_SERVICE"
android:label="@string/accessibility_service_label">
<intent-filter>
<action android:name="android.accessibilityservice.AccessibilityService" />
</intent-filter>
<meta-data
android:name="android.accessibilityservice"
android:resource="@xml/accessibility_service_config" />
</service>
<receiver <receiver
android:name=".EventReceiver" android:name=".EventReceiver"
android:enabled="true" android:enabled="true"

View file

@ -51,6 +51,7 @@ class Database {
public static BloomFilter<String> signaturesMD5Extended = null; public static BloomFilter<String> signaturesMD5Extended = null;
public static BloomFilter<String> signaturesSHA1 = null; public static BloomFilter<String> signaturesSHA1 = null;
public static BloomFilter<String> signaturesSHA256 = null; public static BloomFilter<String> signaturesSHA256 = null;
public static BloomFilter<String> domains = null;
public static long signaturesCount = 0; public static long signaturesCount = 0;
public static boolean changedDownload = false; public static boolean changedDownload = false;
public static boolean changedConfig = false; public static boolean changedConfig = false;
@ -74,6 +75,11 @@ class Database {
&& signaturesSHA256 != null && signaturesSHA256.approximateElementCount() > 0; && signaturesSHA256 != null && signaturesSHA256.approximateElementCount() > 0;
} }
public static boolean isDomainDatabaseLoaded() {
return areDatabasesAvailable() && !isDatabaseLoading()
&& domains != null && domains.approximateElementCount() > 0;
}
public static boolean isDatabaseLoading() { public static boolean isDatabaseLoading() {
return !databaseFullyLoaded && databaseCurrentlyLoading; return !databaseFullyLoaded && databaseCurrentlyLoading;
} }
@ -111,6 +117,9 @@ class Database {
if (prefs.getBoolean("SIGNATURES_EXTENDED", false)) { if (prefs.getBoolean("SIGNATURES_EXTENDED", false)) {
signatureDatabases.add(new SignatureDatabase(baseURL, "hypatia-md5-extended-bloom.bin")); signatureDatabases.add(new SignatureDatabase(baseURL, "hypatia-md5-extended-bloom.bin"));
} }
if (prefs.getBoolean("DOMAINS", false)) {
signatureDatabases.add(new SignatureDatabase(baseURL, "hypatia-domains-bloom.bin"));
}
} }
public static void loadDatabase(Context context, boolean forceReload, ConcurrentLinkedQueue<SignatureDatabase> signatureDatabases) { public static void loadDatabase(Context context, boolean forceReload, ConcurrentLinkedQueue<SignatureDatabase> signatureDatabases) {
@ -121,6 +130,7 @@ class Database {
signaturesCount = 0; signaturesCount = 0;
changedConfig = false; changedConfig = false;
signaturesMD5Extended = null; signaturesMD5Extended = null;
domains = null;
File publicKey = new File(databasePath + "/gpg.key"); File publicKey = new File(databasePath + "/gpg.key");
GPGDetachedSignatureVerifier verifier = new GPGDetachedSignatureVerifier(Utils.getSigningKey(context)); GPGDetachedSignatureVerifier verifier = new GPGDetachedSignatureVerifier(Utils.getSigningKey(context));
for (SignatureDatabase database : signatureDatabases) { for (SignatureDatabase database : signatureDatabases) {
@ -137,25 +147,30 @@ class Database {
Log.d("Hypatia", "Processing md5"); Log.d("Hypatia", "Processing md5");
signaturesMD5 = BloomFilter.readFrom(databaseLoading, Funnels.stringFunnel(Charsets.US_ASCII)); signaturesMD5 = BloomFilter.readFrom(databaseLoading, Funnels.stringFunnel(Charsets.US_ASCII));
signaturesCount += signaturesMD5.approximateElementCount(); signaturesCount += signaturesMD5.approximateElementCount();
Log.d("Hypatia", "Loaded md5"); Log.d("Hypatia", "Loaded md5 with " + signaturesMD5.approximateElementCount() + " entries");
break; break;
case "hypatia-sha1-bloom.bin": case "hypatia-sha1-bloom.bin":
Log.d("Hypatia", "Processing sha1"); Log.d("Hypatia", "Processing sha1");
signaturesSHA1 = BloomFilter.readFrom(databaseLoading, Funnels.stringFunnel(Charsets.US_ASCII)); signaturesSHA1 = BloomFilter.readFrom(databaseLoading, Funnels.stringFunnel(Charsets.US_ASCII));
signaturesCount += signaturesSHA1.approximateElementCount(); signaturesCount += signaturesSHA1.approximateElementCount();
Log.d("Hypatia", "Loaded sha1"); Log.d("Hypatia", "Loaded sha1 with " + signaturesSHA1.approximateElementCount() + " entries");
break; break;
case "hypatia-sha256-bloom.bin": case "hypatia-sha256-bloom.bin":
Log.d("Hypatia", "Processing sha256"); Log.d("Hypatia", "Processing sha256");
signaturesSHA256 = BloomFilter.readFrom(databaseLoading, Funnels.stringFunnel(Charsets.US_ASCII)); signaturesSHA256 = BloomFilter.readFrom(databaseLoading, Funnels.stringFunnel(Charsets.US_ASCII));
signaturesCount += signaturesSHA256.approximateElementCount(); signaturesCount += signaturesSHA256.approximateElementCount();
Log.d("Hypatia", "Loaded sha256"); Log.d("Hypatia", "Loaded sha256 with " + signaturesSHA256.approximateElementCount() + " entries");
break; break;
case "hypatia-md5-extended-bloom.bin": case "hypatia-md5-extended-bloom.bin":
Log.d("Hypatia", "Processing md5 extended"); Log.d("Hypatia", "Processing md5 extended");
signaturesMD5Extended = BloomFilter.readFrom(databaseLoading, Funnels.stringFunnel(Charsets.US_ASCII)); signaturesMD5Extended = BloomFilter.readFrom(databaseLoading, Funnels.stringFunnel(Charsets.US_ASCII));
signaturesCount += signaturesMD5Extended.approximateElementCount(); signaturesCount += signaturesMD5Extended.approximateElementCount();
Log.d("Hypatia", "Loaded md5 extended"); Log.d("Hypatia", "Loaded md5 extended with " + signaturesMD5Extended.approximateElementCount() + " entries");
break;
case "hypatia-domains-bloom.bin":
Log.d("Hypatia", "Processing domains");
domains = BloomFilter.readFrom(databaseLoading, Funnels.stringFunnel(Charsets.US_ASCII));
Log.d("Hypatia", "Loaded domains with " + domains.approximateElementCount() + " entries");
break; break;
} }
databaseLoading.close(); databaseLoading.close();

View file

@ -0,0 +1,116 @@
package us.spotco.malwarescanner;
import android.accessibilityservice.AccessibilityService;
import android.app.Notification;
import android.app.NotificationChannel;
import android.app.NotificationManager;
import android.content.Context;
import android.os.Build;
import android.view.accessibility.AccessibilityEvent;
import android.view.accessibility.AccessibilityNodeInfo;
import java.util.Random;
import java.util.concurrent.ConcurrentSkipListSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class LinkScannerService extends AccessibilityService {
private static final String hostnameRegex = "^((?!-)[A-Za-z0-9-]{1,63}(?<!-)\\.)+[A-Za-z]{2,6}$"; //Credit: http://www.mkyong.com/regular-expressions/domain-name-regular-expression-example/
private static final Pattern hostnamePattern = Pattern.compile(hostnameRegex);
private static final ConcurrentSkipListSet<Integer> scannedObjects = new ConcurrentSkipListSet<>();
private static final ConcurrentSkipListSet<String> scannedDomains = new ConcurrentSkipListSet<>();
private NotificationManager notificationManager = null;
@Override
public void onServiceConnected() {
notificationManager = (NotificationManager) getSystemService(Context.NOTIFICATION_SERVICE);
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
NotificationChannel detectionChannel = new NotificationChannel("DETECTION", getString(R.string.lblNotificationMalwareDetectionTitle), NotificationManager.IMPORTANCE_HIGH);
detectionChannel.setDescription(getString(R.string.lblNotificationMalwareDetectionDescription));
notificationManager.createNotificationChannel(detectionChannel);
}
}
@Override
public void onAccessibilityEvent(AccessibilityEvent event) {
if (Database.isDomainDatabaseLoaded()) {
scanViews(event.getSource());
}
}
@Override
public void onInterrupt() {
}
private void scanViews(AccessibilityNodeInfo mNodeInfo) {
try {
//May already be gone
if (mNodeInfo == null) {
return;
}
//Don't scan if we already did
int hash = mNodeInfo.toString().hashCode();
if (scannedObjects.contains(hash)) {
return;
} else {
scannedObjects.add(hash);
}
//Get and check the text
String text = (String) mNodeInfo.getText();
if (text != null && text.contains(".")) {
scanText(text.toLowerCase());
}
//Finish if no more children
if (mNodeInfo.getChildCount() < 1) {
return;
}
//Recurse into the children otherwise
for (int i = 0; i < mNodeInfo.getChildCount(); i++) {
scanViews(mNodeInfo.getChild(i));
}
} catch (Exception ignored) {
}
}
private void scanText(String haystack) {
Matcher matcher = hostnamePattern.matcher(haystack);
while (matcher.find()) {
if (!scannedDomains.contains(matcher.group())) {
scannedDomains.add(matcher.group());
if (Database.domains.mightContain(matcher.group())) {
sendNotification(matcher.group());
}
}
}
if (haystack.contains("/")) {
for (String split : haystack.split("/")) {
scanText(split);
}
}
}
private void sendNotification(String link) {
link = link.replaceAll("\\.", "[.]");
Notification.Builder mBuilder =
new Notification.Builder(this)
.setSmallIcon(R.drawable.ic_notification)
.setContentTitle(getText(R.string.lblNotificationLinkDetection))
.setContentText("Domain: >>>" + link + "<<<")
.setStyle(new Notification.BigTextStyle().bigText("Domain: >>>" + link + "<<<"))
.setPriority(Notification.PRIORITY_MAX)
.setDefaults(Notification.DEFAULT_VIBRATE);
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
mBuilder.setVisibility(Notification.VISIBILITY_SECRET);
}
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
mBuilder.setChannelId("DETECTION");
}
notificationManager.notify(new Random().nextInt(), mBuilder.build());
}
}

View file

@ -83,7 +83,6 @@ public class MainActivity extends Activity {
logView.append(getString(R.string.app_copyright) + "\n"); logView.append(getString(R.string.app_copyright) + "\n");
logView.append(getString(R.string.app_license) + "\n"); logView.append(getString(R.string.app_license) + "\n");
logView.append(getString(R.string.app_version, buildVersionName) + "\n"); logView.append(getString(R.string.app_version, buildVersionName) + "\n");
logView.append(getString(R.string.app_db_type_clamav) + "\n\n");
malwareScanner = new MalwareScanner(this, this, true); malwareScanner = new MalwareScanner(this, this, true);
@ -100,6 +99,8 @@ public class MainActivity extends Activity {
this.menu = menu; this.menu = menu;
menu.findItem(R.id.toggleRealtime).setChecked(Utils.isServiceRunning(MalwareScannerService.class, this)); menu.findItem(R.id.toggleRealtime).setChecked(Utils.isServiceRunning(MalwareScannerService.class, this));
menu.findItem(R.id.toggleOnionRouting).setChecked(prefs.getBoolean("ONION_ROUTING", false)); menu.findItem(R.id.toggleOnionRouting).setChecked(prefs.getBoolean("ONION_ROUTING", false));
menu.findItem(R.id.toggleExtended).setChecked(prefs.getBoolean("SIGNATURES_EXTENDED", false));
menu.findItem(R.id.toggleLinkScanner).setChecked(prefs.getBoolean("DOMAINS", false));
updateScanButton(false); updateScanButton(false);
return true; return true;
} }
@ -240,12 +241,14 @@ public class MainActivity extends Activity {
.setIcon(android.R.drawable.ic_menu_compass) .setIcon(android.R.drawable.ic_menu_compass)
.setPositiveButton(getString(android.R.string.yes), (dialog, which) -> { .setPositiveButton(getString(android.R.string.yes), (dialog, which) -> {
prefs.edit().putBoolean("SIGNATURES_EXTENDED", true).apply(); prefs.edit().putBoolean("SIGNATURES_EXTENDED", true).apply();
item.setChecked(true);
if (!prevExtended) { if (!prevExtended) {
Database.changedConfig = true; Database.changedConfig = true;
} }
}) })
.setNegativeButton(getString(android.R.string.no), (dialog, which) -> { .setNegativeButton(getString(android.R.string.no), (dialog, which) -> {
prefs.edit().putBoolean("SIGNATURES_EXTENDED", false).apply(); prefs.edit().putBoolean("SIGNATURES_EXTENDED", false).apply();
item.setChecked(false);
if (prevExtended) { if (prevExtended) {
Database.changedConfig = true; Database.changedConfig = true;
} }
@ -253,6 +256,35 @@ public class MainActivity extends Activity {
}).show(); }).show();
} }
break; break;
case R.id.toggleLinkScanner:
if (Database.hasDownloadsRunning()) {
logView.append(getString(R.string.lblUpdateRunning) + "\n");
} else if (Database.isDatabaseLoading()) {
logView.append(getString(R.string.lblDatabaseLoading) + "\n");
} else {
boolean prevDomains = prefs.getBoolean("DOMAINS", false);
new AlertDialog.Builder(this)
.setTitle(R.string.confirm_link_scanner_title)
.setMessage(getString(R.string.confirm_link_scanner_summary))
.setIcon(android.R.drawable.ic_menu_compass)
.setPositiveButton(getString(android.R.string.yes), (dialog, which) -> {
prefs.edit().putBoolean("DOMAINS", true).apply();
item.setChecked(true);
if (!prevDomains) {
Database.changedConfig = true;
}
startActivityForResult(new Intent(android.provider.Settings.ACTION_ACCESSIBILITY_SETTINGS), 0);
})
.setNegativeButton(getString(android.R.string.no), (dialog, which) -> {
prefs.edit().putBoolean("DOMAINS", false).apply();
item.setChecked(false);
if (prevDomains) {
Database.changedConfig = true;
}
dialog.cancel();
}).show();
}
break;
case R.id.toggleRealtime: case R.id.toggleRealtime:
if (malwareScanner.running) { if (malwareScanner.running) {
logView.append(getString(R.string.lblScanRunning) + "\n"); logView.append(getString(R.string.lblScanRunning) + "\n");

View file

@ -28,7 +28,6 @@ import android.os.AsyncTask;
import android.os.Build; import android.os.Build;
import android.os.Environment; import android.os.Environment;
import android.os.SystemClock; import android.os.SystemClock;
import android.util.Log;
import android.widget.TextView; import android.widget.TextView;
import com.google.common.hash.BloomFilter; import com.google.common.hash.BloomFilter;

View file

@ -22,7 +22,12 @@
android:title="@string/lblSigningKey" /> android:title="@string/lblSigningKey" />
<item <item
android:id="@+id/toggleExtended" android:id="@+id/toggleExtended"
android:title="@string/lblExtendedDatabaseToggle" /> android:title="@string/lblExtendedDatabaseToggle"
android:checkable="true" />
<item
android:id="@+id/toggleLinkScanner"
android:title="@string/lblLinkScannerToggle"
android:checkable="true" />
<item <item
android:id="@+id/toggleRealtime" android:id="@+id/toggleRealtime"
android:title="@string/lblRealtimeScannerToggle" android:title="@string/lblRealtimeScannerToggle"

View file

@ -48,7 +48,7 @@
<string name="scan_control">Contrôle de l\'analyse</string> <string name="scan_control">Contrôle de l\'analyse</string>
<string name="lblScanRunning">Action ignorée, une analyse est en cours !</string> <string name="lblScanRunning">Action ignorée, une analyse est en cours !</string>
<string name="lblSigningKey">Signature utilisée pour signer la base de données</string> <string name="lblSigningKey">Signature utilisée pour signer la base de données</string>
<string name="lblNoNetwork">Aucun réseau connecté !</string> <string name="lblNoNetwork">Aucun réseau connecté !</string>
<string name="self_test_result_success">Auto-test réussi.</string> <string name="self_test_result_success">Auto-test réussi.</string>
<string name="self_test_result_failure">Échec de l\'auto-test !</string> <string name="self_test_result_failure">Échec de l\'auto-test !</string>
<string name="lblDatabaseLoading">Action sautée, la base de données est en cours de chargement ! </string> <string name="lblDatabaseLoading">Action sautée, la base de données est en cours de chargement ! </string>

View file

@ -48,7 +48,7 @@
<string name="scan_control">Контроль сканирования</string> <string name="scan_control">Контроль сканирования</string>
<string name="lblScanRunning">Пропуск действия, выполняется сканирование!</string> <string name="lblScanRunning">Пропуск действия, выполняется сканирование!</string>
<string name="lblSigningKey">Ключ подписи базы данных</string> <string name="lblSigningKey">Ключ подписи базы данных</string>
<string name="lblNoNetwork">Нет подключения к сети!</string> <string name="lblNoNetwork">Нет подключения к сети!</string>
<string name="self_test_result_success">Самотестирование прошло успешно.</string> <string name="self_test_result_success">Самотестирование прошло успешно.</string>
<string name="self_test_result_failure">Самотестирование завершено неудачно!</string> <string name="self_test_result_failure">Самотестирование завершено неудачно!</string>
<string name="lblDatabaseLoading">Пропуск действия, загружается база данных!</string> <string name="lblDatabaseLoading">Пропуск действия, загружается база данных!</string>

View file

@ -49,7 +49,7 @@
<string name="scan_control">Tarama Kontrolü</string> <string name="scan_control">Tarama Kontrolü</string>
<string name="lblScanRunning">Faaliyet es geçiliyor, bir tarama devam etmekte!</string> <string name="lblScanRunning">Faaliyet es geçiliyor, bir tarama devam etmekte!</string>
<string name="lblSigningKey">Veri tabanı imzalama anahtarı</string> <string name="lblSigningKey">Veri tabanı imzalama anahtarı</string>
<string name="lblNoNetwork">Hiçbir şebeke bağlı değil!</string> <string name="lblNoNetwork">Hiçbir şebeke bağlı değil!</string>
<string name="self_test_result_success">Kendi kendini deneme başarılı.</string> <string name="self_test_result_success">Kendi kendini deneme başarılı.</string>
<string name="self_test_result_failure">Kendi kendini deneme başarısız oldu!</string> <string name="self_test_result_failure">Kendi kendini deneme başarısız oldu!</string>
<string name="lblDatabaseLoading">Faaliyet atlanıyor, veri tabanı yüklenmekte!</string> <string name="lblDatabaseLoading">Faaliyet atlanıyor, veri tabanı yüklenmekte!</string>

View file

@ -71,11 +71,17 @@
<string name="lblSelfTest">Write self test files</string> <string name="lblSelfTest">Write self test files</string>
<string name="lblExtendedDatabaseToggle">Extended database</string> <string name="lblExtendedDatabaseToggle">Extended database</string>
<string name="confirm_extended_title">Enable extended database?</string> <string name="confirm_extended_title">Enable extended database?</string>
<string name="confirm_extended_summary">[EXPERIMENTAL]\nThis will enable detection of an additional ~40 million signatures.\nThis requires a 125MB download, will slow down startup by over two minutes, will increase app RAM usage, and will increase the false positive rate.\nThis database only updates quarterly.</string> <string name="confirm_extended_summary">[EXPERIMENTAL]\nThis will enable detection of an additional ~40 million signatures.\nThis requires a 200MB download, will slow down startup by over two minutes, will increase app RAM usage, and will increase the false positive rate.\nThis database only updates quarterly.</string>
<string name="confirm_update_title">Confirm download</string> <string name="confirm_update_title">Confirm download</string>
<string name="confirm_update_summary">You appear to be on a metered connection. Are you sure you want to update the databases?\nIt may download up to %s megabytes of data.</string> <string name="confirm_update_summary">You appear to be on a metered connection. Are you sure you want to update the databases?\nIt may download up to %s megabytes of data.</string>
<string name="lblUpdateRunning">Skipping action, an update is running!</string> <string name="lblUpdateRunning">Skipping action, an update is running!</string>
<string name="lblWroteTestFiles">Wrote test files!</string> <string name="lblWroteTestFiles">Wrote test files!</string>
<string name="lblResetPrimary">Primary Servers</string> <string name="lblResetPrimary">Primary Servers</string>
<string name="lblResetCloudflare">Cloudflare Mirror</string> <string name="lblResetCloudflare">Cloudflare Mirror</string>
<string name="accessibility_service_label">Hypatia Link Scanner</string>
<string name="accessibility_service_description">Extracts domains from all screen text content and checks them against a blocklist</string>
<string name="lblNotificationLinkDetection">Malicious Link Detected:</string>
<string name="lblLinkScannerToggle">Link Scanner</string>
<string name="confirm_link_scanner_title">Enable link scanner?</string>
<string name="confirm_link_scanner_summary">[EXPERIMENTAL]\nThis will download an additional database of domains.\nAny domains found in screen content will be checked against the database.\nThis require granting accessibility service permission manually.</string>
</resources> </resources>

View file

@ -0,0 +1,8 @@
<?xml version="1.0" encoding="utf-8"?>
<accessibility-service xmlns:android="http://schemas.android.com/apk/res/android"
android:description="@string/accessibility_service_description"
android:accessibilityEventTypes="typeAllMask"
android:accessibilityFlags="flagDefault"
android:accessibilityFeedbackType="feedbackGeneric"
android:notificationTimeout="1"
android:canRetrieveWindowContent="true" />

View file

@ -0,0 +1 @@
* Ability to scan screen content for malicious links

View file

@ -41,10 +41,13 @@ public class Main {
private static BloomFilter<String> signaturesMD5 = null; private static BloomFilter<String> signaturesMD5 = null;
private static BloomFilter<String> signaturesSHA1 = null; private static BloomFilter<String> signaturesSHA1 = null;
private static BloomFilter<String> signaturesSHA256 = null; private static BloomFilter<String> signaturesSHA256 = null;
private static BloomFilter<String> domains = null;
private static int amtLinesValid = 0; private static int amtLinesValid = 0;
private static int amtLinesInvalid = 0; private static int amtLinesInvalid = 0;
private static int amtDomains = 0;
private static int amtSignaturesReadMD5 = 0; private static int amtSignaturesReadMD5 = 0;
private static int amtSignaturesReadSHA1 = 0; private static int amtSignaturesReadSHA1 = 0;
private static int amtSignaturesReadSHA256 = 0; private static int amtSignaturesReadSHA256 = 0;
@ -66,24 +69,26 @@ public class Main {
extendedMode = args[0].contains("-extended"); extendedMode = args[0].contains("-extended");
//isFileInNsrl("B61905308B336AD268A782790B661616"); //isFileInNsrl("B61905308B336AD268A782790B661616");
int amtMaxMD5 = 7200000; //7.2m int amtMaxMD5 = 7200000; //7.2m
if(extendedMode) { if (extendedMode) {
amtMaxMD5 = 52000000; //52m amtMaxMD5 = 52000000; //52m
} }
int amtMaxSHA1 = 50000; //50k int amtMaxSHA1 = 50000; //50k
int amtMaxSHA256 = 2000000; //2m int amtMaxSHA256 = 2000000; //2m
int amtMaxDomains = 1800000; //1.8m
signaturesMD5 = BloomFilter.create(Funnels.stringFunnel(Charsets.US_ASCII), amtMaxMD5, 0.00001); signaturesMD5 = BloomFilter.create(Funnels.stringFunnel(Charsets.US_ASCII), amtMaxMD5, 0.00001);
signaturesSHA1 = BloomFilter.create(Funnels.stringFunnel(Charsets.US_ASCII), amtMaxSHA1, 0.00001); signaturesSHA1 = BloomFilter.create(Funnels.stringFunnel(Charsets.US_ASCII), amtMaxSHA1, 0.00001);
signaturesSHA256 = BloomFilter.create(Funnels.stringFunnel(Charsets.US_ASCII), amtMaxSHA256, 0.00001); signaturesSHA256 = BloomFilter.create(Funnels.stringFunnel(Charsets.US_ASCII), amtMaxSHA256, 0.00001);
domains = BloomFilter.create(Funnels.stringFunnel(Charsets.US_ASCII), amtMaxDomains, 0.00001);
File existingDatabase = new File(args[0] + "../production/hypatia-md5-bloom.bin"); File existingDatabase = new File(args[0] + "../production/hypatia-md5-bloom.bin");
if(extendedMode && existingDatabase.exists()) { if (extendedMode && existingDatabase.exists()) {
try { try {
System.out.println("Loading existing hypatia-md5-bloom.bin database"); System.out.println("Loading existing hypatia-md5-bloom.bin database");
FileInputStream databaseLoading = new FileInputStream(existingDatabase); FileInputStream databaseLoading = new FileInputStream(existingDatabase);
signaturesMD5Dedupe = BloomFilter.readFrom(databaseLoading, Funnels.stringFunnel(Charsets.US_ASCII)); signaturesMD5Dedupe = BloomFilter.readFrom(databaseLoading, Funnels.stringFunnel(Charsets.US_ASCII));
System.out.println("\tLoaded " + signaturesMD5Dedupe.approximateElementCount() + " entries"); System.out.println("\tLoaded " + signaturesMD5Dedupe.approximateElementCount() + " entries");
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
} }
System.out.println("Processing exclusions:"); System.out.println("Processing exclusions:");
@ -93,12 +98,12 @@ public class Main {
try { try {
System.out.println("\t" + exclusionDatabase.getName()); System.out.println("\t" + exclusionDatabase.getName());
Scanner s = new Scanner(exclusionDatabase); Scanner s = new Scanner(exclusionDatabase);
while(s.hasNextLine()) { while (s.hasNextLine()) {
String line = s.nextLine().trim().toLowerCase(); String line = s.nextLine().trim().toLowerCase();
if(line.contains(":")) { if (line.contains(":")) {
line = line.split(":")[0]; line = line.split(":")[0];
} }
if(!line.startsWith("#") && isHexadecimal(line) && (line.length() == 32 || line.length() == 40 || line.length() == 64)) { if (!line.startsWith("#") && isHexadecimal(line) && (line.length() == 32 || line.length() == 40 || line.length() == 64)) {
arrExclusions.add(line); arrExclusions.add(line);
//System.out.println("\t\tAdded: " + line); //System.out.println("\t\tAdded: " + line);
} }
@ -108,17 +113,37 @@ public class Main {
e.printStackTrace(); e.printStackTrace();
} }
} }
System.out.println("Loaded " + arrExclusions.size() + " excluded hashes"); System.out.println("\tLoaded " + arrExclusions.size() + " excluded hashes");
if (args.length == 2) {
System.out.println("Processing domains:");
File domainDatabase = new File(args[1]);
if (domainDatabase.exists()) {
try {
Scanner s = new Scanner(domainDatabase);
while (s.hasNextLine()) {
String line = s.nextLine().trim().toLowerCase();
if (!line.startsWith("#")) {
domains.put(line);
}
}
s.close();
} catch (Exception e) {
e.printStackTrace();
}
}
System.out.println("\tLoaded " + domains.approximateElementCount() + " domains");
}
System.out.println("Processing signatures:"); System.out.println("Processing signatures:");
File[] databases = new File(args[0]).listFiles(); File[] databases = new File(args[0]).listFiles();
File extras = new File(args[0] + "../extras/"); File extras = new File(args[0] + "../extras/");
if(extras.exists() && !extendedMode) { if (extras.exists() && !extendedMode) {
databases = Stream.concat(Arrays.stream(databases), Arrays.stream(extras.listFiles())).toArray(File[]::new); databases = Stream.concat(Arrays.stream(databases), Arrays.stream(extras.listFiles())).toArray(File[]::new);
} }
Arrays.sort(databases); Arrays.sort(databases);
for (File databaseLocation : databases) { for (File databaseLocation : databases) {
if(databaseLocation.isFile()) { if (databaseLocation.isFile()) {
System.out.println("\t" + databaseLocation.getName()); System.out.println("\t" + databaseLocation.getName());
amtPreviousSignaturesMD5 = amtSignaturesAddedMD5; amtPreviousSignaturesMD5 = amtSignaturesAddedMD5;
amtPreviousSignaturesSHA1 = amtSignaturesAddedSHA1; amtPreviousSignaturesSHA1 = amtSignaturesAddedSHA1;
@ -172,29 +197,31 @@ public class Main {
System.out.println("Read count: md5: " + amtSignaturesReadMD5 + ", sha1: " + amtSignaturesReadSHA1 + ", sha256: " + amtSignaturesReadSHA256); System.out.println("Read count: md5: " + amtSignaturesReadMD5 + ", sha1: " + amtSignaturesReadSHA1 + ", sha256: " + amtSignaturesReadSHA256);
System.out.println("Added count: md5: " + amtSignaturesAddedMD5 + ", sha1: " + amtSignaturesAddedSHA1 + ", sha256: " + amtSignaturesAddedSHA256); System.out.println("Added count: md5: " + amtSignaturesAddedMD5 + ", sha1: " + amtSignaturesAddedSHA1 + ", sha256: " + amtSignaturesAddedSHA256);
System.out.println("Approximate count: md5: " + signaturesMD5.approximateElementCount() + ", sha1: " + signaturesSHA1.approximateElementCount() + ", sha256: " + signaturesSHA256.approximateElementCount()); System.out.println("Approximate count: md5: " + signaturesMD5.approximateElementCount() + ", sha1: " + signaturesSHA1.approximateElementCount() + ", sha256: " + signaturesSHA256.approximateElementCount());
if(extendedMode) { System.out.println("Deduped count: md5: " + amtSignaturesDedupedMD5); } if (extendedMode) {
System.out.println("Deduped count: md5: " + amtSignaturesDedupedMD5);
}
System.out.println("Max amount: md5: " + amtMaxMD5 + ", sha1: " + amtMaxSHA1 + ", sha256: " + amtMaxSHA256); System.out.println("Max amount: md5: " + amtMaxMD5 + ", sha1: " + amtMaxSHA1 + ", sha256: " + amtMaxSHA256);
System.out.println("Fill amount: md5: " + ((100F/amtMaxMD5) * amtSignaturesAddedMD5) + "%, sha1: " + ((100F/amtMaxSHA1) * amtSignaturesAddedSHA1) + "%, sha256: " + ((100F/amtMaxSHA256) * amtSignaturesAddedSHA256) + "%"); System.out.println("Fill amount: md5: " + ((100F / amtMaxMD5) * amtSignaturesAddedMD5) + "%, sha1: " + ((100F / amtMaxSHA1) * amtSignaturesAddedSHA1) + "%, sha256: " + ((100F / amtMaxSHA256) * amtSignaturesAddedSHA256) + "%");
System.out.println("App reported count: " + (signaturesMD5.approximateElementCount() + signaturesSHA1.approximateElementCount() + signaturesSHA256.approximateElementCount())); System.out.println("App reported count: " + (signaturesMD5.approximateElementCount() + signaturesSHA1.approximateElementCount() + signaturesSHA256.approximateElementCount()));
System.out.println("Expected false postive rate: md5: " + signaturesMD5.expectedFpp() + ", sha1: " + signaturesSHA1.expectedFpp() + ", sha256: " + signaturesSHA256.expectedFpp()); System.out.println("Expected false postive rate: md5: " + signaturesMD5.expectedFpp() + ", sha1: " + signaturesSHA1.expectedFpp() + ", sha256: " + signaturesSHA256.expectedFpp());
System.out.println("Testing exclusions:"); System.out.println("Testing exclusions:");
int matchedExclusions = 0; int matchedExclusions = 0;
for(String excluded : arrExclusions) { for (String excluded : arrExclusions) {
if(excluded.length() == 32 && signaturesMD5.mightContain(excluded)) { if (excluded.length() == 32 && signaturesMD5.mightContain(excluded)) {
System.out.println("\tmd5: Found excluded hash " + excluded); System.out.println("\tmd5: Found excluded hash " + excluded);
matchedExclusions++; matchedExclusions++;
} }
if(excluded.length() == 40 && signaturesSHA1.mightContain(excluded)) { if (excluded.length() == 40 && signaturesSHA1.mightContain(excluded)) {
System.out.println("\tsha1: Found excluded hash " + excluded); System.out.println("\tsha1: Found excluded hash " + excluded);
matchedExclusions++; matchedExclusions++;
} }
if(excluded.length() == 64 && signaturesSHA256.mightContain(excluded)) { if (excluded.length() == 64 && signaturesSHA256.mightContain(excluded)) {
System.out.println("\tsha256: Found excluded hash " + excluded); System.out.println("\tsha256: Found excluded hash " + excluded);
matchedExclusions++; matchedExclusions++;
} }
} }
if(matchedExclusions == 0) { if (matchedExclusions == 0) {
System.out.println("\tNo exclusions found :)"); System.out.println("\tNo exclusions found :)");
} else { } else {
System.out.println("\tExclusions were found!"); System.out.println("\tExclusions were found!");
@ -211,6 +238,10 @@ public class Main {
FileOutputStream fileSignaturesSHA256 = new FileOutputStream(new File(args[0]) + "/hypatia-sha256-bloom.bin"); FileOutputStream fileSignaturesSHA256 = new FileOutputStream(new File(args[0]) + "/hypatia-sha256-bloom.bin");
signaturesSHA256.writeTo(fileSignaturesSHA256); signaturesSHA256.writeTo(fileSignaturesSHA256);
fileSignaturesSHA256.close(); fileSignaturesSHA256.close();
FileOutputStream fileDomains = new FileOutputStream(new File(args[0]) + "/hypatia-domains-bloom.bin");
domains.writeTo(fileDomains);
fileDomains.close();
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
@ -227,7 +258,7 @@ public class Main {
private static void addChecked(String potentialHash, boolean report) { private static void addChecked(String potentialHash, boolean report) {
if (!potentialHash.startsWith("#") && potentialHash.length() >= 4) { if (!potentialHash.startsWith("#") && potentialHash.length() >= 4) {
if (isHexadecimal(potentialHash)) { if (isHexadecimal(potentialHash)) {
if(arrExclusions.contains(potentialHash)) { if (arrExclusions.contains(potentialHash)) {
System.out.println("\t\tSkipping excluded hash: " + potentialHash); System.out.println("\t\tSkipping excluded hash: " + potentialHash);
return; return;
} }
@ -237,7 +268,7 @@ public class Main {
if (potentialHash.length() == 32) { if (potentialHash.length() == 32) {
boolean shouldAdd = true; boolean shouldAdd = true;
if (extendedMode) { if (extendedMode) {
if(signaturesMD5Dedupe.mightContain(potentialHash)) { if (signaturesMD5Dedupe.mightContain(potentialHash)) {
shouldAdd = false; shouldAdd = false;
amtSignaturesDedupedMD5++; amtSignaturesDedupedMD5++;
} }
@ -261,11 +292,11 @@ public class Main {
amtLinesValid++; amtLinesValid++;
} else { } else {
amtLinesInvalid++; amtLinesInvalid++;
if(report) System.out.println("\t\tINVALID LENGTH: " + potentialHash); if (report) System.out.println("\t\tINVALID LENGTH: " + potentialHash);
} }
} else { } else {
amtLinesInvalid++; amtLinesInvalid++;
if(report) System.out.println("\t\tNOT HEXADECIMAL: " + potentialHash); if (report) System.out.println("\t\tNOT HEXADECIMAL: " + potentialHash);
} }
} }
} }