From 1c29038125ebbc9356314928983ae44c8c7fa5d1 Mon Sep 17 00:00:00 2001
From: Tad <tad@spotco.us>
Date: Thu, 21 Dec 2023 23:26:19 -0500
Subject: [PATCH] Directly load the pre-processed bloom filters

No more conversion, much quicker loading, much smaller downloads

Signed-off-by: Tad <tad@spotco.us>
---
 .idea/misc.xml                                |   1 -
 app/build.gradle                              |   4 +-
 .../us/spotco/malwarescanner/Database.java    | 107 +++-----------
 .../spotco/malwarescanner/MainActivity.java   |  55 -------
 .../spotco/malwarescanner/MalwareScanner.java |   2 +-
 .../malwarescanner/MalwareScannerService.java |   3 +-
 app/src/main/res/menu/menu_main.xml           |   3 -
 .../metadata/android/en-US/changelogs/112.txt |   2 +
 scripts/0clamav.sh                            |   4 +
 scripts/0eset.sh                              |   4 +-
 scripts/0malware_bazaar-android.sh            |   6 -
 scripts/0sign.sh                              |  14 ++
 scripts/Main.java                             | 139 ++++++++++++++++++
 13 files changed, 184 insertions(+), 160 deletions(-)
 create mode 100644 fastlane/metadata/android/en-US/changelogs/112.txt
 delete mode 100644 scripts/0malware_bazaar-android.sh
 create mode 100644 scripts/0sign.sh
 create mode 100644 scripts/Main.java

diff --git a/.idea/misc.xml b/.idea/misc.xml
index 5c54277..5c7a2a1 100644
--- a/.idea/misc.xml
+++ b/.idea/misc.xml
@@ -1,4 +1,3 @@
-<?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
   <component name="DesignSurface">
     <option name="filePathToZoomLevelMap">
diff --git a/app/build.gradle b/app/build.gradle
index e8e2937..049f33b 100644
--- a/app/build.gradle
+++ b/app/build.gradle
@@ -6,8 +6,8 @@ android {
         applicationId "us.spotco.malwarescanner"
         minSdkVersion 19
         targetSdkVersion 32
-        versionCode 110
-        versionName "2.34"
+        versionCode 112
+        versionName "2.35"
         resConfigs 'en', 'af', 'de', 'el', 'es', 'fi', 'fr', 'it', 'pl', 'pt', 'ru', 'tr', 'zh-rCN'
     }
     buildTypes {
diff --git a/app/src/main/java/us/spotco/malwarescanner/Database.java b/app/src/main/java/us/spotco/malwarescanner/Database.java
index 0583a2c..6e3c341 100644
--- a/app/src/main/java/us/spotco/malwarescanner/Database.java
+++ b/app/src/main/java/us/spotco/malwarescanner/Database.java
@@ -26,12 +26,9 @@ import android.widget.TextView;
 import com.google.common.hash.BloomFilter;
 import com.google.common.hash.Funnels;
 
-import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
-import java.io.FileReader;
-import java.io.InputStreamReader;
 import java.net.HttpURLConnection;
 import java.net.InetSocketAddress;
 import java.net.Proxy;
@@ -40,7 +37,6 @@ import java.nio.charset.StandardCharsets;
 import java.text.DateFormat;
 import java.util.Date;
 import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.zip.GZIPInputStream;
 
 class Database {
 
@@ -52,11 +48,8 @@ class Database {
 
     public final static ConcurrentLinkedQueue<SignatureDatabase> signatureDatabases = new ConcurrentLinkedQueue<>();
     public static BloomFilter<String> signaturesMD5 = null;
-    public static int amtSignaturesMD5 = 0;
     public static BloomFilter<String> signaturesSHA1 = null;
-    public static int amtSignaturesSHA1 = 0;
     public static BloomFilter<String> signaturesSHA256 = null;
-    public static int amtSignaturesSHA256 = 0;
 
     private static final DateFormat dateFormat = DateFormat.getDateInstance();
 
@@ -69,14 +62,9 @@ class Database {
     }
 
     public static boolean isDatabaseLoaded() {
-        return areDatabasesAvailable() && databaseFullyLoaded && (amtSignaturesMD5 > 0 || amtSignaturesSHA1 > 0 || amtSignaturesSHA256 > 0);
+        return areDatabasesAvailable() && databaseFullyLoaded;
     }
 
-    public static int getSignatureCount() {
-        return amtSignaturesMD5 + amtSignaturesSHA1 + amtSignaturesSHA256;
-    }
-
-
     public static void updateDatabase(Context context, ConcurrentLinkedQueue<SignatureDatabase> signatureDatabases) {
         initDatabase(context);
         log.append(context.getString(R.string.main_database_updating, String.valueOf(signatureDatabases.size())) + "\n");
@@ -97,37 +85,11 @@ class Database {
         databasePath.mkdir();
 
         signatureDatabases.clear();
-
         prefs = context.getSharedPreferences(BuildConfig.APPLICATION_ID, Context.MODE_PRIVATE);
         String baseURL = Utils.getDatabaseURL(context);
-        if (prefs.getBoolean("SIGNATURES_TARGETEDTHREATS", true)) {
-            signatureDatabases.add(new SignatureDatabase(baseURL, "targetedthreats.hdb.gz"));
-            signatureDatabases.add(new SignatureDatabase(baseURL, "targetedthreats.hsb.gz"));
-        }
-        if (prefs.getBoolean("SIGNATURES_AMNESTY", true)) {
-            signatureDatabases.add(new SignatureDatabase(baseURL, "amnesty.hsb.gz"));
-        }
-        if (prefs.getBoolean("SIGNATURES_STALKERWARE", true)) {
-            signatureDatabases.add(new SignatureDatabase(baseURL, "stalkerware.hsb.gz"));
-        }
-        if (prefs.getBoolean("SIGNATURES_ESET", true)) {
-            signatureDatabases.add(new SignatureDatabase(baseURL, "eset.hdb.gz"));
-            signatureDatabases.add(new SignatureDatabase(baseURL, "eset.hsb.gz"));
-        }
-        if (prefs.getBoolean("SIGNATURES_MALWAREBAZAAR", false)) {
-            signatureDatabases.add(new SignatureDatabase(baseURL, "malware_bazaar-full.hsb.gz"));
-        } else if (prefs.getBoolean("SIGNATURES_MALWAREBAZAAR-ANDROID", true)) {
-            signatureDatabases.add(new SignatureDatabase(baseURL, "malware_bazaar.hsb.gz"));
-        }
-        if (prefs.getBoolean("SIGNATURES_CLAMAV", false)) {
-            signatureDatabases.add(new SignatureDatabase(baseURL, "main.hdb.gz"));
-            signatureDatabases.add(new SignatureDatabase(baseURL, "main.hsb.gz"));
-            signatureDatabases.add(new SignatureDatabase(baseURL, "daily.hdb.gz"));
-            signatureDatabases.add(new SignatureDatabase(baseURL, "daily.hsb.gz"));
-        } else if (prefs.getBoolean("SIGNATURES_CLAMAV-ANDROID", true)) {
-            signatureDatabases.add(new SignatureDatabase(baseURL, "Android.hdb.gz"));
-            signatureDatabases.add(new SignatureDatabase(baseURL, "Android.hsb.gz"));
-        }
+        signatureDatabases.add(new SignatureDatabase(baseURL, "hypatia-md5-bloom.bin"));
+        signatureDatabases.add(new SignatureDatabase(baseURL, "hypatia-sha1-bloom.bin"));
+        signatureDatabases.add(new SignatureDatabase(baseURL, "hypatia-sha256-bloom.bin"));
     }
 
     public static void loadDatabase(Context context, boolean forceReload, ConcurrentLinkedQueue<SignatureDatabase> signatureDatabases) {
@@ -135,14 +97,6 @@ class Database {
             databaseFullyLoaded = false;
             databaseCurrentlyLoading = true;
             initDatabase(context);
-            //XXX: These must be kept in sync with the databases
-            signaturesMD5 = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000, 0.001); //200k
-            amtSignaturesMD5 = 0;
-            signaturesSHA1 = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 5000000, 0.001); //5m
-            amtSignaturesSHA1 = 0;
-            signaturesSHA256 = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 1000000, 0.001); //1m
-            amtSignaturesSHA256 = 0;
-            System.gc();
             File publicKey = new File(databasePath + "/gpg.key");
             GPGDetachedSignatureVerifier verifier = new GPGDetachedSignatureVerifier(Utils.getSigningKey(context));
             for (SignatureDatabase database : signatureDatabases) {
@@ -153,41 +107,22 @@ class Database {
                         boolean validated = verifier.verify(databaseLocation, databaseSigLocation, publicKey);
                         if (validated) {
                             Log.d("Hypatia", "Successfully validated database");
-                            BufferedReader reader;
-                            if (databaseLocation.getName().endsWith(".gz")) {
-                                reader = new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(databaseLocation))));
-                            } else {
-                                reader = new BufferedReader(new FileReader(databaseLocation));
+                            FileInputStream databaseLoading = new FileInputStream(databaseLocation);
+                            switch (databaseLocation.getName()) {
+                                case "hypatia-md5-bloom.bin":
+                                    Log.d("Hypatia", "Processing md5");
+                                    signaturesMD5 = BloomFilter.readFrom(databaseLoading, Funnels.stringFunnel(StandardCharsets.US_ASCII));
+                                    break;
+                                case "hypatia-sha1-bloom.bin":
+                                    Log.d("Hypatia", "Processing sha1");
+                                    signaturesSHA1 = BloomFilter.readFrom(databaseLoading, Funnels.stringFunnel(StandardCharsets.US_ASCII));
+                                    break;
+                                case "hypatia-sha256-bloom.bin":
+                                    Log.d("Hypatia", "Processing sha256");
+                                    signaturesSHA256 = BloomFilter.readFrom(databaseLoading, Funnels.stringFunnel(StandardCharsets.US_ASCII));
+                                    break;
                             }
-                            String line;
-                            if (database.getName().contains(".hdb")) {//.hdb format: md5, size, name
-                                while ((line = reader.readLine()) != null) {
-                                    if (line.length() > 0) {
-                                        String[] lineS = line.split(":");
-                                        if (lineS[0].length() > 0) {
-                                            if (signaturesMD5.put(lineS[0])) {
-                                                amtSignaturesMD5++;
-                                            }
-                                        }
-                                    }
-                                }
-                            } else if (database.getName().contains(".hsb")) {//.hsb format: sha256, size, name
-                                while ((line = reader.readLine()) != null) {
-                                    if (line.length() > 0) {
-                                        String[] lineS = line.split(":");
-                                        if (lineS[0].length() == 32) {
-                                            if (signaturesSHA1.put(lineS[0])) {
-                                                amtSignaturesSHA1++;
-                                            }
-                                        } else if (lineS[0].length() > 0) {
-                                            if (signaturesSHA256.put(lineS[0])) {
-                                                amtSignaturesSHA256++;
-                                            }
-                                        }
-                                    }
-                                }
-                            }
-                            reader.close();
+                            databaseLoading.close();
                         } else {
                             Log.w("Hypatia", "Failed to validate database");
                         }
@@ -197,10 +132,6 @@ class Database {
                 }
                 System.gc();
             }
-
-            Log.w("Hypatia", "Loaded database - md5: " + amtSignaturesMD5 + ", sha1: " + amtSignaturesSHA1 + ", sha256: " + amtSignaturesSHA256);
-            signaturesMD5.put("44d88612fea8a8f36de82e1278abb02f");
-            signaturesSHA256.put("6a0b4866f143c32e651662cebf7f380d27b0db809db3b6a34cf34c7436ab6bbf");
             System.gc();
             databaseFullyLoaded = true;
             databaseCurrentlyLoading = false;
diff --git a/app/src/main/java/us/spotco/malwarescanner/MainActivity.java b/app/src/main/java/us/spotco/malwarescanner/MainActivity.java
index d2eeea6..600ce3b 100644
--- a/app/src/main/java/us/spotco/malwarescanner/MainActivity.java
+++ b/app/src/main/java/us/spotco/malwarescanner/MainActivity.java
@@ -131,57 +131,6 @@ public class MainActivity extends Activity {
         creditsDialog.show();
     }
 
-    private String localizeDBDescription(String desc) {
-        return desc
-                .replaceAll("AUTHOR", getString(R.string.db_desc_author))
-                .replaceAll("LICENSE", getString(R.string.db_desc_license))
-                .replaceAll("SIZE_SMALL", getString(R.string.db_desc_size_small))
-                .replaceAll("SIZE_MEDIUM", getString(R.string.db_desc_size_medium))
-                .replaceAll("SIZE_LARGE", getString(R.string.db_desc_size_large))
-                .replaceAll("SIZE", getString(R.string.db_desc_size))
-                .replaceAll("SOURCE", getString(R.string.db_desc_source));
-    }
-
-    private void selectDatabases() {
-        final String[] databases = {
-                localizeDBDescription("ClamAV: Android Only\n • SIZE: SIZE_MEDIUM\n • LICENSE: GPL-2.0\n • AUTHOR: Cisco\n • SOURCE: https://clamav.net\n"),
-                localizeDBDescription("ClamAV: Full\n • SIZE: SIZE_LARGE\n • LICENSE: GPL-2.0\n • AUTHOR: Cisco\n • SOURCE: https://clamav.net\n"),
-                localizeDBDescription("ESET\n • SIZE: SIZE_SMALL\n • LICENSE: BSD 2-Clause\n • AUTHOR: ESET\n • SOURCE: https://github.com/eset/malware-ioc\n"),
-                localizeDBDescription("Targeted Threats\n • SIZE: SIZE_SMALL\n • LICENSE: CC BY-SA 4.0\n • AUTHOR: Nex\n • SOURCE: https://github.com/botherder/targetedthreats\n"),
-                localizeDBDescription("Amnesty Tech Investigations\n • SIZE: SIZE_SMALL\n • LICENSE: CC BY 2.0\n • AUTHOR: Amnesty International\n • SOURCE: https://github.com/amnestytech/investigations\n"),
-                localizeDBDescription("Stalkerware\n • SIZE: SIZE_SMALL\n • LICENSE: CC BY 4.0\n • AUTHOR: Echap\n • SOURCE: https://github.com/AssoEchap/stalkerware-indicators\n"),
-                localizeDBDescription("MalwareBazaar: Android Only\n • SIZE: SIZE_SMALL\n • LICENSE: CC0\n • AUTHOR: Abuse.ch\n • SOURCE: https://bazaar.abuse.ch\n"),
-                localizeDBDescription("MalwareBazaar: Full\n • SIZE: SIZE_LARGE\n • LICENSE: CC0\n • AUTHOR: Abuse.ch\n • SOURCE: https://bazaar.abuse.ch")};
-        final boolean[] databaseDefaults = {
-                prefs.getBoolean("SIGNATURES_CLAMAV-ANDROID", true),
-                prefs.getBoolean("SIGNATURES_CLAMAV", false),
-                prefs.getBoolean("SIGNATURES_ESET", true),
-                prefs.getBoolean("SIGNATURES_TARGETEDTHREATS", true),
-                prefs.getBoolean("SIGNATURES_AMNESTY", true),
-                prefs.getBoolean("SIGNATURES_STALKERWARE", true),
-                prefs.getBoolean("SIGNATURES_MALWAREBAZAAR-ANDROID", true),
-                prefs.getBoolean("SIGNATURES_MALWAREBAZAAR", false)};
-
-        Dialog databaseDialog;
-        AlertDialog.Builder databaseBuilder = new AlertDialog.Builder(this);
-        databaseBuilder.setTitle(R.string.lblSelectDatabasesTitle);
-
-        databaseBuilder.setMultiChoiceItems(databases, databaseDefaults, (dialogInterface, i, selected) -> databaseDefaults[i] = selected);
-        databaseBuilder.setPositiveButton("OK", (dialogInterface, i) -> {
-            prefs.edit().putBoolean("SIGNATURES_CLAMAV-ANDROID", databaseDefaults[0]).apply();
-            prefs.edit().putBoolean("SIGNATURES_CLAMAV", databaseDefaults[1]).apply();
-            prefs.edit().putBoolean("SIGNATURES_ESET", databaseDefaults[2]).apply();
-            prefs.edit().putBoolean("SIGNATURES_TARGETEDTHREATS", databaseDefaults[3]).apply();
-            prefs.edit().putBoolean("SIGNATURES_AMNESTY", databaseDefaults[4]).apply();
-            prefs.edit().putBoolean("SIGNATURES_STALKERWARE", databaseDefaults[5]).apply();
-            prefs.edit().putBoolean("SIGNATURES_MALWAREBAZAAR-ANDROID", databaseDefaults[6]).apply();
-            prefs.edit().putBoolean("SIGNATURES_MALWAREBAZAAR", databaseDefaults[7]).apply();
-        });
-
-        databaseDialog = databaseBuilder.create();
-        databaseDialog.show();
-    }
-
     @Override
     public final boolean onOptionsItemSelected(MenuItem item) {
         switch (item.getItemId()) {
@@ -211,9 +160,6 @@ public class MainActivity extends Activity {
                     updateDatabase();
                 }
                 break;
-            case R.id.mnuSelectDatabases:
-                selectDatabases();
-                break;
             case R.id.mnuDatabaseServer:
                 AlertDialog.Builder builderServerOverride = new AlertDialog.Builder(this);
                 builderServerOverride.setTitle(getString(R.string.lblDatabaseServer));
@@ -314,7 +260,6 @@ public class MainActivity extends Activity {
             filesToScan.add(new File("/storage"));
         }
 
-
         malwareScanner.executeOnExecutor(Utils.getThreadPoolExecutor(), filesToScan);
         new Thread(() -> {
             try {
diff --git a/app/src/main/java/us/spotco/malwarescanner/MalwareScanner.java b/app/src/main/java/us/spotco/malwarescanner/MalwareScanner.java
index a20176b..154e01e 100644
--- a/app/src/main/java/us/spotco/malwarescanner/MalwareScanner.java
+++ b/app/src/main/java/us/spotco/malwarescanner/MalwareScanner.java
@@ -135,7 +135,7 @@ class MalwareScanner extends AsyncTask<HashSet<File>, Object, String> {
             }
         }
         if (Database.isDatabaseLoaded()) {
-            publishProgress("\t" + context.getString(R.string.main_database_loaded, NumberFormat.getInstance().format(Database.getSignatureCount())) + "\n", true);
+            publishProgress("\t" + context.getString(R.string.main_database_loaded, "?") + "\n", true);
 
             //Get file hashes
             publishProgress("\t" + context.getString(R.string.main_hashing_files), true);
diff --git a/app/src/main/java/us/spotco/malwarescanner/MalwareScannerService.java b/app/src/main/java/us/spotco/malwarescanner/MalwareScannerService.java
index 4cf1f63..ad5344e 100644
--- a/app/src/main/java/us/spotco/malwarescanner/MalwareScannerService.java
+++ b/app/src/main/java/us/spotco/malwarescanner/MalwareScannerService.java
@@ -108,6 +108,7 @@ public class MalwareScannerService extends Service {
                         .setContentTitle(getText(R.string.lblNotificationRealtimeTitle))
                         .setContentText(getText(R.string.lblNotificationRealtimeText))
                         .setPriority(Notification.PRIORITY_MIN)
+                        .setOngoing(true)
                         .setShowWhen(false);
 
         if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
@@ -119,7 +120,7 @@ public class MalwareScannerService extends Service {
 
     private void updateForegroundNotification() {
         foregroundNotification
-                .setSubText(NumberFormat.getInstance().format(Database.getSignatureCount()) + " sigs" + " • " + getString(R.string.main_files_scanned_count, NumberFormat.getInstance().format(Utils.FILES_SCANNED)));
+                .setSubText(getString(R.string.main_files_scanned_count, NumberFormat.getInstance().format(Utils.FILES_SCANNED)));
         notificationManager.notify(-1, foregroundNotification.build());
     }
 
diff --git a/app/src/main/res/menu/menu_main.xml b/app/src/main/res/menu/menu_main.xml
index 53c684e..90ce992 100644
--- a/app/src/main/res/menu/menu_main.xml
+++ b/app/src/main/res/menu/menu_main.xml
@@ -14,9 +14,6 @@
     <item
         android:id="@+id/mnuUpdateDatabase"
         android:title="@string/lblUpdateDatabase" />
-    <item
-        android:id="@+id/mnuSelectDatabases"
-        android:title="@string/lblSelectDatabases" />
     <item
         android:id="@+id/mnuDatabaseServer"
         android:title="@string/lblDatabaseServer" />
diff --git a/fastlane/metadata/android/en-US/changelogs/112.txt b/fastlane/metadata/android/en-US/changelogs/112.txt
new file mode 100644
index 0000000..9f69044
--- /dev/null
+++ b/fastlane/metadata/android/en-US/changelogs/112.txt
@@ -0,0 +1,2 @@
+* Elimiates all local database conversion
+* Server now serves up the optimized bloom filters directly for smaller downloads
diff --git a/scripts/0clamav.sh b/scripts/0clamav.sh
index 50ead77..df87c9d 100644
--- a/scripts/0clamav.sh
+++ b/scripts/0clamav.sh
@@ -16,6 +16,8 @@ sigtool -u daily.c*d
 #MD5
 grep "Andr\\." main.hdb >> Android.hdb
 grep "Andr\\." daily.hdb >> Android.hdb
+#grep "Java\\." main.hdb >> Android.hdb
+#grep "Java\\." daily.hdb >> Android.hdb
 grep "Unix\\." main.hdb >> Android.hdb
 grep "Unix\\." daily.hdb >> Android.hdb
 grep "Multios\\." main.hdb >> Android.hdb
@@ -24,6 +26,8 @@ grep "Multios\\." daily.hdb >> Android.hdb
 #SHA
 grep "Andr\\." main.hsb >> Android.hsb
 grep "Andr\\." daily.hsb >> Android.hsb
+#grep "Java\\." main.hsb >> Android.hsb
+#grep "Java\\." daily.hsb >> Android.hsb
 grep "Unix\\." main.hsb >> Android.hsb
 grep "Unix\\." daily.hsb >> Android.hsb
 grep "Multios\\." main.hsb >> Android.hsb
diff --git a/scripts/0eset.sh b/scripts/0eset.sh
index a840db8..bd82e44 100644
--- a/scripts/0eset.sh
+++ b/scripts/0eset.sh
@@ -8,7 +8,7 @@ processHashes() {
 		dos2unix $1/samples.$2
 		while IFS= read -r line
 		do
-			echo "$line":0:ESET."$name" >> ./eset.$3;
+			echo "$line" >> ./eset.$2;
 		done < "$1/samples.$2";
 	fi;
 }
@@ -17,5 +17,3 @@ export -f processHashes;
 find . -maxdepth 2 -mindepth 1 -type d -exec bash -c 'processHashes "{}" md5 hdb' \;
 find . -maxdepth 2 -mindepth 1 -type d -exec bash -c 'processHashes "{}" sha1 hsb' \;
 find . -maxdepth 2 -mindepth 1 -type d -exec bash -c 'processHashes "{}" sha256 hsb' \;
-gzip *.hdb;
-gzip *.hsb;
diff --git a/scripts/0malware_bazaar-android.sh b/scripts/0malware_bazaar-android.sh
deleted file mode 100644
index 1e94c3e..0000000
--- a/scripts/0malware_bazaar-android.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/bash
-#License: GPLv3
-#Description: Hypatia conversion script for https://bazaar.abuse.ch/export/csv/full/ (CC0)
-grep -v "^#" full.csv | grep -e "\"apk\"" -e "\"application/x-executable\"" -e "\"application/x-sharedlib\"" | awk '{ print $3 } ' | sed 's/^"//' | sed 's/",$/:0:MB/' | sort -u > malware_bazaar.hsb;
-grep -v "^#" full.csv | awk '{ print $3 } ' | sed 's/^"//' | sed 's/",$/:0:MB/' | sort -u > malware_bazaar-full.hsb;
-gzip *.hsb;
diff --git a/scripts/0sign.sh b/scripts/0sign.sh
new file mode 100644
index 0000000..95db9f9
--- /dev/null
+++ b/scripts/0sign.sh
@@ -0,0 +1,14 @@
+for database in *.bin
+do
+	if [ -f "$database.sig" ]; then
+		#If it does exist sign if it doesn't match
+		if ! gpg --verify "$database.sig"; then
+			rm "$database.sig";
+			gpg --sign --local-user 6395FC9911EDCD6158712DF7BADFCABDDBF5B694 --detach-sign "$database";
+		fi;
+	else
+		#Sign it if it doesn't exist
+		gpg --sign --local-user 6395FC9911EDCD6158712DF7BADFCABDDBF5B694 --detach-sign "$database";
+	fi;
+
+done
diff --git a/scripts/Main.java b/scripts/Main.java
new file mode 100644
index 0000000..e9775cd
--- /dev/null
+++ b/scripts/Main.java
@@ -0,0 +1,139 @@
+/*
+Copyright (c) 2023 Divested Computing Group
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.
+*/
+
+import com.google.common.hash.BloomFilter;
+import com.google.common.hash.Funnels;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.zip.GZIPInputStream;
+
+public class Main {
+
+    public static int amtSignaturesRead = 0;
+    public static BloomFilter<String> signaturesMD5 = null;
+    public static int amtSignaturesMD5 = 0;
+    public static BloomFilter<String> signaturesSHA1 = null;
+    public static int amtSignaturesSHA1 = 0;
+    public static BloomFilter<String> signaturesSHA256 = null;
+    public static int amtSignaturesSHA256 = 0;
+
+    public static void main(String[] args) {
+            signaturesMD5 = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.US_ASCII), 150000, 0.00001); //150k
+            signaturesSHA1 = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.US_ASCII), 4500000, 0.00001); //4.5m
+            signaturesSHA256 = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.US_ASCII), 800000, 0.00001); //800k
+
+           for (File databaseLocation : new File(args[0]).listFiles()) {
+           	 System.out.println("Processing: " + databaseLocation);
+                    try {
+                            BufferedReader reader;
+                            if (databaseLocation.getName().endsWith(".gz")) {
+                                reader = new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(databaseLocation))));
+                            } else {
+                                reader = new BufferedReader(new FileReader(databaseLocation));
+                            }
+                            String line;
+                            if (databaseLocation.getName().contains(".hdb")) {//.hdb format: md5, size, name
+                                while ((line = reader.readLine()) != null) {
+                                    if (line.length() > 0) {
+                                        String[] lineS = line.split(":");
+                                        if (lineS[0].length() > 0) {
+                                            if (signaturesMD5.put(lineS[0])) {
+                                                amtSignaturesMD5++;
+                                            }
+                                            amtSignaturesRead++;
+                                        }
+                                    }
+                                }
+                            } else if (databaseLocation.getName().contains(".hsb")) {//.hsb format: sha256, size, name
+                                while ((line = reader.readLine()) != null) {
+                                    if (line.length() > 0) {
+                                        String[] lineS = line.split(":");
+                                        if (lineS[0].length() == 32) {
+                                            if (signaturesSHA1.put(lineS[0])) {
+                                                amtSignaturesSHA1++;
+                                            }
+                                            amtSignaturesRead++;
+                                        } else if (lineS[0].length() > 0) {
+                                            if (signaturesSHA256.put(lineS[0])) {
+                                                amtSignaturesSHA256++;
+                                            }
+                                            amtSignaturesRead++;
+                                        }
+                                    }
+                                }
+                            } else if (databaseLocation.getName().contains(".md5")) {//one signature per line
+                                while ((line = reader.readLine()) != null) {
+                                    if (line.length() > 0) {
+                                        if (signaturesMD5.put(line)) {
+                                            amtSignaturesMD5++;
+                                        }
+                                        amtSignaturesRead++;
+                                    }
+                                }
+                            } else if (databaseLocation.getName().contains(".sha1")) {//one signature per line
+                                while ((line = reader.readLine()) != null) {
+                                    if (line.length() > 0) {
+                                        if (signaturesSHA1.put(line)) {
+                                            amtSignaturesSHA1++;
+                                        }
+                                        amtSignaturesRead++;
+                                    }
+                                }
+                            } else if (databaseLocation.getName().contains(".sha256")) {//one signature per line
+                                while ((line = reader.readLine()) != null) {
+                                    if (line.length() > 0) {
+                                        if (signaturesSHA256.put(line)) {
+                                            amtSignaturesSHA256++;
+                                        }
+                                        amtSignaturesRead++;
+                                    }
+                                }
+                            }
+                            reader.close();
+                            System.out.println("\tmd5: " + amtSignaturesMD5 + ", sha1: " + amtSignaturesSHA1 + ", sha256: " + amtSignaturesSHA256);
+                    } catch (Exception e) {
+                        e.printStackTrace();
+                    }
+            }
+            signaturesMD5.put("44d88612fea8a8f36de82e1278abb02f"); //Eicar test file
+            signaturesSHA256.put("6a0b4866f143c32e651662cebf7f380d27b0db809db3b6a34cf34c7436ab6bbf"); //Hypatia test file
+
+            System.out.println("Total: " + amtSignaturesRead);
+            System.out.println("Mismatch: " + (amtSignaturesRead-amtSignaturesMD5-amtSignaturesSHA1-amtSignaturesSHA256));
+            System.out.println("Loaded all databases - md5: " + amtSignaturesMD5 + ", sha1: " + amtSignaturesSHA1 + ", sha256: " + amtSignaturesSHA256);
+            System.out.println("Expected false postive rate - md5: " + signaturesMD5.expectedFpp() + ", sha1: " + signaturesSHA1.expectedFpp() + ", sha256: " + signaturesSHA256.expectedFpp());
+            try {
+                    FileOutputStream fileSignaturesMD5 = new FileOutputStream(new File(args[0]) + "/hypatia-md5-bloom.bin");
+                    signaturesMD5.writeTo(fileSignaturesMD5);
+                    fileSignaturesMD5.close();
+
+                    FileOutputStream fileSignaturesSHA1 = new FileOutputStream(new File(args[0]) + "/hypatia-sha1-bloom.bin");
+                    signaturesSHA1.writeTo(fileSignaturesSHA1);
+                    fileSignaturesSHA1.close();
+
+                    FileOutputStream fileSignaturesSHA256 = new FileOutputStream(new File(args[0]) + "/hypatia-sha256-bloom.bin");
+                    signaturesSHA256.writeTo(fileSignaturesSHA256);
+                    fileSignaturesSHA256.close();
+            } catch (Exception e) { e.printStackTrace(); }
+    }
+}