Extended database option

Signed-off-by: Tad <tad@spotco.us>

app/src/main/java/us/spotco/malwarescanner/Database.java

@@ -48,9 +48,11 @@ class Database {
 
     public final static ConcurrentLinkedQueue<SignatureDatabase> signatureDatabases = new ConcurrentLinkedQueue<>();
     public static BloomFilter<String> signaturesMD5 = null;
+    public static BloomFilter<String> signaturesMD5Extended = null;
     public static BloomFilter<String> signaturesSHA1 = null;
     public static BloomFilter<String> signaturesSHA256 = null;
     public static long signaturesCount = 0;
+    public static boolean changed = false;
 
     private static final DateFormat dateFormat = DateFormat.getDateInstance();
 
@@ -85,6 +87,7 @@ class Database {
         if (!Utils.getDatabaseURL(context).equals(Utils.DATABASE_URL_DEFAULT)) {
             log.append(context.getString(R.string.main_database_override, Utils.getDatabaseURL(context)) + "\n");
         }
+        changed = false;
         boolean onionRouting = prefs.getBoolean("ONION_ROUTING", false);
         new Downloader().executeOnExecutor(Utils.getThreadPoolExecutor(), onionRouting, Utils.getDatabaseURL(context) + "gpg.key", databasePath + "/gpg.key", Utils.getDatabaseURL(context));
 
@@ -105,6 +108,9 @@ class Database {
         signatureDatabases.add(new SignatureDatabase(baseURL, "hypatia-md5-bloom.bin"));
         signatureDatabases.add(new SignatureDatabase(baseURL, "hypatia-sha1-bloom.bin"));
         signatureDatabases.add(new SignatureDatabase(baseURL, "hypatia-sha256-bloom.bin"));
+        if (prefs.getBoolean("SIGNATURES_EXTENDED", false)) {
+            signatureDatabases.add(new SignatureDatabase(baseURL, "hypatia-md5-extended-bloom.bin"));
+        }
     }
 
     public static void loadDatabase(Context context, boolean forceReload, ConcurrentLinkedQueue<SignatureDatabase> signatureDatabases) {
@@ -129,16 +135,25 @@ class Database {
                                     Log.d("Hypatia", "Processing md5");
                                     signaturesMD5 = BloomFilter.readFrom(databaseLoading, Funnels.stringFunnel(Charsets.US_ASCII));
                                     signaturesCount += signaturesMD5.approximateElementCount();
+                                    Log.d("Hypatia", "Loaded md5");
                                     break;
                                 case "hypatia-sha1-bloom.bin":
                                     Log.d("Hypatia", "Processing sha1");
                                     signaturesSHA1 = BloomFilter.readFrom(databaseLoading, Funnels.stringFunnel(Charsets.US_ASCII));
                                     signaturesCount += signaturesSHA1.approximateElementCount();
+                                    Log.d("Hypatia", "Loaded sha1");
                                     break;
                                 case "hypatia-sha256-bloom.bin":
                                     Log.d("Hypatia", "Processing sha256");
                                     signaturesSHA256 = BloomFilter.readFrom(databaseLoading, Funnels.stringFunnel(Charsets.US_ASCII));
                                     signaturesCount += signaturesSHA256.approximateElementCount();
+                                    Log.d("Hypatia", "Loaded sha256");
+                                    break;
+                                case "hypatia-md5-extended-bloom.bin":
+                                    Log.d("Hypatia", "Processing md5 extended");
+                                    signaturesMD5Extended = BloomFilter.readFrom(databaseLoading, Funnels.stringFunnel(Charsets.US_ASCII));
+                                    signaturesCount += signaturesMD5Extended.approximateElementCount();
+                                    Log.d("Hypatia", "Loaded md5 extended");
                                     break;
                             }
                             databaseLoading.close();
@@ -212,6 +227,7 @@ class Database {
 
                         fileOutputStream.close();
                         outNew.renameTo(out); //Move the new file into place
+                        changed = true;
 
                         publishProgress(url.replaceAll(baseURL, "")
                                 + "\n\t" + Utils.getContext().getString(R.string.main_database_download_success)

app/src/main/java/us/spotco/malwarescanner/MainActivity.java

@@ -200,6 +200,17 @@ public class MainActivity extends Activity {
                 });
                 builderKey.show();
                 break;
+            case R.id.toggleExtended:
+                new AlertDialog.Builder(this)
+                        .setTitle(R.string.confirm_extended_title)
+                        .setMessage(getString(R.string.confirm_extended_summary))
+                        .setIcon(android.R.drawable.ic_menu_compass)
+                        .setPositiveButton(getString(android.R.string.yes), (dialog, which) -> prefs.edit().putBoolean("SIGNATURES_EXTENDED", true).apply())
+                        .setNegativeButton(getString(android.R.string.no), (dialog, which) -> {
+                            prefs.edit().putBoolean("SIGNATURES_EXTENDED", false).apply();
+                            dialog.cancel();
+                        }).show();
+                break;
             case R.id.toggleRealtime:
                 if (malwareScanner.running) {
                     logView.append(getString(R.string.lblScanRunning) + "\n");
@@ -293,10 +304,10 @@ public class MainActivity extends Activity {
         Utils.getThreadPoolExecutor().execute(() -> {
             try {
                 Thread.sleep(500);
-                Log.w("Hypatia", "Invoking database reload!");
+                Log.d("Hypatia", "Considering database reload!");
                 while (Database.hasDownloadsRunning()) {
                     Thread.sleep(500);
-                    Log.w("Hypatia", "Download in progress, waiting!");
+                    Log.d("Hypatia", "Download in progress, waiting!");
 
                 }
             } catch (InterruptedException e) {
@@ -304,8 +315,14 @@ public class MainActivity extends Activity {
             }
             runOnUiThread(() -> logView.append(getString(R.string.lblDatabasesUpdated) + "\n"));
             if (Database.isDatabaseLoaded()) {
-                Log.w("Hypatia", "Really reloading database!");
+                if(Database.changed) {
+                    Log.d("Hypatia", "Really reloading database!");
                     Database.loadDatabase(getApplicationContext(), true, Database.signatureDatabases);
+                } else {
+                    Log.d("Hypatia", "Database not changed, skipping reload!");
+                }
+            } else {
+                Log.d("Hypatia", "Database not loaded, skipping reload!");
             }
         });
     }

app/src/main/java/us/spotco/malwarescanner/MalwareScanner.java

@@ -229,6 +229,9 @@ class MalwareScanner extends AsyncTask<HashSet<File>, Object, String> {
             checkSignature("MD5", fileHashesMD5, Database.signaturesMD5);
             checkSignature("SHA-1", fileHashesSHA1, Database.signaturesSHA1);
             checkSignature("SHA-256", fileHashesSHA256, Database.signaturesSHA256);
+            if (Database.signaturesMD5Extended != null) {
+                checkSignature("MD5 Extended", fileHashesMD5, Database.signaturesMD5Extended);
+            }
 
             //Scan result
             if (startCount == amtMatchedFiles) {

app/src/main/res/menu/menu_main.xml

@@ -20,6 +20,9 @@
     <item
         android:id="@+id/mnuSigningKey"
         android:title="@string/lblSigningKey" />
+    <item
+        android:id="@+id/toggleExtended"
+        android:title="@string/lblExtendedDatabaseToggle" />
     <item
         android:id="@+id/toggleRealtime"
         android:title="@string/lblRealtimeScannerToggle"

app/src/main/res/values/strings.xml

@@ -69,4 +69,7 @@
     <string name="ignored">Ignored!</string>
     <string name="uninstallApp">Uninstall</string>
     <string name="lblSelfTest">Write self test files</string>
+    <string name="lblExtendedDatabaseToggle">Extended datatabase</string>
+    <string name="confirm_extended_title">Enable extended database?</string>
+    <string name="confirm_extended_summary">This will enable detection of an additional ~40 million signatures.\nThis requires a 125MB download, will slow down startup by over two minutes, will increase app RAM usage, and will increase the false positive rate.\nThis database only updates quarterly.</string>
 </resources>