Many changes

- Fix signature count reseting on update but not changed
- Better handle reloads for extended
- Metered connection warning
- Block all actions when appropriate
- Use a wakelock instead of keeping the screen on
- Increase max scan sizes
- Always check if files can be read before scanning
- Resolve true paths to avoid duplicates from symlinks
- Scan more app paths
- Scan more system paths
- Bump version

Signed-off-by: Tad <tad@spotco.us>

app/build.gradle

@@ -6,8 +6,8 @@ android {
         applicationId "us.spotco.malwarescanner"
         minSdkVersion 16
         targetSdkVersion 32
-        versionCode 302
+        versionCode 305
-        versionName "3.02"
+        versionName "3.05"
         resConfigs 'en', 'af', 'de', 'el', 'es', 'fi', 'fr', 'it', 'pl', 'pt', 'ru', 'tr', 'zh-rCN'
     }
     buildTypes {
@@ -20,7 +20,7 @@ android {
             shrinkResources true
             minifyEnabled true
             zipAlignEnabled true
-            proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
+            proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro'
         }
     }
     lint {

app/src/main/AndroidManifest.xml

@@ -16,6 +16,7 @@
         android:name="android.permission.QUERY_ALL_PACKAGES"
         tools:ignore="QueryAllPackagesPermission" />
     <uses-permission android:name="android.permission.REQUEST_DELETE_PACKAGES" />
+    <uses-permission android:name="android.permission.WAKE_LOCK" />
 
     <queries>
         <package android:name="org.torproject.android" />

app/src/main/java/us/spotco/malwarescanner/Database.java

@@ -52,7 +52,8 @@ class Database {
     public static BloomFilter<String> signaturesSHA1 = null;
     public static BloomFilter<String> signaturesSHA256 = null;
     public static long signaturesCount = 0;
-    public static boolean changed = false;
+    public static boolean changedDownload = false;
+    public static boolean changedConfig = false;
 
     private static final DateFormat dateFormat = DateFormat.getDateInstance();
 
@@ -87,7 +88,7 @@ class Database {
         if (!Utils.getDatabaseURL(context).equals(Utils.DATABASE_URL_DEFAULT)) {
             log.append(context.getString(R.string.main_database_override, Utils.getDatabaseURL(context)) + "\n");
         }
-        changed = false;
+        changedDownload = false;
         boolean onionRouting = prefs.getBoolean("ONION_ROUTING", false);
         new Downloader().executeOnExecutor(Utils.getThreadPoolExecutor(), onionRouting, Utils.getDatabaseURL(context) + "gpg.key", databasePath + "/gpg.key", Utils.getDatabaseURL(context));
 
@@ -102,7 +103,6 @@ class Database {
         databasePath.mkdir();
 
         signatureDatabases.clear();
-        signaturesCount = 0;
         prefs = context.getSharedPreferences(BuildConfig.APPLICATION_ID, Context.MODE_PRIVATE);
         String baseURL = Utils.getDatabaseURL(context);
         signatureDatabases.add(new SignatureDatabase(baseURL, "hypatia-md5-bloom.bin"));
@@ -119,6 +119,8 @@ class Database {
             databaseCurrentlyLoading = true;
             initDatabase(context);
             signaturesCount = 0;
+            changedConfig = false;
+            signaturesMD5Extended = null;
             File publicKey = new File(databasePath + "/gpg.key");
             GPGDetachedSignatureVerifier verifier = new GPGDetachedSignatureVerifier(Utils.getSigningKey(context));
             for (SignatureDatabase database : signatureDatabases) {
@@ -227,7 +229,7 @@ class Database {
 
                         fileOutputStream.close();
                         outNew.renameTo(out); //Move the new file into place
-                        changed = true;
+                        changedDownload = true;
 
                         publishProgress(url.replaceAll(baseURL, "")
                                 + "\n\t" + Utils.getContext().getString(R.string.main_database_download_success)

app/src/main/java/us/spotco/malwarescanner/MainActivity.java

@@ -32,6 +32,7 @@ import android.net.Uri;
 import android.os.Build;
 import android.os.Bundle;
 import android.os.Environment;
+import android.os.PowerManager;
 import android.provider.Settings;
 import android.text.InputType;
 import android.text.method.ScrollingMovementMethod;
@@ -75,8 +76,6 @@ public class MainActivity extends Activity {
         Utils.setContext(getApplicationContext());
         setContentView(R.layout.content_main);
 
-        getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON | WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED);
-
         logView = findViewById(R.id.txtLogOutput);
         logView.setMovementMethod(new ScrollingMovementMethod());
         logView.setTextIsSelectable(true);
@@ -154,19 +153,36 @@ public class MainActivity extends Activity {
                 }
                 break;
             case R.id.mnuUpdateDatabase:
-                if (malwareScanner.running) {
+                if (Database.hasDownloadsRunning()) {
+                    logView.append(getString(R.string.lblUpdateRunning) + "\n");
+                } else if (malwareScanner.running) {
                     logView.append(getString(R.string.lblScanRunning) + "\n");
                 } else if (!Utils.isNetworkAvailable(this)) {
                     logView.append(getString(R.string.lblNoNetwork) + "\n");
+                } else if (Database.isDatabaseLoading()) {
+                    logView.append(getString(R.string.lblDatabaseLoading) + "\n");
+                } else if (Utils.isConnectionMetered(this)) {
+                    int amt = prefs.getBoolean("SIGNATURES_EXTENDED", false) ? 200 : 50;
+                    new AlertDialog.Builder(this)
+                            .setTitle(R.string.confirm_update_title)
+                            .setMessage(getString(R.string.confirm_update_summary, String.valueOf(amt)))
+                            .setIcon(android.R.drawable.ic_dialog_alert)
+                            .setPositiveButton(getString(android.R.string.yes), (dialog, which) -> {
+                                updateDatabase();
+                            })
+                            .setNegativeButton(getString(android.R.string.no), (dialog, which) -> {
+                                dialog.cancel();
+                            }).show();
                 } else {
-                    if (prefs.getBoolean("ONION_ROUTING", false)) {
-                        Utils.requestStartOrbot(this);
-                        logView.append(getString(R.string.lblOnionRoutingEnabledHint) + "\n");
-                    }
                     updateDatabase();
                 }
                 break;
             case R.id.mnuDatabaseServer:
+                if (Database.hasDownloadsRunning()) {
+                    logView.append(getString(R.string.lblUpdateRunning) + "\n");
+                } else if (Database.isDatabaseLoading()) {
+                    logView.append(getString(R.string.lblDatabaseLoading) + "\n");
+                } else {
                     AlertDialog.Builder builderServerOverride = new AlertDialog.Builder(this);
                     builderServerOverride.setTitle(getString(R.string.lblDatabaseServer));
                     final EditText inputServerOverride = new EditText(this);
@@ -185,8 +201,14 @@ public class MainActivity extends Activity {
                         dialog.cancel();
                     });
                     builderServerOverride.show();
+                }
                 break;
             case R.id.mnuSigningKey:
+                if (Database.hasDownloadsRunning()) {
+                    logView.append(getString(R.string.lblUpdateRunning) + "\n");
+                } else if (Database.isDatabaseLoading()) {
+                    logView.append(getString(R.string.lblDatabaseLoading) + "\n");
+                } else {
                     AlertDialog.Builder builderKey = new AlertDialog.Builder(this);
                     builderKey.setTitle(getString(R.string.lblSigningKey));
                     final EditText inputKey = new EditText(this);
@@ -199,17 +221,33 @@ public class MainActivity extends Activity {
                         dialog.cancel();
                     });
                     builderKey.show();
+                }
                 break;
             case R.id.toggleExtended:
+                if (Database.hasDownloadsRunning()) {
+                    logView.append(getString(R.string.lblUpdateRunning) + "\n");
+                } else if (Database.isDatabaseLoading()) {
+                    logView.append(getString(R.string.lblDatabaseLoading) + "\n");
+                } else {
+                    boolean prevExtended = prefs.getBoolean("SIGNATURES_EXTENDED", false);
                     new AlertDialog.Builder(this)
                             .setTitle(R.string.confirm_extended_title)
                             .setMessage(getString(R.string.confirm_extended_summary))
                             .setIcon(android.R.drawable.ic_menu_compass)
-                        .setPositiveButton(getString(android.R.string.yes), (dialog, which) -> prefs.edit().putBoolean("SIGNATURES_EXTENDED", true).apply())
+                            .setPositiveButton(getString(android.R.string.yes), (dialog, which) -> {
+                                prefs.edit().putBoolean("SIGNATURES_EXTENDED", true).apply();
+                                if (!prevExtended) {
+                                    Database.changedConfig = true;
+                                }
+                            })
                             .setNegativeButton(getString(android.R.string.no), (dialog, which) -> {
                                 prefs.edit().putBoolean("SIGNATURES_EXTENDED", false).apply();
+                                if (prevExtended) {
+                                    Database.changedConfig = true;
+                                }
                                 dialog.cancel();
                             }).show();
+                }
                 break;
             case R.id.toggleRealtime:
                 if (malwareScanner.running) {
@@ -250,8 +288,14 @@ public class MainActivity extends Activity {
                 break;
             case R.id.btnStartScan:
                 if (!malwareScanner.running) {
+                    if (Database.hasDownloadsRunning()) {
+                        logView.append(getString(R.string.lblUpdateRunning) + "\n");
+                    } else if (Database.isDatabaseLoading()) {
+                        logView.append(getString(R.string.lblDatabaseLoading) + "\n");
+                    } else {
                         updateScanButton(true);
                         startScanner();
+                    }
                 } else {
                     logView.append("\n" + getString(R.string.main_cancelling_scan) + "\n\n");
                     malwareScanner.cancel(true);
@@ -267,20 +311,41 @@ public class MainActivity extends Activity {
         HashSet<File> filesToScan = new HashSet<>();
         if (scanSystem) {
             filesToScan.add(Environment.getRootDirectory());
+            filesToScan.add(new File("/"));
+            filesToScan.add(new File("/apex"));
+            filesToScan.add(new File("/cache"));
+            filesToScan.add(new File("/data"));
+            filesToScan.add(new File("/data/local/tmp"));
+            filesToScan.add(new File("/firmware"));
+            filesToScan.add(new File("/odm"));
+            filesToScan.add(new File("/odm_dlkm"));
+            filesToScan.add(new File("/product"));
+            filesToScan.add(new File("/system"));
+            filesToScan.add(new File("/system_dlkm"));
+            filesToScan.add(new File("/vendor"));
+            filesToScan.add(new File("/vendor_dlkm"));
         }
         if (scanApps) {
             for (ApplicationInfo packageInfo : getPackageManager().getInstalledApplications(PackageManager.GET_META_DATA)) {
                 filesToScan.add(new File(packageInfo.sourceDir));
-                //Log.d("Hypatia", "Planning to scan " + packageInfo.sourceDir);
+                filesToScan.add(new File(packageInfo.dataDir));
+                filesToScan.add(new File(packageInfo.nativeLibraryDir));
+                filesToScan.add(new File(packageInfo.publicSourceDir));
             }
         }
         if (scanInternal) {
             filesToScan.add(Environment.getExternalStorageDirectory());
         }
         if (scanExternal) {
-            filesToScan.add(new File("/storage"));
+            File externalStorage = new File("/storage");
+            if (externalStorage.exists()) {
+                filesToScan.add(externalStorage);
+            }
         }
 
+        PowerManager powerManager = (PowerManager) getSystemService(POWER_SERVICE);
+        PowerManager.WakeLock wakeLock = powerManager.newWakeLock(PowerManager.PARTIAL_WAKE_LOCK, "Hypatia::ManualScanLock");
+        wakeLock.acquire(10 * 60 * 1000L); /* 10 minutes */
         malwareScanner.executeOnExecutor(Utils.getThreadPoolExecutor(), filesToScan);
         new Thread(() -> {
             try {
@@ -288,6 +353,7 @@ public class MainActivity extends Activity {
                     Thread.sleep(500);
                 }
                 runOnUiThread(() -> updateScanButton(false));
+                wakeLock.release();
             } catch (Exception e) {
                 e.printStackTrace();
             }
@@ -295,12 +361,16 @@ public class MainActivity extends Activity {
     }
 
     private void updateDatabase() {
+        if (prefs.getBoolean("ONION_ROUTING", false)) {
+            Utils.requestStartOrbot(this);
+            logView.append(getString(R.string.lblOnionRoutingEnabledHint) + "\n");
+        }
         new Database(findViewById(R.id.txtLogOutput));
         if (!Database.isDatabaseLoading()) {
+            PowerManager powerManager = (PowerManager) getSystemService(POWER_SERVICE);
+            PowerManager.WakeLock wakeLock = powerManager.newWakeLock(PowerManager.PARTIAL_WAKE_LOCK, "Hypatia::UpdateLock");
+            wakeLock.acquire(3 * 60 * 1000L); /* 3 minutes */
             Database.updateDatabase(this, Database.signatureDatabases);
-        } else {
-            logView.append(getString(R.string.lblDatabaseLoading) + "\n");
-        }
             Utils.getThreadPoolExecutor().execute(() -> {
                 try {
                     Thread.sleep(500);
@@ -308,14 +378,14 @@ public class MainActivity extends Activity {
                     while (Database.hasDownloadsRunning()) {
                         Thread.sleep(500);
                         Log.d("Hypatia", "Download in progress, waiting!");
-
                     }
+                    wakeLock.release();
                 } catch (InterruptedException e) {
                     throw new RuntimeException(e);
                 }
                 runOnUiThread(() -> logView.append(getString(R.string.lblDatabasesUpdated) + "\n"));
                 if (Database.isDatabaseLoaded()) {
-                if(Database.changed) {
+                    if (Database.changedDownload || Database.changedConfig) {
                         Log.d("Hypatia", "Really reloading database!");
                         Database.loadDatabase(getApplicationContext(), true, Database.signatureDatabases);
                     } else {
@@ -325,6 +395,9 @@ public class MainActivity extends Activity {
                     Log.d("Hypatia", "Database not loaded, skipping reload!");
                 }
             });
+        } else {
+            logView.append(getString(R.string.lblDatabaseLoading) + "\n");
+        }
     }
 
     private void updateScanButton(boolean running) {

app/src/main/java/us/spotco/malwarescanner/MalwareScanner.java

@@ -221,6 +221,7 @@ class MalwareScanner extends AsyncTask<HashSet<File>, Object, String> {
                         spinnerCur = " = ";
                     }
                 }
+                //Log.d("Hypatia", "Scanning " + file);
             }
             filesToScanReal.clear();
             publishProgress("\n\t" + context.getString(R.string.main_hashing_done) + "\n", true);

app/src/main/java/us/spotco/malwarescanner/MalwareScannerService.java

@@ -79,7 +79,7 @@ public class MalwareScannerService extends Service {
                     case FileObserver.MOVED_TO:
                     case FileObserver.CLOSE_WRITE:
                         File file = new File(path);
-                        if (file.exists() && /*file.length() > 0 &&*/ file.length() <= Utils.MAX_SCAN_SIZE_REALTIME) {
+                        if (file.exists() && file.length() <= Utils.MAX_SCAN_SIZE_REALTIME) {
                             HashSet<File> filesToScan = new HashSet<>();
                             filesToScan.add(file);
                             new MalwareScanner(null, getApplicationContext(), false).executeOnExecutor(threadPoolExecutor, filesToScan);

app/src/main/java/us/spotco/malwarescanner/Utils.java

@@ -41,8 +41,8 @@ import java.util.concurrent.atomic.AtomicInteger;
 class Utils {
 
     private static Context context = null;
-    public final static int MAX_SCAN_SIZE = (1000 * 1000) * 80; //80MB
+    public final static int MAX_SCAN_SIZE = (1000 * 1000) * 500; //500MB
-    public final static int MAX_SCAN_SIZE_REALTIME = MAX_SCAN_SIZE / 2; //40MB
+    public final static int MAX_SCAN_SIZE_REALTIME = (1000 * 1000) * 250; //250MB
     public final static String DATABASE_URL_DEFAULT = "https://divested.dev/MalwareScannerSignatures/";
     public final static String SIGNING_KEY_DEFAULT = "BADFCABDDBF5B694";
 
@@ -75,8 +75,14 @@ class Utils {
 
     public static HashSet<File> getFilesRecursive(File root) {
         HashSet<File> filesAll = new HashSet<>();
-        if (root.isFile()) { //TODO: Skip this
+        try {
+            root = root.getCanonicalFile();
+        } catch (Exception ignored) {
+        }
+        if (root.isFile()) {
+            if (root.canRead() && root.length() <= MAX_SCAN_SIZE) {
                 filesAll.add(root);
+            }
             return filesAll;
         } else {
             File[] files = root.listFiles();
@@ -85,7 +91,7 @@ class Utils {
                     if (f.isDirectory()) {
                         filesAll.addAll(getFilesRecursive(f));
                     } else {
-                        if (f.length() <= MAX_SCAN_SIZE && f.canRead()) {//Exclude files larger than limit for performance
+                        if (f.isFile() && f.canRead() && f.length() <= MAX_SCAN_SIZE) {//Exclude files larger than limit for performance
                             filesAll.add(f);
                         }
                     }
@@ -177,6 +183,11 @@ class Utils {
         }
     }
 
+    public static boolean isConnectionMetered(Context context) {
+        ConnectivityManager connectivityManager = (ConnectivityManager) context.getSystemService(Context.CONNECTIVITY_SERVICE);
+        return connectivityManager != null && connectivityManager.isActiveNetworkMetered();
+    }
+
     //Credit: https://stackoverflow.com/a/4239019
     public static boolean isNetworkAvailable(Context context) {
         ConnectivityManager connectivityManager = (ConnectivityManager) context.getSystemService(Context.CONNECTIVITY_SERVICE);

app/src/main/res/values/strings.xml

@@ -53,7 +53,7 @@
     <string name="lblNoNetwork">No network connected!</string>
     <string name="self_test_result_success">Self test successful.</string>
     <string name="self_test_result_failure">Self test failed!</string>
-    <string name="lblDatabaseLoading">Database is loading, not updating!</string>
+    <string name="lblDatabaseLoading">Skipping action, database is loading!</string>
     <string name="lblDatabasesUpdated">All databases updated!</string>
     <string name="lookupVT">Lookup</string>
     <string name="deleteFile">Delete</string>
@@ -71,5 +71,8 @@
     <string name="lblSelfTest">Write self test files</string>
     <string name="lblExtendedDatabaseToggle">Extended datatabase</string>
     <string name="confirm_extended_title">Enable extended database?</string>
-    <string name="confirm_extended_summary">This will enable detection of an additional ~40 million signatures.\nThis requires a 125MB download, will slow down startup by over two minutes, will increase app RAM usage, and will increase the false positive rate.\nThis database only updates quarterly.</string>
+    <string name="confirm_extended_summary">[EXPERIMENTAL]\nThis will enable detection of an additional ~40 million signatures.\nThis requires a 125MB download, will slow down startup by over two minutes, will increase app RAM usage, and will increase the false positive rate.\nThis database only updates quarterly.</string>
+    <string name="confirm_update_title">Confirm download</string>
+    <string name="confirm_update_summary">You appear to be on a metered connection. Are you sure you want to update the databases?\nIt may download up to %s megabytes of data.</string>
+    <string name="lblUpdateRunning">Skipping action, an update is running!</string>
 </resources>

fastlane/metadata/android/en-US/changelogs/305.txt

@@ -0,0 +1,10 @@
+* Fix signature count reseting on update but not changed
+* Better handle reloads when extended DB is enabled
+* Metered connection warning
+* Block all actions when appropriate
+* Use a wakelock instead of keeping the screen on
+* Increase max scan sizes: 500MB manual, 250MB realtime
+* Always check if files can be read before scanning
+* Resolve true paths to avoid duplicates from symlinks
+* Scan more app paths
+* Scan more system paths