Many changes

- Implement rewritten signature handling
- Implement rewritten database handler with support for basic remote API
- Add back Downloader

.idea/misc.xml

@@ -24,7 +24,7 @@
       </value>
     </option>
   </component>
-  <component name="ProjectRootManager" version="2" languageLevel="JDK_1_7" default="true" project-jdk-name="JDK" project-jdk-type="JavaSDK">
+  <component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" default="true" project-jdk-name="JDK" project-jdk-type="JavaSDK">
     <output url="file://$PROJECT_DIR$/build/classes" />
   </component>
   <component name="ProjectType">

app/src/main/java/us/spotco/malwarescanner/Database.java

@@ -1,4 +0,0 @@
-package us.spotco.malwarescanner;
-
-public class Database {
-}

app/src/main/java/us/spotco/malwarescanner/DatabaseManager.java

@@ -17,5 +17,127 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
 package us.spotco.malwarescanner;
 
+import android.content.Context;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileReader;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.util.HashSet;
+import java.util.Scanner;
+import java.util.zip.GZIPInputStream;
+
 public class DatabaseManager {
+
+    public static File databasePath = null;
+    public final static HashSet<Signatures.SignatureDatabase> signatureDatabases = new HashSet<>();
+
+    private Context context;
+
+    public DatabaseManager(Context context) {
+        this.context = context;
+        databasePath = new File(context.getFilesDir() + "/signatures/");
+        databasePath.mkdir();
+    }
+
+    public static boolean areDatabasesAvailable() {
+        return databasePath != null && databasePath.listFiles().length > 0;
+    }
+
+    public static boolean isDatabaseAvailable(String name) {
+        return new File(databasePath + "/"  + name).exists();
+    }
+
+    public static void deteleDatabase(String name) {
+        new File(databasePath + "/"  + name).delete();
+    }
+
+    public void loadRemoteDatabases() {
+        try {
+            signatureDatabases.clear();
+            String api = Utils.prefs.getString("database_repo", "https://spotco.us/MalwareScannerSignatures/api.php");
+            HttpURLConnection connection = Utils.getConnection(api);
+            connection.connect();
+            int res = connection.getResponseCode();
+            if (res == 200 || res == 301 || res == 302) {
+                Scanner request = new Scanner(connection.getErrorStream(), connection.getContentEncoding());
+                String line;
+                while (request.hasNextLine()) {
+                    line = request.nextLine();
+                    String[] items = line.split("~~");
+                    Signatures.SignatureDatabase newDatabase = new Signatures.SignatureDatabase(items[0], items[1], false);
+                    newDatabase.setAvailable(isDatabaseAvailable(items[0]));
+                    signatureDatabases.add(newDatabase);
+                }
+            }
+            connection.disconnect();
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    public void updateDatabases() {
+        loadRemoteDatabases();
+        for (Signatures.SignatureDatabase db : signatureDatabases) {
+            if (db.isAvailable()) {
+                new DownloaderTask().execute(db.getUrl(), databasePath + "/" + db.getName());
+            }
+        }
+    }
+
+    public void loadAllAvailableDatabases(boolean forceReload) {
+        if (!Signatures.available() || forceReload) {
+            Signatures.clear();
+            for (Signatures.SignatureDatabase db : signatureDatabases) {
+                if (db.isAvailable()) {
+                    File dbLocation = new File(databasePath + "/" + db.getName());
+                    loadDatabase(dbLocation);
+                }
+            }
+        }
+    }
+
+    private void loadDatabase(File database) {
+        try {
+            if (database.exists()) {
+                BufferedReader reader;
+                if (database.getName().endsWith(".gz")) {
+                    reader = new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(database))));
+                } else {
+                    reader = new BufferedReader(new FileReader(database));
+                }
+                String line;
+                boolean trim = Utils.prefs.getBoolean("database_trim_variants", false);
+                int maxLength = Utils.prefs.getInt("database_hash_length", 12);
+
+                if (database.getName().contains(".hdb")) {//.hdb format: md5, size, name
+                    while ((line = reader.readLine()) != null) {
+                        String[] lineS = line.split(":");
+                        if (trim) {
+                            lineS[2] = lineS[2].split("-")[0];
+                        }
+                        Signatures.MD5.put(lineS[0].substring(0, maxLength), lineS[2]);
+                    }
+                } else if (database.getName().contains(".hsb")) {//.hsb format: sha256, size, name
+                    while ((line = reader.readLine()) != null) {
+                        String[] lineS = line.split(":");
+                        if (trim) {
+                            lineS[2] = lineS[2].split("-")[0];
+                        }
+                        if (lineS[0].length() == 32) {
+                            Signatures.SHA1.put(lineS[0].substring(0, maxLength), lineS[2]);
+                        } else {
+                            Signatures.SHA256.put(lineS[0].substring(0, maxLength), lineS[2]);
+                        }
+                    }
+                }
+                reader.close();
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
 }

app/src/main/java/us/spotco/malwarescanner/DownloaderTask.java

@@ -0,0 +1,69 @@
+/*
+Hypatia: An realtime malware scanner for Android
+Copyright (c) 2017-2018 Divested Computing, Inc.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.
+*/
+package us.spotco.malwarescanner;
+
+import android.os.AsyncTask;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.net.HttpURLConnection;
+
+public class DownloaderTask extends AsyncTask<Object, String, String> {
+
+    @Override
+    protected String doInBackground(Object... objects) {
+        String url = (String) objects[0];
+        File out = new File((String) objects[1]);
+        publishProgress("Downloading " + url);
+        try {
+            HttpURLConnection connection = Utils.getConnection(url);
+            if (out.exists()) {
+                connection.setIfModifiedSince(out.lastModified());
+            }
+            connection.connect();
+            int res = connection.getResponseCode();
+            if (res != 304) {
+                if (res == 200) {
+                    if (out.exists()) {
+                        out.delete();
+                    }
+                    FileOutputStream fileOutputStream = new FileOutputStream(out);
+
+                    final byte data[] = new byte[1024];
+                    int count;
+                    while ((count = connection.getInputStream().read(data, 0, 1024)) != -1) {
+                        fileOutputStream.write(data, 0, count);
+                    }
+
+                    fileOutputStream.close();
+                    publishProgress("Successfully downloaded\n");
+                } else {
+                    publishProgress("File not downloaded, response code " + res + "\n");
+                }
+            } else {
+                publishProgress("File not changed\n");
+            }
+            connection.disconnect();
+        } catch (Exception e) {
+            e.printStackTrace();
+            out.delete();
+            publishProgress("Failed to download, check logcat\n");
+        }
+        return null;
+    }
+}

app/src/main/java/us/spotco/malwarescanner/MainActivity.java

@@ -47,8 +47,11 @@ public class MainActivity extends FragmentActivity {
         fragmentManager.beginTransaction().add(R.id.main_fragment, logFragment, "History").hide(logFragment).commit();
         fragmentManager.beginTransaction().add(R.id.main_fragment, databaseFragment, "Databases").hide(databaseFragment).commit();
         fragmentManager.beginTransaction().add(R.id.main_fragment, settingsFragment, "Settings").hide(settingsFragment).commit();
-        fragmentManager.beginTransaction().add(R.id.main_fragment, scannerFragment, "Scanner").commit();
+        fragmentManager.beginTransaction().add(R.id.main_fragment, scannerFragment, "MalwareScanner").commit();
         activeFragment = scannerFragment;
+
+        new Utils(this);
+        new DatabaseManager(this);
     }
 
     private BottomNavigationView.OnNavigationItemSelectedListener mOnNavigationItemSelectedListener

app/src/main/java/us/spotco/malwarescanner/MalwareScanner.java

@@ -17,5 +17,5 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
 package us.spotco.malwarescanner;
 
-public class Scanner {
+public class MalwareScanner {
 }

app/src/main/java/us/spotco/malwarescanner/MalwareScannerService.java

@@ -17,5 +17,5 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
 package us.spotco.malwarescanner;
 
-public class ScannerService {
+public class MalwareScannerService {
 }

app/src/main/java/us/spotco/malwarescanner/Signatures.java

@@ -0,0 +1,73 @@
+/*
+Hypatia: An realtime malware scanner for Android
+Copyright (c) 2017-2018 Divested Computing, Inc.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.
+*/
+package us.spotco.malwarescanner;
+
+import java.util.HashMap;
+
+public class Signatures {
+
+
+    public final static HashMap<String, String> MD5 = new HashMap<>();
+    public final static HashMap<String, String> SHA1 = new HashMap<>();
+    public final static HashMap<String, String> SHA256 = new HashMap<>();
+
+    public static boolean available() {
+        return MD5.size() > 0 && SHA1.size() > 0 && SHA256.size() > 0;
+    }
+
+    public static int getSignatureCount() {
+        return MD5.size() + SHA1.size() + SHA256.size();
+    }
+
+    public static void clear() {
+        MD5.clear();
+        SHA1.clear();
+        SHA256.clear();
+    }
+
+    public static class SignatureDatabase {
+
+        private String name = null;
+        private String url = null;
+        private boolean available = false;
+
+        public SignatureDatabase(String name, String url, boolean available) {
+            this.name = name;
+            this.url = url;
+            this.available = available;
+        }
+
+        public final String getName() {
+            return name;
+        }
+
+        public final String getUrl() {
+            return url;
+        }
+
+        public final boolean isAvailable() {
+            return available;
+        }
+
+        public final void setAvailable(boolean available) {
+            this.available = available;
+        }
+
+    }
+
+}

app/src/main/java/us/spotco/malwarescanner/Utils.java

@@ -24,14 +24,18 @@ import android.content.SharedPreferences;
 import android.content.pm.PackageManager;
 
 import java.io.File;
+import java.net.HttpURLConnection;
+import java.net.InetSocketAddress;
+import java.net.Proxy;
 import java.net.Socket;
+import java.net.URL;
 import java.util.HashSet;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ThreadPoolExecutor;
 
 public class Utils {
 
-    private static SharedPreferences prefs = null;
+    public static SharedPreferences prefs = null;
 
     public Utils(Context context) {
         prefs = context.getSharedPreferences(BuildConfig.APPLICATION_ID, Context.MODE_PRIVATE);
@@ -141,4 +145,24 @@ public class Utils {
         }
         return listening;
     }
+
+    public static HttpURLConnection getConnection(String url) {
+        try {
+            HttpURLConnection connection;
+            if (prefs.getBoolean("network_use_tor", false)) {
+                waitUntilOrbotIsAvailable();
+                Proxy orbot = new Proxy(Proxy.Type.SOCKS, new InetSocketAddress("127.0.0.1", 9050));
+                connection = (HttpURLConnection) new URL(url).openConnection(orbot);
+            } else {
+                connection = (HttpURLConnection) new URL(url).openConnection();
+            }
+            connection.setConnectTimeout(90000);
+            connection.setReadTimeout(30000);
+            connection.addRequestProperty("User-Agent", "Hypatia");
+            return connection;
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+        return null;
+    }
 }

app/src/main/res/values/strings.xml

@@ -35,5 +35,7 @@
     <string name="settings_category_other">Other</string>
     <string name="settings_pref_other_third_party">Enable third party access</string>
     <string name="settings_pref_other_third_party_summary">Allow other apps to invoke the scanner on objects</string>
+    <string name="settings_pref_network_database_repo">Database repo</string>
+    <string name="settings_pref_network_database_repo_summary">API to request databases from</string>
     <!-- END SettingsFragment -->
 </resources>

app/src/main/res/xml/fragment_settings.xml

@@ -46,6 +46,12 @@
     </PreferenceCategory>
 
     <PreferenceCategory android:title="@string/settings_category_network">
+        <EditTextPreference
+            android:key="database_repo"
+            android:title="@string/settings_pref_network_database_repo"
+            android:summary="@string/settings_pref_network_database_repo_summary"
+            android:defaultValue="https://spotco.us/MalwareScannerSignatures/api.php"
+            android:inputType="number" />
         <CheckBoxPreference
             android:key="network_use_tor"
             android:title="@string/settings_pref_network_tor"