MaintainTeam/Hypatia
1
0
Fork 0
mirror of https://github.com/MaintainTeam/Hypatia.git synced 2025-03-01 05:48:23 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

Hash all files in one pass, dramatic speedup

Browse source
This commit is contained in:
Tad 2017-12-14 21:10:51 -05:00
parent b663a11ee9
commit 8199d43c6d
3 changed files with 56 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.idea/misc.xml generated
View file

@ -24,7 +24,7 @@
</value> </value>
</option> </option>
</component> </component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" default="true" project-jdk-name="JDK" project-jdk-type="JavaSDK"> <component name="ProjectRootManager" version="2" languageLevel="JDK_1_7" default="true" project-jdk-name="JDK" project-jdk-type="JavaSDK">
<output url="file://$PROJECT_DIR$/build/classes" /> <output url="file://$PROJECT_DIR$/build/classes" />
</component> </component>
<component name="ProjectType"> <component name="ProjectType">

12
app/src/main/java/us/spotco/malwarescanner/Database.java
View file

@ -33,9 +33,9 @@ public class Database {
signatureDatabases.add(new SignatureDatabase("https://spotco.us/clamav-main.hdb", "clamav-main.hdb")); signatureDatabases.add(new SignatureDatabase("https://spotco.us/clamav-main.hdb", "clamav-main.hdb"));
signatureDatabases.add(new SignatureDatabase("https://spotco.us/clamav-main.hsb", "clamav-main.hsb")); signatureDatabases.add(new SignatureDatabase("https://spotco.us/clamav-main.hsb", "clamav-main.hsb"));
signatureDatabases.add(new SignatureDatabase("http://clamav.bofhland.org/bofhland_malware_attach.hdb", "bofhland_malware_attach.hdb")); //signatureDatabases.add(new SignatureDatabase("http://clamav.bofhland.org/bofhland_malware_attach.hdb", "bofhland_malware_attach.hdb"));
signatureDatabases.add(new SignatureDatabase("http://cdn.malware.expert/malware.expert.hdb", "malware.expert.hdb")); //signatureDatabases.add(new SignatureDatabase("http://cdn.malware.expert/malware.expert.hdb", "malware.expert.hdb"));
signatureDatabases.add(new SignatureDatabase("http://cdn.rfxn.com/downloads/rfxn.hdb", "rfxn.hdb")); //signatureDatabases.add(new SignatureDatabase("http://cdn.rfxn.com/downloads/rfxn.hdb", "rfxn.hdb"));
} }
public static boolean doesDatabaseExist() { public static boolean doesDatabaseExist() {
@ -49,9 +49,9 @@ public class Database {
} }
public static void loadDatabase(ArrayList<SignatureDatabase> signatureDatabases) { public static void loadDatabase(ArrayList<SignatureDatabase> signatureDatabases) {
signaturesMD5 = new HashMap<String, String>(); signaturesMD5.clear();
signaturesSHA1 = new HashMap<String, String>(); signaturesSHA1.clear();
signaturesSHA256 = new HashMap<String, String>(); signaturesSHA256.clear();
System.gc(); System.gc();
for (SignatureDatabase database : signatureDatabases) { for (SignatureDatabase database : signatureDatabases) {
File databaseLocation = new File(databasePath + "/" + database.getName()); File databaseLocation = new File(databasePath + "/" + database.getName());

72
app/src/main/java/us/spotco/malwarescanner/MalwareScanner.java
View file

@ -12,7 +12,11 @@ import com.google.common.hash.Hashing;
import com.google.common.io.Files; import com.google.common.io.Files;
import java.io.File; import java.io.File;
import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream;
import java.math.BigInteger;
import java.security.MessageDigest;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.Map; import java.util.Map;
@ -20,16 +24,20 @@ import java.util.Set;
public class MalwareScanner { public class MalwareScanner {
private Context context = null; private static Context context = null;
private TextView log = null; private static TextView log = null;
private AsyncTask<Boolean, String, String> malwareScannerTask = null; private static AsyncTask<Boolean, String, String> malwareScannerTask = null;
private static HashMap<String, File> fileHashesMD5 = new HashMap<String, File>();
private static HashMap<String, File> fileHashesSHA1 = new HashMap<String, File>();
private static HashMap<String, File> fileHashesSHA256 = new HashMap<String, File>();
public MalwareScanner(Context context, TextView log) { public MalwareScanner(Context context, TextView log) {
this.context = context; this.context = context;
this.log = log; this.log = log;
} }
public void startScanner(boolean scanSystem, boolean scanApps, boolean scanInternal, boolean scanExternal) { public static void startScanner(boolean scanSystem, boolean scanApps, boolean scanInternal, boolean scanExternal) {
if(Database.doesDatabaseExist()) { if(Database.doesDatabaseExist()) {
malwareScannerTask = new MalwareScannerTask().execute(scanSystem, scanApps, scanInternal, scanExternal); malwareScannerTask = new MalwareScannerTask().execute(scanSystem, scanApps, scanInternal, scanExternal);
} else { } else {
@ -48,11 +56,14 @@ public class MalwareScanner {
return malwareScannerTask.getStatus().equals(AsyncTask.Status.PENDING) || malwareScannerTask.getStatus().equals(AsyncTask.Status.RUNNING); return malwareScannerTask.getStatus().equals(AsyncTask.Status.PENDING) || malwareScannerTask.getStatus().equals(AsyncTask.Status.RUNNING);
} }
public class MalwareScannerTask extends AsyncTask<Boolean, String, String> { public static class MalwareScannerTask extends AsyncTask<Boolean, String, String> {
@Override @Override
protected void onPreExecute() { protected void onPreExecute() {
log.append("Initiating scan...\n"); log.append("Initiating scan...\n");
fileHashesMD5.clear();
fileHashesSHA1.clear();
fileHashesSHA256.clear();
} }
@Override @Override
@ -85,20 +96,19 @@ public class MalwareScanner {
} }
publishProgress(filesToScan.size() + " files pending scan"); publishProgress(filesToScan.size() + " files pending scan");
HashMap<String, File> fileHashesMD5 = new HashMap<String, File>();
HashMap<String, File> fileHashesSHA1 = new HashMap<String, File>();
HashMap<String, File> fileHashesSHA256 = new HashMap<String, File>();
Database.loadDatabase(Database.signatureDatabases); Database.loadDatabase(Database.signatureDatabases);
publishProgress("Loaded " + Database.signaturesMD5.size() + " MD5 signatures"); publishProgress("Loaded " + Database.signaturesMD5.size() + " MD5 signatures");
publishProgress("Loaded " + Database.signaturesSHA1.size() + " SHA-1 signatures"); publishProgress("Loaded " + Database.signaturesSHA1.size() + " SHA-1 signatures");
publishProgress("Loaded " + Database.signaturesSHA256.size() + " SHA-256 signatures\n"); publishProgress("Loaded " + Database.signaturesSHA256.size() + " SHA-256 signatures\n");
if (Database.signaturesMD5.size() > 0) { publishProgress("Hashing files...");
for (File file : filesToScan) { for (File file : filesToScan) {
fileHashesMD5.put(getFileHashSum(file, Hashing.md5()), file); getFileHashes(file);
} }
publishProgress("Calculated MD5 hashes for all files"); publishProgress("Calculated MD5/SHA-1/SHA-256 hashes for all files\n");
if (Database.signaturesMD5.size() > 0) {
for (Map.Entry<String, File> file : fileHashesMD5.entrySet()) { for (Map.Entry<String, File> file : fileHashesMD5.entrySet()) {
String result = Database.checkInDatabase(file.getKey()); String result = Database.checkInDatabase(file.getKey());
if (result != null) { if (result != null) {
@ -111,10 +121,6 @@ public class MalwareScanner {
} }
if (Database.signaturesSHA1.size() > 0) { if (Database.signaturesSHA1.size() > 0) {
for (File file : filesToScan) {
fileHashesSHA1.put(getFileHashSum(file, Hashing.sha1()), file);
}
publishProgress("Calculated SHA-1 hashes for all files");
for (Map.Entry<String, File> file : fileHashesSHA1.entrySet()) { for (Map.Entry<String, File> file : fileHashesSHA1.entrySet()) {
String result = Database.checkInDatabase(file.getKey()); String result = Database.checkInDatabase(file.getKey());
if (result != null) { if (result != null) {
@ -127,9 +133,6 @@ public class MalwareScanner {
} }
if (Database.signaturesSHA256.size() > 0) { if (Database.signaturesSHA256.size() > 0) {
for (File file : filesToScan) {
fileHashesSHA256.put(getFileHashSum(file, Hashing.sha256()), file);
}
publishProgress("Calculated SHA-256 hashes for all files"); publishProgress("Calculated SHA-256 hashes for all files");
for (Map.Entry<String, File> file : fileHashesSHA256.entrySet()) { for (Map.Entry<String, File> file : fileHashesSHA256.entrySet()) {
String result = Database.checkInDatabase(file.getKey()); String result = Database.checkInDatabase(file.getKey());
@ -159,7 +162,7 @@ public class MalwareScanner {
} }
private Set<File> getFilesRecursive(File root) { private static Set<File> getFilesRecursive(File root) {
Set<File> filesAll = new HashSet<>(); Set<File> filesAll = new HashSet<>();
File[] files = root.listFiles(); File[] files = root.listFiles();
@ -179,13 +182,34 @@ public class MalwareScanner {
return filesAll; return filesAll;
} }
private String getFileHashSum(File file, HashFunction hash) { private static void getFileHashes(File file) {
try { try {
return Files.asByteSource(file).hash(hash).toString(); InputStream fis = new FileInputStream(file);
} catch (IOException e) {
byte[] buffer = new byte[4096];
int numRead;
MessageDigest digestMD5 = MessageDigest.getInstance("MD5");
MessageDigest digestSHA1 = MessageDigest.getInstance("SHA-1");
MessageDigest digestSHA256 = MessageDigest.getInstance("SHA-256");
do {
numRead = fis.read(buffer);
if(numRead > 0) {
digestMD5.update(buffer, 0, numRead);
digestSHA1.update(buffer, 0, numRead);
digestSHA256.update(buffer, 0, numRead);
}
} while (numRead != -1);
fis.close();
fileHashesMD5.put(String.format("%032x", new BigInteger(1, digestMD5.digest())), file);
fileHashesSHA1.put(String.format("%032x", new BigInteger(1, digestSHA1.digest())), file);
fileHashesSHA256.put(String.format( "%064x", new BigInteger(1, digestSHA256.digest())), file);
} catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
return null;
} }
} }