MaintainTeam/Hypatia
1
0
Fork 0
mirror of https://github.com/MaintainTeam/Hypatia.git synced 2025-02-28 21:38:21 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

Script updates

Browse source 
Signed-off-by: Tavi <tavi@divested.dev>
This commit is contained in:
Tavi 2024-05-09 08:05:52 -04:00
parent 4a8013921c
commit 5f0899a87e
No known key found for this signature in database
GPG key ID: E599F62ECBAEAF2E
3 changed files with 56 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
scripts/0clamav.sh
View file

@ -11,6 +11,8 @@ cp /var/lib/clamav/daily.c*d .
sigtool -u main.c*d sigtool -u main.c*d
sigtool -u daily.c*d sigtool -u daily.c*d
mv /tmp/mss/*.fp* exclusions/ mv /tmp/mss/*.fp* "$origDir/exclusions/"
mv /tmp/mss/*.hsb raw/ mv /tmp/mss/*.hsb "$origDir/raw/"
mv /tmp/mss/*.hdb raw/ mv /tmp/mss/*.hdb "$origDir/raw/"
mv /tmp/mss/*.hdu "$origDir/raw/"
mv /tmp/mss/*.hsu "$origDir/raw/"

17
scripts/0sanesecurity.sh Normal file
View file

@ -0,0 +1,17 @@
cd sanesecurity-real/
#rsync -av rsync://rsync.sanesecurity.net/sanesecurity .
for f in *.hsb; do
echo "" >> "../raw/sanesecurity-$f";
cat "$f" >> "../raw/sanesecurity-$f";
sort -u -o "../raw/sanesecurity-$f" "../raw/sanesecurity-$f";
done
for f in *.hdb; do
echo "" >> "../raw/sanesecurity-$f";
cat "$f" >> "../raw/sanesecurity-$f";
sort -u -o "../raw/sanesecurity-$f" "../raw/sanesecurity-$f";
done
rm -v ../raw/sanesecurity-crdfam.clamav.hdb ../raw/sanesecurity-doppelstern.hdb ../raw/sanesecurity-malware.expert.hdb

39
scripts/Main.java
View file

@ -60,9 +60,12 @@ public class Main {
public static void main(String[] args) { public static void main(String[] args) {
//isFileInNsrl("B61905308B336AD268A782790B661616"); //isFileInNsrl("B61905308B336AD268A782790B661616");
signaturesMD5 = BloomFilter.create(Funnels.stringFunnel(Charsets.US_ASCII), 6000000, 0.00001); //6m int amtMaxMD5 = 7000000; //7m
signaturesSHA1 = BloomFilter.create(Funnels.stringFunnel(Charsets.US_ASCII), 50000, 0.00001); //50k int amtMaxSHA1 = 50000; //50k
signaturesSHA256 = BloomFilter.create(Funnels.stringFunnel(Charsets.US_ASCII), 2000000, 0.00001); //2m int amtMaxSHA256 = 2000000; //2m
signaturesMD5 = BloomFilter.create(Funnels.stringFunnel(Charsets.US_ASCII), amtMaxMD5, 0.00001);
signaturesSHA1 = BloomFilter.create(Funnels.stringFunnel(Charsets.US_ASCII), amtMaxSHA1, 0.00001);
signaturesSHA256 = BloomFilter.create(Funnels.stringFunnel(Charsets.US_ASCII), amtMaxSHA256, 0.00001);
System.out.println("Processing exclusions:"); System.out.println("Processing exclusions:");
File[] exclusions = new File(args[0] + "../exclusions/").listFiles(); File[] exclusions = new File(args[0] + "../exclusions/").listFiles();
@ -110,7 +113,9 @@ public class Main {
} }
String line; String line;
if (databaseLocation.getName().endsWith(".hdb") //.hdb/.hsb format: hash:size:name:version if (databaseLocation.getName().endsWith(".hdb") //.hdb/.hsb format: hash:size:name:version
|| databaseLocation.getName().endsWith(".hsb")) { || databaseLocation.getName().endsWith(".hsb")
|| databaseLocation.getName().endsWith(".hdu")
|| databaseLocation.getName().endsWith(".hsu")) {
while ((line = reader.readLine()) != null) { while ((line = reader.readLine()) != null) {
if (line.length() > 0 && line.contains(":")) { if (line.length() > 0 && line.contains(":")) {
String[] lineS = line.trim().toLowerCase().split(":"); String[] lineS = line.trim().toLowerCase().split(":");
@ -147,8 +152,32 @@ public class Main {
System.out.println("Lines read: valid: " + amtLinesValid + ", invalid: " + amtLinesInvalid); System.out.println("Lines read: valid: " + amtLinesValid + ", invalid: " + amtLinesInvalid);
System.out.println("Read count: md5: " + amtSignaturesReadMD5 + ", sha1: " + amtSignaturesReadSHA1 + ", sha256: " + amtSignaturesReadSHA256); System.out.println("Read count: md5: " + amtSignaturesReadMD5 + ", sha1: " + amtSignaturesReadSHA1 + ", sha256: " + amtSignaturesReadSHA256);
System.out.println("Added count: md5: " + amtSignaturesAddedMD5 + ", sha1: " + amtSignaturesAddedSHA1 + ", sha256: " + amtSignaturesAddedSHA256); System.out.println("Added count: md5: " + amtSignaturesAddedMD5 + ", sha1: " + amtSignaturesAddedSHA1 + ", sha256: " + amtSignaturesAddedSHA256);
System.out.println("Max amount: md5: " + amtMaxMD5 + ", sha1: " + amtMaxSHA1 + ", sha256: " + amtMaxSHA256);
System.out.println("Fill amount: md5: " + ((100F/amtMaxMD5) * amtSignaturesAddedMD5) + "%, sha1: " + ((100F/amtMaxSHA1) * amtSignaturesAddedSHA1) + "%, sha256: " + ((100F/amtMaxSHA256) * amtSignaturesAddedSHA256));
System.out.println("Approximate count: md5: " + signaturesMD5.approximateElementCount() + ", sha1: " + signaturesSHA1.approximateElementCount() + ", sha256: " + signaturesSHA256.approximateElementCount()); System.out.println("Approximate count: md5: " + signaturesMD5.approximateElementCount() + ", sha1: " + signaturesSHA1.approximateElementCount() + ", sha256: " + signaturesSHA256.approximateElementCount());
System.out.println("App reported count: " + (signaturesMD5.approximateElementCount() + signaturesSHA1.approximateElementCount() + signaturesSHA256.approximateElementCount()));
System.out.println("Expected false postive rate: md5: " + signaturesMD5.expectedFpp() + ", sha1: " + signaturesSHA1.expectedFpp() + ", sha256: " + signaturesSHA256.expectedFpp()); System.out.println("Expected false postive rate: md5: " + signaturesMD5.expectedFpp() + ", sha1: " + signaturesSHA1.expectedFpp() + ", sha256: " + signaturesSHA256.expectedFpp());
System.out.println("Testing exclusions:");
int matchedExclusions = 0;
for(String excluded : arrExclusions) {
if(excluded.length() == 32 && signaturesMD5.mightContain(excluded)) {
System.out.println("\tmd5: Found excluded hash " + excluded);
matchedExclusions++;
}
if(excluded.length() == 40 && signaturesSHA1.mightContain(excluded)) {
System.out.println("\tsha1: Found excluded hash " + excluded);
matchedExclusions++;
}
if(excluded.length() == 64 && signaturesSHA256.mightContain(excluded)) {
System.out.println("\tsha256: Found excluded hash " + excluded);
matchedExclusions++;
}
}
if(matchedExclusions == 0) {
System.out.println("\tNo exclusions found :)");
} else {
System.out.println("\tExclusions were found!");
}
try { try {
FileOutputStream fileSignaturesMD5 = new FileOutputStream(new File(args[0]) + "/hypatia-md5-bloom.bin"); FileOutputStream fileSignaturesMD5 = new FileOutputStream(new File(args[0]) + "/hypatia-md5-bloom.bin");
signaturesMD5.writeTo(fileSignaturesMD5); signaturesMD5.writeTo(fileSignaturesMD5);
@ -181,7 +210,7 @@ public class Main {
System.out.println("\t\tSkipping excluded hash: " + potentialHash); System.out.println("\t\tSkipping excluded hash: " + potentialHash);
return; return;
} }
//if(isFileInNsrl(potentialHash)) { //if(isFileInNsrl(potentialHash)) {
// return; // return;
//} //}
if (potentialHash.length() == 32) { if (potentialHash.length() == 32) {