From eb1e565806e24debab54914e5e072583755f0ad4 Mon Sep 17 00:00:00 2001
From: Tavi <tavi@divested.dev>
Date: Fri, 12 Jan 2024 03:30:10 -0500
Subject: [PATCH] Script updates

Signed-off-by: Tavi <tavi@divested.dev>
---
 scripts/0genbloom.sh |  1 +
 scripts/Main.java    | 23 +++++++++++++++--------
 scripts/optimize.py  |  8 --------
 3 files changed, 16 insertions(+), 16 deletions(-)
 delete mode 100755 scripts/optimize.py

diff --git a/scripts/0genbloom.sh b/scripts/0genbloom.sh
index d13aade..8302bf6 100644
--- a/scripts/0genbloom.sh
+++ b/scripts/0genbloom.sh
@@ -7,4 +7,5 @@ echo "<pre>" >> production/index.html
 sed -i "s/\[DATE\]/$(date -u)/" production/index.html
 java -jar HypatiaDatabaseConverter-0.1__.jar raw/ >> production/index.html
 echo "</pre>" >> production/index.html
+cat extended.html >> production/index.html
 mv raw/hypatia-*-bloom.bin production/
diff --git a/scripts/Main.java b/scripts/Main.java
index 2c8d80c..447f20a 100644
--- a/scripts/Main.java
+++ b/scripts/Main.java
@@ -60,28 +60,35 @@ public class Main {
         signaturesSHA1 = BloomFilter.create(Funnels.stringFunnel(Charsets.US_ASCII), 50000, 0.00001); //50k
         signaturesSHA256 = BloomFilter.create(Funnels.stringFunnel(Charsets.US_ASCII), 2000000, 0.00001); //2m
 
-        try {
-            File exclusionDatabase = new File(args[0] + "../excluded.hashes");
-            if(exclusionDatabase.exists()) {
+        System.out.println("Processing exclusions:");
+        File[] exclusions = new File(args[0] + "../exclusions/").listFiles();
+        Arrays.sort(exclusions);
+        for (File exclusionDatabase : exclusions) {
+            try {
+                System.out.println("\t" + exclusionDatabase.getName());
                 Scanner s = new Scanner(exclusionDatabase);
                 while(s.hasNextLine()) {
                     String line = s.nextLine().trim().toLowerCase();
+                    if(line.contains(":")) {
+                        line = line.split(":")[0];
+                    }
                     if(!line.startsWith("#") && isHexadecimal(line) && (line.length() == 32 || line.length() == 40 || line.length() == 64)) {
                         arrExclusions.add(line);
+                        //System.out.println("\t\tAdded: " + line);
                     }
                 }
                 s.close();
-                System.out.println("Loaded " + arrExclusions.size() + " excluded hashes");
+            } catch (Exception e) {
+                e.printStackTrace();
             }
-        } catch (Exception e) {
-            e.printStackTrace();
         }
+        System.out.println("Loaded " + arrExclusions.size() + " excluded hashes");
 
-        System.out.println("Processing:");
+        System.out.println("Processing signatures:");
         File[] databases = new File(args[0]).listFiles();
         Arrays.sort(databases);
         for (File databaseLocation : databases) {
-            System.out.println("\t" + databaseLocation);
+            System.out.println("\t" + databaseLocation.getName());
             amtPreviousSignaturesMD5 = amtSignaturesAddedMD5;
             amtPreviousSignaturesSHA1 = amtSignaturesAddedSHA1;
             amtPreviousSignaturesSHA256 = amtSignaturesAddedSHA256;
diff --git a/scripts/optimize.py b/scripts/optimize.py
deleted file mode 100755
index bba9e36..0000000
--- a/scripts/optimize.py
+++ /dev/null
@@ -1,8 +0,0 @@
-import sys
-
-database = open(sys.argv[1], "r");
-for line in database:
-	arrSplit = line.strip().split(":");
-	strHash = arrSplit[0];
-	strName = arrSplit[2].split("-")[0];
-	print(strHash + ":0:" + strName);