From b82476ba9f996095c149a8b4b3287adc879d0114 Mon Sep 17 00:00:00 2001
From: Tad <tad@spotco.us>
Date: Wed, 21 Jul 2021 13:44:19 -0400
Subject: [PATCH] Update conversion scripts

---
 scripts/{clamav.sh => 0clamav.sh} | 16 +++++++++++---
 scripts/{eset.sh => 0eset.sh}     |  1 +
 scripts/0targetedthreats.sh       | 35 +++++++++++++++++++++++++++++++
 3 files changed, 49 insertions(+), 3 deletions(-)
 rename scripts/{clamav.sh => 0clamav.sh} (60%)
 rename scripts/{eset.sh => 0eset.sh} (84%)
 create mode 100644 scripts/0targetedthreats.sh

diff --git a/scripts/clamav.sh b/scripts/0clamav.sh
similarity index 60%
rename from scripts/clamav.sh
rename to scripts/0clamav.sh
index 22118f3..b8e3a2e 100644
--- a/scripts/clamav.sh
+++ b/scripts/0clamav.sh
@@ -1,13 +1,15 @@
 #!/bin/sh
-#License: GPLv3
+#License: GPL-3.0
+#Description: Hypatia conversion script for ClamAV databases (GPL-2.0)
 
 #sudo freshclam
 mkdir /tmp/mss
+mkdir /tmp/mss/processed
 cd /tmp/mss
 cp /var/lib/clamav/main.cvd .
-cp /var/lib/clamav/daily.cld .
+cp /var/lib/clamav/daily.cvd .
 sigtool -u main.cvd
-sigtool -u daily.cld
+sigtool -u daily.cvd
 
 #MD5
 grep "Andr\\." main.hdb >> Android.hdb
@@ -25,5 +27,13 @@ grep "Unix\\." daily.hsb >> Android.hsb
 grep "Multios\\." main.hsb >> Android.hsb
 grep "Multios\\." daily.hsb >> Android.hsb
 
+databases=("Android.hdb" "Android.hsb" "main.hdb" "main.hsb" "daily.hdb" "daily.hsb");
+for db in "${databases[@]}"
+do
+	sort --parallel=$(nproc) --unique "$db" --output processed/"$db";
+done;
+
 gzip /tmp/mss/*.hdb
 gzip /tmp/mss/*.hsb
+gzip /tmp/mss/processed/*.hdb
+gzip /tmp/mss/processed/*.hsb
diff --git a/scripts/eset.sh b/scripts/0eset.sh
similarity index 84%
rename from scripts/eset.sh
rename to scripts/0eset.sh
index 74934ce..d6774f4 100644
--- a/scripts/eset.sh
+++ b/scripts/0eset.sh
@@ -1,5 +1,6 @@
 #!/bin/bash
 #License: GPLv3
+#Description: Hypatia conversion script for https://github.com/eset/malware-ioc (BSD-2-Clause)
 
 processHashes() {
 	local name=$(basename $1);
diff --git a/scripts/0targetedthreats.sh b/scripts/0targetedthreats.sh
new file mode 100644
index 0000000..6a5915d
--- /dev/null
+++ b/scripts/0targetedthreats.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+#License: GPLv3
+#Description: Hypatia conversion script for https://github.com/botherder/targetedthreats (CC BY-SA 4.0)
+
+while IFS=, read -r col1MD5 col2SHA256 col3Family col4Actor col5Country col6Report
+do
+	col4Actor=$(sed -e 's/^"//' -e 's/"$//' <<< "$col4Actor");
+	col3Family=$(sed -e 's/^"//' -e 's/"$//' <<< "$col3Family");
+	if [ -n "$col4Actor" ] && [ -n "$col3Family" ]; then
+		description="$col4Actor-$col3Family";
+	elif [ -n "$col4Actor" ] && [ -z "$col3Family" ]; then
+		description="$col4Actor";
+	elif [ -z "$col4Actor" ] && [ -n "$col3Family" ]; then
+		description="$col3Family";
+	fi;
+	if [ -z "$description" ]; then
+		description="targetedthreats";
+	fi;
+
+	col1MD5=$(sed -e 's/^"//' -e 's/"$//' <<< "$col1MD5");
+	if [ -n "$col1MD5" ]; then
+		echo "$col1MD5:0:$description" >> ./targetedthreats.hdb;
+	fi;
+
+	col2SHA256=$(sed -e 's/^"//' -e 's/"$//' <<< "$col2SHA256");
+	if [ -n "$col2SHA256" ]; then
+		echo "$col2SHA256:0:$description" >> ./targetedthreats.hsb;
+	fi;
+done < samples.csv;
+
+sed -i '1d' targetedthreats.hdb;
+sed -i '1d' targetedthreats.hsb;
+
+gzip targetedthreats.hdb;
+gzip targetedthreats.hsb;