diff --git a/scripts/0avast-covid19.sh b/scripts/0avast-covid19.sh new file mode 100644 index 0000000..9ca11fe --- /dev/null +++ b/scripts/0avast-covid19.sh @@ -0,0 +1,5 @@ +#!/bin/sh +#License: CC0 +#Description: Hypatia conversion script for https://github.com/avast/covid-19-ioc + +tail -n +2 */*.csv | sed 's/,/ , /' | awk '{ print $1 }' | sort -u >> avast-covid19.sha256 diff --git a/scripts/0clamav.sh b/scripts/0clamav.sh index df87c9d..15a7433 100644 --- a/scripts/0clamav.sh +++ b/scripts/0clamav.sh @@ -1,48 +1,12 @@ #!/bin/sh -#License: GPL-3.0 +#License: CC0 #Description: Hypatia conversion script for ClamAV databases (GPL-2.0) #sudo -i freshclam origDir="$PWD" mkdir /tmp/mss -mkdir /tmp/mss/optimized -mkdir /tmp/mss/processed cd /tmp/mss cp /var/lib/clamav/main.c*d . cp /var/lib/clamav/daily.c*d . sigtool -u main.c*d sigtool -u daily.c*d - -#MD5 -grep "Andr\\." main.hdb >> Android.hdb -grep "Andr\\." daily.hdb >> Android.hdb -#grep "Java\\." main.hdb >> Android.hdb -#grep "Java\\." daily.hdb >> Android.hdb -grep "Unix\\." main.hdb >> Android.hdb -grep "Unix\\." daily.hdb >> Android.hdb -grep "Multios\\." main.hdb >> Android.hdb -grep "Multios\\." daily.hdb >> Android.hdb - -#SHA -grep "Andr\\." main.hsb >> Android.hsb -grep "Andr\\." daily.hsb >> Android.hsb -#grep "Java\\." main.hsb >> Android.hsb -#grep "Java\\." daily.hsb >> Android.hsb -grep "Unix\\." main.hsb >> Android.hsb -grep "Unix\\." daily.hsb >> Android.hsb -grep "Multios\\." main.hsb >> Android.hsb -grep "Multios\\." daily.hsb >> Android.hsb - -databases=("Android.hdb" "Android.hsb" "main.hdb" "main.hsb" "daily.hdb" "daily.hsb"); -for db in "${databases[@]}" -do - #remove unnecessary bits to reduce file size and app memory usage - python "$origDir"/optimize.py "$db" >> optimized/"$db"; - #sort to increase compression efficiency - sort -k3 -t ":" --parallel=$(nproc) --output processed/"$db" optimized/"$db"; -done; - -gzip -k /tmp/mss/*.hdb -gzip -k /tmp/mss/*.hsb -gzip -k /tmp/mss/processed/*.hdb -gzip -k /tmp/mss/processed/*.hsb diff --git a/scripts/0eset.sh b/scripts/0eset.sh index bd82e44..fd8ab1c 100644 --- a/scripts/0eset.sh +++ b/scripts/0eset.sh @@ -1,5 +1,5 @@ #!/bin/bash -#License: GPLv3 +#License: CC0 #Description: Hypatia conversion script for https://github.com/eset/malware-ioc (BSD-2-Clause) processHashes() { diff --git a/scripts/0genbloom.sh b/scripts/0genbloom.sh new file mode 100644 index 0000000..d13aade --- /dev/null +++ b/scripts/0genbloom.sh @@ -0,0 +1,10 @@ +#!/bin/sh +#License: CC0 + +rm production/index.html +cp template.html production/index.html +echo "
" >> production/index.html +sed -i "s/\[DATE\]/$(date -u)/" production/index.html +java -jar HypatiaDatabaseConverter-0.1__.jar raw/ >> production/index.html +echo "" >> production/index.html +mv raw/hypatia-*-bloom.bin production/ diff --git a/scripts/0sign.sh b/scripts/0sign.sh deleted file mode 100644 index 95db9f9..0000000 --- a/scripts/0sign.sh +++ /dev/null @@ -1,14 +0,0 @@ -for database in *.bin -do - if [ -f "$database.sig" ]; then - #If it does exist sign if it doesn't match - if ! gpg --verify "$database.sig"; then - rm "$database.sig"; - gpg --sign --local-user 6395FC9911EDCD6158712DF7BADFCABDDBF5B694 --detach-sign "$database"; - fi; - else - #Sign it if it doesn't exist - gpg --sign --local-user 6395FC9911EDCD6158712DF7BADFCABDDBF5B694 --detach-sign "$database"; - fi; - -done diff --git a/scripts/0stalkerware.sh b/scripts/0stalkerware.sh index 28298b3..c546afc 100644 --- a/scripts/0stalkerware.sh +++ b/scripts/0stalkerware.sh @@ -1,5 +1,5 @@ #!/bin/bash -#License: GPLv3 +#License: CC0 #Description: Hypatia conversion script for https://github.com/AssoEchap/stalkerware-indicators (CC BY 4.0) while IFS=, read -r col1SHA col2Package col3Certificate col4Version col5Name @@ -10,5 +10,3 @@ do done < samples.csv; sed -i '1d' stalkerware.hsb; - -gzip stalkerware.hsb; diff --git a/scripts/0targetedthreats.sh b/scripts/0targetedthreats.sh index 6a5915d..93f07a3 100644 --- a/scripts/0targetedthreats.sh +++ b/scripts/0targetedthreats.sh @@ -1,5 +1,5 @@ #!/bin/bash -#License: GPLv3 +#License: CC0 #Description: Hypatia conversion script for https://github.com/botherder/targetedthreats (CC BY-SA 4.0) while IFS=, read -r col1MD5 col2SHA256 col3Family col4Actor col5Country col6Report @@ -30,6 +30,3 @@ done < samples.csv; sed -i '1d' targetedthreats.hdb; sed -i '1d' targetedthreats.hsb; - -gzip targetedthreats.hdb; -gzip targetedthreats.hsb; diff --git a/scripts/0threatfox.sh b/scripts/0threatfox.sh new file mode 100644 index 0000000..2bf8133 --- /dev/null +++ b/scripts/0threatfox.sh @@ -0,0 +1,5 @@ +#!/bin/sh +#License: CC0 +#Description: Hypatia conversion script for https://threatfox.abuse.ch/export/csv/sha256/full/ (CC0) + +tail -n +10 full_sha256.csv | awk '{ print $4 } ' | sed 's/^"//' | sed 's/",$//' > threatfox.sha256 diff --git a/scripts/Main.java b/scripts/Main.java index 76f81ba..9784e75 100644 --- a/scripts/Main.java +++ b/scripts/Main.java @@ -26,39 +26,61 @@ import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.FileReader; import java.io.InputStreamReader; +import java.util.*; +import java.util.Arrays; import java.util.zip.GZIPInputStream; import java.util.regex.Matcher; import java.util.regex.Pattern; public class Main { - public static BloomFilter