-
Notifications
You must be signed in to change notification settings - Fork 4
159 lines (149 loc) · 7.61 KB
/
main.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
# workflow de descarga de feeds
name: downloadEDLs
# defino el tigger del workflow. En este caso se ejecutaran unas tares
on:
# on demand ( manually)
workflow_dispatch:
# scheduled cada 13 horas y 3 minutos, conviene que no sea primo.
schedule:
# every 12:30
- cron: '30 12 * * *'
jobs:
feeds:
# Descarga automatica de feeds. Hasta ahora de:
# - Abuse.ch las recomendadas. IPs.
# - CERT-PA. URLs.
# - Phishunt.io: URLs.
# - URLhaus: Proyecto de abuse.ch. URL maliciosas.
# - http://www.botvrij.eu
# - Download feeds https://feodotracker.abuse.ch/downloads/ipblocklist_recommended.txt
# - 28-08-22 borrado => https://infosec.cert-pa.it/analyze/listurls.txt eliminada por problemas de conexion
# - 28-08-22 borrado => https://www.binarydefense.com/banlist.txt -O feeds/download/binarydefense-ips.txt problemas conexion
# - 28-08-22 añadido => Nodos de salida de TOR
# - 28-08-22 añadido => lista negra de certificados, SSLBL SSL Certificate Blacklist (SHA1 Fingerprints)
# CSV: Listing date (UTC),SHA1 Fingerprint of the blacklisted SSL certificate,Listing reason
# - 28-08-22 añadido Suricata IDS SSL Certificate Ruleset
# - 28-08-22 añadido "https://sslbl.abuse.ch/blacklist/sslipblacklist.csv" fichero de ips cuyo certificado esta incluido en la lista anterior
# El formato del CSV Firstseen,DstIP,DstPort
# - 28-08-22 añadido SSLBL RPZ (DNS Reponse Policy Zone (RPZ or DNS Firewall)), para bloquear la resolución de
# ciertos dominios dns. Contiene la IP que se ejecutan con un certificado bloqueado en SSLBL. Se bloqueará cualquier dominio que
# resuelva en esas direcciones.
name: Feeds
runs-on: ubuntu-latest
steps:
- name: IOCs
uses: actions/checkout@v3
- name: Run a wget
env:
API: ${{ secrets.MALSHARE_API }}
API_B: ${{ secrets.BITCOINT_API }}
run: |-
wget "https://feodotracker.abuse.ch/downloads/ipblocklist_recommended.txt" -O feeds/download/ipblocklist_recommended-ips.txt
wget "https://phishunt.io/feed.txt" -O feeds/download/feed-phishunt-url.txt
wget "https://urlhaus.abuse.ch/downloads/text/" -O feeds/download/urlhaus-abuse-sh-url.txt
wget "https://malshare.com/api.php?api_key=$API&action=getlistraw" -O feeds/download/malshare-hash.txt
wget "https://www.botvrij.eu/data/ioclist.ip-dst.raw" -O feeds/download/botvrij-ioc-ips.txt
wget "https://www.botvrij.eu/data/ioclist.url.raw" -O feeds/download/botvrij-ioc-url.txt
wget "https://www.botvrij.eu/data/ioclist.domain.raw" -O feeds/download/botvrij-ioc-domain.txt
wget "https://www.botvrij.eu/data/ioclist.filename" -O feeds/download/botvrij-ioc-filename.txt
wget "https://www.botvrij.eu/data/ioclist.hostname.raw" -O feeds/download/botvrij-ioc-hostname.txt
wget "https://www.botvrij.eu/data/ioclist.md5.raw" -O feeds/download/botvrij-ioc-md5-txt
wget "https://www.botvrij.eu/data/ioclist.sha256.raw" -O feeds/download/botvrij-ioc-sha256.txt
curl "http://reputation.alienvault.com/reputation.data" | cut -d"#" -f1 > feeds/download/alienvault.reputation-ips.txt
wget "https://lists.blocklist.de/lists/all.txt" -O feeds/download/blocklist-de-ips.txt
curl https://www.bitcoinabuse.com/api/download/1d?api_token={$API_B}|cut -d, -f2 | egrep "[^13][a-km-zA-HJ-NP-Z1-9]{25,34}$"|sort|uniq > feeds/download/BTCabuse-hash.txt
wget "https://www.dan.me.uk/torlist/?exit" -O feeds/download/TOR-exit.txt
curl "https://sslbl.abuse.ch/blacklist/sslblacklist.csv" | grep -v "#" > feeds/download/sslblacklist.csv
curl "https://sslbl.abuse.ch/blacklist/sslipblacklist.csv" | grep -v "#" > feeds/download/sslipblacklist.csv
curl "https://sslbl.abuse.ch/blacklist/sslblacklist_tls_cert.rules" | grep -v "#" > feeds/download/sslblacklist_tls_cert.rules
curl "https://sslbl.abuse.ch/blacklist/sslipblacklist.csv" | grep -v "#" > feeds/download/sslipblacklist.csv
curl "https://sslbl.abuse.ch/blacklist/sslbl.rpz" > feeds/download/sslbl.rpz
curl "https://openphish.com/feed.txt" > feeds/download/openphish_feed.txt
curl "https://threatview.io/Downloads/DOMAIN-High-Confidence-Feed.txt" > feeds/download/threatview_domain_High_Confidence.txt
- name: Commit
run: |-
git config user.name "Automated"
git config user.email "actions@users.noreply.github.com"
git add -A
timestamp=$(date -u)
git commit -m "Latest data: ${timestamp}" || exit 0
- name: Push
if: steps.check_changes.outputs.bigChange != 'true'
run: |-
git push
# tareas. Se debe desplegar un entorno virtual de ubuntu para que se ejecute esta accion. Como maquina virtual.
# accion que se conectará a una API externa y en base a lo que diga el fichero de configuración cloud-servicecs-mining-config.yml
# aplicará un filtrado y la salida se almacenará en el repositorio de la maquina ubuntu. Luego haremos un push para colocarla.
#edlaction:
edl:
runs-on: ubuntu-latest
steps:
# checkout
- name: Check out this repo
uses: actions/checkout@v3
# Fetch Accion de Luigi Mori ( PaloAlto - Minemeld )
- name: Fetch the endpoint lists
uses: jtschichold/mm-cloud-services-miners@v0
with:
config: feeds/cloud-services-mining-config.yml
configSection: azure
#- name: Fetch 0365
#uses: jtschichold/mm-cloud-services-miners@v0
#with:
#config: feeds/cloud-services-mining-config.yml
#configSection: o365
# para bajarse las listas de officee365 la API requiere un GUID a.k.a UUID (Globally Unique IDentifier or Universally Unique Identifier)
# eso es así pq Microsoft de tracking
#env:
#0365MINER_CLIENT_GUID: ${{ secrets.GUID }}
# Filter & Transform
- name: Process IP lists
# we pin a specific commit, for increased security
uses: jtschichold/mm-process-ip-list@v0
with:
# process al the lists matching the glob
list: feeds/download/**/*ips.txt
# do not aggregate, just filter lists in place
filterInPlace: true
# remove overlaps with reserved IP addresses
filterReservedIPs: true
# filter IPv4 CIDRs where the subnet mask is smaller than 8
minIPv4Mask: 8
# filter IPv6 CIDRs where the subnet mask is smaller than 8
minIPv6Mask: 8
# remove overlaps with CIDRs contained in the
# file ip-exclusion-list in the repo
filter: feeds/download/ip-exclusion-list
# Check & Update
# check before commit
- name: Check changes
id: check_changes
uses: jtschichold/mm-check-changes@v0
with:
includeGlob: '**/*.txt'
# commit
- name: Commit
run: |-
git config user.name "Automated"
git config user.email "actions@users.noreply.github.com"
git status
git add -A
git status
timestamp=$(date -u)
git commit -m "Latest data: ${timestamp}" || exit 0
# not a big change, let's push automatically
- name: Push
# this is the important part!
# this step happens if this condition is verified
# in this case if the check_changes step has *not* set bigChange to 'true'
if: steps.check_changes.outputs.bigChange != 'true'
run: |-
git push
# big change detected, open a PR
- name: Create PR
# same as above, but reversed logci
# we want this step to be executed only if bigChange is 'true'
if: steps.check_changes.outputs.bigChange == 'true'
uses: peter-evans/create-pull-request@v3.6.0
with:
branch: please-check-changes