downloadEDLs #1497
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # workflow de descarga de feeds | |
| name: downloadEDLs | |
| # defino el tigger del workflow. En este caso se ejecutaran unas tares | |
| on: | |
| # on demand ( manually) | |
| workflow_dispatch: | |
| # scheduled cada 13 horas y 3 minutos, conviene que no sea primo. | |
| schedule: | |
| # every 12:30 | |
| - cron: '30 12 * * *' | |
| jobs: | |
| feeds: | |
| # Descarga automatica de feeds. Hasta ahora de: | |
| # - Abuse.ch las recomendadas. IPs. | |
| # - CERT-PA. URLs. | |
| # - Phishunt.io: URLs. | |
| # - URLhaus: Proyecto de abuse.ch. URL maliciosas. | |
| # - http://www.botvrij.eu | |
| # - Download feeds https://feodotracker.abuse.ch/downloads/ipblocklist_recommended.txt | |
| # - 28-08-22 borrado => https://infosec.cert-pa.it/analyze/listurls.txt eliminada por problemas de conexion | |
| # - 28-08-22 borrado => https://www.binarydefense.com/banlist.txt -O feeds/download/binarydefense-ips.txt problemas conexion | |
| # - 28-08-22 añadido => Nodos de salida de TOR | |
| # - 28-08-22 añadido => lista negra de certificados, SSLBL SSL Certificate Blacklist (SHA1 Fingerprints) | |
| # CSV: Listing date (UTC),SHA1 Fingerprint of the blacklisted SSL certificate,Listing reason | |
| # - 28-08-22 añadido Suricata IDS SSL Certificate Ruleset | |
| # - 28-08-22 añadido "https://sslbl.abuse.ch/blacklist/sslipblacklist.csv" fichero de ips cuyo certificado esta incluido en la lista anterior | |
| # El formato del CSV Firstseen,DstIP,DstPort | |
| # - 28-08-22 añadido SSLBL RPZ (DNS Reponse Policy Zone (RPZ or DNS Firewall)), para bloquear la resolución de | |
| # ciertos dominios dns. Contiene la IP que se ejecutan con un certificado bloqueado en SSLBL. Se bloqueará cualquier dominio que | |
| # resuelva en esas direcciones. | |
| name: Feeds | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: IOCs | |
| uses: actions/checkout@v3 | |
| - name: Run a wget | |
| env: | |
| API: ${{ secrets.MALSHARE_API }} | |
| API_B: ${{ secrets.BITCOINT_API }} | |
| run: |- | |
| wget "https://feodotracker.abuse.ch/downloads/ipblocklist_recommended.txt" -O feeds/download/ipblocklist_recommended-ips.txt | |
| wget "https://phishunt.io/feed.txt" -O feeds/download/feed-phishunt-url.txt | |
| wget "https://urlhaus.abuse.ch/downloads/text/" -O feeds/download/urlhaus-abuse-sh-url.txt | |
| wget "https://malshare.com/api.php?api_key=$API&action=getlistraw" -O feeds/download/malshare-hash.txt | |
| wget "https://www.botvrij.eu/data/ioclist.ip-dst.raw" -O feeds/download/botvrij-ioc-ips.txt | |
| wget "https://www.botvrij.eu/data/ioclist.url.raw" -O feeds/download/botvrij-ioc-url.txt | |
| wget "https://www.botvrij.eu/data/ioclist.domain.raw" -O feeds/download/botvrij-ioc-domain.txt | |
| wget "https://www.botvrij.eu/data/ioclist.filename" -O feeds/download/botvrij-ioc-filename.txt | |
| wget "https://www.botvrij.eu/data/ioclist.hostname.raw" -O feeds/download/botvrij-ioc-hostname.txt | |
| wget "https://www.botvrij.eu/data/ioclist.md5.raw" -O feeds/download/botvrij-ioc-md5-txt | |
| wget "https://www.botvrij.eu/data/ioclist.sha256.raw" -O feeds/download/botvrij-ioc-sha256.txt | |
| curl "http://reputation.alienvault.com/reputation.data" | cut -d"#" -f1 > feeds/download/alienvault.reputation-ips.txt | |
| wget "https://lists.blocklist.de/lists/all.txt" -O feeds/download/blocklist-de-ips.txt | |
| curl https://www.bitcoinabuse.com/api/download/1d?api_token={$API_B}|cut -d, -f2 | egrep "[^13][a-km-zA-HJ-NP-Z1-9]{25,34}$"|sort|uniq > feeds/download/BTCabuse-hash.txt | |
| wget "https://www.dan.me.uk/torlist/?exit" -O feeds/download/TOR-exit.txt | |
| curl "https://sslbl.abuse.ch/blacklist/sslblacklist.csv" | grep -v "#" > feeds/download/sslblacklist.csv | |
| curl "https://sslbl.abuse.ch/blacklist/sslipblacklist.csv" | grep -v "#" > feeds/download/sslipblacklist.csv | |
| curl "https://sslbl.abuse.ch/blacklist/sslblacklist_tls_cert.rules" | grep -v "#" > feeds/download/sslblacklist_tls_cert.rules | |
| curl "https://sslbl.abuse.ch/blacklist/sslipblacklist.csv" | grep -v "#" > feeds/download/sslipblacklist.csv | |
| curl "https://sslbl.abuse.ch/blacklist/sslbl.rpz" > feeds/download/sslbl.rpz | |
| curl "https://openphish.com/feed.txt" > feeds/download/openphish_feed.txt | |
| curl "https://threatview.io/Downloads/DOMAIN-High-Confidence-Feed.txt" > feeds/download/threatview_domain_High_Confidence.txt | |
| - name: Commit | |
| run: |- | |
| git config user.name "Automated" | |
| git config user.email "actions@users.noreply.github.com" | |
| git add -A | |
| timestamp=$(date -u) | |
| git commit -m "Latest data: ${timestamp}" || exit 0 | |
| - name: Push | |
| if: steps.check_changes.outputs.bigChange != 'true' | |
| run: |- | |
| git push | |
| # tareas. Se debe desplegar un entorno virtual de ubuntu para que se ejecute esta accion. Como maquina virtual. | |
| # accion que se conectará a una API externa y en base a lo que diga el fichero de configuración cloud-servicecs-mining-config.yml | |
| # aplicará un filtrado y la salida se almacenará en el repositorio de la maquina ubuntu. Luego haremos un push para colocarla. | |
| #edlaction: | |
| edl: | |
| runs-on: ubuntu-latest | |
| steps: | |
| # checkout | |
| - name: Check out this repo | |
| uses: actions/checkout@v3 | |
| # Fetch Accion de Luigi Mori ( PaloAlto - Minemeld ) | |
| - name: Fetch the endpoint lists | |
| uses: jtschichold/mm-cloud-services-miners@v0 | |
| with: | |
| config: feeds/cloud-services-mining-config.yml | |
| configSection: azure | |
| #- name: Fetch 0365 | |
| #uses: jtschichold/mm-cloud-services-miners@v0 | |
| #with: | |
| #config: feeds/cloud-services-mining-config.yml | |
| #configSection: o365 | |
| # para bajarse las listas de officee365 la API requiere un GUID a.k.a UUID (Globally Unique IDentifier or Universally Unique Identifier) | |
| # eso es así pq Microsoft de tracking | |
| #env: | |
| #0365MINER_CLIENT_GUID: ${{ secrets.GUID }} | |
| # Filter & Transform | |
| - name: Process IP lists | |
| # we pin a specific commit, for increased security | |
| uses: jtschichold/mm-process-ip-list@v0 | |
| with: | |
| # process al the lists matching the glob | |
| list: feeds/download/**/*ips.txt | |
| # do not aggregate, just filter lists in place | |
| filterInPlace: true | |
| # remove overlaps with reserved IP addresses | |
| filterReservedIPs: true | |
| # filter IPv4 CIDRs where the subnet mask is smaller than 8 | |
| minIPv4Mask: 8 | |
| # filter IPv6 CIDRs where the subnet mask is smaller than 8 | |
| minIPv6Mask: 8 | |
| # remove overlaps with CIDRs contained in the | |
| # file ip-exclusion-list in the repo | |
| filter: feeds/download/ip-exclusion-list | |
| # Check & Update | |
| # check before commit | |
| - name: Check changes | |
| id: check_changes | |
| uses: jtschichold/mm-check-changes@v0 | |
| with: | |
| includeGlob: '**/*.txt' | |
| # commit | |
| - name: Commit | |
| run: |- | |
| git config user.name "Automated" | |
| git config user.email "actions@users.noreply.github.com" | |
| git status | |
| git add -A | |
| git status | |
| timestamp=$(date -u) | |
| git commit -m "Latest data: ${timestamp}" || exit 0 | |
| # not a big change, let's push automatically | |
| - name: Push | |
| # this is the important part! | |
| # this step happens if this condition is verified | |
| # in this case if the check_changes step has *not* set bigChange to 'true' | |
| if: steps.check_changes.outputs.bigChange != 'true' | |
| run: |- | |
| git push | |
| # big change detected, open a PR | |
| - name: Create PR | |
| # same as above, but reversed logci | |
| # we want this step to be executed only if bigChange is 'true' | |
| if: steps.check_changes.outputs.bigChange == 'true' | |
| uses: peter-evans/create-pull-request@v3.6.0 | |
| with: | |
| branch: please-check-changes |