mkdir dags logs plugins config
echo -e "AIRFLOW_UID=$(id -u)\nAIRFLOW_GID=0" > .env

docker compose up airflow-init
docker compose up

http://localhost:8080

user/pass airflow

docker exec -it airflow-docker-airflow-webserver-1 airflow version
docker exec -it airflow-docker-airflow-webserver-1 bash

curl -X GET --user "airflow:airflow" "http//:localhost:8080/api/v1/dags"

Apache Spark

Master http://localhost:9090/

SparkUI http://localhost:4040/

Necesitamos configurar la conexión:

Lanzar pyspark básico

spark-submit python/wordcountjob.py arg1 arg2

Para compilar con sbt básico

cd jobs/scala/wordcount
sbt compile publishLocal

cd jobs/scala/basicjob/target/scala-2.12
sbt compile publishLocal
spark-submit --class es.david.WordCount basic-job-scala_2.12-0.1.jar
spark-submit --class es.david.ReduceJoins basic-job-scala_2.12-0.1.jar

Podemos lanzar con 3 worker

docker-compose up --scale spark-worker=3 -d

HDFS

Añadimos HDFS a nuestro cluster para usar con Spark

Tenemos un generador de datos y luego los subiremos a hdfs, para ello nos con

docker exec -it learnairflow-namenode-1 /bin/bash
python generar_ficeros.py
hdfs dfs -put xyz_grande.csv /xyz_grande.csv
hdfs dfs -put xyz_medio.csv  /xyz_medio.csv

Lanzamos desde Spark

docker exec -it learnairflow-spark-master-1 bash
spark-submit --name hdfs_spark python/hdfs.py