feat(data-catalog): Adding Hue

hue/hue/base/hive-site-xml-secret.yaml

@@ -0,0 +1,38 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: hue-hive-site-xml
+type: Opaque
+stringData:
+  hive-site.xml: |
+    <?xml version="1.0"?>
+    <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+    <configuration>
+      <property>
+        <name>hive.server2.transport.mode</name>
+        <value>binary</value>
+        <description>Server transport mode. binary or http.</description>
+      </property>
+
+      <property>
+        <name>hive.server2.thrift.http.port</name>
+        <value>10000</value>
+        <description>Port number when in HTTP mode.</description>
+      </property>
+
+      <property>
+        <name>fs.s3a.awsAccessKeyId</name>
+        <value></value>
+      </property>
+
+      <property>
+        <name>fs.s3a.awsSecretAccessKey</name>
+        <value></value>
+      </property>
+
+      <property>
+        <name>fs.s3a.endpoint</name>
+        <value>${s3_endpoint}</value>
+      </property>
+    </configuration>

hue/hue/base/hue-db-init-job.yaml

@@ -0,0 +1,28 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: hue-db-init
+spec:
+  parallelism: 1
+  completions: 1
+  template:
+    metadata:
+      name: hue-db-init
+    spec:
+      containers:
+        - name: db-init
+          image: hue
+          command: ["build/env/bin/hue", "migrate"]
+          imagePullPolicy: Always
+          volumeMounts:
+            - name: "hue-ini"
+              mountPath: /hue/desktop/conf/
+      volumes:
+        - name: "hue-ini"
+          secret:
+            secretName: hue-ini
+            items:
+              - key: hue.ini
+                path: hue.ini
+      restartPolicy: OnFailure

hue/hue/base/hue-dc.yaml

@@ -0,0 +1,68 @@
+---
+apiVersion: v1
+kind: DeploymentConfig
+metadata:
+  name: hue
+spec:
+  template:
+    metadata:
+      labels:
+        name: hue
+    spec:
+      containers:
+        - name: hue
+          image: hue
+          command: ["build/env/bin/hue", "runcpserver"]
+          imagePullPolicy: Always
+          resources:
+            requests:
+              cpu: 300m
+              memory: 500Mi
+            limits:
+              cpu: 500m
+              memory: 1Gi
+          env:
+            - name: AWS_ACCESS_KEY_ID
+              valueFrom:
+                secretKeyRef:
+                  name: hue-secret
+                  key: AWS_ACCESS_KEY_ID
+            - name: AWS_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: hue-secret
+                  key: AWS_SECRET_ACCESS_KEY
+          volumeMounts:
+            - name: "hue-ini"
+              mountPath: /hue/desktop/conf/
+            - name: "hive-site-xml"
+              mountPath: /etc/hive/conf/
+          livenessProbe:
+            failureThreshold: 4
+            httpGet:
+              path: /
+              port: 8000
+              scheme: HTTP
+            periodSeconds: 30
+            initialDelaySeconds: 300
+            successThreshold: 1
+            timeoutSeconds: 5
+      restartPolicy: Always
+      volumes:
+        - name: "hue-ini"
+          secret:
+            secretName: hue-ini
+            items:
+              - key: hue.ini
+                path: hue.ini
+        - name: "hive-site-xml"
+          secret:
+            secretName: hue-hive-site-xml
+            items:
+              - key: hive-site.xml
+                path: hive-site.xml
+  replicas: 1
+  triggers:
+    - type: ConfigChange
+  strategy:
+    type: Rolling

hue/hue/base/hue-ini-secret.yaml

@@ -0,0 +1,84 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: hue-ini
+type: Opaque
+stringData:
+  hue.ini: |
+    [desktop]
+      # Hide unused apps
+      app_blacklist=impala,security,jobbrowser,jobsub,pig,hbase,sqoop,zookeeper,spark,oozie,search
+      secret_key=thisISaSECRET_1234
+      http_host=0.0.0.0
+      http_port=8000
+      is_hue_4=true
+      time_zone=America/Los_Angeles
+      django_debug_mode=false
+      dev=false
+      database_logging=false
+      send_dbug_messages=false
+      http_500_debug_mode=false
+
+      [[database]]
+        engine=django.db.backends.mysql
+        host=hue-mysql.$(namespace).svc
+        port=3306
+        user=datacatalog
+        password=datacatalog
+        name=datacatalog
+
+      [[smtp]]
+        host=localhost
+        port=25
+        user=
+        password=
+        tls=no
+
+    [notebook]
+      [[interpreters]]
+        [[[hive]]]
+          name=Hive
+          interface=hiveserver2
+
+        [[[impala]]]
+          name=Impala
+          interface=hiveserver2
+
+        [[[sparksql]]]
+          name=SparkSql
+          interface=hiveserver2
+
+        [[[text]]]
+          name=Text
+          interface=text
+
+        [[[markdown]]]
+          name=Markdown
+          interface=text
+
+    [dashboard]
+      is_enabled=true
+      has_sql_enabled=true
+      has_query_builder_enabled=true
+      has_report_enabled=true
+
+      [[engines]]
+        [[[sql]]]
+        analytics=true
+        nesting=false
+
+        [[[hive]]]
+        analytics=true
+        nesting=true
+
+    [beeswax]
+      hive_server_host=thriftserver.$(namespace).svc
+      hive_server_port=10000
+      auth_username=hue
+      auth_password=password
+      hive_conf_dir=/etc/hive/conf
+
+    [metastore]
+      enable_new_create_table=true
+      force_hs2_metadata=false

hue/hue/base/hue-mysql-dc.yaml

@@ -0,0 +1,114 @@
+---
+apiVersion: v1
+kind: DeploymentConfig
+metadata:
+  name: hue-mysql
+  annotations:
+    template.alpha.openshift.io/wait-for-ready: "true"
+spec:
+  strategy:
+    type: Recreate
+  triggers:
+    - type: ImageChange
+      imageChangeParams:
+        automatic: true
+        containerNames:
+          - mysql
+        from:
+          kind: ImageStreamTag
+    - type: ConfigChange
+  replicas: 1
+  selector:
+    name: hue-mysql
+  template:
+    metadata:
+      labels:
+        name: hue-mysql
+    spec:
+      containers:
+        - name: mysqld-exporter
+          image: mysqld-exporter
+          ports:
+            - containerPort: 9104
+              protocol: TCP
+          env:
+            - name: DATA_SOURCE_NAME
+              valueFrom:
+                secretKeyRef:
+                  name: "hue-mysqld-exporter-secret"
+                  key: DATA_SOURCE_NAME
+          livenessProbe:
+            httpGet:
+              path: /metrics
+              port: 9104
+          readinessProbe:
+            httpGet:
+              path: /metrics
+              port: 9104
+        - name: mysql
+          image: mysql
+          ports:
+            - containerPort: 3306
+              protocol: TCP
+          readinessProbe:
+            exec:
+              command:
+                - /bin/sh
+                - "-i"
+                - "-c"
+                - >-
+                  MYSQL_PWD=$MYSQL_PASSWORD mysql -h 127.0.0.1 -u $MYSQL_USER -D
+                  $MYSQL_DATABASE -e 'SELECT 1'
+            failureThreshold: 3
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 1
+          livenessProbe:
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 10
+            successThreshold: 1
+            tcpSocket:
+              port: 3306
+            timeoutSeconds: 1
+          env:
+            - name: MYSQL_USER
+              valueFrom:
+                secretKeyRef:
+                  name: "hue-mysql-secret"
+                  key: database-user
+            - name: MYSQL_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: "hue-mysql-secret"
+                  key: database-password
+            - name: MYSQL_DATABASE
+              valueFrom:
+                secretKeyRef:
+                  name: "hue-mysql-secret"
+                  key: database-name
+            - name: MYSQL_ROOT_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: "hue-mysql-secret"
+                  key: database-root-password
+          resources:
+            limits:
+              memory: "512Mi"
+          volumeMounts:
+            - name: "mysql"
+              mountPath: "/var/lib/mysql/data"
+          terminationMessagePath: "/dev/termination-log"
+          terminationMessagePolicy: File
+          imagePullPolicy: IfNotPresent
+          capabilities: {}
+          securityContext:
+            capabilities: {}
+            privileged: false
+      volumes:
+        - name: mysql
+          persistentVolumeClaim:
+            claimName: hue-mysql
+      restartPolicy: Always
+      dnsPolicy: ClusterFirst

hue/hue/base/hue-mysql-pvc.yaml

@@ -0,0 +1,13 @@
+---
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: hue-mysql
+  annotations:
+    volume.beta.kubernetes.io/storage-class: "$(storage_class)"
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: "1Gi"

hue/hue/base/hue-mysql-secret.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: hue-mysql-secret
+  annotations:
+    template.openshift.io/expose-username: "{.data['database-user']}"
+    template.openshift.io/expose-password: "{.data['database-password']}"
+    template.openshift.io/expose-database_name: "{.data['database-name']}"
+    template.openshift.io/expose-root_password: |
+      {.data['database-root-password']}
+stringData:
+  database-user: datacatalog
+  database-password: datacatalog
+  database-name: datacatalog
+  database-root-password: root

hue/hue/base/hue-mysql-service.yaml

@@ -0,0 +1,20 @@
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: hue-mysql
+  annotations:
+    template.openshift.io/expose-uri: |
+      'mysql://{.spec.clusterIP}:{.spec.ports[?(.name=="mysql")].port}'
+spec:
+  ports:
+    - name: hue-mysql
+      protocol: TCP
+      port: 3306
+      targetPort: 3306
+  selector:
+    name: "hue-mysql"
+  type: ClusterIP
+  sessionAffinity: None
+status:
+  loadBalancer: {}

hue/hue/base/hue-mysqld-exporter-secret.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: hue-mysqld-exporter-secret
+stringData:
+  DATA_SOURCE_NAME: "root:root@(localhost:3306)/"