diff --git a/.flake8 b/.flake8 index ca686841a0a..9ca88cd7ba5 100644 --- a/.flake8 +++ b/.flake8 @@ -17,7 +17,6 @@ per-file-ignores = exclude = __init__.py, build/* - mindsdb/microservices_grpc/* # TODO: clean up this code mindsdb/integrations/handlers/druid_handler/druid_handler.py mindsdb/integrations/handlers/dynamodb_handler/dynamodb_handler.py diff --git a/MANIFEST.in b/MANIFEST.in index 5bf5aca2a99..a35d38ff0a3 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,4 @@ include requirements/requirements.txt -include requirements/requirements-grpc.txt include requirements/requirements-telemetry.txt include mindsdb/migrations/alembic.ini recursive-include mindsdb/integrations/utilities/datasets *.csv diff --git a/helm/README.md b/helm/README.md deleted file mode 100644 index 5248e5d4e06..00000000000 --- a/helm/README.md +++ /dev/null @@ -1,94 +0,0 @@ -# mindsdb - -[MindsDB](https://mindsdb.com?utm_medium=community&utm_source=github&utm_campaign=mindsdb%20repo) enables you to use ML predictions in your database using SQL. - -- Developers can quickly add AI capabilities to your applications. -- Data Scientists can streamline MLOps by deploying ML models as AI Tables. -- Data Analysts can easily make forecasts on complex data (like multivariate time-series with high cardinality) and visualize them in BI tools like Tableau. - -# Pre-requisites - -- Kubernetes -- Helm 3.0+ - -# Installing the Chart - -```bash -###FIRST CLONE THE WHOLE REPO cd helm/mindsdb and then run this command (namespace is option you can add it or if you remove it than it goes default) -helm upgrade -i \ - mindsdb mindsdb \ - --namespace mindsdb \ - --create-namespace -``` - -# Configuration - -All the configurations can be done in the [values.yaml](./mindsdb/values.yaml) file or you can create a separate YAML file with only the values that you want to override and pass it with a `-f` to the `helm install` command - -### Image - -| Parameter | Description | Default | -| ------------------ | --------------------------------------------------------------------------------------- | ----------------- | -| `image.repository` | Image to start for this pod | `mindsdb/mindsbd` | -| `image.tag` | [Image tag](https://hub.docker.com/r/mindsdb/mindsdb/tags?page=1&ordering=last_updated) | `latest` | -| `image.pullPolicy` | Image pull policy | `Always` | - -### Ingress - -| Parameter | Description | Default | -| ------------------------------------ | --------------------------------------------------------------------------- | -------------------------------------------------- | -| `ingress.enabled` | enable ingress | `false` | -| `ingress.annotations` | add ingress annotations | | -| `ingress.hosts[0].host` | add hosts for ingress | `chart-example.local` | -| `ingress.hosts[0].paths[0].path` | add path for each ingress host | `/` | -| `ingress.hosts[0].paths[0].pathType` | add ingress path type | `ImplementationSpecific` | -| `ingress.tls` | add ingress tls settings | `[]` | -| `ingress.className` | add ingress class name. Only used in k8s 1.19+ | | -| `ingress.apiVersion` | specify APIVersion of ingress object. Mostly would only be used for argocd. | version indicated by helm's `Capabilities` object. | - -### Service - -#### Web - -| Parameter | Description | Default | -| --------------------------------------- | ------------------------------------------------------------------------------------------------------------ | ----------- | -| `service.http.enabled` | Enable Web service | `true` | -| `service.http.type` | Kubernetes service type for web traffic | `ClusterIP` | -| `service.http.port` | Port for web traffic | `47334` | -| `service.http.clusterIP` | ClusterIP setting for http autosetup for statefulset is None | `None` | -| `service.http.loadBalancerIP` | LoadBalancer Ip setting | | -| `service.http.nodePort` | NodePort for http service | | -| `service.http.externalTrafficPolicy` | If `service.http.type` is `NodePort` or `LoadBalancer`, set this to `Local` to enable source IP preservation | | -| `service.http.externalIPs` | http service external IP addresses | | -| `service.http.loadBalancerSourceRanges` | Source range filter for http loadbalancer | `[]` | -| `service.http.annotations` | http service annotations | | - -#### MySQL - -| Parameter | Description | Default | -| --------------------------------------- | ------------------------------------------------------------------------------------------------------------ | ----------- | -| `service.mysql.enabled` | Enable MySQL service | `true` | -| `service.mysql.type` | Kubernetes service type for MySQL traffic | `ClusterIP` | -| `service.mysql.port` | Port for MySQL traffic | `47335` | -| `service.mysql.clusterIP` | ClusterIP setting for MySQL autosetup for statefulset is None | `None` | -| `service.mysql.loadBalancerIP` | LoadBalancer Ip setting | | -| `service.mysql.nodePort` | NodePort for http service | | -| `service.mysql.externalTrafficPolicy` | If `service.mysql.type` is `NodePort` or `LoadBalancer`, set this to `Local` to enable source IP preservation | | -| `service.mysql.externalIPs` | MySQL service external IP addresses | | -| `service.mysql.loadBalancerSourceRanges` | Source range filter for MySQL loadbalancer | `[]` | -| `service.mysql.annotations` | MySQL service annotations | | - -#### MongoDB - -| Parameter | Description | Default | -| --------------------------------------- | ------------------------------------------------------------------------------------------------------------ | ----------- | -| `service.mongodb.enabled` | Enable MongoDB service | `true` | -| `service.mongodb.type` | Kubernetes service type for MongoDB traffic | `ClusterIP` | -| `service.mongodb.port` | Port for MongoDB traffic | `47336` | -| `service.mongodb.clusterIP` | ClusterIP setting for MongoDB autosetup for statefulset is None | `None` | -| `service.mongodb.loadBalancerIP` | LoadBalancer Ip setting | | -| `service.mongodb.nodePort` | NodePort for http service | | -| `service.mongodb.externalTrafficPolicy` | If `service.mongodb.type` is `NodePort` or `LoadBalancer`, set this to `Local` to enable source IP preservation | | -| `service.mongodb.externalIPs` | MongoDB service external IP addresses | | -| `service.mongodb.loadBalancerSourceRanges` | Source range filter for MongoDB loadbalancer | `[]` | -| `service.mongodb.annotations` | MongoDB service annotations | | diff --git a/helm/mindsdb/.helmignore b/helm/mindsdb/.helmignore deleted file mode 100644 index 0e8a0eb36f4..00000000000 --- a/helm/mindsdb/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/helm/mindsdb/Chart.yaml b/helm/mindsdb/Chart.yaml deleted file mode 100644 index a678c35a416..00000000000 --- a/helm/mindsdb/Chart.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: v2 -name: mindsdb -description: A Helm chart for Kubernetes - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "1.16.0" diff --git a/helm/mindsdb/templates/NOTES.txt b/helm/mindsdb/templates/NOTES.txt deleted file mode 100644 index 969bdc0f413..00000000000 --- a/helm/mindsdb/templates/NOTES.txt +++ /dev/null @@ -1,22 +0,0 @@ -1. Get the application URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range $host := .Values.ingress.hosts }} - {{- range .paths }} - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} - {{- end }} -{{- end }} -{{- else if contains "NodePort" .Values.service.http.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "mindsdb.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.http.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "mindsdb.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "mindsdb.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - echo http://$SERVICE_IP:{{ .Values.service.http.port }} -{{- else if contains "ClusterIP" .Values.service.http.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "mindsdb.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT -{{- end }} diff --git a/helm/mindsdb/templates/_helpers.tpl b/helm/mindsdb/templates/_helpers.tpl deleted file mode 100644 index 577d8afd8c0..00000000000 --- a/helm/mindsdb/templates/_helpers.tpl +++ /dev/null @@ -1,62 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "mindsdb.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "mindsdb.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "mindsdb.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "mindsdb.labels" -}} -helm.sh/chart: {{ include "mindsdb.chart" . }} -{{ include "mindsdb.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "mindsdb.selectorLabels" -}} -app.kubernetes.io/name: {{ include "mindsdb.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "mindsdb.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "mindsdb.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/helm/mindsdb/templates/deployment.yaml b/helm/mindsdb/templates/deployment.yaml deleted file mode 100644 index fa005e3ecd4..00000000000 --- a/helm/mindsdb/templates/deployment.yaml +++ /dev/null @@ -1,59 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "mindsdb.fullname" . }} - labels: - {{- include "mindsdb.labels" . | nindent 4 }} -spec: - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} - {{- end }} - selector: - matchLabels: - {{- include "mindsdb.selectorLabels" . | nindent 6 }} - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "mindsdb.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "mindsdb.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - name: http - containerPort: {{ .Values.service.http.port }} - protocol: TCP - - name: mysql - containerPort: {{ .Values.service.mysql.port }} - protocol: TCP - - name: mongodb - containerPort: {{ .Values.service.mongodb.port }} - protocol: TCP - resources: - {{- toYaml .Values.resources | nindent 12 }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/helm/mindsdb/templates/hpa.yaml b/helm/mindsdb/templates/hpa.yaml deleted file mode 100644 index cc99ca764b6..00000000000 --- a/helm/mindsdb/templates/hpa.yaml +++ /dev/null @@ -1,28 +0,0 @@ -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "mindsdb.fullname" . }} - labels: - {{- include "mindsdb.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "mindsdb.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/helm/mindsdb/templates/http-service.yaml b/helm/mindsdb/templates/http-service.yaml deleted file mode 100644 index 053c14da075..00000000000 --- a/helm/mindsdb/templates/http-service.yaml +++ /dev/null @@ -1,38 +0,0 @@ -{{- if .Values.service.http.enabled -}} -apiVersion: v1 -kind: Service -metadata: - name: {{ include "mindsdb.fullname" . }}-http - labels: - {{- include "mindsdb.labels" . | nindent 4 }} -spec: - type: {{ .Values.service.http.type }} - {{- if and .Values.service.http.loadBalancerIP (eq .Values.service.http.type "LoadBalancer") }} - loadBalancerIP: {{ .Values.service.http.loadBalancerIP }} - {{- end }} - {{- if .Values.service.http.loadBalancerSourceRanges }} - loadBalancerSourceRanges: - {{- range .Values.service.http.loadBalancerSourceRanges }} - - {{ . }} - {{- end }} - {{- end }} - {{- if .Values.service.http.externalIPs }} - externalIPs: - {{- toYaml .Values.service.http.externalIPs | nindent 4 }} - {{- end }} - {{- if .Values.service.http.externalTrafficPolicy }} - externalTrafficPolicy: {{ .Values.service.http.externalTrafficPolicy }} - {{- end }} - {{- if and .Values.service.http.clusterIP (eq .Values.service.http.type "ClusterIP") }} - clusterIP: {{ .Values.service.http.clusterIP }} - {{- end }} - ports: - - name: http - port: {{ .Values.service.http.port }} - {{- if .Values.service.http.nodePort }} - nodePort: {{ .Values.service.http.nodePort }} - {{- end }} - targetPort: {{ .Values.service.http.port }} - selector: - {{- include "mindsdb.selectorLabels" . | nindent 4 }} -{{- end }} diff --git a/helm/mindsdb/templates/ingress.yaml b/helm/mindsdb/templates/ingress.yaml deleted file mode 100644 index 831c8bc5104..00000000000 --- a/helm/mindsdb/templates/ingress.yaml +++ /dev/null @@ -1,61 +0,0 @@ -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "mindsdb.fullname" . -}} -{{- $svcPort := .Values.service.http.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "mindsdb.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/helm/mindsdb/templates/mongodb-service.yaml b/helm/mindsdb/templates/mongodb-service.yaml deleted file mode 100644 index 07674d168c4..00000000000 --- a/helm/mindsdb/templates/mongodb-service.yaml +++ /dev/null @@ -1,38 +0,0 @@ -{{- if .Values.service.mongodb.enabled -}} -apiVersion: v1 -kind: Service -metadata: - name: {{ include "mindsdb.fullname" . }}-mongodb - labels: - {{- include "mindsdb.labels" . | nindent 4 }} -spec: - type: {{ .Values.service.mongodb.type }} - {{- if and .Values.service.mongodb.loadBalancerIP (eq .Values.service.mongodb.type "LoadBalancer") }} - loadBalancerIP: {{ .Values.service.mongodb.loadBalancerIP }} - {{- end }} - {{- if .Values.service.mongodb.loadBalancerSourceRanges }} - loadBalancerSourceRanges: - {{- range .Values.service.mongodb.loadBalancerSourceRanges }} - - {{ . }} - {{- end }} - {{- end }} - {{- if .Values.service.mongodb.externalIPs }} - externalIPs: - {{- toYaml .Values.service.mongodb.externalIPs | nindent 4 }} - {{- end }} - {{- if .Values.service.mongodb.externalTrafficPolicy }} - externalTrafficPolicy: {{ .Values.service.mongodb.externalTrafficPolicy }} - {{- end }} - {{- if and .Values.service.mongodb.clusterIP (eq .Values.service.mongodb.type "ClusterIP") }} - clusterIP: {{ .Values.service.mongodb.clusterIP }} - {{- end }} - ports: - - name: http - port: {{ .Values.service.mongodb.port }} - {{- if .Values.service.mongodb.nodePort }} - nodePort: {{ .Values.service.mongodb.nodePort }} - {{- end }} - targetPort: {{ .Values.service.mongodb.port }} - selector: - {{- include "mindsdb.selectorLabels" . | nindent 4 }} -{{- end }} diff --git a/helm/mindsdb/templates/mysql-service.yaml b/helm/mindsdb/templates/mysql-service.yaml deleted file mode 100644 index 62361f41bf3..00000000000 --- a/helm/mindsdb/templates/mysql-service.yaml +++ /dev/null @@ -1,38 +0,0 @@ -{{- if .Values.service.mysql.enabled -}} -apiVersion: v1 -kind: Service -metadata: - name: {{ include "mindsdb.fullname" . }}-mysql - labels: - {{- include "mindsdb.labels" . | nindent 4 }} -spec: - type: {{ .Values.service.mysql.type }} - {{- if and .Values.service.mysql.loadBalancerIP (eq .Values.service.mysql.type "LoadBalancer") }} - loadBalancerIP: {{ .Values.service.mysql.loadBalancerIP }} - {{- end }} - {{- if .Values.service.mysql.loadBalancerSourceRanges }} - loadBalancerSourceRanges: - {{- range .Values.service.mysql.loadBalancerSourceRanges }} - - {{ . }} - {{- end }} - {{- end }} - {{- if .Values.service.mysql.externalIPs }} - externalIPs: - {{- toYaml .Values.service.mysql.externalIPs | nindent 4 }} - {{- end }} - {{- if .Values.service.mysql.externalTrafficPolicy }} - externalTrafficPolicy: {{ .Values.service.mysql.externalTrafficPolicy }} - {{- end }} - {{- if and .Values.service.mysql.clusterIP (eq .Values.service.mysql.type "ClusterIP") }} - clusterIP: {{ .Values.service.mysql.clusterIP }} - {{- end }} - ports: - - name: http - port: {{ .Values.service.mysql.port }} - {{- if .Values.service.mysql.nodePort }} - nodePort: {{ .Values.service.mysql.nodePort }} - {{- end }} - targetPort: {{ .Values.service.mysql.port }} - selector: - {{- include "mindsdb.selectorLabels" . | nindent 4 }} -{{- end }} diff --git a/helm/mindsdb/templates/serviceaccount.yaml b/helm/mindsdb/templates/serviceaccount.yaml deleted file mode 100644 index 27c2b57e380..00000000000 --- a/helm/mindsdb/templates/serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "mindsdb.serviceAccountName" . }} - labels: - {{- include "mindsdb.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} diff --git a/helm/mindsdb/templates/tests/test-connection.yaml b/helm/mindsdb/templates/tests/test-connection.yaml deleted file mode 100644 index cc95fcdab86..00000000000 --- a/helm/mindsdb/templates/tests/test-connection.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "{{ include "mindsdb.fullname" . }}-test-connection" - labels: - {{- include "mindsdb.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['wget'] - args: ['{{ include "mindsdb.fullname" . }}:{{ .Values.service.port }}'] - restartPolicy: Never diff --git a/helm/mindsdb/values.yaml b/helm/mindsdb/values.yaml deleted file mode 100644 index f4e537908fe..00000000000 --- a/helm/mindsdb/values.yaml +++ /dev/null @@ -1,114 +0,0 @@ -# Default values for mindsdb. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -replicaCount: 1 - -image: - repository: mindsdb/mindsdb - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - tag: "latest" - -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" - -serviceAccount: - # Specifies whether a service account should be created - create: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - -podAnnotations: {} - -podSecurityContext: {} - # fsGroup: 2000 - -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - -service: - http: - enabled: true - type: ClusterIP - port: 47334 - clusterIP: None - #loadBalancerIP: - #nodePort: - #externalTrafficPolicy: - #externalIPs: - loadBalancerSourceRanges: [] - annotations: - mysql: - enabled: true - type: ClusterIP - port: 47335 - clusterIP: None - #loadBalancerIP: - #nodePort: - #externalTrafficPolicy: - #externalIPs: - #hostPort: - loadBalancerSourceRanges: [] - annotations: - mongodb: - enabled: true - type: ClusterIP - port: 47336 - clusterIP: None - #loadBalancerIP: - #nodePort: - #externalTrafficPolicy: - #externalIPs: - #hostPort: - loadBalancerSourceRanges: [] - annotations: -ingress: - enabled: false - className: "" - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: chart-example.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - -resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - -nodeSelector: {} - -tolerations: [] - -affinity: {} diff --git a/mindsdb/api/executor/command_executor.py b/mindsdb/api/executor/command_executor.py index c0a3bc17118..7cd315adfc9 100644 --- a/mindsdb/api/executor/command_executor.py +++ b/mindsdb/api/executor/command_executor.py @@ -1092,7 +1092,9 @@ def _create_integration(self, name: str, engine: str, connection_args: dict): connection_args[arg_name] = path handler = self.session.integration_controller.create_tmp_handler( - handler_type=engine, connection_data=connection_args + name=name, + engine=engine, + connection_args=connection_args ) status = handler.check_connection() if status.copy_storage: diff --git a/mindsdb/api/http/namespaces/config.py b/mindsdb/api/http/namespaces/config.py index 04c0eea37d7..9aa4dc2da92 100644 --- a/mindsdb/api/http/namespaces/config.py +++ b/mindsdb/api/http/namespaces/config.py @@ -15,7 +15,7 @@ from mindsdb.api.http.utils import http_error from mindsdb.metrics.metrics import api_endpoint_metrics from mindsdb.utilities import log -from mindsdb.utilities.functions import encrypt, decrypt +from mindsdb.utilities.functions import decrypt from mindsdb.utilities.log_controller import get_logs from mindsdb.utilities.config import Config @@ -139,36 +139,19 @@ def put(self, name): params[key] = str(file_path) is_test = params.get('test', False) - - config = Config() - secret_key = config.get('secret_key', 'dummy-key') - + # TODO: Move this to new Endpoint if is_test: del params['test'] - handler_type = params.pop('type', None) params.pop('publish', None) - handler = ca.integration_controller.create_tmp_handler( - handler_type=handler_type, - connection_data=params - ) - + handler = ca.integration_controller.create_tmp_handler(name, handler_type, params) status = handler.check_connection() - if temp_dir is not None: - shutil.rmtree(temp_dir) - resp = status.to_json() - if status.success and 'code' in params: - if hasattr(handler, 'handler_storage'): - # attach storage if exists - export = handler.handler_storage.export_files() - if export: - # encrypt with flask secret key - encrypted = encrypt(export, secret_key) - resp['storage'] = encrypted.decode() - return resp, 200 + config = Config() + secret_key = config.get('secret_key', 'dummy-key') + integration = ca.integration_controller.get(name, sensitive_info=False) if integration is not None: abort(400, f"Integration with name '{name}' already exists") @@ -235,18 +218,6 @@ def post(self, name): return "", 200 -@ns_conf.route('/integrations//check') -@ns_conf.param('name', 'Database integration checks') -class Check(Resource): - @ns_conf.doc('check') - @api_endpoint_metrics('GET', '/config/integrations/integration/check') - def get(self, name): - if ca.integration_controller.get(name) is None: - abort(404, f'Can\'t find database integration: {name}') - connections = ca.integration_controller.check_connections() - return connections.get(name, False), 200 - - @ns_conf.route('/vars') class Vars(Resource): @api_endpoint_metrics('GET', '/config/vars') diff --git a/mindsdb/api/mongo/responders/insert.py b/mindsdb/api/mongo/responders/insert.py index f291ed7bae4..a186de6db52 100644 --- a/mindsdb/api/mongo/responders/insert.py +++ b/mindsdb/api/mongo/responders/insert.py @@ -47,7 +47,9 @@ def _insert_database(self, query, request_env, mindsdb_env): status = HandlerStatusResponse(success=False) try: handler = mindsdb_env["integration_controller"].create_tmp_handler( - handler_type=doc["engine"], connection_data=doc["connection_args"] + name=doc['name'], + engine=doc["engine"], + connection_args=doc["connection_args"] ) status = handler.check_connection() except Exception as e: diff --git a/mindsdb/api/mysql/mysql_proxy/executor/__init__.py b/mindsdb/api/mysql/mysql_proxy/executor/__init__.py index 427aa31a9e5..8c34740bda0 100644 --- a/mindsdb/api/mysql/mysql_proxy/executor/__init__.py +++ b/mindsdb/api/mysql/mysql_proxy/executor/__init__.py @@ -1,9 +1,2 @@ from .mysql_executor import Executor -# from .executor_client import ExecutorClient -# from .executor_client_factory import ExecutorClient -# -# -# # In case of Executor Service need to use ExecutorClient instead of Executor -# # so here we hide the difference between two objects -# # if os.environ.get("MINDSDB_EXECUTOR_URL"): -# Executor = ExecutorClient + diff --git a/mindsdb/api/mysql/mysql_proxy/executor/executor_client_factory.py b/mindsdb/api/mysql/mysql_proxy/executor/executor_client_factory.py deleted file mode 100644 index 2c881e0ba5e..00000000000 --- a/mindsdb/api/mysql/mysql_proxy/executor/executor_client_factory.py +++ /dev/null @@ -1,38 +0,0 @@ -# import os -# -# from mindsdb.api.mysql.mysql_proxy.executor.mysql_executor import Executor -# from mindsdb.utilities import log -# -# -# logger = log.getLogger(__name__) -# -# -# class ExecutorClientFactory: -# def __init__(self): -# self.host = os.environ.get("MINDSDB_EXECUTOR_SERVICE_HOST") -# self.port = os.environ.get("MINDSDB_EXECUTOR_SERVICE_PORT") -# if self.host is not None and self.port is not None: -# try: -# from mindsdb.api.mysql.mysql_proxy.executor.executor_grpc_client import ExecutorClientGRPC -# self.client_class = ExecutorClientGRPC -# except (ImportError, ModuleNotFoundError): -# logger.error("to use microservice mode please install 'pip install mindsdb[grpc]'") -# self.host = None -# self.port = None -# -# def __call__(self, session, sqlserver): -# if self.host is None or self.port is None: -# logger.info( -# "%s.__call__: no post/port to ExecutorService have provided. Local Executor instance will use", -# self.__class__.__name__, -# ) -# return Executor(session, sqlserver) -# -# logger.info("%s.__call__: api to communicate with db services - gRPC", -# self.__class__.__name__, -# ) -# -# return self.client_class(session, sqlserver) -# -# -# ExecutorClient = ExecutorClientFactory() diff --git a/mindsdb/api/mysql/mysql_proxy/executor/executor_grpc_client.py b/mindsdb/api/mysql/mysql_proxy/executor/executor_grpc_client.py deleted file mode 100644 index 641c26da543..00000000000 --- a/mindsdb/api/mysql/mysql_proxy/executor/executor_grpc_client.py +++ /dev/null @@ -1,145 +0,0 @@ -# import os -# import json -# import pickle -# from uuid import uuid4 -# -# from mindsdb.utilities import log -# from mindsdb.utilities.context import context as ctx -# from mindsdb.integrations.libs.handler_helpers import action_logger -# -# logger = log.getLogger(__name__) -# -# -# try: -# import grpc -# from mindsdb.microservices_grpc.executor import executor_pb2_grpc -# from mindsdb.microservices_grpc.executor import executor_pb2 -# except ImportError: -# logger.error("to user microservice mode please install 'pip install mindsdb[grpc]'") -# -# -# class ExecutorClientGRPC: -# -# def __init__(self, session, sqlserver): -# self.id = f"executor_{uuid4()}" -# -# self.sqlserver = sqlserver -# self.session = session -# self.query = None -# -# # returns -# self.columns = [] -# self.params = [] -# self.data = None -# self.state_track = None -# self.server_status = None -# self.is_executed = False -# -# # additional attributes used in handling query process -# self.sql = "" -# self.sql_lower = "" -# -# host = os.environ.get("MINDSDB_EXECUTOR_SERVICE_HOST", None) -# port = os.environ.get("MINDSDB_EXECUTOR_SERVICE_PORT", None) -# self.channel = grpc.insecure_channel(f"{host}:{port}") -# self.stub = executor_pb2_grpc.ExecutorServiceStub(self.channel) -# -# @property -# def _context(self): -# logger.info("%s._context: preparing context. id - %s, session - %s", self.__class__.__name__, self.id, self.session) -# _id = self.id -# connection_id = self.sqlserver.connection_id if hasattr(self.sqlserver, "connection_id") else -1 -# session_id = self.session.id -# session = json.dumps(self.session.to_json()) -# context = json.dumps(ctx.dump()) -# -# return executor_pb2.ExecutorContext( -# id=_id, -# connection_id=connection_id, -# session_id=session_id, -# session=session, -# context=context, -# ) -# -# def __del__(self): -# self.stub.DeleteExecutor(self._context) -# -# def _update_attrs(self, response: executor_pb2.ExecutorResponse): -# self.columns = pickle.loads(response.columns) -# self.params = pickle.loads(response.params) -# self.data = pickle.loads(response.data) -# self.state_track = response.state_track -# self.is_executed = response.is_executed -# self.session.from_json(json.loads(response.session)) -# self.error_code = response.error_code -# self.error_message = response.error_message -# logger.info("%s._update_attrs: got data from service - %s", self.__class__.__name__, self.data) -# -# @action_logger(logger) -# def stmt_prepare(self, sql): -# params = executor_pb2.ExecutionContext(context=self._context, sql=sql) -# resp = self.stub.StatementPrepare(params) -# if resp.error_message != "": -# raise Exception(resp.error_message) -# self._update_attrs(resp) -# -# @action_logger(logger) -# def stmt_execute(self, param_values): -# if self.is_executed: -# return -# -# params = executor_pb2.StatementExecuteContext(context=self._context, param_values=json.dumps(param_values)) -# resp = self.stub.StatementExecute(params) -# if resp.error_message != "": -# raise Exception(resp.error_message) -# self._update_attrs(resp) -# -# @action_logger(logger) -# def query_execute(self, sql): -# params = executor_pb2.ExecutionContext(context=self._context, sql=sql) -# resp = self.stub.QueryExecute(params) -# if resp.error_message != "": -# raise Exception(resp.error_message) -# self._update_attrs(resp) -# -# @action_logger(logger) -# def binary_query_execute(self, sql): -# sql = pickle.dumps(sql) -# params = executor_pb2.BinaryExecutionContext(context=self._context, sql=sql) -# resp = self.stub.BinaryQueryExecute(params) -# if resp.error_message != "": -# raise Exception(resp.error_message) -# self._update_attrs(resp) -# -# @action_logger(logger) -# def execute_external(self, sql): -# params = executor_pb2.ExecutionContext(context=self._context, sql=sql) -# resp = self.stub.ExecuteExternal(params) -# if resp.error_message != "": -# raise Exception(resp.error_message) -# self._update_attrs(resp) -# -# @action_logger(logger) -# def parse(self, sql): -# params = executor_pb2.ExecutionContext(context=self._context, sql=sql) -# resp = self.stub.Parse(params) -# if resp.error_message != "": -# raise Exception(resp.error_message) -# self._update_attrs(resp) -# -# @action_logger(logger) -# def do_execute(self): -# if self.is_executed: -# return -# resp = self.stub.DoExecute(self._context) -# if resp.error_message != "": -# raise Exception(resp.error_message) -# self._update_attrs(resp) -# -# @action_logger(logger) -# def change_default_db(self, new_db): -# params = executor_pb2.DefaultDBContext(context=self._context, new_db=new_db) -# resp = self.stub.ChangeDefaultDB(params) -# if resp.error_message != "": -# raise Exception(resp.error_message) -# self._update_attrs(resp) diff --git a/mindsdb/api/mysql/mysql_proxy/executor/executor_grpc_wrapper.py b/mindsdb/api/mysql/mysql_proxy/executor/executor_grpc_wrapper.py deleted file mode 100644 index d1cf34e35b0..00000000000 --- a/mindsdb/api/mysql/mysql_proxy/executor/executor_grpc_wrapper.py +++ /dev/null @@ -1,252 +0,0 @@ -# import json -# import pickle -# import traceback -# from concurrent import futures -# -# import grpc -# from mindsdb.microservices_grpc.executor import executor_pb2_grpc -# from mindsdb.microservices_grpc.executor import executor_pb2 -# -# from mindsdb.utilities.context import context as ctx -# from mindsdb.api.mysql.mysql_proxy.executor.mysql_executor import Executor -# from mindsdb.api.executor.controllers.session_controller import ( -# SessionController, -# ) -# from mindsdb.utilities import log -# -# logger = log.getLogger(__name__) -# -# -# class SqlServerStub: -# """This class is just an emulation of Server object, -# used by Executor. -# In 'monolithic' mode of MindsDB work the Executor takes -# some information from the sql server which. Here we emulate -# this object.""" -# -# def __init__(self, **kwargs): -# for arg in kwargs: -# setattr(self, arg, kwargs[arg]) -# -# -# class ExecutorServiceServicer(executor_pb2_grpc.ExecutorServiceServicer): -# -# def __init__(self): -# logger.info( -# "%s.__init__: ", self.__class__.__name__ -# ) -# self.executors_cache = {} -# self.sessions_cache = {} -# -# def run(self, **kwargs): -# host = kwargs.get("host", "127.0.0.1") -# port = kwargs.get("port", 50051) -# addr = f"{host}:{port}" -# server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) -# executor_pb2_grpc.add_ExecutorServiceServicer_to_server( -# ExecutorServiceServicer(), server) -# server.add_insecure_port(addr) -# server.start() -# server.wait_for_termination() -# -# def _get_executor(self, params: executor_pb2.ExecutorContext): -# # We have to send context between client and server -# # here we load the context json received from the client(mindsdb) -# # to the local context instance in this Flask thread -# ctx.load(json.loads(params.context)) -# exec_id = params.id -# if exec_id in self.executors_cache: -# logger.info( -# "%s: executor %s found in cache", self.__class__.__name__, exec_id -# ) -# return self.executors_cache[exec_id] -# session_id = params.session_id -# if session_id in self.sessions_cache: -# logger.info( -# "%s: session %s found in cache", self.__class__.__name__, session_id -# ) -# session = self.sessions_cache[session_id] -# else: -# logger.info( -# "%s: creating new session. id - %s, params - %s", -# self.__class__.__name__, -# session_id, -# params.session, -# ) -# session = SessionController() -# self.sessions_cache[session_id] = session -# session_params = json.loads(params.session) -# session.database = session_params["database"] -# session.username = session_params["username"] -# session.auth = session_params["auth"] -# session.prepared_stmts = session_params["prepared_stmts"] -# session.packet_sequence_number = session_params["packet_sequence_number"] -# sqlserver = SqlServerStub(connection_id=params.connection_id) -# -# logger.info( -# "%s: session info - id=%s, params=%s", -# self.__class__.__name__, -# session_id, -# session.to_json(), -# ) -# logger.info( -# "%s: creating new executor. id - %s, session_id - %s", -# self.__class__.__name__, -# exec_id, -# session_id, -# ) -# executor = Executor(session, sqlserver) -# self.executors_cache[exec_id] = executor -# return executor -# -# @staticmethod -# def _prepare_response(executor): -# columns = pickle.dumps(executor.columns) -# params = pickle.dumps(executor.params) -# data = pickle.dumps(executor.data) -# state_track = executor.state_track -# server_status = executor.server_status -# is_executed = executor.is_executed -# session = json.dumps(executor.session.to_json()) -# -# res = executor_pb2.ExecutorResponse( -# columns=columns, -# params=params, -# data=data, -# state_track=state_track, -# server_status=server_status, -# is_executed=is_executed, -# session=session, -# ) -# return res -# -# def DeleteExecutor(self, request, context): -# # to delete executors -# exec_id = request.id -# session_id = request.session_id -# logger.info( -# "%s: removing executor instance. id - %s", self.__class__.__name__, exec_id -# ) -# if exec_id is not None and exec_id in self.executors_cache: -# del self.executors_cache[exec_id] -# -# if session_id is not None and session_id in self.sessions_cache: -# del self.sessions_cache[session_id] -# -# return executor_pb2.ExecutorStatusResponse(success=True, error_message="") -# -# def StatementPrepare(self, request, context): -# result = None -# try: -# params = request.json -# executor = self._get_executor(params) -# sql = params.get("sql") -# executor.stmt_prepare(sql) -# result = self._prepare_response(executor) -# except Exception: -# err_msg = traceback.format_exc() -# result = executor_pb2.ExecutorResponse(error_message=err_msg) -# return result -# -# def StatementExecute(self, request, context): -# result = None -# try: -# executor = self._get_executor(request.context) -# param_values = json.loads(request.param_values) -# executor.stmt_execute(param_values) -# result = self._prepare_response(executor) -# except Exception: -# err_msg = traceback.format_exc() -# result = executor_pb2.ExecutorResponse(error_message=err_msg) -# return result -# -# def QueryExecute(self, request, context): -# result = None -# try: -# executor = self._get_executor(request.context) -# sql = request.sql -# executor.query_execute(sql) -# result = self._prepare_response(executor) -# result.error_code = executor.error_code if executor.error_code else 0 -# result.error_message = executor.error_message if executor.error_message else "" -# except Exception: -# err_msg = traceback.format_exc() -# result = executor_pb2.ExecutorResponse(error_message=err_msg) -# return result -# -# def BinaryQueryExecute(self, request, context): -# result = None -# try: -# executor = self._get_executor(request.context) -# sql = pickle.loads(request.sql) -# executor.binary_query_execute(sql) -# result = self._prepare_response(executor) -# result.error_code = executor.error_code if executor.error_code else 0 -# result.error_message = executor.error_message if executor.error_message else "" -# except Exception: -# err_msg = traceback.format_exc() -# result = executor_pb2.ExecutorResponse(error_message=err_msg) -# return result -# -# def ExecuteExternal(self, request, context): -# result = None -# try: -# executor = self._get_executor(request.context) -# sql = request.sql -# executor.execute_external(sql) -# result = self._prepare_response(executor) -# except Exception: -# err_msg = traceback.format_exc() -# result = executor_pb2.ExecutorResponse(error_message=err_msg) -# return result -# -# def Parse(self, request, context): -# result = None -# try: -# executor = self._get_executor(request.context) -# sql = request.sql -# executor.parse(sql) -# result = self._prepare_response(executor) -# except Exception: -# err_msg = traceback.format_exc() -# result = executor_pb2.ExecutorResponse(error_message=err_msg) -# return result -# -# def DoExecute(self, request, context): -# result = None -# try: -# executor = self._get_executor(request.context) -# executor.do_execute() -# result = self._prepare_response(executor) -# except Exception: -# err_msg = traceback.format_exc() -# result = executor_pb2.ExecutorResponse(error_message=err_msg) -# return result -# -# def ChangeDefaultDB(self, request, context): -# result = None -# try: -# executor = self._get_executor(request.context) -# new_db = request.new_db -# executor.change_default_db(new_db) -# result = self._prepare_response(executor) -# except Exception: -# err_msg = traceback.format_exc() -# result = executor_pb2.ExecutorResponse(error_message=err_msg) -# return result -# -# -# def serve(): -# host = "127.0.0.1" -# port = 50052 -# addr = f"{host}:{port}" -# server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) -# executor_pb2_grpc.add_ExecutorServiceServicer_to_server( -# ExecutorServiceServicer(), server) -# server.add_insecure_port(addr) -# server.start() -# server.wait_for_termination() -# -# -# if __name__ == '__main__': -# serve() diff --git a/mindsdb/api/mysql/mysql_proxy/executor/executor_service.py b/mindsdb/api/mysql/mysql_proxy/executor/executor_service.py deleted file mode 100644 index b221171f8a6..00000000000 --- a/mindsdb/api/mysql/mysql_proxy/executor/executor_service.py +++ /dev/null @@ -1,18 +0,0 @@ -# import os -# -# import mindsdb.interfaces.storage.db as db -# from mindsdb.api.mysql.mysql_proxy.executor.executor_grpc_wrapper import ( -# ExecutorServiceServicer, -# ) -# from mindsdb.utilities.config import Config -# from mindsdb.utilities import log -# -# if __name__ == "__main__": -# config = Config() -# db.init() -# logger = log.getLogger(__name__) -# app = ExecutorServiceServicer() -# port = int(os.environ.get("PORT", 5500)) -# host = os.environ.get("HOST", "0.0.0.0") -# logger.info("Running Executor service: host=%s, port=%s", host, port) -# app.run(debug=True, host=host, port=port) diff --git a/mindsdb/api/nlp/__init__.py b/mindsdb/api/nlp/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/mindsdb/integrations/handlers_client/__init__.py b/mindsdb/integrations/handlers_client/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/mindsdb/integrations/handlers_client/db_client_factory.py b/mindsdb/integrations/handlers_client/db_client_factory.py deleted file mode 100644 index d3c8c47b962..00000000000 --- a/mindsdb/integrations/handlers_client/db_client_factory.py +++ /dev/null @@ -1,38 +0,0 @@ -import os -from mindsdb.utilities import log - - -logger = log.getLogger(__name__) - - -class DBClientFactory: - def __init__(self): - self.host = os.environ.get("MINDSDB_DB_SERVICE_HOST", None) - self.port = os.environ.get("MINDSDB_DB_SERVICE_PORT", None) - if self.host is not None and self.port is not None: - try: - from mindsdb.integrations.handlers_client.db_grpc_client import DBClientGRPC - self.client_class = DBClientGRPC - except (ImportError, ModuleNotFoundError): - logger.info("to work with microservice mode please install 'pip install mindsdb[grpc]'") - self.host = None - self.port = None - - def __call__(self, handler_type: str, handler: type, **kwargs: dict): - if self.host is None or self.port is None: - logger.info( - "%s.__call__: no post/port to DBService have provided. Handle all db request locally", - self.__class__.__name__, - ) - return handler(**kwargs) - - logger.info("%s.__call__: api to communicate with db services - gRPC, host - %s, port - %s", - self.__class__.__name__, - self.host, - self.port, - ) - - return self.client_class(handler_type, **kwargs) - - -DBClient = DBClientFactory() diff --git a/mindsdb/integrations/handlers_client/db_grpc_client.py b/mindsdb/integrations/handlers_client/db_grpc_client.py deleted file mode 100644 index d49df349b66..00000000000 --- a/mindsdb/integrations/handlers_client/db_grpc_client.py +++ /dev/null @@ -1,141 +0,0 @@ -import os -import pickle -import json - -import grpc -from mindsdb.microservices_grpc.db import db_pb2_grpc -from mindsdb.microservices_grpc.db import db_pb2, common_pb2 - -from mindsdb.integrations.libs.response import ( - HandlerStatusResponse as StatusResponse, - HandlerResponse as Response, -) -from mindsdb.integrations.libs.handler_helpers import action_logger -from mindsdb.utilities.context import context as ctx -from mindsdb.utilities import log - - -logger = log.getLogger(__name__) - - -class DBClientGRPC: - - def __init__(self, handler_type: str, **kwargs: dict): - self.handler_type = handler_type - self.handler_params = kwargs - host = os.environ.get("MINDSDB_DB_SERVICE_HOST", None) - port = os.environ.get("MINDSDB_DB_SERVICE_PORT", None) - for a in ("fs_store", "file_storage"): - if a in self.handler_params: - del self.handler_params[a] - - # FileController is an object - # so it is not a good idea to send it - # to service side as parameter - # will create a separate instace instead - if self.handler_type == "files": - del self.handler_params["file_controller"] - self.channel = grpc.insecure_channel(f"{host}:{port}") - self.stub = db_pb2_grpc.DBServiceStub(self.channel) - - def __del__(self): - self.channel.close() - - @property - def context(self): - ctx_str = json.dumps(ctx.dump()) - return db_pb2.HandlerContext(handler_type=self.handler_type, - handler_params=json.dumps(self.handler_params), - context=ctx_str) - - @staticmethod - def _to_status_response(response: common_pb2.StatusResponse): - return StatusResponse(success=response.success, - error_message=response.error_message) - - @staticmethod - def _to_response(response: common_pb2.Response): - data = pickle.loads(response.data_frame) - return Response( - resp_type=response.type, - data_frame=data, - query=response.query, - error_code=response.error_code, - error_message=response.error_message, - ) - - @action_logger(logger) - def connect(self): - resp = self.stub.Connect(self.context) - logger.info( - "%s.connect: returns success - %s, error - %s", - self.__class__.__name__, - resp.success, - resp.error_message - ) - - return self._to_status_response(resp) - - @action_logger(logger) - def check_connection(self): - resp = self.stub.CheckConnection(self.context) - logger.info( - "%s.check_connection: returns success - %s, error - %s", - self.__class__.__name__, - resp.success, - resp.error_message - ) - - return self._to_status_response(resp) - - @action_logger(logger) - def disconnect(self): - resp = self.stub.Disconnect(self.context) - logger.info( - "%s.disconnect: returns success - %s, error - %s", - self.__class__.__name__, - resp.success, - resp.error_message, - ) - - return self._to_status_response(resp) - - @action_logger(logger) - def native_query(self, query): - request = db_pb2.NativeQueryContext(context=self.context, query=query) - resp = self.stub.NativeQuery(request) - logger.info("%s.native_query: returned error - %s, error_message - %s", self.__class__.__name__, resp.error_code, resp.error_message) - data = pickle.loads(resp.data_frame) - logger.info("%s.native_query: returned data(type of %s) - %s", self.__class__.__name__, type(data), data) - - return self._to_response(resp) - - @action_logger(logger) - def query(self, query): - query = pickle.dumps(query) - request = db_pb2.BinaryQueryContext(context=self.context, query=query) - resp = self.stub.BinaryQuery(request) - logger.info("%s.query: returned error - %s, error_message - %s", self.__class__.__name__, resp.error_code, resp.error_message) - data = pickle.loads(resp.data_frame) - logger.info("%s.query: returned data(type of %s) - %s", self.__class__.__name__, type(data), data) - - return self._to_response(resp) - - @action_logger(logger) - def get_tables(self): - resp = self.stub.GetTables(self.context) - logger.info("%s.get_tables: returned error - %s, error_message - %s", self.__class__.__name__, resp.error_code, resp.error_message) - data = pickle.loads(resp.data_frame) - logger.info("%s.get_tables: returned data(type of %s) - %s", self.__class__.__name__, type(data), data) - - return self._to_response(resp) - - @action_logger(logger) - def get_columns(self, table): - request = db_pb2.ColumnsContext(context=self.context, table=table) - resp = self.stub.GetColumns(request) - logger.info("%s.get_columns: returned error - %s, error_message - %s", self.__class__.__name__, resp.error_code, resp.error_message) - data = pickle.loads(resp.data_frame) - logger.info("%s.get_columns: returned data(type of %s) - %s", self.__class__.__name__, type(data), data) - - return self._to_response(resp) diff --git a/mindsdb/integrations/handlers_wrapper/__init__.py b/mindsdb/integrations/handlers_wrapper/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/mindsdb/integrations/handlers_wrapper/db_grpc_wrapper.py b/mindsdb/integrations/handlers_wrapper/db_grpc_wrapper.py deleted file mode 100644 index 956a5ba2262..00000000000 --- a/mindsdb/integrations/handlers_wrapper/db_grpc_wrapper.py +++ /dev/null @@ -1,241 +0,0 @@ -import json -import pickle -import traceback -from concurrent import futures - -import grpc -from mindsdb.microservices_grpc.db import db_pb2_grpc -from mindsdb.microservices_grpc.db import db_pb2, common_pb2 - -from mindsdb.interfaces.file.file_controller import FileController -from mindsdb.integrations.libs.handler_helpers import get_handler -from mindsdb.api.executor.data_types.response_type import RESPONSE_TYPE -from mindsdb.utilities.context import context as ctx -from mindsdb.utilities import log - - -logger = log.getLogger(__name__) - - -class DBServiceServicer(db_pb2_grpc.DBServiceServicer): - - def __init__(self): - logger.info( - "%s.__init__: ", self.__class__.__name__ - ) - - def _get_handler(self, handler_ctx: db_pb2.HandlerContext): - ctx.load(json.loads(handler_ctx.context)) - handler_class = get_handler(handler_ctx.handler_type) - logger.info( - "%s._get_handler: requested instance of %s handler", - self.__class__.__name__, - handler_class, - ) - handler_kwargs = json.loads(handler_ctx.handler_params) - # Create an instance of FileController for - # 'files' type of handler - if handler_ctx.handler_type == "files": - handler_kwargs["file_controller"] = FileController() - return handler_class(**handler_kwargs) - - def CheckConnection(self, request, context): - - result = None - logger.info( - "%s.check_connection calling", self.__class__.__name__ - ) - try: - handler = self._get_handler(request) - res = handler.check_connection() - result = common_pb2.StatusResponse(success=res.success, error_message=res.error_message) - except Exception: - msg = traceback.format_exc() - result = common_pb2.StatusResponse(success=False, error_message=msg) - return result - - def Connect(self, request, context): - - result = None - logger.info( - "%s.connect calling", self.__class__.__name__ - ) - try: - handler = self._get_handler(request) - handler.connect() - result = common_pb2.StatusResponse(success=True, error_message="") - except Exception: - msg = traceback.format_exc() - result = common_pb2.StatusResponse(success=False, error_message=msg) - return result - - def Disconnect(self, request, context): - result = None - logger.info( - "%s.disconnect calling", self.__class__.__name__ - ) - try: - handler = self._get_handler(request) - handler.disconnect() - result = common_pb2.StatusResponse(success=True, error_message="") - except Exception: - msg = traceback.format_exc() - result = common_pb2.StatusResponse(success=False, error_message=msg) - return result - - def NativeQuery(self, request, context): - - result = None - query = request.query - - logger.info( - "%s.native_query: calling 'native_query' with query - %s", - self.__class__.__name__, - query, - ) - try: - handler = self._get_handler(request.context) - res = handler.native_query(query) - data = pickle.dumps(res.data_frame) - result = common_pb2.Response( - type=res.resp_type, - data_frame=data, - query=res.query, - error_code=res.error_code, - error_message=res.error_message - ) - - except Exception: - msg = traceback.format_exc() - logger.error("%s.native_query: error - %s", self.__class__.__name__, msg) - result = common_pb2.Response( - type=RESPONSE_TYPE.ERROR, - data_frame=None, - query=0, - error_code=1, - error_message=msg, - ) - return result - - def BinaryQuery(self, request, context): - - result = None - try: - query = pickle.loads(request.query) - - logger.info( - "%s.query: calling 'query' with query - %s", - self.__class__.__name__, - query, - ) - handler = self._get_handler(request.context) - res = handler.query(query) - data = pickle.dumps(res.data_frame) - result = common_pb2.Response( - type=res.resp_type, - data_frame=data, - query=res.query, - error_code=res.error_code, - error_message=res.error_message - ) - - except Exception: - msg = traceback.format_exc() - logger.error("%s.query: error - %s", self.__class__.__name__, msg) - result = common_pb2.Response( - type=RESPONSE_TYPE.ERROR, - data_frame=None, - query=0, - error_code=1, - error_message=msg - ) - return result - - def GetTables(self, request, context): - - logger.info( - "%s.get_tables: calling", - self.__class__.__name__, - ) - result = None - try: - handler = self._get_handler(request) - res = handler.get_tables() - data = pickle.dumps(res.data_frame) - result = common_pb2.Response( - type=res.resp_type, - data_frame=data, - query=res.query, - error_code=res.error_code, - error_message=res.error_message, - ) - - except Exception: - msg = traceback.format_exc() - logger.error("%s.get_tables: error - %s", self.__class__.__name__, msg) - result = common_pb2.Response( - type=RESPONSE_TYPE.ERROR, - data_frame=None, - query=0, - error_code=1, - error_message=msg, - ) - return result - - def GetColumns(self, request, context): - - logger.info( - "%s.get_columns: calling for table - %s", - self.__class__.__name__, - request.table, - ) - result = None - try: - handler = self._get_handler(request.context) - res = handler.get_columns(request.table) - data = pickle.dumps(res.data_frame) - result = common_pb2.Response( - type=res.resp_type, - data_frame=data, - query=res.query, - error_code=res.error_code, - error_message=res.error_message, - ) - - except Exception: - msg = traceback.format_exc() - logger.error("%s.get_tables: error - %s", self.__class__.__name__, msg) - result = common_pb2.Response( - type=RESPONSE_TYPE.ERROR, - data_frame=None, - query=0, - error_code=1, - error_message=msg, - ) - return result - - def run(self, **kwargs): - host = kwargs.get("host", "127.0.0.1") - port = kwargs.get("port", 50051) - addr = f"{host}:{port}" - server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) - db_pb2_grpc.add_DBServiceServicer_to_server( - DBServiceServicer(), server) - server.add_insecure_port(addr) - # logger.info("staring rpc server on [::]:50051") - server.start() - server.wait_for_termination() - - -def serve(): - server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) - db_pb2_grpc.add_DBServiceServicer_to_server( - DBServiceServicer(), server) - server.add_insecure_port('[::]:50051') - logger.info("staring rpc server on [::]:50051") - server.start() - server.wait_for_termination() - - -if __name__ == '__main__': - serve() diff --git a/mindsdb/integrations/handlers_wrapper/db_handler_service.py b/mindsdb/integrations/handlers_wrapper/db_handler_service.py deleted file mode 100644 index 4d10348a545..00000000000 --- a/mindsdb/integrations/handlers_wrapper/db_handler_service.py +++ /dev/null @@ -1,16 +0,0 @@ -import os - -import mindsdb.interfaces.storage.db as db -from mindsdb.integrations.handlers_wrapper.db_grpc_wrapper import DBServiceServicer -from mindsdb.utilities.config import Config -from mindsdb.utilities import log - -if __name__ == "__main__": - config = Config() - db.init() - logger = log.getLogger(__name__) - app = DBServiceServicer() - port = int(os.environ.get("PORT", 5001)) - host = os.environ.get("HOST", "0.0.0.0") - logger.info("Running dbservice(%s): host=%s, port=%s", type(app), host, port) - app.run(debug=True, host=host, port=port) diff --git a/mindsdb/integrations/handlers_wrapper/ml_grpc_wrapper.py b/mindsdb/integrations/handlers_wrapper/ml_grpc_wrapper.py deleted file mode 100644 index 61fc063f762..00000000000 --- a/mindsdb/integrations/handlers_wrapper/ml_grpc_wrapper.py +++ /dev/null @@ -1,179 +0,0 @@ -import json -import pickle -import traceback -from concurrent import futures - -import grpc -from mindsdb.microservices_grpc.ml import ml_pb2_grpc -from mindsdb.microservices_grpc.ml import ml_pb2, common_pb2 - -from mindsdb.interfaces.storage.model_fs import ModelStorage, HandlerStorage - -from mindsdb.integrations.libs.response import ( - RESPONSE_TYPE, -) -from mindsdb.interfaces.database.integrations import integration_controller -from mindsdb.integrations.libs.handler_helpers import get_handler -from mindsdb.utilities.context import context as ctx -from mindsdb.utilities import log - - -logger = log.getLogger(__name__) - - -class MLServiceServicer(ml_pb2_grpc.MLServiceServicer): - - def __init__(self): - logger.info( - "%s.__init__: ", self.__class__.__name__ - ) - - def run(self, **kwargs): - host = kwargs.get("host", "127.0.0.1") - port = kwargs.get("port", 50052) - addr = f"{host}:{port}" - server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) - ml_pb2_grpc.add_MLServiceServicer_to_server( - MLServiceServicer(), server) - server.add_insecure_port(addr) - server.start() - server.wait_for_termination() - - def _get_handler_controller(self): - return integration_controller - - def _get_handler(self, handler_context: ml_pb2.HandlerContextML): - ctx.load(json.loads(handler_context.context)) - params = json.loads(handler_context.handler_params) - logger.info( - "%s._get_handler: create handler. params - %s", - self.__class__.__name__, - params, - ) - integration_id = handler_context.integration_id - predictor_id = handler_context.predictor_id - _type = params.get("engine") - del params["engine"] - logger.info( - "%s.get_handler: request handler of type - %s, context - %s", - self.__class__.__name__, - _type, - params, - ) - - HandlerClass = get_handler(_type) - handlerStorage = HandlerStorage(integration_id) - modelStorage = ModelStorage(predictor_id) - - ml_handler = HandlerClass( - engine_storage=handlerStorage, - model_storage=modelStorage, - **params - ) - return ml_handler - - def Predict(self, request, context): - result = None - try: - args = json.loads(request.args) - df = pickle.loads(request.df) - logger.info( - "%s.Predict: args - %s", - self.__class__.__name__, - args, - ) - - handler = self._get_handler(request.context) - predictions = handler.predict(df, args) - handler.close() - - logger.info( - "%s.Predict: got predictions - %s(type - %s)", - self.__class__.__name__, - predictions, - type(predictions), - ) - predictions = pickle.dumps(predictions) - result = common_pb2.Response( - type=RESPONSE_TYPE.OK, - data_frame=predictions, - query=0, - error_code=0, - error_message="", - ) - - except Exception: - msg = traceback.format_exc() - logger.error("%s.Predict: error - %s", self.__class__.__name__, msg) - result = common_pb2.Response( - type=RESPONSE_TYPE.ERROR, - data_frame=None, - query=0, - error_code=1, - error_message=msg, - ) - return result - - def Create(self, request, context): - result = None - try: - args = json.loads(request.args) - target = request.target - df = pickle.loads(request.df) - logger.info( - "%s.Create: target - %s df - %s, args - %s", - self.__class__.__name__, - target, - df, - args, - ) - - handler = self._get_handler(request.context) - handler.create(target, df=df, args=args) - - result = common_pb2.StatusResponse(success=True, error_message="") - - except Exception: - msg = traceback.format_exc() - logger.error("%s.Create: error - %s", self.__class__.__name__, msg) - result = common_pb2.StatusResponse(success=False, error_message=msg) - - return result - - def Update(self, request, context): - - result = None - try: - args = json.loads(request.args) - df = pickle.loads(request.df) - logger.info( - "%s.Update: args - %s", - self.__class__.__name__, - args, - ) - - handler = self._get_handler(request.context) - handler.update(df, args) - - result = common_pb2.StatusResponse(success=True, error_message="") - - except Exception: - msg = traceback.format_exc() - logger.error("%s.Update: error - %s", self.__class__.__name__, msg) - result = common_pb2.StatusResponse(success=False, error_message=msg) - - return result - - -def serve(): - server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) - ml_pb2_grpc.add_MLServiceServicer_to_server( - MLServiceServicer(), server) - server.add_insecure_port('[::]:50052') - logger.info("staring rpc server on [::]:50052") - server.start() - server.wait_for_termination() - - -if __name__ == '__main__': - serve() diff --git a/mindsdb/integrations/handlers_wrapper/ml_handler_service.py b/mindsdb/integrations/handlers_wrapper/ml_handler_service.py deleted file mode 100644 index 8f2887fed58..00000000000 --- a/mindsdb/integrations/handlers_wrapper/ml_handler_service.py +++ /dev/null @@ -1,30 +0,0 @@ -import os - -import mindsdb.interfaces.storage.db as db -from mindsdb.integrations.handlers_wrapper.ml_grpc_wrapper import MLServiceServicer -from mindsdb.integrations.libs.handler_helpers import registry -from mindsdb.utilities import log -from mindsdb.utilities.config import Config - -if __name__ == "__main__": - config = Config() - db.init() - logger = log.getLogger(__name__) - - app = MLServiceServicer() - port = int(os.environ.get("PORT", 5001)) - host = os.environ.get("HOST") - - # If there is a handler discovery service - # start a thread to send service info - # to the discovery service - registry_url = os.environ.get("REGISTRY_URL") - if registry_url: - registry_url = f"{registry_url}/register" - interval = os.environ.get("REGISTRY_INTERVAL", 50) - service_type = os.environ.get("MINDSDB_SERVICE_TYPE", "lightwood") - data = {"host": host, "port": port, "type": service_type} - registry(registry_url, data, interval) - - logger.info("Running ML service: host=%s, port=%s", host, port) - app.run(debug=True, host="0.0.0.0", port=port) diff --git a/mindsdb/integrations/libs/ml_handler_process/learn_process.py b/mindsdb/integrations/libs/ml_handler_process/learn_process.py index b0fb8ce3d79..dd37fab6c95 100644 --- a/mindsdb/integrations/libs/ml_handler_process/learn_process.py +++ b/mindsdb/integrations/libs/ml_handler_process/learn_process.py @@ -94,7 +94,6 @@ def learn_process(data_integration_ref: dict, problem_definition: dict, fetch_da if base_model_id is not None: kwargs['base_model_storage'] = ModelStorage(base_model_id) kwargs['base_model_storage'].fileStorage.pull() - ml_handler = module.Handler( engine_storage=handlerStorage, model_storage=modelStorage, diff --git a/mindsdb/integrations/libs/net_helpers.py b/mindsdb/integrations/libs/net_helpers.py deleted file mode 100644 index ce74c9eae2a..00000000000 --- a/mindsdb/integrations/libs/net_helpers.py +++ /dev/null @@ -1,26 +0,0 @@ -"""This module contains some useful features which -help to increase solution stability in network layer""" - -import time - - -def sending_attempts(exception_type=Exception, attempts_number=3, delay=0.5): - """Decorator factory for functions where several call attempts may require for success""" - def decorator(func): - def wrapper(*args, **kwargs): - to_raise = None - res = None - for _ in range(attempts_number): - try: - res = func(*args, **kwargs) - break - # Save catched exception - except Exception as e: - to_raise = e - time.sleep(delay) - # Else executed if there was no loop breaks - else: - raise exception_type(f"{type(to_raise)}: {to_raise}") from to_raise # noqa: pylint: disable=raising-bad-type - return res - return wrapper - return decorator diff --git a/mindsdb/interfaces/database/integrations.py b/mindsdb/interfaces/database/integrations.py index b6141f94744..80fc68ef470 100644 --- a/mindsdb/interfaces/database/integrations.py +++ b/mindsdb/interfaces/database/integrations.py @@ -26,7 +26,6 @@ from mindsdb.integrations.libs.base import BaseMLEngine from mindsdb.integrations.libs.api_handler import APIHandler from mindsdb.integrations.libs.const import HANDLER_CONNECTION_ARG_TYPE as ARG_TYPE, HANDLER_TYPE -from mindsdb.integrations.handlers_client.db_client_factory import DBClient from mindsdb.interfaces.model.functions import get_model_records from mindsdb.utilities.context import context as ctx from mindsdb.utilities import log @@ -382,20 +381,9 @@ def get_all(self, sensitive_info=True): integration_dict[record.name] = self._get_integration_record_data(record, sensitive_info) return integration_dict - def check_connections(self): - connections = {} - for integration_name, integration_meta in self.get_all().items(): - handler = self.create_tmp_handler( - handler_type=integration_meta['engine'], - connection_data=integration_meta['connection_data'] - ) - status = handler.check_connection() - connections[integration_name] = status.get('success', False) - return connections - def _make_handler_args(self, name: str, handler_type: str, connection_data: dict, integration_id: int = None, file_storage: FileStorage = None, handler_storage: HandlerStorage = None): - handler_ars = dict( + handler_args = dict( name=name, integration_id=integration_id, connection_data=connection_data, @@ -404,47 +392,33 @@ def _make_handler_args(self, name: str, handler_type: str, connection_data: dict ) if handler_type == 'files': - handler_ars['file_controller'] = FileController() + handler_args['file_controller'] = FileController() elif self.handler_modules.get(handler_type, False).type == HANDLER_TYPE.ML: - handler_ars['handler_controller'] = self - handler_ars['company_id'] = ctx.company_id + handler_args['handler_controller'] = self + handler_args['company_id'] = ctx.company_id - return handler_ars + return handler_args - def create_tmp_handler(self, handler_type: str, connection_data: dict) -> object: - """ Returns temporary handler. That handler does not exist in database. + def create_tmp_handler(self, name: str, engine: str, connection_args: dict) -> dict: + """Create temporary handler, mostly for testing connections - Args: - handler_type (str) - connection_data (dict) + Args: + name (str): Integration name + engine (str): Integration engine name + connection_args (dict): Connection arguments - Returns: - Handler object + Returns: + HandlerClass: Handler class instance """ - handler_meta = self.handlers_import_status[handler_type] - if not handler_meta["import"]["success"]: - logger.info(f"to use {handler_type} please install 'pip install mindsdb[{handler_type}]'") - - logger.debug("%s.create_tmp_handler: connection args - %s", self.__class__.__name__, connection_data) - integration_id = int(time() * 10000) - file_storage = FileStorage( - resource_group=RESOURCE_GROUP.INTEGRATION, - resource_id=integration_id, - root_dir='tmp', - sync=False - ) - handler_storage = HandlerStorage(integration_id, root_dir='tmp', is_temporal=True) - handler_ars = self._make_handler_args( - name='tmp_handler', - handler_type=handler_type, - connection_data=connection_data, - integration_id=integration_id, - file_storage=file_storage, - handler_storage=handler_storage, + HandlerClass = self.handler_modules[engine].Handler + handler_args = self._make_handler_args( + name=name, + handler_type=engine, + connection_data=connection_args, + integration_id=int(time() * 10000) ) - - logger.debug("%s.create_tmp_handler: create a client to db of %s type", self.__class__.__name__, handler_type) - return DBClient(handler_type, self.handler_modules[handler_type].Handler, **handler_ars) + handler = HandlerClass(**handler_args) + return handler def copy_integration_storage(self, integration_id_from, integration_id_to): storage_from = HandlerStorage(integration_id_from) diff --git a/mindsdb/microservices_grpc/__init__.py b/mindsdb/microservices_grpc/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/mindsdb/microservices_grpc/db/__init__.py b/mindsdb/microservices_grpc/db/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/mindsdb/microservices_grpc/db/common_pb2.py b/mindsdb/microservices_grpc/db/common_pb2.py deleted file mode 100644 index 9d6c07c00dd..00000000000 --- a/mindsdb/microservices_grpc/db/common_pb2.py +++ /dev/null @@ -1,27 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: common.proto -"""Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder -from google.protobuf import descriptor as _descriptor -from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import symbol_database as _symbol_database -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0c\x63ommon.proto\"8\n\x0eStatusResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x15\n\rerror_message\x18\x02 \x01(\t\"f\n\x08Response\x12\x0c\n\x04type\x18\x01 \x01(\t\x12\x12\n\ndata_frame\x18\x02 \x01(\x0c\x12\r\n\x05query\x18\x03 \x01(\x05\x12\x12\n\nerror_code\x18\x04 \x01(\x05\x12\x15\n\rerror_message\x18\x05 \x01(\tb\x06proto3') - -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'common_pb2', globals()) -if _descriptor._USE_C_DESCRIPTORS == False: - - DESCRIPTOR._options = None - _STATUSRESPONSE._serialized_start=16 - _STATUSRESPONSE._serialized_end=72 - _RESPONSE._serialized_start=74 - _RESPONSE._serialized_end=176 -# @@protoc_insertion_point(module_scope) diff --git a/mindsdb/microservices_grpc/db/common_pb2.pyi b/mindsdb/microservices_grpc/db/common_pb2.pyi deleted file mode 100644 index 2e87ce41185..00000000000 --- a/mindsdb/microservices_grpc/db/common_pb2.pyi +++ /dev/null @@ -1,27 +0,0 @@ -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from typing import ClassVar as _ClassVar, Optional as _Optional - -DESCRIPTOR: _descriptor.FileDescriptor - -class Response(_message.Message): - __slots__ = ["data_frame", "error_code", "error_message", "query", "type"] - DATA_FRAME_FIELD_NUMBER: _ClassVar[int] - ERROR_CODE_FIELD_NUMBER: _ClassVar[int] - ERROR_MESSAGE_FIELD_NUMBER: _ClassVar[int] - QUERY_FIELD_NUMBER: _ClassVar[int] - TYPE_FIELD_NUMBER: _ClassVar[int] - data_frame: bytes - error_code: int - error_message: str - query: int - type: str - def __init__(self, type: _Optional[str] = ..., data_frame: _Optional[bytes] = ..., query: _Optional[int] = ..., error_code: _Optional[int] = ..., error_message: _Optional[str] = ...) -> None: ... - -class StatusResponse(_message.Message): - __slots__ = ["error_message", "success"] - ERROR_MESSAGE_FIELD_NUMBER: _ClassVar[int] - SUCCESS_FIELD_NUMBER: _ClassVar[int] - error_message: str - success: bool - def __init__(self, success: bool = ..., error_message: _Optional[str] = ...) -> None: ... diff --git a/mindsdb/microservices_grpc/db/common_pb2_grpc.py b/mindsdb/microservices_grpc/db/common_pb2_grpc.py deleted file mode 100644 index 2daafffebfc..00000000000 --- a/mindsdb/microservices_grpc/db/common_pb2_grpc.py +++ /dev/null @@ -1,4 +0,0 @@ -# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! -"""Client and server classes corresponding to protobuf-defined services.""" -import grpc - diff --git a/mindsdb/microservices_grpc/db/db_pb2.py b/mindsdb/microservices_grpc/db/db_pb2.py deleted file mode 100644 index 4ce02df9bab..00000000000 --- a/mindsdb/microservices_grpc/db/db_pb2.py +++ /dev/null @@ -1,34 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: db.proto -"""Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder -from google.protobuf import descriptor as _descriptor -from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import symbol_database as _symbol_database -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -import mindsdb.microservices_grpc.db.common_pb2 as common__pb2 - - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x08\x64\x62.proto\x1a\x0c\x63ommon.proto\"O\n\x0eHandlerContext\x12\x0f\n\x07\x63ontext\x18\x01 \x01(\t\x12\x14\n\x0chandler_type\x18\x02 \x01(\t\x12\x16\n\x0ehandler_params\x18\x03 \x01(\t\"E\n\x12NativeQueryContext\x12 \n\x07\x63ontext\x18\x01 \x01(\x0b\x32\x0f.HandlerContext\x12\r\n\x05query\x18\x02 \x01(\t\"E\n\x12\x42inaryQueryContext\x12 \n\x07\x63ontext\x18\x01 \x01(\x0b\x32\x0f.HandlerContext\x12\r\n\x05query\x18\x02 \x01(\x0c\"A\n\x0e\x43olumnsContext\x12 \n\x07\x63ontext\x18\x01 \x01(\x0b\x32\x0f.HandlerContext\x12\r\n\x05table\x18\x02 \x01(\t2\xdc\x02\n\tDBService\x12-\n\x07\x43onnect\x12\x0f.HandlerContext\x1a\x0f.StatusResponse\"\x00\x12\x35\n\x0f\x43heckConnection\x12\x0f.HandlerContext\x1a\x0f.StatusResponse\"\x00\x12\x30\n\nDisconnect\x12\x0f.HandlerContext\x1a\x0f.StatusResponse\"\x00\x12/\n\x0bNativeQuery\x12\x13.NativeQueryContext\x1a\t.Response\"\x00\x12/\n\x0b\x42inaryQuery\x12\x13.BinaryQueryContext\x1a\t.Response\"\x00\x12)\n\tGetTables\x12\x0f.HandlerContext\x1a\t.Response\"\x00\x12*\n\nGetColumns\x12\x0f.ColumnsContext\x1a\t.Response\"\x00\x62\x06proto3') - -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'db_pb2', globals()) -if _descriptor._USE_C_DESCRIPTORS == False: - - DESCRIPTOR._options = None - _HANDLERCONTEXT._serialized_start=26 - _HANDLERCONTEXT._serialized_end=105 - _NATIVEQUERYCONTEXT._serialized_start=107 - _NATIVEQUERYCONTEXT._serialized_end=176 - _BINARYQUERYCONTEXT._serialized_start=178 - _BINARYQUERYCONTEXT._serialized_end=247 - _COLUMNSCONTEXT._serialized_start=249 - _COLUMNSCONTEXT._serialized_end=314 - _DBSERVICE._serialized_start=317 - _DBSERVICE._serialized_end=665 -# @@protoc_insertion_point(module_scope) diff --git a/mindsdb/microservices_grpc/db/db_pb2.pyi b/mindsdb/microservices_grpc/db/db_pb2.pyi deleted file mode 100644 index f2a91f9ce7b..00000000000 --- a/mindsdb/microservices_grpc/db/db_pb2.pyi +++ /dev/null @@ -1,40 +0,0 @@ -import mindsdb.microservices_grpc.db.common_pb2 as _common_pb2 -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from typing import ClassVar as _ClassVar, Mapping as _Mapping, Optional as _Optional, Union as _Union - -DESCRIPTOR: _descriptor.FileDescriptor - -class BinaryQueryContext(_message.Message): - __slots__ = ["context", "query"] - CONTEXT_FIELD_NUMBER: _ClassVar[int] - QUERY_FIELD_NUMBER: _ClassVar[int] - context: HandlerContext - query: bytes - def __init__(self, context: _Optional[_Union[HandlerContext, _Mapping]] = ..., query: _Optional[bytes] = ...) -> None: ... - -class ColumnsContext(_message.Message): - __slots__ = ["context", "table"] - CONTEXT_FIELD_NUMBER: _ClassVar[int] - TABLE_FIELD_NUMBER: _ClassVar[int] - context: HandlerContext - table: str - def __init__(self, context: _Optional[_Union[HandlerContext, _Mapping]] = ..., table: _Optional[str] = ...) -> None: ... - -class HandlerContext(_message.Message): - __slots__ = ["context", "handler_params", "handler_type"] - CONTEXT_FIELD_NUMBER: _ClassVar[int] - HANDLER_PARAMS_FIELD_NUMBER: _ClassVar[int] - HANDLER_TYPE_FIELD_NUMBER: _ClassVar[int] - context: str - handler_params: str - handler_type: str - def __init__(self, context: _Optional[str] = ..., handler_type: _Optional[str] = ..., handler_params: _Optional[str] = ...) -> None: ... - -class NativeQueryContext(_message.Message): - __slots__ = ["context", "query"] - CONTEXT_FIELD_NUMBER: _ClassVar[int] - QUERY_FIELD_NUMBER: _ClassVar[int] - context: HandlerContext - query: str - def __init__(self, context: _Optional[_Union[HandlerContext, _Mapping]] = ..., query: _Optional[str] = ...) -> None: ... diff --git a/mindsdb/microservices_grpc/db/db_pb2_grpc.py b/mindsdb/microservices_grpc/db/db_pb2_grpc.py deleted file mode 100644 index 535fbe3fd3b..00000000000 --- a/mindsdb/microservices_grpc/db/db_pb2_grpc.py +++ /dev/null @@ -1,354 +0,0 @@ -# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! -"""Client and server classes corresponding to protobuf-defined services.""" -import grpc - -import mindsdb.microservices_grpc.db.common_pb2 as common__pb2 -import mindsdb.microservices_grpc.db.db_pb2 as db__pb2 - - -class DBServiceStub(object): - """// Represents server ouput without - // data. like 'OK' or 'ERROR' - message StatusResponse { - bool success = 1; - string error_message = 2; - } - - // Represents server output with - // some data inside. SELECT result for instance - message Response { - string type = 1; - bytes data_frame = 2; - int32 query = 3; - int32 error_code = 4; - string error_message = 5; - } - - Interface exported by the server. - """ - - def __init__(self, channel): - """Constructor. - - Args: - channel: A grpc.Channel. - """ - self.Connect = channel.unary_unary( - '/DBService/Connect', - request_serializer=db__pb2.HandlerContext.SerializeToString, - response_deserializer=common__pb2.StatusResponse.FromString, - ) - self.CheckConnection = channel.unary_unary( - '/DBService/CheckConnection', - request_serializer=db__pb2.HandlerContext.SerializeToString, - response_deserializer=common__pb2.StatusResponse.FromString, - ) - self.Disconnect = channel.unary_unary( - '/DBService/Disconnect', - request_serializer=db__pb2.HandlerContext.SerializeToString, - response_deserializer=common__pb2.StatusResponse.FromString, - ) - self.NativeQuery = channel.unary_unary( - '/DBService/NativeQuery', - request_serializer=db__pb2.NativeQueryContext.SerializeToString, - response_deserializer=common__pb2.Response.FromString, - ) - self.BinaryQuery = channel.unary_unary( - '/DBService/BinaryQuery', - request_serializer=db__pb2.BinaryQueryContext.SerializeToString, - response_deserializer=common__pb2.Response.FromString, - ) - self.GetTables = channel.unary_unary( - '/DBService/GetTables', - request_serializer=db__pb2.HandlerContext.SerializeToString, - response_deserializer=common__pb2.Response.FromString, - ) - self.GetColumns = channel.unary_unary( - '/DBService/GetColumns', - request_serializer=db__pb2.ColumnsContext.SerializeToString, - response_deserializer=common__pb2.Response.FromString, - ) - - -class DBServiceServicer(object): - """// Represents server ouput without - // data. like 'OK' or 'ERROR' - message StatusResponse { - bool success = 1; - string error_message = 2; - } - - // Represents server output with - // some data inside. SELECT result for instance - message Response { - string type = 1; - bytes data_frame = 2; - int32 query = 3; - int32 error_code = 4; - string error_message = 5; - } - - Interface exported by the server. - """ - - def Connect(self, request, context): - """A simple RPC. - - Establish connection to the specified database - - by creating an appropriate type of Handler instance - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def CheckConnection(self, request, context): - """A simple RPC. - - Check connection to the specified database - - by creating an appropriate type of Handler instance - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Disconnect(self, request, context): - """A simple RPC. - - Drop the connection to the specified database - - by creating an appropriate type of Handler instance - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def NativeQuery(self, request, context): - """A simple RPC. - - Execute native query (string) and returns the result - - wrapped into Response object - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def BinaryQuery(self, request, context): - """A simple RPC. - - Execute query (object) and returns the result - - wrapped into Response object - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def GetTables(self, request, context): - """A simple RPC. - - Performs 'get_tables' request and returns the result - - wrapped into Response object - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def GetColumns(self, request, context): - """A simple RPC. - - Performs 'get_columns' request and returns the result - - wrapped into Response object - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - -def add_DBServiceServicer_to_server(servicer, server): - rpc_method_handlers = { - 'Connect': grpc.unary_unary_rpc_method_handler( - servicer.Connect, - request_deserializer=db__pb2.HandlerContext.FromString, - response_serializer=common__pb2.StatusResponse.SerializeToString, - ), - 'CheckConnection': grpc.unary_unary_rpc_method_handler( - servicer.CheckConnection, - request_deserializer=db__pb2.HandlerContext.FromString, - response_serializer=common__pb2.StatusResponse.SerializeToString, - ), - 'Disconnect': grpc.unary_unary_rpc_method_handler( - servicer.Disconnect, - request_deserializer=db__pb2.HandlerContext.FromString, - response_serializer=common__pb2.StatusResponse.SerializeToString, - ), - 'NativeQuery': grpc.unary_unary_rpc_method_handler( - servicer.NativeQuery, - request_deserializer=db__pb2.NativeQueryContext.FromString, - response_serializer=common__pb2.Response.SerializeToString, - ), - 'BinaryQuery': grpc.unary_unary_rpc_method_handler( - servicer.BinaryQuery, - request_deserializer=db__pb2.BinaryQueryContext.FromString, - response_serializer=common__pb2.Response.SerializeToString, - ), - 'GetTables': grpc.unary_unary_rpc_method_handler( - servicer.GetTables, - request_deserializer=db__pb2.HandlerContext.FromString, - response_serializer=common__pb2.Response.SerializeToString, - ), - 'GetColumns': grpc.unary_unary_rpc_method_handler( - servicer.GetColumns, - request_deserializer=db__pb2.ColumnsContext.FromString, - response_serializer=common__pb2.Response.SerializeToString, - ), - } - generic_handler = grpc.method_handlers_generic_handler( - 'DBService', rpc_method_handlers) - server.add_generic_rpc_handlers((generic_handler,)) - - - # This class is part of an EXPERIMENTAL API. -class DBService(object): - """// Represents server ouput without - // data. like 'OK' or 'ERROR' - message StatusResponse { - bool success = 1; - string error_message = 2; - } - - // Represents server output with - // some data inside. SELECT result for instance - message Response { - string type = 1; - bytes data_frame = 2; - int32 query = 3; - int32 error_code = 4; - string error_message = 5; - } - - Interface exported by the server. - """ - - @staticmethod - def Connect(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/DBService/Connect', - db__pb2.HandlerContext.SerializeToString, - common__pb2.StatusResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) - - @staticmethod - def CheckConnection(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/DBService/CheckConnection', - db__pb2.HandlerContext.SerializeToString, - common__pb2.StatusResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) - - @staticmethod - def Disconnect(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/DBService/Disconnect', - db__pb2.HandlerContext.SerializeToString, - common__pb2.StatusResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) - - @staticmethod - def NativeQuery(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/DBService/NativeQuery', - db__pb2.NativeQueryContext.SerializeToString, - common__pb2.Response.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) - - @staticmethod - def BinaryQuery(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/DBService/BinaryQuery', - db__pb2.BinaryQueryContext.SerializeToString, - common__pb2.Response.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) - - @staticmethod - def GetTables(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/DBService/GetTables', - db__pb2.HandlerContext.SerializeToString, - common__pb2.Response.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) - - @staticmethod - def GetColumns(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/DBService/GetColumns', - db__pb2.ColumnsContext.SerializeToString, - common__pb2.Response.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/mindsdb/microservices_grpc/executor/__init__.py b/mindsdb/microservices_grpc/executor/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/mindsdb/microservices_grpc/executor/executor_pb2.py b/mindsdb/microservices_grpc/executor/executor_pb2.py deleted file mode 100644 index 8ada7a18f3a..00000000000 --- a/mindsdb/microservices_grpc/executor/executor_pb2.py +++ /dev/null @@ -1,39 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: executor.proto -"""Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder -from google.protobuf import descriptor as _descriptor -from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import symbol_database as _symbol_database -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0e\x65xecutor.proto\"j\n\x0f\x45xecutorContext\x12\n\n\x02id\x18\x01 \x01(\t\x12\x15\n\rconnection_id\x18\x02 \x01(\x05\x12\x12\n\nsession_id\x18\x03 \x01(\t\x12\x0f\n\x07session\x18\x04 \x01(\t\x12\x0f\n\x07\x63ontext\x18\x05 \x01(\t\"B\n\x10\x45xecutionContext\x12!\n\x07\x63ontext\x18\x01 \x01(\x0b\x32\x10.ExecutorContext\x12\x0b\n\x03sql\x18\x02 \x01(\t\"H\n\x16\x42inaryExecutionContext\x12!\n\x07\x63ontext\x18\x01 \x01(\x0b\x32\x10.ExecutorContext\x12\x0b\n\x03sql\x18\x02 \x01(\x0c\"L\n\x17StatementExecuteContext\x12!\n\x07\x63ontext\x18\x01 \x01(\x0b\x32\x10.ExecutorContext\x12\x0e\n\x06params\x18\x02 \x01(\t\"E\n\x10\x44\x65\x66\x61ultDBContext\x12!\n\x07\x63ontext\x18\x01 \x01(\x0b\x32\x10.ExecutorContext\x12\x0e\n\x06new_db\x18\x02 \x01(\t\"\xbe\x01\n\x10\x45xecutorResponse\x12\x0f\n\x07\x63olumns\x18\x01 \x01(\x0c\x12\x0e\n\x06params\x18\x02 \x01(\x0c\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\x12\x13\n\x0bstate_track\x18\x04 \x01(\t\x12\x15\n\rserver_status\x18\x05 \x01(\t\x12\x13\n\x0bis_executed\x18\x06 \x01(\x08\x12\x0f\n\x07session\x18\x07 \x01(\t\x12\x12\n\nerror_code\x18\x08 \x01(\x05\x12\x15\n\rerror_message\x18\t \x01(\t\"@\n\x16\x45xecutorStatusResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x15\n\rerror_message\x18\x02 \x01(\t2\xa6\x04\n\x0f\x45xecutorService\x12:\n\x10StatementPrepare\x12\x11.ExecutionContext\x1a\x11.ExecutorResponse\"\x00\x12\x41\n\x10StatementExecute\x12\x18.StatementExecuteContext\x1a\x11.ExecutorResponse\"\x00\x12\x36\n\x0cQueryExecute\x12\x11.ExecutionContext\x1a\x11.ExecutorResponse\"\x00\x12\x42\n\x12\x42inaryQueryExecute\x12\x17.BinaryExecutionContext\x1a\x11.ExecutorResponse\"\x00\x12\x39\n\x0f\x45xecuteExternal\x12\x11.ExecutionContext\x1a\x11.ExecutorResponse\"\x00\x12/\n\x05Parse\x12\x11.ExecutionContext\x1a\x11.ExecutorResponse\"\x00\x12\x32\n\tDoExecute\x12\x10.ExecutorContext\x1a\x11.ExecutorResponse\"\x00\x12\x39\n\x0f\x43hangeDefaultDB\x12\x11.DefaultDBContext\x1a\x11.ExecutorResponse\"\x00\x12=\n\x0e\x44\x65leteExecutor\x12\x10.ExecutorContext\x1a\x17.ExecutorStatusResponse\"\x00\x62\x06proto3') - -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'executor_pb2', globals()) -if _descriptor._USE_C_DESCRIPTORS == False: - - DESCRIPTOR._options = None - _EXECUTORCONTEXT._serialized_start=18 - _EXECUTORCONTEXT._serialized_end=124 - _EXECUTIONCONTEXT._serialized_start=126 - _EXECUTIONCONTEXT._serialized_end=192 - _BINARYEXECUTIONCONTEXT._serialized_start=194 - _BINARYEXECUTIONCONTEXT._serialized_end=266 - _STATEMENTEXECUTECONTEXT._serialized_start=268 - _STATEMENTEXECUTECONTEXT._serialized_end=344 - _DEFAULTDBCONTEXT._serialized_start=346 - _DEFAULTDBCONTEXT._serialized_end=415 - _EXECUTORRESPONSE._serialized_start=418 - _EXECUTORRESPONSE._serialized_end=608 - _EXECUTORSTATUSRESPONSE._serialized_start=610 - _EXECUTORSTATUSRESPONSE._serialized_end=674 - _EXECUTORSERVICE._serialized_start=677 - _EXECUTORSERVICE._serialized_end=1227 -# @@protoc_insertion_point(module_scope) diff --git a/mindsdb/microservices_grpc/executor/executor_pb2.pyi b/mindsdb/microservices_grpc/executor/executor_pb2.pyi deleted file mode 100644 index ba88330344a..00000000000 --- a/mindsdb/microservices_grpc/executor/executor_pb2.pyi +++ /dev/null @@ -1,81 +0,0 @@ -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from typing import ClassVar as _ClassVar, Mapping as _Mapping, Optional as _Optional, Union as _Union - -DESCRIPTOR: _descriptor.FileDescriptor - -class BinaryExecutionContext(_message.Message): - __slots__ = ["context", "sql"] - CONTEXT_FIELD_NUMBER: _ClassVar[int] - SQL_FIELD_NUMBER: _ClassVar[int] - context: ExecutorContext - sql: bytes - def __init__(self, context: _Optional[_Union[ExecutorContext, _Mapping]] = ..., sql: _Optional[bytes] = ...) -> None: ... - -class DefaultDBContext(_message.Message): - __slots__ = ["context", "new_db"] - CONTEXT_FIELD_NUMBER: _ClassVar[int] - NEW_DB_FIELD_NUMBER: _ClassVar[int] - context: ExecutorContext - new_db: str - def __init__(self, context: _Optional[_Union[ExecutorContext, _Mapping]] = ..., new_db: _Optional[str] = ...) -> None: ... - -class ExecutionContext(_message.Message): - __slots__ = ["context", "sql"] - CONTEXT_FIELD_NUMBER: _ClassVar[int] - SQL_FIELD_NUMBER: _ClassVar[int] - context: ExecutorContext - sql: str - def __init__(self, context: _Optional[_Union[ExecutorContext, _Mapping]] = ..., sql: _Optional[str] = ...) -> None: ... - -class ExecutorContext(_message.Message): - __slots__ = ["connection_id", "context", "id", "session", "session_id"] - CONNECTION_ID_FIELD_NUMBER: _ClassVar[int] - CONTEXT_FIELD_NUMBER: _ClassVar[int] - ID_FIELD_NUMBER: _ClassVar[int] - SESSION_FIELD_NUMBER: _ClassVar[int] - SESSION_ID_FIELD_NUMBER: _ClassVar[int] - connection_id: int - context: str - id: str - session: str - session_id: str - def __init__(self, id: _Optional[str] = ..., connection_id: _Optional[int] = ..., session_id: _Optional[str] = ..., session: _Optional[str] = ..., context: _Optional[str] = ...) -> None: ... - -class ExecutorResponse(_message.Message): - __slots__ = ["columns", "data", "error_code", "error_message", "is_executed", "params", "server_status", "session", "state_track"] - COLUMNS_FIELD_NUMBER: _ClassVar[int] - DATA_FIELD_NUMBER: _ClassVar[int] - ERROR_CODE_FIELD_NUMBER: _ClassVar[int] - ERROR_MESSAGE_FIELD_NUMBER: _ClassVar[int] - IS_EXECUTED_FIELD_NUMBER: _ClassVar[int] - PARAMS_FIELD_NUMBER: _ClassVar[int] - SERVER_STATUS_FIELD_NUMBER: _ClassVar[int] - SESSION_FIELD_NUMBER: _ClassVar[int] - STATE_TRACK_FIELD_NUMBER: _ClassVar[int] - columns: bytes - data: bytes - error_code: int - error_message: str - is_executed: bool - params: bytes - server_status: str - session: str - state_track: str - def __init__(self, columns: _Optional[bytes] = ..., params: _Optional[bytes] = ..., data: _Optional[bytes] = ..., state_track: _Optional[str] = ..., server_status: _Optional[str] = ..., is_executed: bool = ..., session: _Optional[str] = ..., error_code: _Optional[int] = ..., error_message: _Optional[str] = ...) -> None: ... - -class ExecutorStatusResponse(_message.Message): - __slots__ = ["error_message", "success"] - ERROR_MESSAGE_FIELD_NUMBER: _ClassVar[int] - SUCCESS_FIELD_NUMBER: _ClassVar[int] - error_message: str - success: bool - def __init__(self, success: bool = ..., error_message: _Optional[str] = ...) -> None: ... - -class StatementExecuteContext(_message.Message): - __slots__ = ["context", "params"] - CONTEXT_FIELD_NUMBER: _ClassVar[int] - PARAMS_FIELD_NUMBER: _ClassVar[int] - context: ExecutorContext - params: str - def __init__(self, context: _Optional[_Union[ExecutorContext, _Mapping]] = ..., params: _Optional[str] = ...) -> None: ... diff --git a/mindsdb/microservices_grpc/executor/executor_pb2_grpc.py b/mindsdb/microservices_grpc/executor/executor_pb2_grpc.py deleted file mode 100644 index a82fb75ed2e..00000000000 --- a/mindsdb/microservices_grpc/executor/executor_pb2_grpc.py +++ /dev/null @@ -1,330 +0,0 @@ -# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! -"""Client and server classes corresponding to protobuf-defined services.""" -import grpc - -import mindsdb.microservices_grpc.executor.executor_pb2 as executor__pb2 - - -class ExecutorServiceStub(object): - """Missing associated documentation comment in .proto file.""" - - def __init__(self, channel): - """Constructor. - - Args: - channel: A grpc.Channel. - """ - self.StatementPrepare = channel.unary_unary( - '/ExecutorService/StatementPrepare', - request_serializer=executor__pb2.ExecutionContext.SerializeToString, - response_deserializer=executor__pb2.ExecutorResponse.FromString, - ) - self.StatementExecute = channel.unary_unary( - '/ExecutorService/StatementExecute', - request_serializer=executor__pb2.StatementExecuteContext.SerializeToString, - response_deserializer=executor__pb2.ExecutorResponse.FromString, - ) - self.QueryExecute = channel.unary_unary( - '/ExecutorService/QueryExecute', - request_serializer=executor__pb2.ExecutionContext.SerializeToString, - response_deserializer=executor__pb2.ExecutorResponse.FromString, - ) - self.BinaryQueryExecute = channel.unary_unary( - '/ExecutorService/BinaryQueryExecute', - request_serializer=executor__pb2.BinaryExecutionContext.SerializeToString, - response_deserializer=executor__pb2.ExecutorResponse.FromString, - ) - self.ExecuteExternal = channel.unary_unary( - '/ExecutorService/ExecuteExternal', - request_serializer=executor__pb2.ExecutionContext.SerializeToString, - response_deserializer=executor__pb2.ExecutorResponse.FromString, - ) - self.Parse = channel.unary_unary( - '/ExecutorService/Parse', - request_serializer=executor__pb2.ExecutionContext.SerializeToString, - response_deserializer=executor__pb2.ExecutorResponse.FromString, - ) - self.DoExecute = channel.unary_unary( - '/ExecutorService/DoExecute', - request_serializer=executor__pb2.ExecutorContext.SerializeToString, - response_deserializer=executor__pb2.ExecutorResponse.FromString, - ) - self.ChangeDefaultDB = channel.unary_unary( - '/ExecutorService/ChangeDefaultDB', - request_serializer=executor__pb2.DefaultDBContext.SerializeToString, - response_deserializer=executor__pb2.ExecutorResponse.FromString, - ) - self.DeleteExecutor = channel.unary_unary( - '/ExecutorService/DeleteExecutor', - request_serializer=executor__pb2.ExecutorContext.SerializeToString, - response_deserializer=executor__pb2.ExecutorStatusResponse.FromString, - ) - - -class ExecutorServiceServicer(object): - """Missing associated documentation comment in .proto file.""" - - def StatementPrepare(self, request, context): - """Missing associated documentation comment in .proto file.""" - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def StatementExecute(self, request, context): - """Missing associated documentation comment in .proto file.""" - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def QueryExecute(self, request, context): - """Missing associated documentation comment in .proto file.""" - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def BinaryQueryExecute(self, request, context): - """Missing associated documentation comment in .proto file.""" - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def ExecuteExternal(self, request, context): - """Missing associated documentation comment in .proto file.""" - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Parse(self, request, context): - """Missing associated documentation comment in .proto file.""" - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def DoExecute(self, request, context): - """Missing associated documentation comment in .proto file.""" - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def ChangeDefaultDB(self, request, context): - """Missing associated documentation comment in .proto file.""" - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def DeleteExecutor(self, request, context): - """Missing associated documentation comment in .proto file.""" - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - -def add_ExecutorServiceServicer_to_server(servicer, server): - rpc_method_handlers = { - 'StatementPrepare': grpc.unary_unary_rpc_method_handler( - servicer.StatementPrepare, - request_deserializer=executor__pb2.ExecutionContext.FromString, - response_serializer=executor__pb2.ExecutorResponse.SerializeToString, - ), - 'StatementExecute': grpc.unary_unary_rpc_method_handler( - servicer.StatementExecute, - request_deserializer=executor__pb2.StatementExecuteContext.FromString, - response_serializer=executor__pb2.ExecutorResponse.SerializeToString, - ), - 'QueryExecute': grpc.unary_unary_rpc_method_handler( - servicer.QueryExecute, - request_deserializer=executor__pb2.ExecutionContext.FromString, - response_serializer=executor__pb2.ExecutorResponse.SerializeToString, - ), - 'BinaryQueryExecute': grpc.unary_unary_rpc_method_handler( - servicer.BinaryQueryExecute, - request_deserializer=executor__pb2.BinaryExecutionContext.FromString, - response_serializer=executor__pb2.ExecutorResponse.SerializeToString, - ), - 'ExecuteExternal': grpc.unary_unary_rpc_method_handler( - servicer.ExecuteExternal, - request_deserializer=executor__pb2.ExecutionContext.FromString, - response_serializer=executor__pb2.ExecutorResponse.SerializeToString, - ), - 'Parse': grpc.unary_unary_rpc_method_handler( - servicer.Parse, - request_deserializer=executor__pb2.ExecutionContext.FromString, - response_serializer=executor__pb2.ExecutorResponse.SerializeToString, - ), - 'DoExecute': grpc.unary_unary_rpc_method_handler( - servicer.DoExecute, - request_deserializer=executor__pb2.ExecutorContext.FromString, - response_serializer=executor__pb2.ExecutorResponse.SerializeToString, - ), - 'ChangeDefaultDB': grpc.unary_unary_rpc_method_handler( - servicer.ChangeDefaultDB, - request_deserializer=executor__pb2.DefaultDBContext.FromString, - response_serializer=executor__pb2.ExecutorResponse.SerializeToString, - ), - 'DeleteExecutor': grpc.unary_unary_rpc_method_handler( - servicer.DeleteExecutor, - request_deserializer=executor__pb2.ExecutorContext.FromString, - response_serializer=executor__pb2.ExecutorStatusResponse.SerializeToString, - ), - } - generic_handler = grpc.method_handlers_generic_handler( - 'ExecutorService', rpc_method_handlers) - server.add_generic_rpc_handlers((generic_handler,)) - - - # This class is part of an EXPERIMENTAL API. -class ExecutorService(object): - """Missing associated documentation comment in .proto file.""" - - @staticmethod - def StatementPrepare(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/ExecutorService/StatementPrepare', - executor__pb2.ExecutionContext.SerializeToString, - executor__pb2.ExecutorResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) - - @staticmethod - def StatementExecute(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/ExecutorService/StatementExecute', - executor__pb2.StatementExecuteContext.SerializeToString, - executor__pb2.ExecutorResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) - - @staticmethod - def QueryExecute(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/ExecutorService/QueryExecute', - executor__pb2.ExecutionContext.SerializeToString, - executor__pb2.ExecutorResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) - - @staticmethod - def BinaryQueryExecute(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/ExecutorService/BinaryQueryExecute', - executor__pb2.BinaryExecutionContext.SerializeToString, - executor__pb2.ExecutorResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) - - @staticmethod - def ExecuteExternal(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/ExecutorService/ExecuteExternal', - executor__pb2.ExecutionContext.SerializeToString, - executor__pb2.ExecutorResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) - - @staticmethod - def Parse(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/ExecutorService/Parse', - executor__pb2.ExecutionContext.SerializeToString, - executor__pb2.ExecutorResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) - - @staticmethod - def DoExecute(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/ExecutorService/DoExecute', - executor__pb2.ExecutorContext.SerializeToString, - executor__pb2.ExecutorResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) - - @staticmethod - def ChangeDefaultDB(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/ExecutorService/ChangeDefaultDB', - executor__pb2.DefaultDBContext.SerializeToString, - executor__pb2.ExecutorResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) - - @staticmethod - def DeleteExecutor(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/ExecutorService/DeleteExecutor', - executor__pb2.ExecutorContext.SerializeToString, - executor__pb2.ExecutorStatusResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/mindsdb/microservices_grpc/ml/__init__.py b/mindsdb/microservices_grpc/ml/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/mindsdb/microservices_grpc/ml/common_pb2.py b/mindsdb/microservices_grpc/ml/common_pb2.py deleted file mode 100644 index 9d6c07c00dd..00000000000 --- a/mindsdb/microservices_grpc/ml/common_pb2.py +++ /dev/null @@ -1,27 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: common.proto -"""Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder -from google.protobuf import descriptor as _descriptor -from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import symbol_database as _symbol_database -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0c\x63ommon.proto\"8\n\x0eStatusResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x15\n\rerror_message\x18\x02 \x01(\t\"f\n\x08Response\x12\x0c\n\x04type\x18\x01 \x01(\t\x12\x12\n\ndata_frame\x18\x02 \x01(\x0c\x12\r\n\x05query\x18\x03 \x01(\x05\x12\x12\n\nerror_code\x18\x04 \x01(\x05\x12\x15\n\rerror_message\x18\x05 \x01(\tb\x06proto3') - -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'common_pb2', globals()) -if _descriptor._USE_C_DESCRIPTORS == False: - - DESCRIPTOR._options = None - _STATUSRESPONSE._serialized_start=16 - _STATUSRESPONSE._serialized_end=72 - _RESPONSE._serialized_start=74 - _RESPONSE._serialized_end=176 -# @@protoc_insertion_point(module_scope) diff --git a/mindsdb/microservices_grpc/ml/common_pb2.pyi b/mindsdb/microservices_grpc/ml/common_pb2.pyi deleted file mode 100644 index 2e87ce41185..00000000000 --- a/mindsdb/microservices_grpc/ml/common_pb2.pyi +++ /dev/null @@ -1,27 +0,0 @@ -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from typing import ClassVar as _ClassVar, Optional as _Optional - -DESCRIPTOR: _descriptor.FileDescriptor - -class Response(_message.Message): - __slots__ = ["data_frame", "error_code", "error_message", "query", "type"] - DATA_FRAME_FIELD_NUMBER: _ClassVar[int] - ERROR_CODE_FIELD_NUMBER: _ClassVar[int] - ERROR_MESSAGE_FIELD_NUMBER: _ClassVar[int] - QUERY_FIELD_NUMBER: _ClassVar[int] - TYPE_FIELD_NUMBER: _ClassVar[int] - data_frame: bytes - error_code: int - error_message: str - query: int - type: str - def __init__(self, type: _Optional[str] = ..., data_frame: _Optional[bytes] = ..., query: _Optional[int] = ..., error_code: _Optional[int] = ..., error_message: _Optional[str] = ...) -> None: ... - -class StatusResponse(_message.Message): - __slots__ = ["error_message", "success"] - ERROR_MESSAGE_FIELD_NUMBER: _ClassVar[int] - SUCCESS_FIELD_NUMBER: _ClassVar[int] - error_message: str - success: bool - def __init__(self, success: bool = ..., error_message: _Optional[str] = ...) -> None: ... diff --git a/mindsdb/microservices_grpc/ml/common_pb2_grpc.py b/mindsdb/microservices_grpc/ml/common_pb2_grpc.py deleted file mode 100644 index 2daafffebfc..00000000000 --- a/mindsdb/microservices_grpc/ml/common_pb2_grpc.py +++ /dev/null @@ -1,4 +0,0 @@ -# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! -"""Client and server classes corresponding to protobuf-defined services.""" -import grpc - diff --git a/mindsdb/microservices_grpc/ml/ml_pb2.py b/mindsdb/microservices_grpc/ml/ml_pb2.py deleted file mode 100644 index 87d58b9ffd6..00000000000 --- a/mindsdb/microservices_grpc/ml/ml_pb2.py +++ /dev/null @@ -1,34 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: ml.proto -"""Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder -from google.protobuf import descriptor as _descriptor -from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import symbol_database as _symbol_database -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -import mindsdb.microservices_grpc.ml.common_pb2 as common__pb2 - - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x08ml.proto\x1a\x0c\x63ommon.proto\"i\n\x10HandlerContextML\x12\x16\n\x0eintegration_id\x18\x01 \x01(\x05\x12\x14\n\x0cpredictor_id\x18\x02 \x01(\x05\x12\x0f\n\x07\x63ontext\x18\x03 \x01(\t\x12\x16\n\x0ehandler_params\x18\x04 \x01(\t\"K\n\x0bPredictCall\x12\"\n\x07\x63ontext\x18\x01 \x01(\x0b\x32\x11.HandlerContextML\x12\n\n\x02\x64\x66\x18\x02 \x01(\x0c\x12\x0c\n\x04\x61rgs\x18\x03 \x01(\t\"Z\n\nCreateCall\x12\"\n\x07\x63ontext\x18\x01 \x01(\x0b\x32\x11.HandlerContextML\x12\x0e\n\x06target\x18\x02 \x01(\t\x12\n\n\x02\x64\x66\x18\x03 \x01(\x0c\x12\x0c\n\x04\x61rgs\x18\x04 \x01(\t\"J\n\nUpdateCall\x12\"\n\x07\x63ontext\x18\x01 \x01(\x0b\x32\x11.HandlerContextML\x12\n\n\x02\x64\x66\x18\x03 \x01(\x0c\x12\x0c\n\x04\x61rgs\x18\x04 \x01(\t2\x85\x01\n\tMLService\x12$\n\x07Predict\x12\x0c.PredictCall\x1a\t.Response\"\x00\x12(\n\x06\x43reate\x12\x0b.CreateCall\x1a\x0f.StatusResponse\"\x00\x12(\n\x06Update\x12\x0b.UpdateCall\x1a\x0f.StatusResponse\"\x00\x62\x06proto3') - -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'ml_pb2', globals()) -if _descriptor._USE_C_DESCRIPTORS == False: - - DESCRIPTOR._options = None - _HANDLERCONTEXTML._serialized_start=26 - _HANDLERCONTEXTML._serialized_end=131 - _PREDICTCALL._serialized_start=133 - _PREDICTCALL._serialized_end=208 - _CREATECALL._serialized_start=210 - _CREATECALL._serialized_end=300 - _UPDATECALL._serialized_start=302 - _UPDATECALL._serialized_end=376 - _MLSERVICE._serialized_start=379 - _MLSERVICE._serialized_end=512 -# @@protoc_insertion_point(module_scope) diff --git a/mindsdb/microservices_grpc/ml/ml_pb2.pyi b/mindsdb/microservices_grpc/ml/ml_pb2.pyi deleted file mode 100644 index 90521e4cd14..00000000000 --- a/mindsdb/microservices_grpc/ml/ml_pb2.pyi +++ /dev/null @@ -1,50 +0,0 @@ -import mindsdb.microservices_grpc.ml.common_pb2 as _common_pb2 -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from typing import ClassVar as _ClassVar, Mapping as _Mapping, Optional as _Optional, Union as _Union - -DESCRIPTOR: _descriptor.FileDescriptor - -class CreateCall(_message.Message): - __slots__ = ["args", "context", "df", "target"] - ARGS_FIELD_NUMBER: _ClassVar[int] - CONTEXT_FIELD_NUMBER: _ClassVar[int] - DF_FIELD_NUMBER: _ClassVar[int] - TARGET_FIELD_NUMBER: _ClassVar[int] - args: str - context: HandlerContextML - df: bytes - target: str - def __init__(self, context: _Optional[_Union[HandlerContextML, _Mapping]] = ..., target: _Optional[str] = ..., df: _Optional[bytes] = ..., args: _Optional[str] = ...) -> None: ... - -class HandlerContextML(_message.Message): - __slots__ = ["context", "handler_params", "integration_id", "predictor_id"] - CONTEXT_FIELD_NUMBER: _ClassVar[int] - HANDLER_PARAMS_FIELD_NUMBER: _ClassVar[int] - INTEGRATION_ID_FIELD_NUMBER: _ClassVar[int] - PREDICTOR_ID_FIELD_NUMBER: _ClassVar[int] - context: str - handler_params: str - integration_id: int - predictor_id: int - def __init__(self, integration_id: _Optional[int] = ..., predictor_id: _Optional[int] = ..., context: _Optional[str] = ..., handler_params: _Optional[str] = ...) -> None: ... - -class PredictCall(_message.Message): - __slots__ = ["args", "context", "df"] - ARGS_FIELD_NUMBER: _ClassVar[int] - CONTEXT_FIELD_NUMBER: _ClassVar[int] - DF_FIELD_NUMBER: _ClassVar[int] - args: str - context: HandlerContextML - df: bytes - def __init__(self, context: _Optional[_Union[HandlerContextML, _Mapping]] = ..., df: _Optional[bytes] = ..., args: _Optional[str] = ...) -> None: ... - -class UpdateCall(_message.Message): - __slots__ = ["args", "context", "df"] - ARGS_FIELD_NUMBER: _ClassVar[int] - CONTEXT_FIELD_NUMBER: _ClassVar[int] - DF_FIELD_NUMBER: _ClassVar[int] - args: str - context: HandlerContextML - df: bytes - def __init__(self, context: _Optional[_Union[HandlerContextML, _Mapping]] = ..., df: _Optional[bytes] = ..., args: _Optional[str] = ...) -> None: ... diff --git a/mindsdb/microservices_grpc/ml/ml_pb2_grpc.py b/mindsdb/microservices_grpc/ml/ml_pb2_grpc.py deleted file mode 100644 index 3912a97817d..00000000000 --- a/mindsdb/microservices_grpc/ml/ml_pb2_grpc.py +++ /dev/null @@ -1,148 +0,0 @@ -# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! -"""Client and server classes corresponding to protobuf-defined services.""" -import grpc - -import mindsdb.microservices_grpc.ml.common_pb2 as common__pb2 -import mindsdb.microservices_grpc.ml.ml_pb2 as ml__pb2 - - -class MLServiceStub(object): - """Missing associated documentation comment in .proto file.""" - - def __init__(self, channel): - """Constructor. - - Args: - channel: A grpc.Channel. - """ - self.Predict = channel.unary_unary( - '/MLService/Predict', - request_serializer=ml__pb2.PredictCall.SerializeToString, - response_deserializer=common__pb2.Response.FromString, - ) - self.Create = channel.unary_unary( - '/MLService/Create', - request_serializer=ml__pb2.CreateCall.SerializeToString, - response_deserializer=common__pb2.StatusResponse.FromString, - ) - self.Update = channel.unary_unary( - '/MLService/Update', - request_serializer=ml__pb2.UpdateCall.SerializeToString, - response_deserializer=common__pb2.StatusResponse.FromString, - ) - - -class MLServiceServicer(object): - """Missing associated documentation comment in .proto file.""" - - def Predict(self, request, context): - """A simple RPC. - - Performs 'predict' request and returns the result - - wrapped into Response object - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Create(self, request, context): - """A simple RPC. - - Performs 'learn' request and returns the result - - wrapped into Response object - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Update(self, request, context): - """A simple RPC. - - Performs 'update' request and returns the result - - wrapped into Response object - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - -def add_MLServiceServicer_to_server(servicer, server): - rpc_method_handlers = { - 'Predict': grpc.unary_unary_rpc_method_handler( - servicer.Predict, - request_deserializer=ml__pb2.PredictCall.FromString, - response_serializer=common__pb2.Response.SerializeToString, - ), - 'Create': grpc.unary_unary_rpc_method_handler( - servicer.Create, - request_deserializer=ml__pb2.CreateCall.FromString, - response_serializer=common__pb2.StatusResponse.SerializeToString, - ), - 'Update': grpc.unary_unary_rpc_method_handler( - servicer.Update, - request_deserializer=ml__pb2.UpdateCall.FromString, - response_serializer=common__pb2.StatusResponse.SerializeToString, - ), - } - generic_handler = grpc.method_handlers_generic_handler( - 'MLService', rpc_method_handlers) - server.add_generic_rpc_handlers((generic_handler,)) - - - # This class is part of an EXPERIMENTAL API. -class MLService(object): - """Missing associated documentation comment in .proto file.""" - - @staticmethod - def Predict(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/MLService/Predict', - ml__pb2.PredictCall.SerializeToString, - common__pb2.Response.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) - - @staticmethod - def Create(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/MLService/Create', - ml__pb2.CreateCall.SerializeToString, - common__pb2.StatusResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) - - @staticmethod - def Update(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/MLService/Update', - ml__pb2.UpdateCall.SerializeToString, - common__pb2.StatusResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/protos/common.proto b/protos/common.proto deleted file mode 100644 index 9ac2ae15ae5..00000000000 --- a/protos/common.proto +++ /dev/null @@ -1,18 +0,0 @@ -syntax = "proto3"; - -// Represents server ouput without -// data. like 'OK' or 'ERROR' -message StatusResponse { - bool success = 1; - string error_message = 2; -} - -// Represents server output with -// some data inside. SELECT result for instance -message Response { - string type = 1; - bytes data_frame = 2; - int32 query = 3; - int32 error_code = 4; - string error_message = 5; -} diff --git a/protos/db.proto b/protos/db.proto deleted file mode 100644 index 96519ae768a..00000000000 --- a/protos/db.proto +++ /dev/null @@ -1,83 +0,0 @@ -syntax = "proto3"; - -import "common.proto"; - - -// Data needs to be sent each time -// Server can't create a Hanlder -// instance without it -message HandlerContext { - string context = 1; - string handler_type = 2; - string handler_params = 3; -} - -// Data of 'native_query' request -message NativeQueryContext { - HandlerContext context = 1; - string query = 2; -} - -// Data of 'query' request -message BinaryQueryContext { - HandlerContext context = 1; - bytes query = 2; -} - -// Data of 'get_columns' request -message ColumnsContext { - HandlerContext context = 1; - string table = 2; -} - -// Interface exported by the server. -service DBService { - // A simple RPC. - // - // Establish connection to the specified database - // - // by creating an appropriate type of Handler instance - rpc Connect(HandlerContext) returns (StatusResponse) {} - - // A simple RPC. - // - // Check connection to the specified database - // - // by creating an appropriate type of Handler instance - rpc CheckConnection(HandlerContext) returns (StatusResponse) {} - - // A simple RPC. - // - // Drop the connection to the specified database - // - // by creating an appropriate type of Handler instance - rpc Disconnect(HandlerContext) returns (StatusResponse) {} - - // A simple RPC. - // - // Execute native query (string) and returns the result - // - // wrapped into Response object - rpc NativeQuery(NativeQueryContext) returns (Response) {} - - // A simple RPC. - // - // Execute query (object) and returns the result - // - // wrapped into Response object - rpc BinaryQuery(BinaryQueryContext) returns (Response) {} - - // A simple RPC. - // - // Performs 'get_tables' request and returns the result - // - // wrapped into Response object - rpc GetTables(HandlerContext) returns (Response) {} - - // A simple RPC. - // - // Performs 'get_columns' request and returns the result - // - // wrapped into Response object - rpc GetColumns(ColumnsContext) returns (Response) {} -} diff --git a/protos/executor.proto b/protos/executor.proto deleted file mode 100644 index 91a3d23d07f..00000000000 --- a/protos/executor.proto +++ /dev/null @@ -1,70 +0,0 @@ -syntax = "proto3"; - -// Data needs to be sent each time -// Server can't create a Hanlder -// instance without it -message ExecutorContext { - string id = 1; - int32 connection_id = 2; - string session_id = 3; - string session = 4; - string context = 5; -} - -message ExecutionContext{ - ExecutorContext context = 1; - string sql = 2; -} - -message BinaryExecutionContext{ - ExecutorContext context = 1; - bytes sql = 2; -} - -message StatementExecuteContext{ - ExecutorContext context = 1; - string params = 2; -} - -message DefaultDBContext{ - ExecutorContext context = 1; - string new_db = 2; -} - -message ExecutorResponse { - bytes columns = 1; - bytes params = 2; - bytes data = 3; - string state_track = 4; - string server_status = 5; - bool is_executed = 6; - string session = 7; - int32 error_code = 8; - string error_message = 9; -} - -message ExecutorStatusResponse { - bool success = 1; - string error_message = 2; -} - -service ExecutorService { - - rpc StatementPrepare(ExecutionContext) returns (ExecutorResponse) {} - - rpc StatementExecute(StatementExecuteContext) returns (ExecutorResponse) {} - - rpc QueryExecute(ExecutionContext) returns (ExecutorResponse) {} - - rpc BinaryQueryExecute(BinaryExecutionContext) returns (ExecutorResponse) {} - - rpc ExecuteExternal(ExecutionContext) returns (ExecutorResponse) {} - - rpc Parse(ExecutionContext) returns (ExecutorResponse) {} - - rpc DoExecute(ExecutorContext) returns (ExecutorResponse) {} - - rpc ChangeDefaultDB(DefaultDBContext) returns (ExecutorResponse) {} - - rpc DeleteExecutor(ExecutorContext) returns (ExecutorStatusResponse) {} -} diff --git a/protos/ml.proto b/protos/ml.proto deleted file mode 100644 index ce04bb2a24b..00000000000 --- a/protos/ml.proto +++ /dev/null @@ -1,56 +0,0 @@ -syntax = "proto3"; - -import "common.proto"; - - -// Data needs to be sent each time -// Server can't create a Hanlder -// instance without it -message HandlerContextML { - int32 integration_id = 1; - int32 predictor_id = 2; - string context = 3; - string handler_params = 4; -} - -message PredictCall { - HandlerContextML context = 1; - bytes df = 2; - string args = 3; -} - -message CreateCall { - HandlerContextML context = 1; - string target = 2; - bytes df = 3; - string args = 4; -} - -message UpdateCall { - HandlerContextML context = 1; - bytes df = 3; - string args = 4; -} - -service MLService { - // A simple RPC. - // - // Performs 'predict' request and returns the result - // - // wrapped into Response object - rpc Predict(PredictCall) returns (Response) {} - - // A simple RPC. - // - // Performs 'learn' request and returns the result - // - // wrapped into Response object - rpc Create(CreateCall) returns (StatusResponse) {} - - // A simple RPC. - // - // Performs 'update' request and returns the result - // - // wrapped into Response object - rpc Update(UpdateCall) returns (StatusResponse) {} -} diff --git a/requirements/requirements-grpc.txt b/requirements/requirements-grpc.txt deleted file mode 100644 index 4a2666ebe76..00000000000 --- a/requirements/requirements-grpc.txt +++ /dev/null @@ -1 +0,0 @@ -grpcio-tools \ No newline at end of file diff --git a/tests/scripts/check_requirements.py b/tests/scripts/check_requirements.py index 36851d3f1e3..fa8b81ff461 100644 --- a/tests/scripts/check_requirements.py +++ b/tests/scripts/check_requirements.py @@ -23,7 +23,6 @@ def get_requirements_from_file(path): MAIN_REQS_PATH = "requirements/requirements.txt" DEV_REQS_PATH = "requirements/requirements-dev.txt" TEST_REQS_PATH = "requirements/requirements-test.txt" -GRPC_REQS_PATH = "requirements/requirements-grpc.txt" DOCKER_REQS_PATH = "docker/handler_discovery/requirements.txt" HANDLER_REQS_PATHS = list( @@ -54,7 +53,7 @@ def get_requirements_from_file(path): # List of rules we can ignore for specific packages # Here we ignore any packages in the main requirements.txt for "listed but not used" errors, because they will be used for the core code but not necessarily in a given handler MAIN_REQUIREMENTS_DEPS = get_requirements_from_file(MAIN_REQS_PATH) + get_requirements_from_file( - TEST_REQS_PATH) + get_requirements_from_file(GRPC_REQS_PATH) + TEST_REQS_PATH) BYOM_HANLDER_DEPS = ["pyarrow"] @@ -101,10 +100,8 @@ def get_requirements_from_file(path): "pymupdf": ["fitz"], "ibm-db": ["ibm_db_dbi"], "python-dateutil": ["dateutil"], - "grpcio": ["grpc"], "sqlalchemy-redshift": ["redshift_sqlalchemy"], "sqlalchemy-vertica-python": ["sqla_vertica_python"], - "grpcio-tools": ["grpc"], "psycopg2-binary": ["psycopg2"], "psycopg-binary": ["psycopg"], "pymongo": ["pymongo", "bson"], @@ -271,7 +268,7 @@ def check_requirements_imports(): # Run against the main codebase errors = run_deptry( - ','.join([MAIN_REQS_PATH, GRPC_REQS_PATH, DOCKER_REQS_PATH]), + ','.join([MAIN_REQS_PATH, DOCKER_REQS_PATH]), get_ignores_str(MAIN_RULE_IGNORES), ".", f"--extend-exclude \"{'|'.join(MAIN_EXCLUDE_PATHS)}\"",