diff --git a/x-pack/legacy/plugins/ml/public/application/data_frame_analytics/common/analytics.ts b/x-pack/legacy/plugins/ml/public/application/data_frame_analytics/common/analytics.ts
index 3dd98395ef701d..ce832513c4adc1 100644
--- a/x-pack/legacy/plugins/ml/public/application/data_frame_analytics/common/analytics.ts
+++ b/x-pack/legacy/plugins/ml/public/application/data_frame_analytics/common/analytics.ts
@@ -374,6 +374,7 @@ interface LoadEvalDataConfig {
   searchQuery?: ResultsSearchQuery;
   ignoreDefaultQuery?: boolean;
   jobType: ANALYSIS_CONFIG_TYPE;
+  requiresKeyword?: boolean;
 }
 
 export const loadEvalData = async ({
@@ -385,6 +386,7 @@ export const loadEvalData = async ({
   searchQuery,
   ignoreDefaultQuery,
   jobType,
+  requiresKeyword,
 }: LoadEvalDataConfig) => {
   const results: LoadEvaluateResult = { success: false, eval: null, error: null };
   const defaultPredictionField = `${dependentVariable}_prediction`;
@@ -392,7 +394,7 @@ export const loadEvalData = async ({
     predictionFieldName ? predictionFieldName : defaultPredictionField
   }`;
 
-  if (jobType === ANALYSIS_CONFIG_TYPE.CLASSIFICATION) {
+  if (jobType === ANALYSIS_CONFIG_TYPE.CLASSIFICATION && requiresKeyword === true) {
     predictedField = `${predictedField}.keyword`;
   }
 
diff --git a/x-pack/legacy/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/classification_exploration/evaluate_panel.tsx b/x-pack/legacy/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/classification_exploration/evaluate_panel.tsx
index ddf52943c2feb2..7bb6949db1a99f 100644
--- a/x-pack/legacy/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/classification_exploration/evaluate_panel.tsx
+++ b/x-pack/legacy/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/classification_exploration/evaluate_panel.tsx
@@ -35,8 +35,13 @@ import {
   ResultsSearchQuery,
   ANALYSIS_CONFIG_TYPE,
 } from '../../../../common/analytics';
+import { IIndexPattern } from '../../../../../../../../../../../src/plugins/data/common/index_patterns';
+import { ES_FIELD_TYPES } from '../../../../../../../../../../../src/plugins/data/public';
 import { LoadingPanel } from '../loading_panel';
 import { getColumnData } from './column_data';
+import { useKibanaContext } from '../../../../../contexts/kibana';
+import { newJobCapsService } from '../../../../../services/new_job_capabilities_service';
+import { getIndexPatternIdFromName } from '../../../../../util/index_utils';
 
 const defaultPanelWidth = 500;
 
@@ -55,17 +60,19 @@ export const EvaluatePanel: FC<Props> = ({ jobConfig, jobStatus, searchQuery })
   const [docsCount, setDocsCount] = useState<null | number>(null);
   const [error, setError] = useState<null | string>(null);
   const [panelWidth, setPanelWidth] = useState<number>(defaultPanelWidth);
-
   // Column visibility
   const [visibleColumns, setVisibleColumns] = useState(() =>
     columns.map(({ id }: { id: string }) => id)
   );
+  const kibanaContext = useKibanaContext();
 
   const index = jobConfig.dest.index;
+  const sourceIndex = jobConfig.source.index[0];
   const dependentVariable = getDependentVar(jobConfig.analysis);
   const predictionFieldName = getPredictionFieldName(jobConfig.analysis);
   // default is 'ml'
   const resultsField = jobConfig.dest.results_field;
+  let requiresKeyword = false;
 
   const loadData = async ({
     isTrainingClause,
@@ -76,6 +83,31 @@ export const EvaluatePanel: FC<Props> = ({ jobConfig, jobStatus, searchQuery })
   }) => {
     setIsLoading(true);
 
+    try {
+      const indexPatternId = getIndexPatternIdFromName(sourceIndex) || sourceIndex;
+      const indexPattern: IIndexPattern = await kibanaContext.indexPatterns.get(indexPatternId);
+
+      if (indexPattern !== undefined) {
+        await newJobCapsService.initializeFromIndexPattern(indexPattern, false, false);
+        // If dependent_variable is of type keyword and text .keyword suffix is required for evaluate endpoint
+        const { fields } = newJobCapsService;
+        const depVarFieldType = fields.find(field => field.name === dependentVariable)?.type;
+
+        // If it's a keyword type - check if it has a corresponding text type
+        if (depVarFieldType !== undefined && depVarFieldType === ES_FIELD_TYPES.KEYWORD) {
+          const field = newJobCapsService.getFieldById(dependentVariable.replace(/\.keyword$/, ''));
+          requiresKeyword = field !== null && field.type === ES_FIELD_TYPES.TEXT;
+        } else if (depVarFieldType !== undefined && depVarFieldType === ES_FIELD_TYPES.TEXT) {
+          // If text, check if has corresponding keyword type
+          const field = newJobCapsService.getFieldById(`${dependentVariable}.keyword`);
+          requiresKeyword = field !== null && field.type === ES_FIELD_TYPES.KEYWORD;
+        }
+      }
+    } catch (e) {
+      // Additional error handling due to missing field type is handled by loadEvalData
+      console.error('Unable to load new field types', error); // eslint-disable-line no-console
+    }
+
     const evalData = await loadEvalData({
       isTraining: false,
       index,
@@ -85,6 +117,7 @@ export const EvaluatePanel: FC<Props> = ({ jobConfig, jobStatus, searchQuery })
       searchQuery,
       ignoreDefaultQuery,
       jobType: ANALYSIS_CONFIG_TYPE.CLASSIFICATION,
+      requiresKeyword,
     });
 
     const docsCountResp = await loadDocsCount({
diff --git a/x-pack/legacy/plugins/ml/public/application/services/new_job_capabilities_service.ts b/x-pack/legacy/plugins/ml/public/application/services/new_job_capabilities_service.ts
index 9d5c33d6cfc5c0..d78c9298c60736 100644
--- a/x-pack/legacy/plugins/ml/public/application/services/new_job_capabilities_service.ts
+++ b/x-pack/legacy/plugins/ml/public/application/services/new_job_capabilities_service.ts
@@ -14,6 +14,7 @@ import {
 } from '../../../common/types/fields';
 import {
   ES_FIELD_TYPES,
+  IIndexPattern,
   IndexPattern,
   IndexPatternsContract,
 } from '../../../../../../../src/plugins/data/public';
@@ -89,7 +90,7 @@ class NewJobCapsService {
   }
 
   public async initializeFromIndexPattern(
-    indexPattern: IndexPattern,
+    indexPattern: IIndexPattern,
     includeEventRateField = true,
     removeTextFields = true
   ) {