Bad accuracy of CNN-SVM vs CNN for MNIST Classification on DPU-PYNQ

[Afef00]

After successfully running the MNIST classifier example on the ZCU104 board, I wanted to try running a hybrid CNN-SVM model
using the extracted features of all the test samples ( the output features of the MNIST classifier example) to be used as training and test examples for the SVM classifier using PYNQ-DPU
as shown in the code below.
However, the obtained accuracy of this model was about 34.34.% compared to CNN model provided in the MNIST classifier example (98.61%)

from time import time
import sklearn
import numpy as np
import mnist
import cv2
from sklearn import svm
import matplotlib.pyplot as plt
%matplotlib inline
from six.moves import urllib
opener = urllib.request.build_opener()
opener.addheaders = [('User-agent', 'Mozilla/5.0')]
urllib.request.install_opener(opener)
from pynq_dpu import DpuOverlay
overlay = DpuOverlay("dpu.bit")
overlay.load_model("dpu_mnist_classifier.xmodel")
raw_data = mnist.test_images()
normalized_data = np.asarray(raw_data/255, dtype=np.float32)
test_data = np.expand_dims(normalized_data, axis=3)
test_label = mnist.test_labels()
dpu = overlay.runner

inputTensors = dpu.get_input_tensors()
outputTensors = dpu.get_output_tensors()

shapeIn = tuple(inputTensors[0].dims)
shapeOut = tuple(outputTensors[0].dims)
outputSize = int(outputTensors[0].get_data_size() / shapeIn[0])
output_data = [np.empty(shapeOut, dtype=np.float32, order="C")]
input_data = [np.empty(shapeIn, dtype=np.float32, order="C")]
image = input_data[0]
total = test_data.shape[0]
array2 = np.empty_like (output_data)
print("Classifying {} digit pictures ...",(total))
for i in range(total):
    image[0,...] = test_data[i]
    job_id = dpu.execute_async(input_data, output_data)
    dpu.wait(job_id)
     array2 = np.vstack((array2, output_data))

array2 = array2.reshape(array2.shape[0], -1)
exTrain = array2[:9900]
y_train = test_label[:9900]
exTest = array2[9901:10000]
y_test = test_label[9901:10000]
from sklearn import svm
clf = svm.SVC(kernel='linear') # Linear Kernel
clf.fit(exTrain, y_train)
predictions = clf.predict(exTest)

I don't get what's the reason for this bad accuracy. Any help please?
Thanks