Tensorflow embedding function error

[sahand68]

Hi,
While running the tensorflow code provided in the code, i faced this error:

import os
import gc
import joblib
import pandas as pd 
import numpy as np
from sklearn import metrics, preprocessing
from tensorflow.keras import layers
from tensorflow.keras import optimizers
from tensorflow.keras.models import Model, load_model
from tensorflow.keras import callbacks
from tensorflow.keras import backend as k
from tensorflow.keras import utils


def create_model(data, catcols):

    '''
    this funciton returns a compiled tf.keras model for entitiy embeddings
    :param data: this is a pandas dataframe
    :param catcols: list of categorical column names
    :return: complied tf.keras model
    '''
    #init the list of inputs for embedding
    inputs =[]
    #init the list of outputs for embedding
    outputs= []
    #loop over all categorical columns
    for c in catcols:

        #find the number of unique values in the column
        num_unique_values= int(data[c].nunique())
        #simple dimension of embedding calculator
        #min size is half the number of unbique values
        #max size is 50. max size depends on the number of values
        #categories too. 50 is quite sufficient most of the times
        #but if you have millions of unique values, you might need a larger dimenion

        embed_dim = int(min(np.ceil((num_unique_values)/2), 50))
        #simple keras input layer with size 1

        inp = layers.Input(shape = (1,))
        #add embedding layer to raw input
        #embedding size is alwasy 1 more than unique values in input

        out = layers.Embedding(num_unique_values + 1, embed_dim, name = c)(inp)

        #1-d spatial dropout is the standard for embedding layers
        #it can be used in nlp tasks as well

        out = layers.SpatialDropout1D(0.3)(out)

        #reshape the input to the dimensions of embedding
        #this becomes our output layer for current feature

        out = layers.Reshape(target_shape = (embed_dim,))(out)

        #add input to input list
        inputs.append(inp)
        #add output to output list
        outputs.append(out)


    #concatenate all output layers
    X = layers.Concatenate()(outputs)
    # add a batchnorm layer
    # from here, everything is up to you
    # you can try different architecture
    # add numerical features here or in concatonate layer
    X = layers.BatchNormalization()(X)

    # a bunch of dense layers with dropout
    # start with 1 or two layers only
    X = layers.Dense(300,activation = 'relu')(X)
    X = layers.Dropout(0.3)(X)
    X = layers.BatchNormalization()(X)

    #using softmax and treating it as a two class problem
    # sigmoid can also be used but then we need only 1 output class
    y = layers.Dense(2, activation = 'softmax')(X)

    model = Model(inputs = inputs ,outputs = y)
    #compile the model
    # we use adam and binary cross entropy
    model.compile(loss = 'binary_crossentropy', optimizer = 'adam')

    return model 
def run(fold):
    df = pd.read_csv('../input/cat_train_folds.csv')
    features = [
        f for f in df.columns if f not in ("id","target","kfold")
    ]
    #fill all Na with NONE
    for col in features:
        df.loc[:,col] = df[col].astype(str).fillna("NONE")
        #encode all features with label encoder individually
        #in a live setting all label encoders need to be saved
        
        for feat in features:
            df.loc[:,feat] = df[feat].astype(str)
            lbl_enc = preprocessing.LabelEncoder()
            lbl_enc = lbl_enc.fit(df[feat].values)
            df.loc[:, feat] = lbl_enc.fit_transform(df[feat].astype(str).values)

        #get trainign data using folds

        df_train= df[df.kfold != fold].reset_index(drop = True)
        df_valid = df[df.kfold ==fold].reset_index(drop = True)

        model = create_model(df, features)
        #our features are a list of list
        Xtrain = [df_train[features].values[:,k] for k in range(len(features))]
        Xvalid = [df_valid[features].values[:,k] for k in range(len(features))]

        ytrain = df_train.target.values
        yvalid = df_train.target.values

        #concert target columns to categories
        #this is just binarization

        ytrain_cat = utils.to_categorical(ytrain)
        yvalid_cat = utils.to_categorical(yvalid)

        #fit the model

        model.fit(Xtrain,ytrain_cat, validation_data = (Xvalid, yvalid_cat), verbose = 1, batch_size =1024, epochs = 3)


        valid_preds = model.predict(Xvalid)[:,1]
        print(metrics.roc_auc_score(yvalid, valid_preds))

        #clear session to free gpu memory

        k.clear_session()

if __name__ == "__main__":
    run(0)
    run(1)
    run(2)
    run(3)
    run(4)

The error:
(ml) sahand@sahand-System-Product-Name:~/ApproachingML/cat-in-the-dat/src$ python neural_embedding.py
2020-09-30 00:50:35.370138: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
2020-09-30 00:50:42.254838: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcuda.so.1
2020-09-30 00:50:42.275733: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-09-30 00:50:42.276275: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1716] Found device 0 with properties:
pciBusID: 0000:01:00.0 name: TITAN RTX computeCapability: 7.5
coreClock: 1.77GHz coreCount: 72 deviceMemorySize: 23.65GiB deviceMemoryBandwidth: 625.94GiB/s
2020-09-30 00:50:42.276297: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
2020-09-30 00:50:42.277275: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcublas.so.10
2020-09-30 00:50:42.278136: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcufft.so.10
2020-09-30 00:50:42.278320: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcurand.so.10
2020-09-30 00:50:42.279236: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusolver.so.10
2020-09-30 00:50:42.279669: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusparse.so.10
2020-09-30 00:50:42.279787: W tensorflow/stream_executor/platform/default/dso_loader.cc:59] Could not load dynamic library 'libcudnn.so.7'; dlerror: libcudnn.so.7: cannot open shared object file: No such file or directory
2020-09-30 00:50:42.279796: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1753] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
2020-09-30 00:50:42.279962: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN)to use the following CPU instructions in performance-critical operations: AVX2 FMA
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
2020-09-30 00:50:42.283430: I tensorflow/core/platform/profile_utils/cpu_utils.cc:104] CPU Frequency: 3699850000 Hz
2020-09-30 00:50:42.283675: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x5629cad878a0 initialized for platform Host (this does not guarantee that XLA will be used). Devices:
2020-09-30 00:50:42.283685: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version
2020-09-30 00:50:42.284397: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1257] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-09-30 00:50:42.284406: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1263]
Epoch 1/3
465/469 [============================>.] - ETA: 0s - loss: 0.4713Traceback (most recent call last):
File "neural_embedding.py", line 135, in
run(0)
File "neural_embedding.py", line 124, in run
model.fit(Xtrain,ytrain_cat, validation_data = (Xvalid, yvalid_cat), verbose = 1, batch_size =1024, epochs = 3)
File "/home/sahand/anaconda3/envs/ml/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py", line 108, in _method_wrapper
return method(self, *args, **kwargs)
File "/home/sahand/anaconda3/envs/ml/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py", line 1110, in fit
self._eval_data_handler = data_adapter.DataHandler(
File "/home/sahand/anaconda3/envs/ml/lib/python3.8/site-packages/tensorflow/python/keras/engine/data_adapter.py", line 1105, in init
self._adapter = adapter_cls(
File "/home/sahand/anaconda3/envs/ml/lib/python3.8/site-packages/tensorflow/python/keras/engine/data_adapter.py", line 282, in init
raise ValueError(msg)
ValueError: Data cardinality is ambiguous:
x sizes: 120000, 120000, 120000, 120000, 120000, 120000, 120000, 120000, 120000, 120000, 120000, 120000, 120000, 120000, 120000, 120000, 120000, 120000, 120000, 120000, 120000, 120000, 120000
y sizes: 480000
Please provide data which shares the same first dimension.

[sahand68]

At this point, i think this error is due to tensorflow removing the support from list inputs. I tried to fix it by giving it a list of numpy arrays as Xtrain but i got a new error.

Xtrain = []
for k in range(len(features)):
        Xtrain.append([])
Xvalid = []
for k in range(len(features)):
      Xvalid.append([])


for k in range(len(features)):
      Xtrain[k].append(np.asarray(df_train[features].values[:,k]))

error:

y_traincat shape 480000
y_traincat shape (480000, 2)
Epoch 1/3
467/469 [============================>.] - ETA: 0s - loss: 0.4757
IndexError Traceback (most recent call last)
in
----> 1 run(1)

in run(fold)
151 print(f'y_traincat shape {ytrain_cat.shape}')
152
--> 153 model.fit(Xtrain,ytrain_cat, validation_data = (Xvalid, yvalid_cat), verbose = 1, batch_size =1024, epochs = 3)
154
155

~/anaconda3/envs/ml/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py in _method_wrapper(self, *args, **kwargs)
64 from tensorflow.python.ops import math_ops
65 from tensorflow.python.ops import sparse_ops
---> 66 from tensorflow.python.ops import summary_ops_v2
67 from tensorflow.python.ops import variables
68 from tensorflow.python.ops.ragged import ragged_concat_ops

~/anaconda3/envs/ml/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)
860 A more detailed description of unpacking behavior for iterator types
861 (Dataset, generator, Sequence) is given below.
--> 862 y: Target data. Like the input data x,
863 it could be either Numpy array(s) or TensorFlow tensor(s).
864 It should be consistent with x (you cannot have Numpy inputs and

~/anaconda3/envs/ml/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py in _method_wrapper(self, *args, **kwargs)
64 from tensorflow.python.ops import math_ops
65 from tensorflow.python.ops import sparse_ops
---> 66 from tensorflow.python.ops import summary_ops_v2
67 from tensorflow.python.ops import variables
68 from tensorflow.python.ops.ragged import ragged_concat_ops

~/anaconda3/envs/ml/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py in evaluate(self, x, y, batch_size, verbose, sample_weight, steps, callbacks, max_queue_size, workers, use_multiprocessing, return_dict)
1044 data_adapter.unpack_x_y_sample_weight(validation_data))
1045
-> 1046 with self.distribute_strategy.scope(),
1047 training_utils.RespectCompiledTrainableState(self):
1048 # Creates a tf.data.Dataset and handles batch and epoch iteration.

~/anaconda3/envs/ml/lib/python3.8/site-packages/tensorflow/python/keras/engine/data_adapter.py in init(self, x, y, sample_weight, batch_size, steps_per_epoch, initial_epoch, epochs, shuffle, class_weight, max_queue_size, workers, use_multiprocessing, model)
1097 if steps_per_execution is None:
1098 self._steps_per_execution = 1
-> 1099 self._steps_per_execution_value = 1
1100 else:
1101 self._steps_per_execution = steps_per_execution

~/anaconda3/envs/ml/lib/python3.8/site-packages/tensorflow/python/keras/engine/data_adapter.py in select_data_adapter(x, y)
955
956 ALL_ADAPTER_CLS = [
--> 957 ListsOfScalarsDataAdapter, TensorLikeDataAdapter,
958 GenericArrayLikeDataAdapter, DatasetAdapter,
959 GeneratorDataAdapter, KerasSequenceAdapter, CompositeTensorDataAdapter,

~/anaconda3/envs/ml/lib/python3.8/site-packages/tensorflow/python/keras/engine/data_adapter.py in (.0)
955
956 ALL_ADAPTER_CLS = [
--> 957 ListsOfScalarsDataAdapter, TensorLikeDataAdapter,
958 GenericArrayLikeDataAdapter, DatasetAdapter,
959 GeneratorDataAdapter, KerasSequenceAdapter, CompositeTensorDataAdapter,

~/anaconda3/envs/ml/lib/python3.8/site-packages/tensorflow/python/keras/engine/data_adapter.py in can_handle(x, y)
617 @staticmethod
618 def can_handle(x, y=None):
--> 619 handles_x = ListsOfScalarsDataAdapter._is_list_of_scalars(x)
620 handles_y = True
621 if y is not None:

~/anaconda3/envs/ml/lib/python3.8/site-packages/tensorflow/python/keras/engine/data_adapter.py in _is_list_of_scalars(inp)
628 return True
629 if isinstance(inp, (list, tuple)):
--> 630 return ListsOfScalarsDataAdapter._is_list_of_scalars(inp[0])
631 return False
632

~/anaconda3/envs/ml/lib/python3.8/site-packages/tensorflow/python/keras/engine/data_adapter.py in _is_list_of_scalars(inp)
628 return True
629 if isinstance(inp, (list, tuple)):
--> 630 return ListsOfScalarsDataAdapter._is_list_of_scalars(inp[0])
631 return False
632

IndexError: list index out of range

[abhishekkrthakur]

The book used tf==1.15

[Anand-0220]

I get a Value Error on running the above code

ValueError: A Concatenate layer should be called on a list of at least 2 inputs

ValueError Traceback (most recent call last)
in
64
65 if name == "main":
---> 66 run(0)
67 run(1)
68 run(2)

in run(fold)
25
26 # create tf.keras model
---> 27 model = create_model(df,features)
28
29 # our features are lists of lists

in create_model(data, catcols)
47
48 # concatenate all output layers
---> 49 x = layers.Concatenate()(outputs)
50
51 # add a batchnorm layer

~/miniconda3/envs/ml/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py in call(self, *args, **kwargs)
895 # Build layer if applicable (if the build method has been
896 # overridden).
--> 897 self._maybe_build(inputs)
898 cast_inputs = self._maybe_cast_inputs(inputs)
899

~/miniconda3/envs/ml/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py in _maybe_build(self, inputs)
2414 # operations.
2415 with tf_utils.maybe_init_scope(self):
-> 2416 self.build(input_shapes) # pylint:disable=not-callable
2417 # We must set also ensure that the layer is marked as built, and the build
2418 # shape is stored since user defined build functions may not be calling

~/miniconda3/envs/ml/lib/python3.7/site-packages/tensorflow/python/keras/utils/tf_utils.py in wrapper(instance, input_shape)
314 if input_shape is not None:
315 input_shape = convert_shapes(input_shape, to_tuples=True)
--> 316 output_shape = fn(instance, input_shape)
317 # Return shapes from fn as TensorShapes.
318 if output_shape is not None:

~/miniconda3/envs/ml/lib/python3.7/site-packages/tensorflow/python/keras/layers/merge.py in build(self, input_shape)
491 # Used purely for shape validation.
492 if not isinstance(input_shape[0], tuple) or len(input_shape) < 2:
--> 493 raise ValueError('A Concatenate layer should be called '
494 'on a list of at least 2 inputs')
495 if all(shape is None for shape in input_shape):