Skip to content
Permalink
Browse files

Modify dtype to tf.string for ODPS reader (#1309)

* Fix dtype to tf.string for ODPS reader

* fix dtype in data_reader_test for odps iris data

* convert records from string to float in odps reader test case

* fix dtype to float from odps reader in test case
  • Loading branch information...
workingloong authored and terrytangyuan committed Oct 15, 2019
1 parent 8ba335e commit bf52115a934e4c5c99ae0007f6235fe83e907f58
@@ -141,7 +141,7 @@ def create_shards(self):

@property
def records_output_types(self):
return tf.float32
return tf.string

@property
def metadata(self):
@@ -221,7 +221,7 @@ def read_batch(self, start, end, columns=None, max_retries=3):
start=start, count=end - start, columns=columns
):
batch_record.append(
[record[column] for column in columns]
[str(record[column]) for column in columns]
)
return batch_record
except Exception as e:
@@ -5,6 +5,7 @@
import unittest
from collections import namedtuple

import numpy as np
import tensorflow as tf
from odps import ODPS

@@ -98,6 +99,7 @@ def test_odps_data_reader_records_reading(self):
_MockedTask(0, 2, self.test_table + ":shard_0")
)
)
records = np.array(records, dtype="float").tolist()
self.assertEqual(
[[6.4, 2.8, 5.6, 2.2, 2], [5.0, 2.3, 3.3, 1.0, 1]], records
)
@@ -136,7 +138,7 @@ def _gen():
if data is not None:
yield data

dataset = tf.data.Dataset.from_generator(_gen, tf.float32)
dataset = tf.data.Dataset.from_generator(_gen, tf.string)
dataset = dataset_fn(
dataset, None, Metadata(column_names=IRIS_TABLE_COLUMN_NAMES)
)
@@ -25,6 +25,7 @@ def optimizer(lr=0.1):
def dataset_fn(dataset, mode, metadata):
def _parse_data(record):
label_col_name = "class"
record = tf.strings.to_number(record, tf.float32)

def _get_features_without_labels(
record, label_col_ind, features_shape

0 comments on commit bf52115

Please sign in to comment.
You can’t perform that action at this time.