Permalink
Browse files

Wide Deep refactor and deep movies (#4506)

* begin branch

* finish download script

* rename download to dataset

* intermediate commit

* intermediate commit

* misc tweaks

* intermediate commit

* intermediate commit

* intermediate commit

* delint and update census test.

* add movie tests

* delint

* fix py2 issue

* address PR comments

* intermediate commit

* intermediate commit

* intermediate commit

* finish wide deep transition to vanilla movielens

* delint

* intermediate commit

* intermediate commit

* intermediate commit

* intermediate commit

* fix import

* add default ncf csv construction

* change default on download_if_missing

* shard and vectorize example serialization

* fix import

* update ncf data unittests

* delint

* delint

* more delinting

* fix wide-deep movielens serialization

* address PR comments

* add file_io tests

* investigate wide-deep test failure

* remove hard coded path and properly use flags.

* address file_io test PR comments

* missed a hash_bucked_size
  • Loading branch information...
robieta committed Jun 20, 2018
1 parent 713228f commit 20070ca46f27964710a4f187d0437164e5a732c7
Showing with 2,095 additions and 1,028 deletions.
  1. +2 −1 official/boosted_trees/train_higgs.py
  2. 0 official/datasets/__init__.py
  3. +309 −0 official/datasets/movielens.py
  4. +3 −1 official/mnist/mnist.py
  5. +2 −0 official/mnist/mnist_eager.py
  6. +0 −23 official/recommendation/constants.py
  7. +0 −365 official/recommendation/data_download.py
  8. +0 −213 official/recommendation/dataset.py
  9. +29 −11 official/recommendation/dataset_test.py
  10. +439 −0 official/recommendation/movielens_dataset.py
  11. +29 −28 official/recommendation/ncf_main.py
  12. +4 −4 official/recommendation/neumf_model.py
  13. +5 −3 official/requirements.txt
  14. +3 −0 official/resnet/resnet_run_loop.py
  15. +3 −0 official/transformer/transformer_main.py
  16. 0 official/utils/data/__init__.py
  17. +203 −0 official/utils/data/file_io.py
  18. +193 −0 official/utils/data/file_io_test.py
  19. +7 −1 official/utils/flags/_base.py
  20. +3 −2 official/utils/logs/hooks_helper.py
  21. +4 −3 official/utils/logs/hooks_helper_test.py
  22. +7 −0 official/utils/misc/model_helpers.py
  23. +204 −0 official/wide_deep/census_dataset.py
  24. +112 −0 official/wide_deep/census_main.py
  25. 0 official/wide_deep/{wide_deep_test.csv → census_test.csv}
  26. +21 −11 official/wide_deep/{wide_deep_test.py → census_test.py}
  27. +0 −71 official/wide_deep/data_download.py
  28. +163 −0 official/wide_deep/movielens_dataset.py
  29. +109 −0 official/wide_deep/movielens_main.py
  30. +117 −0 official/wide_deep/movielens_test.py
  31. +0 −291 official/wide_deep/wide_deep.py
  32. +124 −0 official/wide_deep/wide_deep_run_loop.py
@@ -258,7 +258,8 @@ def main(_):

def define_train_higgs_flags():
"""Add tree related flags as well as training/eval configuration."""
flags_core.define_base(stop_threshold=False, batch_size=False, num_gpu=False)
flags_core.define_base(clean=False, stop_threshold=False, batch_size=False,
num_gpu=False)
flags_core.define_benchmark()
flags.adopt_module_key_flags(flags_core)

No changes.
@@ -0,0 +1,309 @@
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Download and extract the MovieLens dataset from GroupLens website.
Download the dataset, and perform basic preprocessing.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import sys
import tempfile
import zipfile

# pylint: disable=g-bad-import-order
import numpy as np
import pandas as pd
import six
from six.moves import urllib # pylint: disable=redefined-builtin
from absl import app as absl_app
from absl import flags
import tensorflow as tf
# pylint: enable=g-bad-import-order

from official.utils.flags import core as flags_core


ML_1M = "ml-1m"
ML_20M = "ml-20m"
DATASETS = [ML_1M, ML_20M]

RATINGS_FILE = "ratings.csv"
MOVIES_FILE = "movies.csv"

# URL to download dataset
_DATA_URL = "http://files.grouplens.org/datasets/movielens/"

GENRE_COLUMN = "genres"
ITEM_COLUMN = "item_id" # movies
RATING_COLUMN = "rating"
TIMESTAMP_COLUMN = "timestamp"
TITLE_COLUMN = "titles"
USER_COLUMN = "user_id"

GENRES = [
'Action', 'Adventure', 'Animation', "Children", 'Comedy', 'Crime',
'Documentary', 'Drama', 'Fantasy', 'Film-Noir', 'Horror', "IMAX", 'Musical',
'Mystery', 'Romance', 'Sci-Fi', 'Thriller', 'War', 'Western'
]
N_GENRE = len(GENRES)

RATING_COLUMNS = [USER_COLUMN, ITEM_COLUMN, RATING_COLUMN, TIMESTAMP_COLUMN]
MOVIE_COLUMNS = [ITEM_COLUMN, TITLE_COLUMN, GENRE_COLUMN]

# Note: Users are indexed [1, k], not [0, k-1]
NUM_USER_IDS = {
ML_1M: 6040,
ML_20M: 138493,
}

# Note: Users are indexed [1, k], not [0, k-1]
# Both the 1m and 20m datasets use the same movie set.
NUM_ITEM_IDS = 3952

MAX_RATING = 5

NUM_RATINGS = {
ML_1M: 1000209,
ML_20M: 20000263
}


def _download_and_clean(dataset, data_dir):
"""Download MovieLens dataset in a standard format.
This function downloads the specified MovieLens format and coerces it into a
standard format. The only difference between the ml-1m and ml-20m datasets
after this point (other than size, of course) is that the 1m dataset uses
whole number ratings while the 20m dataset allows half integer ratings.
"""
if dataset not in DATASETS:
raise ValueError("dataset {} is not in {{{}}}".format(
dataset, ",".join(DATASETS)))

data_subdir = os.path.join(data_dir, dataset)

expected_files = ["{}.zip".format(dataset), RATINGS_FILE, MOVIES_FILE]

tf.gfile.MakeDirs(data_subdir)
if set(expected_files).intersection(
tf.gfile.ListDirectory(data_subdir)) == set(expected_files):
tf.logging.info("Dataset {} has already been downloaded".format(dataset))
return

url = "{}{}.zip".format(_DATA_URL, dataset)

temp_dir = tempfile.mkdtemp()
try:
zip_path = os.path.join(temp_dir, "{}.zip".format(dataset))
def _progress(count, block_size, total_size):
sys.stdout.write("\r>> Downloading {} {:.1f}%".format(
zip_path, 100.0 * count * block_size / total_size))
sys.stdout.flush()

zip_path, _ = urllib.request.urlretrieve(url, zip_path, _progress)
statinfo = os.stat(zip_path)
# A new line to clear the carriage return from download progress
# tf.logging.info is not applicable here
print()
tf.logging.info(
"Successfully downloaded {} {} bytes".format(
zip_path, statinfo.st_size))

zipfile.ZipFile(zip_path, "r").extractall(temp_dir)

if dataset == ML_1M:
_regularize_1m_dataset(temp_dir)
else:
_regularize_20m_dataset(temp_dir)

for fname in tf.gfile.ListDirectory(temp_dir):
tf.gfile.Copy(os.path.join(temp_dir, fname),
os.path.join(data_subdir, fname))

finally:
tf.gfile.DeleteRecursively(temp_dir)


def _transform_csv(input_path, output_path, names, skip_first, separator=","):
"""Transform csv to a regularized format.
Args:
input_path: The path of the raw csv.
output_path: The path of the cleaned csv.
names: The csv column names.
skip_first: Boolean of whether to skip the first line of the raw csv.
separator: Character used to separate fields in the raw csv.
"""
if six.PY2:
names = [n.decode("utf-u") for n in names]

with tf.gfile.Open(output_path, "wb") as f_out, \
tf.gfile.Open(input_path, "rb") as f_in:

# Write column names to the csv.
f_out.write(",".join(names).encode("utf-8"))
f_out.write(b"\n")
for i, line in enumerate(f_in):
if i == 0 and skip_first:
continue # ignore existing labels in the csv

line = line.decode("utf-8", errors="ignore")
fields = line.split(separator)
if separator != ",":
fields = ['"{}"'.format(field) if "," in field else field
for field in fields]
f_out.write(",".join(fields).encode("utf-8"))


def _regularize_1m_dataset(temp_dir):
"""
ratings.dat
The file has no header row, and each line is in the following format:
UserID::MovieID::Rating::Timestamp
- UserIDs range from 1 and 6040
- MovieIDs range from 1 and 3952
- Ratings are made on a 5-star scale (whole-star ratings only)
- Timestamp is represented in seconds since midnight Coordinated Universal
Time (UTC) of January 1, 1970.
- Each user has at least 20 ratings
movies.dat
Each line has the following format:
MovieID::Title::Genres
- MovieIDs range from 1 and 3952
"""
working_dir = os.path.join(temp_dir, ML_1M)

_transform_csv(
input_path=os.path.join(working_dir, "ratings.dat"),
output_path=os.path.join(temp_dir, RATINGS_FILE),
names=RATING_COLUMNS, skip_first=False, separator="::")

_transform_csv(
input_path=os.path.join(working_dir, "movies.dat"),
output_path=os.path.join(temp_dir, MOVIES_FILE),
names=MOVIE_COLUMNS, skip_first=False, separator="::")

tf.gfile.DeleteRecursively(working_dir)


def _regularize_20m_dataset(temp_dir):
"""
ratings.csv
Each line of this file after the header row represents one rating of one
movie by one user, and has the following format:
userId,movieId,rating,timestamp
- The lines within this file are ordered first by userId, then, within user,
by movieId.
- Ratings are made on a 5-star scale, with half-star increments
(0.5 stars - 5.0 stars).
- Timestamps represent seconds since midnight Coordinated Universal Time
(UTC) of January 1, 1970.
- All the users had rated at least 20 movies.
movies.csv
Each line has the following format:
MovieID,Title,Genres
- MovieIDs range from 1 and 3952
"""
working_dir = os.path.join(temp_dir, ML_20M)

_transform_csv(
input_path=os.path.join(working_dir, "ratings.csv"),
output_path=os.path.join(temp_dir, RATINGS_FILE),
names=RATING_COLUMNS, skip_first=True, separator=",")

_transform_csv(
input_path=os.path.join(working_dir, "movies.csv"),
output_path=os.path.join(temp_dir, MOVIES_FILE),
names=MOVIE_COLUMNS, skip_first=True, separator=",")

tf.gfile.DeleteRecursively(working_dir)


def download(dataset, data_dir):
if dataset:
_download_and_clean(dataset, data_dir)
else:
_ = [_download_and_clean(d, data_dir) for d in DATASETS]


def ratings_csv_to_dataframe(data_dir, dataset):
with tf.gfile.Open(os.path.join(data_dir, dataset, RATINGS_FILE)) as f:
return pd.read_csv(f, encoding="utf-8")


def csv_to_joint_dataframe(data_dir, dataset):
ratings = ratings_csv_to_dataframe(data_dir, dataset)

with tf.gfile.Open(os.path.join(data_dir, dataset, MOVIES_FILE)) as f:
movies = pd.read_csv(f, encoding="utf-8")

df = ratings.merge(movies, on=ITEM_COLUMN)
df[RATING_COLUMN] = df[RATING_COLUMN].astype(np.float32)

return df


def integerize_genres(dataframe):
"""Replace genre string with a binary vector.
Args:
dataframe: a pandas dataframe of movie data.
Returns:
The transformed dataframe.
"""
def _map_fn(entry):
entry.replace("Children's", "Children") # naming difference.
movie_genres = entry.split("|")
output = np.zeros((len(GENRES),), dtype=np.int64)
for i, genre in enumerate(GENRES):
if genre in movie_genres:
output[i] = 1
return output

dataframe[GENRE_COLUMN] = dataframe[GENRE_COLUMN].apply(_map_fn)

return dataframe


def define_data_download_flags():
"""Add flags specifying data download arguments."""
flags.DEFINE_string(
name="data_dir", default="/tmp/movielens-data/",
help=flags_core.help_wrap(
"Directory to download and extract data."))

flags.DEFINE_enum(
name="dataset", default=None,
enum_values=DATASETS, case_sensitive=False,
help=flags_core.help_wrap("Dataset to be trained and evaluated."))


def main(_):
"""Download and extract the data from GroupLens website."""
download(flags.FLAGS.dataset, flags.FLAGS.data_dir)


if __name__ == "__main__":
tf.logging.set_verbosity(tf.logging.INFO)
define_data_download_flags()
FLAGS = flags.FLAGS
absl_app.run(main)
@@ -159,6 +159,7 @@ def run_mnist(flags_obj):
Args:
flags_obj: An object containing parsed flag values.
"""
model_helpers.apply_clean(flags_obj)
model_function = model_fn

# Get number of GPUs as defined by the --num_gpus flags and the number of
@@ -210,7 +211,8 @@ def eval_input_fn():

# Set up hook that outputs training logs every 100 steps.
train_hooks = hooks_helper.get_train_hooks(
flags_obj.hooks, batch_size=flags_obj.batch_size)
flags_obj.hooks, model_dir=flags_obj.model_dir,
batch_size=flags_obj.batch_size)

# Train and evaluate model.
for _ in range(flags_obj.train_epochs // flags_obj.epochs_between_evals):
@@ -39,6 +39,7 @@
from official.mnist import dataset as mnist_dataset
from official.mnist import mnist
from official.utils.flags import core as flags_core
from official.utils.misc import model_helpers


def loss(logits, labels):
@@ -104,6 +105,7 @@ def run_mnist_eager(flags_obj):
flags_obj: An object containing parsed flag values.
"""
tf.enable_eager_execution()
model_helpers.apply_clean(flags.FLAGS)

# Automatically determine device and data_format
(device, data_format) = ('/gpu:0', 'channels_first')

This file was deleted.

Oops, something went wrong.
Oops, something went wrong.

0 comments on commit 20070ca

Please sign in to comment.