Я изучаю, как использовать API tf.data.Dataset. Я использую пример кода, предоставленный Google для их класса tensorflow для курса. В частности, я работаю с блокнотом c_dataset.ipynb здесь.
В этом блокноте есть подпрограмма model.train, которая выглядит следующим образом:
model.train(input_fn = get_train(), steps = 1000)
Подпрограмма get_train() в конечном итоге вызывает код, который использует API tf.data.Dataset с этим фрагментом кода:
filenames_dataset = tf.data.Dataset.list_files(filename)
# read lines from text files
# this results in a dataset of textlines from all files
textlines_dataset = filenames_dataset.flat_map(tf.data.TextLineDataset)
# Parse text lines as comma-separated values (CSV)
# this does the decoder function for each textline
dataset = textlines_dataset.map(decode_csv)
Комментарии хорошо объясняют происходящее. Позже эта процедура вернется так:
# return the features and label as a tensorflow node, these
# will trigger file load operations progressively only when
# needed.
return dataset.make_one_shot_iterator().get_next()
Можно ли как-то оценить результат за одну итерацию? Я пытался что-то вроде этого, но это не удается.
# Try to read what its using from the cvs file.
one_batch_the_csv_file = get_train()
with tf.Session() as sess:
result = sess.run(one_batch_the_csv_file)
print(one_batch_the_csv_file)
Согласно предложению Рубена ниже, я добавил это
Я перешел к следующему набору лабораторных работ в этом классе, где они представили тензорную доску, и я получил несколько графиков, но по-прежнему никаких входных или выходных данных. С учетом сказанного, вот более полный набор исходников.
# curious he did not do this
# I am guessing because the output is so verbose
tf.logging.set_verbosity(tf.logging.INFO) # putting back in since, tf.train.LoggingTensorHook mentions it
def train_and_evaluate(output_dir, num_train_steps):
# Added this while trying to get input vals from csv.
# This gives an error about scafolding
# summary_hook = tf.train.SummarySaverHook()
# SAVE_EVERY_N_STEPS,
# summary_op=tf.summary.merge_all())
# To convert a model to distributed train and evaluate do four things
estimator = tf.estimator.DNNClassifier( # 1. Estimator
model_dir = output_dir,
feature_columns = feature_cols,
hidden_units=[160, 80, 40, 20],
n_classes=2,
config=tf.estimator.RunConfig().replace(save_summary_steps=2) # 2. run config
# ODD. he mentions we need a run config in the videos, but it was missing in the lab
# notebook. Later I found the bug report which gave me this bit of code.
# I got a working TensorBoard when I changed this from save_summary_steps=10 to 2.
)#
# .. also need the trainspec to tell the estimator how to get training data
train_spec = tf.estimator.TrainSpec(
input_fn = read_dataset('./taxi-train.csv', mode = tf.estimator.ModeKeys.TRAIN), # make sure you use the dataset api
max_steps = num_train_steps)
# training_hook=[summary_hook]) # Added this while trying to get input vals from csv.
# ... also need this
# serving and training-time inputs are often very different
exporter = tf.estimator.LatestExporter('exporter', serving_input_receiver_fn = serving_input_fn)
# .. also need an EvalSpec which controls the evaluation and
# the checkpointing of the model since they happen at the same time
eval_spec = tf.estimator.EvalSpec(
input_fn = read_dataset('./taxi-valid.csv', mode = tf.estimator.ModeKeys.EVAL), # make sure you use the dataset api
steps=None, # evals on 100 batches
start_delay_secs = 1, # start evaluating after N secoonds. orig was 1. 3 seemed to fail?
throttle_secs = 10, # eval no more than every 10 secs. Can not be more frequent than the checkpoint config specified in the run config.
exporters = exporter) # how to export the model for production.
tf.estimator.train_and_evaluate(
estimator,
train_spec, # 3. Train Spec
eval_spec) # 4. Eval Spec
OUTDIR = './model_trained'
shutil.rmtree(OUTDIR, ignore_errors = True) # start fresh each time
TensorBoard().start(OUTDIR)
# need to let this complete before running next cell
# call the above routine
train_and_evaluate(OUTDIR, num_train_steps = 6000) # originally 2000. 1000 after reset shows only projectors
