#写libsvm格式 数据 write libsvm
#!/usr/bin/env python
#coding=gbk
# ==============================================================================
# file gen-records.py
# author chenghuige
# date 2016-08-12 11:52:01.952044
# Description
# ==============================================================================
from __future__ import absolute_import
from __future__ import division
#from __future__ import print_function
import sys,os
import tensorflow as tf
import numpy as np
flags = tf.app.flags
FLAGS = flags.FLAGS
_float_feature = lambda v: tf.train.Feature(float_list=tf.train.FloatList(value=v))
_int_feature = lambda v: tf.train.Feature(int64_list=tf.train.Int64List(value=v))
#how to store global info, using sequence example?
def main(argv):
writer = tf.python_io.TFRecordWriter(argv[2])
for line in open(argv[1]):
l = line.rstrip().split()
label = int(l[0])
start = 1
num_features = 0
if ':' not in l[1]:
num_features = int(l[1])
start += 1
indexes = []
values = []
for item in l[start:]:
index,value = item.split(':')
indexes.append(int(index))
values.append(float(value))
example = tf.train.Example(features=tf.train.Features(feature={
'label': _int_feature([label]),
'num_features': _int_feature
'index': _int_feature(indexes),
'value': _float_feature(values)
}))
writer.write(example.SerializeToString())
if __name__ == '__main__':
tf.app.run()
#读libsvm格式 数据 read libsvm
#!/usr/bin/env python
#coding=gbk
# ==============================================================================
# file read-records.py
# author chenghuige
# date 2016-07-19 17:09:07.466651
# Description
# ==============================================================================
#@TODO treat comment as sparse input ?
from __future__ import absolute_import
from __future__ import division
#from __future__ import print_function
import sys, os, time
import tensorflow as tf
import numpy as np
flags = tf.app.flags
FLAGS = flags.FLAGS
flags.DEFINE_integer('batch_size', 5, 'Batch size.')
flags.DEFINE_integer('num_epochs', 10, 'Number of epochs to run trainer.')
flags.DEFINE_integer('num_preprocess_threads', 12, '')
MIN_AFTER_DEQUEUE = 10000
def read(filename_queue):
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
return serialized_example
def decode(batch_serialized_examples):
features = tf.parse_example(
batch_serialized_examples,
features={
'label' : tf.FixedLenFeature([], tf.int64),
'index' : tf.VarLenFeature(tf.int64),
'value' : tf.VarLenFeature(tf.float32),
})
label = features['label']
index = features['index']
value = features['value']
return label, index, value
def batch_inputs(files, batch_size, num_epochs = None, num_preprocess_threads=1):
"""Reads input data num_epochs times.
"""
if not num_epochs: num_epochs = None
with tf.name_scope('input'):
filename_queue = tf.train.string_input_producer(
files, num_epochs=num_epochs)
serialized_example = read(filename_queue)
batch_serialized_examples = tf.train.shuffle_batch(
[serialized_example],
batch_size=batch_size,
num_threads=num_preprocess_threads,
capacity=MIN_AFTER_DEQUEUE + (num_preprocess_threads + 1) * batch_size,
# Ensures a minimum amount of shuffling of examples.
min_after_dequeue=MIN_AFTER_DEQUEUE)
return decode(batch_serialized_examples)
def read_records():
# Tell TensorFlow that the model will be built into the default Graph.
with tf.Graph().as_default():
# Input images and labels.
tf_record_pattern = sys.argv[1]
data_files = tf.gfile.Glob(tf_record_pattern)
label, index, value = batch_inputs(data_files,
batch_size=FLAGS.batch_size,
num_epochs=FLAGS.num_epochs,
num_preprocess_threads=FLAGS.num_preprocess_threads)
# The op for initializing the variables.
init_op = tf.group(tf.initialize_all_variables(),
tf.initialize_local_variables())
# Create a session for running operations in the Graph.
#sess = tf.Session()
sess = tf.InteractiveSession()
#init_op = tf.initialize_all_variables()
#self.session.run(init)
# Initialize the variables (the trained variables and the
# epoch counter).
sess.run(init_op)
# Start input enqueue threads.
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
try:
step = 0
while not coord.should_stop():
start_time = time.time()
label_, index_, value_ = sess.run([label, index, value])
print label_
print index_
print value_
print index_[0]
print index_[1]
print index_[2]
duration = time.time() - start_time
step += 1
except tf.errors.OutOfRangeError:
print('Done training for %d epochs, %d steps.' % (FLAGS.num_epochs, step))
finally:
# When done, ask the threads to stop.
coord.request_stop()
# Wait for threads to finish.
coord.join(threads)
sess.close()
def main(_):
read_records()
if __name__ == '__main__':
tf.app.run()
#文本分类 text classification
https://github.com/chenghuige/tensorflow-example
using TfRecord only need small modification, like below, I will update the code in github soon.
class SparseClassificationTrainer(object):
"""General framework for Sparse BinaryClassificationTrainer
Sparse BinaryClassfiction will use sparse embedding look up trick
see https://github.com/tensorflow/tensorflow/issues/342
"""
def __init__(self, dataset = None, num_features = 0):
if dataset is not None and type(dataset) != TfDataSet:
self.labels = dataset.labels
self.features = dataset.features
self.num_features = dataset.num_features
self.num_classes = dataset.num_classes
else:
self.features = SparseFeatures()
self.num_features = num_features
self.num_classes = None
self.index_only = False
self.total_features = self.num_features
if type(dataset) != TfDataSet:
self.sp_indices = tf.placeholder(tf.int64, name = 'sp_indices')
self.sp_shape = tf.placeholder(tf.int64, name = 'sp_shape')
self.sp_ids_val = tf.placeholder(tf.int64, name = 'sp_ids_val')
self.sp_weights_val = tf.placeholder(tf.float32, name = 'sp_weights_val')
self.sp_ids = tf.SparseTensor(self.sp_indices, self.sp_ids_val, self.sp_shape)
self.sp_weights = tf.SparseTensor(self.sp_indices, self.sp_weights_val, self.sp_shape)
self.X = (self.sp_ids, self.sp_weights)
self.Y = tf.placeholder(tf.int32) #same as batch size
else:
self.X = (dataset.index, dataset.value)
self.Y = dataset.label
self.type = 'sparse'
MIN_AFTER_DEQUEUE = 10000
def read(filename_queue):
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
return serialized_example
def decode(batch_serialized_examples):
features = tf.parse_example(
batch_serialized_examples,
features={
'label' : tf.FixedLenFeature([], tf.int64),
'index' : tf.VarLenFeature(tf.int64),
'value' : tf.VarLenFeature(tf.float32),
})
label = features['label']
index = features['index']
value = features['value']
return label, index, value
def batch_inputs(files, batch_size, num_epochs=None, num_preprocess_threads=12):
if not num_epochs: num_epochs = None
with tf.name_scope('input'):
filename_queue = tf.train.string_input_producer(
files, num_epochs=num_epochs)
serialized_example = read(filename_queue)
batch_serialized_examples = tf.train.shuffle_batch(
[serialized_example],
batch_size=batch_size,
num_threads=num_preprocess_threads,
capacity=MIN_AFTER_DEQUEUE + (num_preprocess_threads + 1) * batch_size,
# Ensures a minimum amount of shuffling of examples.
min_after_dequeue=MIN_AFTER_DEQUEUE)
return decode(batch_serialized_examples
class TfDataSet(object):
def __init__(self, data_files):
self.data_files = data_files
#@TODO now only deal sparse input
self.features = SparseFeatures()
self.label = None
def build_read_graph(self, batch_size):
tf_record_pattern = self.data_files
data_files = tf.gfile.Glob(tf_record_pattern)
self.label, self.index, self.value = batch_inputs(data_files, batch_size)
def next_batch(self, sess):
label, index, value = sess.run([self.label, self.index, self.value])
trX = (index, value)
trY = label
return trX, trY
trainset = melt.load_dataset(trainset_file, is_record=FLAGS.is_record)
if FLAGS.is_record:
trainset.build_read_graph(batch_size)
step = 0
while not coord.should_stop():
#self.trainer.X, self.trainer.Y = trainset.next_batch(self.session)
_, cost_, accuracy_ = self.session.run([self.train_op, self.cost, self.accuracy])
if step % 100 == 0:
print 'step:', step, 'train precision@1:', accuracy_,'cost:', cost_
if step % 1000 == 0:
pass
step +=