Tensorflow a2.0.0: преобразование CSV в tfrecord, создание модели Keras, которая использует конвейерные данные из большого источника, сохранение весов в файл CSV?

После 24 часов непрерывных исследований, я наконец приклеил все кусочки к головоломке. API потрясающий, но документация отсутствует.

Для преобразования CSV в tfrecord:

import tensorflow as tf
import numpy as np
import pandas as pd # For reading .csv
from datetime import datetime # For knowing how long does each read/write take

def _bytes_feature(value):
    # Returns a bytes_list from a string / byte.
    if isinstance(value, type(tf.constant(0))):
        value = value.numpy()  # BytesList won't unpack a string from an EagerTensor.
    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))


def _float_feature(value):
    # Returns a float_list from a float / double.
    # If a list of values was passed, a float list feature with the entire list will be returned
    if isinstance(value, list):
        return tf.train.Feature(float_list=tf.train.FloatList(value=value))

    return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))


def _int64_feature(value):
    # Returns an int64_list from a bool / enum / int / uint.
    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))


def serialize_example(pandabase):
    # Serializes inputs from a pandas dataset (read in chunks)

    # Creates a mapping of the features from the header row of the file
    base_chunk = pandabase.get_chunk(0)
    num_features = len(base_chunk.columns)
    features_map = {}
    for i in range(num_features):
        features_map.update({'feature' + str(i): _float_feature(0)})

    # Set writing options with compression
    options = tf.io.TFRecordOptions(compression_type=tf.io.TFRecordCompressionType.ZLIB,
                                    compression_level=9)
    with tf.io.TFRecordWriter('test2.tfrecord.zip', options=options) as writer:
    # Convert the chunk to a numpy array, and write each row to the file in a double for loop
        for chunk in pandabase:
            nump = chunk.to_numpy()
            for row in nump:
                ii = 0
                for elem in row:
                    features_map['feature' + str(ii)] = _float_feature(float(elem))
                    ii += 1
                myProto = tf.train.Example(features=tf.train.Features(feature=features_map))
                writer.write(myProto.SerializeToString())


start = datetime.now()
bk1 = pd.read_csv("Book2.csv", chunksize=2048, engine='c', iterator=True)    
serialize_example(bk1)
end = datetime.now()
print("- consumed time: %ds" % (end-start).seconds)

Для машинного обучения из tfrecords и использования графического процессора: следуйте этому руководству по правильной настройке, затем используйте этот код:

# Recreate the feature mappings (Must be similar to the one used to write the tfrecords)
_NUMCOL = 5
feature_description = {}
for i in range(_NUMCOL):
    feature_description.update({'feature' + str(i): tf.io.FixedLenFeature([], tf.float32)})

# Parse the tfrecords into the form (x, y) or (x, y, weights) to be used with keras
def _parse_function(example_proto):
    dic = tf.io.parse_single_example(example_proto, feature_description)
    y = dic['feature0']
    x = tf.stack([dic['feature1'],
                   dic['feature2'],
                   dic['feature3'],
                   dic['feature4']], axis=0)
    return x, y

# Let tensorflow autotune the training speed
AUTOTUNE = tf.data.experimental.AUTOTUNE
# creat a tfdataset from the recorded file, set parallel reads to number of cores for best running speed
myData = tf.data.TFRecordDataset('test.tfrecord.zip', compression_type='ZLIB',
                                 num_parallel_reads=2)
# Map the data to a form useable by keras (using _parse_function), cache the data, shuffle, and read the data in batches  
myData = myData.map(_parse_function, num_parallel_calls=AUTOTUNE)
myData = myData.cache()
myData = myData.shuffle(buffer_size=8192)
batches = 16385
myData = myData.batch(batches).prefetch(buffer_size=AUTOTUNE)

model = keras.Sequential([
    keras.layers.Dense(100, activation='softmax', kernel_regularizer=keras.regularizers.l2(lambd)),
    keras.layers.Dense(10, activation='softmax', kernel_regularizer=keras.regularizers.l2(lambd)),
    keras.layers.Dense(1, activation='linear', kernel_regularizer=keras.regularizers.l2(lambd))])

model.compile(optimizer='adam',
              loss='mean_squared_error')
model.save('keras.HD5F')