Struct tensorflow::io::RecordReader

pub struct RecordReader<R: Read + Seek> { /* private fields */ }

A type for deserializing TFRecord formats

Implementations

impl<R> RecordReader<R>where R: Read + Seek,

pub fn new(reader: R) -> Self

Construct a new RecordReader from an underlying Read.

pub fn peek_next_len(&mut self) -> Result<Option<u64>, RecordReadError>

The length of the next record. Does not checksum the length. Use this to find out how large the byte slice needs to be to read the next record.

pub fn read_next( &mut self, buf: &mut [u8] ) -> Result<Option<usize>, RecordReadError>

Read the next record into a byte slice. Returns the number of bytes read, if successful. Returns None, if it could read exactly 0 bytes (indicating EOF)

Examples

// When we are sure of the max item size, we can just stack allocate an array to hold
use tensorflow::io::{RecordReadError, RecordReader, RecordWriter};
use std::{io::Cursor, rc::Rc};
let mut buf = Vec::new();
let mut rc = Rc::new(&mut buf);
let records = vec!["foo", "barr", "baz"];
{
    let mut writer = RecordWriter::new(Rc::get_mut(&mut rc).unwrap());
    for rec in records.iter() {
        writer.write_record(rec.as_bytes()).unwrap();
    }
}
let read = std::io::BufReader::new(Cursor::new(buf));
let mut reader = RecordReader::new(read);
let mut ary = [0u8; 4];
let mut i = 0;
loop {
    let next = reader.read_next(&mut ary);
    match next {
        Ok(res) => match res {
            Some(len) => assert_eq!(&ary[0..len], records[i].as_bytes()),
            None => break,
        },
        Err(RecordReadError::CorruptFile) | Err(RecordReadError::IoError { .. }) => {
            break;
        }
        _ => {}
    }
    i += 1;
}

When we may need to dynamically resize our buffer, use this peek_next_len()

use tensorflow::io::{RecordReadError, RecordReader, RecordWriter};
use std::{io::Cursor, rc::Rc};
let mut buf = Vec::new();
let mut rc = Rc::new(&mut buf);
let records = vec!["foo", "barr", "baz"];
{
    let mut writer = RecordWriter::new(Rc::get_mut(&mut rc).unwrap());
    for rec in records.iter() {
        writer.write_record(rec.as_bytes()).unwrap();
    }
}
let read = std::io::BufReader::new(Cursor::new(buf));
let mut reader = RecordReader::new(read);
let mut vec = Vec::new();
while let Ok(Some(len)) = reader.peek_next_len() {
    let len = len as usize;
    if vec.len() < len {
        vec.resize(len, 0);
    }
    let next = reader.read_next(&mut vec[0..len]);
    assert_eq!(next.unwrap().unwrap(), len);
    // &vec[0..len] contains the bytes of this record
}
assert_eq!(vec.len(), 4);

pub fn read_next_owned(&mut self) -> Result<Option<Vec<u8>>, RecordReadError>

Allocate a Vec on the heap and read the next record into it. Returns the filled Vec, if successful. Returns None, if it could read exactly 0 bytes (indicating EOF)

Example

use tensorflow::io::{RecordReadError, RecordReader, RecordWriter};
use std::{io::Cursor, rc::Rc};
let mut buf = Vec::new();
let mut rc = Rc::new(&mut buf);
let records = vec!["foo", "barr", "baz"];
{
    let mut writer = RecordWriter::new(Rc::get_mut(&mut rc).unwrap());
    for rec in records.iter() {
        writer.write_record(rec.as_bytes()).unwrap();
    }
}
let read = std::io::BufReader::new(Cursor::new(buf));
let mut reader = RecordReader::new(read);
let mut i = 0;
loop {
    let next = reader.read_next_owned();
    match next {
        Ok(res) => match res {
            Some(vec) => assert_eq!(&vec[..], records[i].as_bytes()),
            None => break,
        },
        Err(RecordReadError::CorruptFile) | Err(RecordReadError::IoError { .. }) => {
            break;
        }
        _ => {}
    }
    i += 1;
}

pub fn into_iter_owned( self ) -> impl Iterator<Item = Result<Vec<u8>, RecordReadError>>

Convert the Reader into an Iterator<Item = Result<Vec, RecordReadError>, which iterates the whole Read. Stops if it finds whole-file corruption.

Example

use tensorflow::io::{RecordWriter, RecordReader};
let records = vec!["Foo bar baz", "boom bing bang", "sum soup shennaninganner"];
let path = "test_resources/io/roundtrip.tfrecord";
let out = ::std::fs::OpenOptions::new()
    .write(true)
    .create(true)
    .open(path)
    .unwrap();
{
    let mut writer = RecordWriter::new(out);
    for rec in records.iter() {
        writer.write_record(rec.as_bytes()).unwrap();
    }
}
{
    let actual = ::std::fs::OpenOptions::new().read(true).open(path).unwrap();
    let reader = RecordReader::new(actual);
    for (actual, expected) in reader.into_iter_owned().zip(records) {
        assert_eq!(actual.unwrap(), expected.as_bytes());
    }
}
{
    let actual = ::std::fs::OpenOptions::new().read(true).open(path).unwrap();
    let reader = RecordReader::new(actual);
    assert_eq!(reader.into_iter_owned().count(), 3);
}
let _ = std::fs::remove_file(path);

Trait Implementations

impl<R: Debug + Read + Seek> Debug for RecordReader<R>

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.