io/reader.hpp

Go to the documentation of this file.

/*
 
Copyright 2025 Matthew Tolman
 
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
 
   http://www.apache.org/licenses/LICENSE-2.0
 
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
 
*/
 
#ifndef MTCORE_READER_HPP
#define MTCORE_READER_HPP
 
#include "mtcore/colls/result.hpp"
#include "mtcore/colls/segmented_list.hpp"
#include "mtcore/colls/slice.hpp"
#include "mtcore/traits.hpp"


namespace mtcore::io {

  template<ReaderImpl Impl>
  struct Reader {
    using ReadElem = typename Impl::ReadElem;
    using ErrType  = typename Impl::ErrType;
 
    using BuffSlice = Slice<std::remove_const_t<ReadElem>>;
    using ResSlice  = Slice<std::add_const_t<ReadElem>>;
 
    Impl underlying;

    Result<ResSlice, ErrType> read(BuffSlice buff) { return underlying.read(buff); }

    Result<ResSlice, ErrType> read_no_eof(BuffSlice buff) {
      auto res = read(buff);
      if (res.is_success() && res.value().size() < buff.size()) {
        return error(ErrType::END_OF_FILE);
      }
      return res;
    }

    Result<ResSlice, ErrType> read_all(BuffSlice buff) {
      auto res = read(buff);
      if (res.is_error()) {
        return res.error();
      }
      auto resSlice = res.value();
      if (resSlice.size() < buff.size()) {
        return resSlice;
      }
 
      // If there's another element not read, then we didn't read everything
      if (auto ro = read_one(); ro.is_success() || ro.error().code != ErrType::END_OF_FILE) {
        return error(ErrType::SIZE_EXCEEDED);
      }
      return resSlice;
    }

    Result<BuffSlice, ErrType> read_all(Allocator &alloc, size_t maxSize = std::numeric_limits<size_t>::max()) {
      ensure(maxSize > 0, "BAD MAX SIZE");
      // If we have no limit or a really high limit (> 100MB), we're going to allocate in chunks
      if (maxSize == std::numeric_limits<size_t>::max() || maxSize >= 100000000) {
        // A segmented list will be use to track the chunks
        // We'll store the actual chunks here rather than copy the individual items
        SegmentedList<BuffSlice> fullBuff;
        if (auto res = fullBuff.init(alloc, 10); res.is_error()) {
          return error(ErrType::ALLOCATION_FAILED);
        }
        // We will do a full copy before a successful return
        // So we can safely set it up to always clean up the full buff
        mtdefer {
          BuffSlice curSlice;
          auto iter = fullBuff.iter();
          while (iter.next().copy_if_present(curSlice)) {
            alloc.destroy_many(curSlice);
          }
          fullBuff.deinit(alloc);
        };
 
        size_t count = 0;
 
        bool atEnd = false;
        do {
          bool added = false;
          // We do chunks of around 8,000 items
          auto curRes = alloc.create_many<std::remove_const_t<ReadElem>>(8000);
          if (curRes.is_error()) {
            return error(ErrType::ALLOCATION_FAILED);
          }
 
          auto cur = curRes.value();
          mtdefer {
            if (!added) {
              alloc.destroy_many(cur);
            }
          };
 
          auto readRes = read(cur);
          if (readRes.is_error()) {
            return error(ErrType::ALLOCATION_FAILED);
          }
 
          auto readData = readRes.value();
          if (readData.empty()) {
            atEnd = true;
          }
          else {
            auto toPush = cur.sub(0, readData.size());
            if (auto pushRes = fullBuff.push(alloc, toPush); pushRes.is_error()) {
              return error(ErrType::ALLOCATION_FAILED);
            }
 
            added = true;
            count += readData.size();
 
            if (readData.size() < cur.size()) {
              atEnd = true;
            }
          }
        } while (!atEnd);
 
        if (count == 0) {
          return BuffSlice{nullptr, 0};
        }
 
        // We have everything chunked in memory, but our contract is to return a contiguous set of bytes
        // To do this, we need to do one final copy into contiguous memory
        auto finalRes = alloc.create_many<std::remove_const_t<ReadElem>>(count);
        if (finalRes.is_error()) {
          return error(ErrType::ALLOCATION_FAILED);
        }
 
        auto finalSlice = finalRes.value();
        // We'll use a Slice writer to make things a little simpler
        auto finalWriter = slice_writer(finalSlice);
        // Since we stored the chunks, we can just iterate each chunk and write the entire chunk at once
        auto iter = fullBuff.iter();
        BuffSlice curBuff;
        while (iter.next().copy_if_present(curBuff)) {
          if (auto r = finalWriter.write_all(curBuff); r.is_error()) {
            alloc.destroy_many(finalSlice);
            return error(ErrType::ALLOCATION_FAILED);
          }
        }
 
        // Return our Result
        return finalSlice;
      }
      else {
        // If we have a smal enough max size, just read it all into one chunk of memory
        auto outRes = alloc.create_many<std::remove_const_t<ReadElem>>(maxSize);
        if (outRes.is_error()) {
          return error(ErrType::ALLOCATION_FAILED);
        }
        auto out = outRes.value();
        auto res = read_all(out);
        if (res.is_error()) {
          alloc.destroy_many(out);
          return res.error();
        }
        auto finalRes = res.value();
 
        // Check to see if we can save memory if we shrink things down
        // This is more CPU time, but less memory pressure
        if (finalRes.size() + alignof(ReadElem) < out.size()) {
          if (finalRes.size() == 0) {
            alloc.destroy_many(out);
            return BuffSlice{nullptr, 0};
          }
          // Try to create a sub optimal buffer
          auto smallerRes = alloc.create_many<std::remove_const_t<ReadElem>>(finalRes.size());
          if (smallerRes.is_error()) {
            // we have it already in memory, it's just suboptimal. Return it anyway
            return out.sub(0, finalRes.size());
          }
          auto smaller = smallerRes.value();
          auto w       = slice_writer(smaller);
          ensure(w.write_all(finalRes).is_success(), "SOMETHING IS WRONG WITH SLICE WRITER!");
          alloc.destroy_many(out);
          return success(smaller);
        }
        return success(out.sub(0, finalRes.size()));
      }
    }

    Result<ReadElem, ErrType> read_one() {
      std::remove_const_t<ReadElem> elem;
      auto s       = Slice<std::remove_const_t<ReadElem>>{&elem, 1};
      auto readRes = read(s);
      if (readRes.is_error()) {
        return readRes.error();
      }
      auto res = readRes.value();
      if (res.empty()) {
        return error(ErrType::END_OF_FILE);
      }
      return elem;
    }

    Result<void, ErrType> skip(size_t n) {
      for (size_t i = 0; i < n; ++i) {
        auto r = read_one();
        if (r.is_error()) {
          return r.error();
        }
      }
      return success();
    }
  };

  namespace readers {
    template<ReaderImpl RI>
    using Reader = Reader<RI>;
    template<typename T>
    Result<typename Reader<T>::ResSlice, typename Reader<T>::ErrType>
    read_until_or_eof(Reader<T> &self, typename Reader<T>::BuffSlice buff,
                      const std::remove_const_t<typename Reader<T>::ReadElem> &delim) {
      using ErrType = typename Reader<T>::ErrType;
      for (size_t i = 0; i < buff.size(); ++i) {
        auto chRes = self.read_one();
        if (chRes.is_error()) {
          if (chRes.error().code == ErrType::END_OF_FILE) {
            return buff.sub(0, i).to_const();
          }
          return chRes.error();
        }
        auto ch = chRes.value();
        if (ch == delim) {
          return buff.sub(0, i).to_const();
        }
        buff[i] = ch;
      }
      auto nxt = self.read_one();
      if (nxt.is_error()) {
        if (nxt.error().code == ErrType::END_OF_FILE) {
          return buff.to_const();
        }
        return nxt.error();
      }
      if (nxt.value() != delim) {
        return error(ErrType::SIZE_EXCEEDED);
      }
      return buff.to_const();
    }

    template<typename T>
    Result<typename Reader<T>::BuffSlice, typename Reader<T>::ErrType>
    read_until_or_eof(Reader<T> &self, Allocator &alloc, const std::remove_const_t<typename Reader<T>::ReadElem> &delim,
                      size_t maxSize = std::numeric_limits<size_t>::max()) {
      using ErrType   = typename Reader<T>::ErrType;
      using BuffSlice = typename Reader<T>::BuffSlice;
      using ReadElem  = typename Reader<T>::ReadElem;
      ensure(maxSize > 0, "BAD MAX SIZE");
      // If we have no limit or a really high limit (> 100MB), we're going to allocate in chunks
      if (maxSize == std::numeric_limits<size_t>::max() || maxSize >= 100000000) {
        // A segmented list will be use to track the data
        SegmentedList<std::remove_const_t<ReadElem>> fullBuff;
        if (auto res = fullBuff.init(alloc, 4000); res.is_error()) {
          return error(ErrType::ALLOCATION_FAILED);
        }
        // We will do a full copy before a successful return
        // So we can safely set it up to always clean up the full buff
        mtdefer { fullBuff.deinit(alloc); };
 
        size_t count = 0;
 
        while (true) {
          auto readRes = self.read_one();
          if (readRes.is_error()) {
            if (readRes.error().code == ErrType::END_OF_FILE) {
              break;
            }
            return error(ErrType::ALLOCATION_FAILED);
          }
 
          auto cur = readRes.value();
          if (cur == delim) {
            break;
          }
          if (auto pushRes = fullBuff.push(alloc, cur); pushRes.is_error()) {
            return error(ErrType::ALLOCATION_FAILED);
          }
          ++count;
        }
 
        if (count == 0) {
          return BuffSlice{nullptr, 0};
        }
 
        // We have everything chunked in memory, but our contract is to return a contiguous set of bytes
        // To do this, we need to do one final copy into contiguous memory
        auto finalRes = alloc.create_many<std::remove_const_t<ReadElem>>(count);
        if (finalRes.is_error()) {
          return error(ErrType::ALLOCATION_FAILED);
        }
 
        auto finalSlice = finalRes.value();
        // We'll use a Slice writer to make things a little simpler
        auto finalWriter = slice_writer(finalSlice);
        // Since we stored the chunks, we can just iterate each chunk and write the entire chunk at once
        auto iter = fullBuff.iter();
        std::remove_const_t<ReadElem> curElem;
        while (iter.next().copy_if_present(curElem)) {
          if (auto r = finalWriter.write(curElem); r.is_error()) {
            alloc.destroy_many(finalSlice);
            return error(ErrType::ALLOCATION_FAILED);
          }
        }
 
        // Return our Result
        return finalSlice;
      }
      else {
        // If we have a smal enough max size, just read it all into one chunk of memory
        auto outRes = alloc.create_many<std::remove_const_t<ReadElem>>(maxSize);
        if (outRes.is_error()) {
          return error(ErrType::ALLOCATION_FAILED);
        }
        auto out = outRes.value();
        auto res = read_until_or_eof(self, out, delim);
        if (res.is_error()) {
          alloc.destroy_many(out);
          return res.error();
        }
        auto finalRes = res.value();
 
        // Check to see if we can save memory if we shrink things down
        // This is more CPU time, but less memory pressure
        if (finalRes.size() + alignof(ReadElem) < out.size()) {
          if (finalRes.size() == 0) {
            alloc.destroy_many(out);
            return BuffSlice{nullptr, 0};
          }
          // Try to create a sub optimal buffer
          auto smallerRes = alloc.create_many<std::remove_const_t<ReadElem>>(finalRes.size());
          if (smallerRes.is_error()) {
            // we have it already in memory, it's just suboptimal. Return it anyway
            return out.sub(0, finalRes.size());
          }
          auto smaller = smallerRes.value();
          auto w       = slice_writer(smaller);
          ensure(w.write_all(finalRes).is_success(), "SOMETHING IS WRONG WITH SLICE WRITER!");
          alloc.destroy_many(out);
          return success(smaller);
        }
        return success(out.sub(0, finalRes.size()));
      }
    }

    template<typename T>
    Result<typename Reader<T>::ResSlice, typename Reader<T>::ErrType>
    read_until_no_eof(Reader<T> &self, typename Reader<T>::BuffSlice buff,
                      const std::remove_const_t<typename Reader<T>::ReadElem> &delim) {
      using ErrType = typename Reader<T>::ErrType;
      for (size_t i = 0; i < buff.size(); ++i) {
        auto chRes = self.read_one();
        if (chRes.is_error()) {
          if (chRes.error().code == ErrType::END_OF_FILE) {
            return error(ErrType::END_OF_FILE);
          }
          return chRes.error();
        }
        auto ch = chRes.value();
        if (ch == delim) {
          return buff.sub(0, i).to_const();
        }
        buff[i] = ch;
      }
      auto nxt = self.read_one();
      if (nxt.is_error()) {
        return nxt.error();
      }
      if (nxt.value() != delim) {
        return error(ErrType::SIZE_EXCEEDED);
      }
      return buff.to_const();
    }

    template<typename T>
    Result<typename Reader<T>::BuffSlice, typename Reader<T>::ErrType>
    read_until_no_eof(Reader<T> &self, Allocator &alloc, const std::remove_const_t<typename Reader<T>::ReadElem> &delim,
                      size_t maxSize = std::numeric_limits<size_t>::max()) {
      using ErrType   = typename Reader<T>::ErrType;
      using BuffSlice = typename Reader<T>::BuffSlice;
      using ReadElem  = typename Reader<T>::ReadElem;
      ensure(maxSize > 0, "BAD MAX SIZE");
      // If we have no limit or a really high limit (> 100MB), we're going to allocate in chunks
      if (maxSize == std::numeric_limits<size_t>::max() || maxSize >= 100000000) {
        // A segmented list will be use to track the data
        SegmentedList<std::remove_const_t<ReadElem>> fullBuff;
        if (auto res = fullBuff.init(alloc, 4000); res.is_error()) {
          return error(ErrType::ALLOCATION_FAILED);
        }
        // We will do a full copy before a successful return
        // So we can safely set it up to always clean up the full buff
        mtdefer { fullBuff.deinit(alloc); };
 
        size_t count = 0;
 
        while (true) {
          auto readRes = self.read_one();
          if (readRes.is_error()) {
            if (readRes.error().code == ErrType::END_OF_FILE) {
              return error(ErrType::END_OF_FILE);
            }
            return error(ErrType::ALLOCATION_FAILED);
          }
 
          auto cur = readRes.value();
          if (cur == delim) {
            break;
          }
          if (auto pushRes = fullBuff.push(alloc, cur); pushRes.is_error()) {
            return error(ErrType::ALLOCATION_FAILED);
          }
          ++count;
        }
 
        if (count == 0) {
          return BuffSlice{nullptr, 0};
        }
 
        // We have everything chunked in memory, but our contract is to return a contiguous set of bytes
        // To do this, we need to do one final copy into contiguous memory
        auto finalRes = alloc.create_many<std::remove_const_t<ReadElem>>(count);
        if (finalRes.is_error()) {
          return error(ErrType::ALLOCATION_FAILED);
        }
 
        auto finalSlice = finalRes.value();
        // We'll use a Slice writer to make things a little simpler
        auto finalWriter = slice_writer(finalSlice);
        // Since we stored the chunks, we can just iterate each chunk and write the entire chunk at once
        auto iter = fullBuff.iter();
        std::remove_const_t<ReadElem> curElem;
        while (iter.next().copy_if_present(curElem)) {
          if (auto r = finalWriter.write(curElem); r.is_error()) {
            alloc.destroy_many(finalSlice);
            return error(ErrType::ALLOCATION_FAILED);
          }
        }
 
        // Return our Result
        return finalSlice;
      }
      else {
        // If we have a smal enough max size, just read it all into one chunk of memory
        auto outRes = alloc.create_many<std::remove_const_t<ReadElem>>(maxSize);
        if (outRes.is_error()) {
          return error(ErrType::ALLOCATION_FAILED);
        }
        auto out = outRes.value();
        auto res = read_until_no_eof(self, out, delim);
        if (res.is_error()) {
          alloc.destroy_many(out);
          return res.error();
        }
        auto finalRes = res.value();
 
        // Check to see if we can save memory if we shrink things down
        // This is more CPU time, but less memory pressure
        if (finalRes.size() + alignof(ReadElem) < out.size()) {
          if (finalRes.size() == 0) {
            alloc.destroy_many(out);
            return BuffSlice{nullptr, 0};
          }
          // Try to create a sub optimal buffer
          auto smallerRes = alloc.create_many<std::remove_const_t<ReadElem>>(finalRes.size());
          if (smallerRes.is_error()) {
            // we have it already in memory, it's just suboptimal. Return it anyway
            return out.sub(0, finalRes.size());
          }
          auto smaller = smallerRes.value();
          auto w       = slice_writer(smaller);
          ensure(w.write_all(finalRes).is_success(), "SOMETHING IS WRONG WITH SLICE WRITER!");
          alloc.destroy_many(out);
          return success(smaller);
        }
        return success(out.sub(0, finalRes.size()));
      }
    }
  }  // namespace readers

  enum class SliceReaderError { END_OF_FILE, ALLOCATION_FAILED, SIZE_EXCEEDED };
 
  namespace impl {
    template<typename T>
    struct SliceReaderImpl {
      Slice<T> buff;
      size_t curReadIndex = 0;
      bool first          = true;
 
      using ReadElem = T;
      using ErrType  = SliceReaderError;
 
      using BuffSlice = Slice<std::remove_const_t<ReadElem>>;
      using ResSlice  = Slice<std::add_const_t<ReadElem>>;
 
      Result<ResSlice, ErrType> read(BuffSlice out) {
        // Return at least an empty Slice when the Slice is empty
        if (!first && curReadIndex >= buff.size()) {
          return error(ErrType::END_OF_FILE);
        }
        first        = false;
        size_t count = 0;
        for (; curReadIndex < buff.size() && count < out.size(); ++curReadIndex, ++count) {
          out[count] = buff[curReadIndex];
        }
        return success(out.sub(0, count).to_const());
      }
    };
  }  // namespace impl

  template<typename T>
  Reader<impl::SliceReaderImpl<T>> slice_reader(Slice<T> buff) {
    return Reader<impl::SliceReaderImpl<T>>{.underlying = impl::SliceReaderImpl<T>{.buff = buff}};
  }
}  // namespace mtcore::io
 
#endif  // MTCORE_READER_HPP