cache.hpp

// This file is part of the AliceVision project.
// Copyright (c) 2016 AliceVision contributors.
// Copyright (c) 2012 openMVG contributors.
// This Source Code Form is subject to the terms of the Mozilla Public License,
// v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
 
#pragma once
 
#include "aliceVision/numeric/numeric.hpp"
#include <memory>
 
#include <unordered_map>
#include <boost/multi_index_container.hpp>
#include <boost/multi_index/hashed_index.hpp>
#include <boost/multi_index/identity.hpp>
#include <boost/multi_index/sequenced_index.hpp>
#include <boost/multi_index/member.hpp>
 
#include <fstream>
#include <list>
#include <sstream>
#include <queue>
 
namespace aliceVision {
namespace image {
 
class TileCacheManager;
 
class CachedTile
{
  public:
    using weak_pointer = std::weak_ptr<CachedTile>;
    using smart_pointer = std::shared_ptr<CachedTile>;
 
  public:
    CachedTile() = delete;
 
    CachedTile(const std::shared_ptr<TileCacheManager>& manager,
               size_t uid,
               size_t tileWidth,
               size_t tileHeight,
               size_t width,
               size_t height,
               size_t depth)
      : _manager(manager),
        _uid(uid),
        _tileWidth(tileWidth),
        _tileHeight(tileHeight),
        _requiredWidth(width),
        _requiredHeight(height),
        _depth(depth)
    {
        /*Make sure the required size is less or equal than the tile size*/
        _requiredWidth = std::min(_requiredWidth, _tileWidth);
        _requiredHeight = std::min(_requiredHeight, _tileHeight);
    }
 
    ~CachedTile();
 
    size_t getUid() const { return _uid; }
 
    size_t getTileWidth() const { return _tileWidth; }
 
    size_t getTileHeight() const { return _tileHeight; }
 
    size_t getRequiredWidth() const { return _requiredWidth; }
 
    size_t getRequiredHeight() const { return _requiredHeight; }
 
    size_t getDepth() const { return _depth; }
 
    /*
    Tells the system that we need the data for this tile.
    This means that the data is out of core, we want it back.
    @return false if the process failed to grab data.
    */
    bool acquire();
 
    void setData(std::unique_ptr<unsigned char>&& data) { _data = std::move(data); }
 
    unsigned char* getDataPointer() const
    {
        if (!_data)
        {
            return nullptr;
        }
 
        return _data.get();
    }
 
    std::unique_ptr<unsigned char> getData() { return std::move(_data); }
 
  private:
    std::unique_ptr<unsigned char> _data = nullptr;
    std::weak_ptr<TileCacheManager> _manager;
 
    size_t _uid;
    size_t _tileWidth;
    size_t _tileHeight;
    size_t _requiredWidth;
    size_t _requiredHeight;
    size_t _depth;
};
 
/*
An abstract concept of cache management for generic objects
*/
class CacheManager
{
  public:
    using IndexedStoragePaths = std::unordered_map<size_t, std::string>;
    using IndexedFreeBlocks = std::unordered_map<size_t, std::list<size_t>>;
 
    /*
    An item of the Most Recently used container
    */
    struct MRUItem
    {
        size_t objectId;
        size_t objectSize;
    };
 
    /*
    An item of the object to memory block associative array
    */
    struct MemoryItem
    {
        size_t startBlockId;
        size_t countBlock;
    };
 
    using MemoryMap = std::map<size_t, MemoryItem>;
 
    using MRUType = boost::multi_index::multi_index_container<
      MRUItem,
      boost::multi_index::indexed_by<boost::multi_index::sequenced<>,
                                     boost::multi_index::hashed_unique<boost::multi_index::member<MRUItem, size_t, &MRUItem::objectId>>>>;
 
  public:
    CacheManager() = delete;
 
    CacheManager(const std::string& pathStorage, size_t blockSize, size_t maxBlocksPerIndex);
    virtual ~CacheManager();
 
    void setMaxMemory(size_t maxMemorySize);
 
    void setInCoreMaxObjectCount(size_t max);
 
    bool createObject(size_t& objectId, size_t blockCount);
 
    bool acquireObject(std::unique_ptr<unsigned char>& data, size_t objectId);
 
    size_t getActiveBlocks() const;
 
  protected:
    std::string getPathForIndex(size_t indexId);
    void deleteIndexFiles();
    void wipe();
 
    bool prepareBlockGroup(size_t startBlockId, size_t blocksCount);
    std::unique_ptr<unsigned char> load(size_t startBlockId, size_t blocksCount);
    bool save(std::unique_ptr<unsigned char>&& data, size_t startBlockId, size_t blockCount);
    bool saveObject(std::unique_ptr<unsigned char>&& data, size_t objectId);
 
    virtual void onRemovedFromMRU(size_t objectId) = 0;
 
    void addFreeBlock(size_t blockId, size_t blockCount);
    size_t getFreeBlockId(size_t blockCount);
 
  protected:
    size_t _blockSize{0};
    size_t _incoreBlockUsageCount{0};
    size_t _incoreBlockUsageMax{0};
    size_t _blockCountPerIndex{0};
    size_t _nextStartBlockId{0};
    size_t _nextObjectId{0};
 
    std::string _basePathStorage;
    IndexedStoragePaths _indexPaths;
    IndexedFreeBlocks _freeBlocks;
 
    MRUType _mru;
    MemoryMap _memoryMap;
};
 
class TileCacheManager : public CacheManager, public std::enable_shared_from_this<TileCacheManager>
{
  public:
    using shared_ptr = std::shared_ptr<TileCacheManager>;
    using MapCachedTile = std::map<size_t, CachedTile::weak_pointer>;
 
  public:
    TileCacheManager() = delete;
 
    static std::shared_ptr<TileCacheManager> create(const std::string& pathStorage, size_t tileWidth, size_t tileHeight, size_t maxTilesPerIndex);
 
    void notifyDestroy(size_t tileId);
 
    bool acquire(size_t tileId);
 
    std::shared_ptr<CachedTile> requireNewCachedTile(size_t width, size_t height, size_t blockCount);
 
    template<class T>
    std::shared_ptr<CachedTile> requireNewCachedTile(size_t width, size_t height)
    {
        return requireNewCachedTile(width, height, sizeof(T));
    }
 
    size_t getTileWidth() const { return _tileWidth; }
 
    size_t getTileHeight() const { return _tileHeight; }
 
  protected:
    TileCacheManager(const std::string& pathStorage, size_t tileWidth, size_t tileHeight, size_t maxTilesPerIndex);
 
    virtual void onRemovedFromMRU(size_t objectId);
 
  protected:
    size_t _tileWidth;
    size_t _tileHeight;
 
    MapCachedTile _objectMap;
};
 
}  // namespace image
}  // namespace aliceVision