kalmar_runtime.h

#pragma once

#include "hc_defines.h"
#include "kalmar_aligned_alloc.h"

namespace hc {
class AmPointerInfo;
class completion_future;
}; // end namespace hc

typedef struct hsa_kernel_dispatch_packet_s hsa_kernel_dispatch_packet_t;

namespace Kalmar {
namespace enums {

enum access_type
{
    access_type_none = 0,
    access_type_read = (1 << 0),
    access_type_write = (1 << 1),
    access_type_read_write = access_type_read | access_type_write,
    access_type_auto = (1 << 31)
};

enum queuing_mode
{
    queuing_mode_immediate,
    queuing_mode_automatic
};

enum execute_order
{
    execute_in_order,
    execute_any_order
};


// Flags to specify visibility of previous commands after a marker is executed.
enum memory_scope
{
    no_scope=0,           // No release operation applied
    accelerator_scope=1,  // Release to current accelerator
    system_scope=2,       // Release to system (CPU + all accelerators)
};

static inline memory_scope greater_scope(memory_scope scope1, memory_scope scope2)
{
    if ((scope1==system_scope) || (scope2 == system_scope)) {
        return system_scope;
    } else if ((scope1==accelerator_scope) || (scope2 == accelerator_scope)) {
        return accelerator_scope;
    } else {
        return no_scope;
    }
}


enum hcCommandKind {
    hcCommandInvalid= -1,

    hcMemcpyHostToHost = 0,
    hcMemcpyHostToDevice = 1,
    hcMemcpyDeviceToHost = 2,
    hcMemcpyDeviceToDevice = 3,
    hcCommandKernel = 4,
    hcCommandMarker = 5,
};


// Commands sent to copy queues:
static inline bool isCopyCommand(hcCommandKind k) 
{
    switch (k) {
        case hcMemcpyHostToHost:
        case hcMemcpyHostToDevice:
        case hcMemcpyDeviceToHost:
        case hcMemcpyDeviceToDevice:
            return true;
        default:
            return false;
    };
};


// Commands sent to compute queue:
static inline bool isComputeQueueCommand(hcCommandKind k) {
    return (k == hcCommandKernel) || (k == hcCommandMarker);
};




enum hcWaitMode {
    hcWaitModeBlocked = 0,
    hcWaitModeActive = 1
};

enum hcAgentProfile {
    hcAgentProfileNone = 0,
    hcAgentProfileBase = 1,
    hcAgentProfileFull = 2
};

} // namespace enums
} // namespace Kalmar

 
namespace Kalmar {

using namespace Kalmar::enums;

class KalmarDevice;
class KalmarQueue;
struct rw_info;

class KalmarAsyncOp {
public:
  KalmarAsyncOp(KalmarQueue *xqueue, hcCommandKind xCommandKind) : queue(xqueue), commandKind(xCommandKind), seqNum(0) {} 

  virtual ~KalmarAsyncOp() {} 
  virtual std::shared_future<void>* getFuture() { return nullptr; }
  virtual void* getNativeHandle() { return nullptr;}

  virtual uint64_t getBeginTimestamp() { return 0L; }

  virtual uint64_t getEndTimestamp() { return 0L; }

  virtual uint64_t getTimestampFrequency() { return 0L; }

  virtual bool isReady() { return false; }

  virtual void setWaitMode(hcWaitMode mode) {}

  void setSeqNumFromQueue();
  uint64_t getSeqNum () const { return seqNum;};

  hcCommandKind getCommandKind() const { return commandKind; };
  void          setCommandKind(hcCommandKind xCommandKind) { commandKind = xCommandKind; };

  KalmarQueue  *getQueue() const { return queue; };

private:
  KalmarQueue    *queue;

  // Kind of this command - copy, kernel, barrier, etc:
  hcCommandKind  commandKind;


  // Sequence number of this op in the queue it is dispatched into.
  uint64_t       seqNum;

};

class KalmarQueue
{
public:

  KalmarQueue(KalmarDevice* pDev, queuing_mode mode = queuing_mode_automatic, execute_order order = execute_in_order)
      : pDev(pDev), mode(mode), order(order), opSeqNums(0) {}

  virtual ~KalmarQueue() {}

  virtual void flush() {}
  virtual void wait(hcWaitMode mode = hcWaitModeBlocked) {}

  // sync kernel launch with dynamic group memory
  virtual void LaunchKernelWithDynamicGroupMemory(void *kernel, size_t dim_ext, size_t *ext, size_t *local_size, size_t dynamic_group_size) {}

  // async kernel launch with dynamic group memory
  virtual std::shared_ptr<KalmarAsyncOp> LaunchKernelWithDynamicGroupMemoryAsync(void *kernel, size_t dim_ext, size_t *ext, size_t *local_size, size_t dynamic_group_size) { return nullptr; }

  // sync kernel launch
  virtual void LaunchKernel(void *kernel, size_t dim_ext, size_t *ext, size_t *local_size) {}

  // async kernel launch
  virtual std::shared_ptr<KalmarAsyncOp> LaunchKernelAsync(void *kernel, size_t dim_ext, size_t *ext, size_t *local_size) { return LaunchKernelWithDynamicGroupMemoryAsync(kernel, dim_ext, ext, local_size, 0); }

  virtual void read(void* device, void* dst, size_t count, size_t offset) = 0;

  virtual void write(void* device, const void* src, size_t count, size_t offset, bool blocking) = 0;

  virtual void copy(void* src, void* dst, size_t count, size_t src_offset, size_t dst_offset, bool blocking) = 0;



  virtual void* map(void* device, size_t count, size_t offset, bool modify) = 0;

  virtual void unmap(void* device, void* addr, size_t count, size_t offset, bool modify) = 0;

  virtual void Push(void *kernel, int idx, void* device, bool modify) = 0;

  virtual uint32_t GetGroupSegmentSize(void *kernel) { return 0; }

  KalmarDevice* getDev() const { return pDev; }
  queuing_mode get_mode() const { return mode; }
  void set_mode(queuing_mode mod) { mode = mod; }

  execute_order get_execute_order() const { return order; }

  virtual int getPendingAsyncOps() { return 0; }

  virtual bool isEmpty() { return 0; }

  virtual void* getHSAQueue() { return nullptr; }

  virtual void* getHSAAgent() { return nullptr; }

  virtual void* getHSAAMRegion() { return nullptr; }

  virtual void* getHSAAMHostRegion() { return nullptr; }
  
  virtual void* getHSACoherentAMHostRegion() { return nullptr; }

  virtual void* getHSAKernargRegion() { return nullptr; }

  virtual bool hasHSAInterOp() { return false; }

  virtual std::shared_ptr<KalmarAsyncOp> EnqueueMarker(memory_scope) { return nullptr; }

  virtual std::shared_ptr<KalmarAsyncOp> EnqueueMarkerWithDependency(int count, std::shared_ptr <KalmarAsyncOp> *depOps, memory_scope scope) { return nullptr; }

  virtual std::shared_ptr<KalmarAsyncOp> detectStreamDeps(hcCommandKind commandKind, KalmarAsyncOp *newCopyOp) { return nullptr; };


  virtual std::shared_ptr<KalmarAsyncOp> EnqueueAsyncCopy(const void* src, void* dst, size_t size_bytes) { return nullptr; }
  virtual std::shared_ptr<KalmarAsyncOp> EnqueueAsyncCopyExt(const void* src, void* dst, size_t size_bytes, 
                                                             hcCommandKind copyDir, const hc::AmPointerInfo &srcInfo, const hc::AmPointerInfo &dstInfo, 
                                                             const Kalmar::KalmarDevice *copyDevice) { return nullptr; };

  // Copy src to dst synchronously
  virtual void copy(const void *src, void *dst, size_t size_bytes) { }

  virtual void copy_ext(const void *src, void *dst, size_t size_bytes, hcCommandKind copyDir, const hc::AmPointerInfo &srcInfo, const hc::AmPointerInfo &dstInfo, bool forceUnpinnedCopy) { };
  virtual void copy_ext(const void *src, void *dst, size_t size_bytes, hcCommandKind copyDir, const hc::AmPointerInfo &srcInfo, const hc::AmPointerInfo &dstInfo, 
                        const Kalmar::KalmarDevice *copyDev, bool forceUnpinnedCopy) { };

  virtual void dispose() {}

  virtual void dispatch_hsa_kernel(const hsa_kernel_dispatch_packet_t *aql, 
                                   const void * args, size_t argsize,
                                   hc::completion_future *cf, const char *kernel_name)  { };
 
  virtual bool set_cu_mask(const std::vector<bool>& cu_mask) { return false; };


  uint64_t assign_op_seq_num() { return ++opSeqNums; };

private:
  KalmarDevice* pDev;
  queuing_mode mode;
  execute_order order;

  uint64_t      opSeqNums; // last seqnum assigned to an op in this queue
};

class KalmarDevice
{
private:
    access_type cpu_type;

    // Set true if the device has large bar

#if !TLS_QUEUE
    std::shared_ptr<KalmarQueue> def;
    std::once_flag flag;
#else
    std::map< std::thread::id, std::shared_ptr<KalmarQueue> > tlsDefaultQueueMap;
    std::mutex tlsDefaultQueueMap_mutex;
#endif

protected:
    // True if the device memory is mapped into CPU address space and can be
    // directly accessed with CPU memory operations.
    bool cpu_accessible_am;


    KalmarDevice(access_type type = access_type_read_write)
        : cpu_type(type),
#if !TLS_QUEUE
          def(), flag()
#else
          tlsDefaultQueueMap(), tlsDefaultQueueMap_mutex()
#endif
          {}
public:
    access_type get_access() const { return cpu_type; }
    void set_access(access_type type) { cpu_type = type; }

    virtual std::wstring get_path() const = 0;
    virtual std::wstring get_description() const = 0;
    virtual size_t get_mem() const = 0;
    virtual bool is_double() const = 0;
    virtual bool is_lim_double() const = 0;
    virtual bool is_unified() const = 0;
    virtual bool is_emulated() const = 0;
    virtual uint32_t get_version() const = 0;

    //       key can used to avoid duplicate allocation
    virtual void* create(size_t count, struct rw_info* key) = 0;

    virtual void release(void* ptr, struct rw_info* key) = 0;

    virtual void BuildProgram(void* size, void* source) {}

    virtual void* CreateKernel(const char* fun, KalmarQueue *queue) { return nullptr; }

    virtual bool IsCompatibleKernel(void* size, void* source) { return true; }

    virtual bool check(size_t* size, size_t dim_ext) { return true; }

    virtual std::shared_ptr<KalmarQueue> createQueue(execute_order order = execute_in_order) = 0;
    virtual ~KalmarDevice() {}

    std::shared_ptr<KalmarQueue> get_default_queue() {
#if !TLS_QUEUE
        std::call_once(flag, [&]() { 
            def = createQueue();
        });
        return def;
#else
        std::thread::id tid = std::this_thread::get_id();
        tlsDefaultQueueMap_mutex.lock();
        if (tlsDefaultQueueMap.find(tid) == tlsDefaultQueueMap.end()) {
            tlsDefaultQueueMap[tid] = createQueue();
        }
        std::shared_ptr<KalmarQueue> result = tlsDefaultQueueMap[tid];
        tlsDefaultQueueMap_mutex.unlock();
        return result;
#endif
    }

    virtual size_t GetMaxTileStaticSize() { return 0; }

    virtual std::vector< std::shared_ptr<KalmarQueue> > get_all_queues() { return std::vector< std::shared_ptr<KalmarQueue> >(); }

    virtual void memcpySymbol(const char* symbolName, void* hostptr, size_t count, size_t offset = 0, hcCommandKind kind = hcMemcpyHostToDevice) {}

    virtual void memcpySymbol(void* symbolAddr, void* hostptr, size_t count, size_t offset = 0, hcCommandKind kind = hcMemcpyHostToDevice) {}

    virtual void* getSymbolAddress(const char* symbolName) { return nullptr; }

    virtual void* getHSAAgent() { return nullptr; }

    virtual hcAgentProfile getProfile() { return hcAgentProfileNone; }

    virtual bool is_peer(const KalmarDevice* other) {return false;}

    virtual unsigned int get_compute_unit_count() {return 0;}

    virtual int get_seqnum() const {return -1;}

    virtual bool has_cpu_accessible_am() {return false;}

};

class CPUQueue final : public KalmarQueue
{
public:

  CPUQueue(KalmarDevice* pDev) : KalmarQueue(pDev) {}

  void read(void* device, void* dst, size_t count, size_t offset) override {
      if (dst != device)
          memmove(dst, (char*)device + offset, count);
  }

  void write(void* device, const void* src, size_t count, size_t offset, bool blocking) override {
      if (src != device)
          memmove((char*)device + offset, src, count);
  }

  void copy(void* src, void* dst, size_t count, size_t src_offset, size_t dst_offset, bool blocking) override {
      if (src != dst)
          memmove((char*)dst + dst_offset, (char*)src + src_offset, count);
  }

  void* map(void* device, size_t count, size_t offset, bool modify) override {
      return (char*)device + offset;
  }

  void unmap(void* device, void* addr, size_t count, size_t offset, bool modify) override {}

  void Push(void *kernel, int idx, void* device, bool modify) override {}
};

class CPUDevice final : public KalmarDevice
{
public:
    std::wstring get_path() const override { return L"cpu"; }
    std::wstring get_description() const override { return L"CPU Device"; }
    size_t get_mem() const override { return 0; }
    bool is_double() const override { return true; }
    bool is_lim_double() const override { return true; }
    bool is_unified() const override { return true; }
    bool is_emulated() const override { return true; }
    uint32_t get_version() const override { return 0; }

    std::shared_ptr<KalmarQueue> createQueue(execute_order order = execute_in_order) override { return std::shared_ptr<KalmarQueue>(new CPUQueue(this)); }
    void* create(size_t count, struct rw_info* /* not used */ ) override { return kalmar_aligned_alloc(0x1000, count); }
    void release(void* ptr, struct rw_info* /* nout used */) override { kalmar_aligned_free(ptr); }
    void* CreateKernel(const char* fun, KalmarQueue *queue) { return nullptr; }
};

class KalmarContext
{
private:
    //TODO: Think about a system which has multiple CPU socket, e.g. server. In this case,
    //We might be able to assume that only the first device is CPU, or we only mimic one cpu
    //device when constructing KalmarContext.
    KalmarDevice* get_default_dev() {
        if (!def) {
            if (Devices.size() <= 1) {
                fprintf(stderr, "There is no device can be used to do the computation\n");
                exit(-1);
            }
            def = Devices[1];
        }
        return def;
    }
protected:
    KalmarDevice* def;
    std::vector<KalmarDevice*> Devices;
    KalmarContext() : def(nullptr), Devices() { Devices.push_back(new CPUDevice); }

    bool init_success = false; 

public:
    virtual ~KalmarContext() {}

    std::vector<KalmarDevice*> getDevices() { return Devices; }

    bool set_default(const std::wstring& path) {
        auto result = std::find_if(std::begin(Devices), std::end(Devices),
                                   [&] (const KalmarDevice* pDev)
                                   { return pDev->get_path() == path; });
        if (result == std::end(Devices))
            return false;
        else {
            def = *result;
            return true;
        }
    }

    std::shared_ptr<KalmarQueue> auto_select() {
        return get_default_dev()->get_default_queue();
    }

    KalmarDevice* getDevice(std::wstring path = L"") {
        if (path == L"default" || path == L"")
            return get_default_dev();
        auto result = std::find_if(std::begin(Devices), std::end(Devices),
                                   [&] (const KalmarDevice* dev)
                                   { return dev->get_path() == path; });
        if (result != std::end(Devices))
            return *result;
        else
            return get_default_dev();
    }

    virtual uint64_t getSystemTicks() { return 0L; };

    virtual uint64_t getSystemTickFrequency() { return 0L; };

    // initialize the printf buffer
    virtual void initPrintfBuffer() {};

    // flush the device printf buffer
    virtual void flushPrintfBuffer() {};

    // get the locked printf buffer VA
    virtual void* getPrintfBufferPointerVA() { return nullptr; };
};

KalmarContext *getContext();

namespace CLAMP {
// used in parallel_for_each.h
#if __KALMAR_ACCELERATOR__ == 2 || __KALMAR_CPU__ == 2
extern bool is_cpu();
extern bool in_cpu_kernel();
extern void enter_kernel();
extern void leave_kernel();
#endif

extern void *CreateKernel(std::string, KalmarQueue*);

extern void PushArg(void *, int, size_t, const void *);
extern void PushArgPtr(void *, int, size_t, const void *);

} // namespace CLAMP

static inline const std::shared_ptr<KalmarQueue> get_cpu_queue() {
    static auto cpu_queue = getContext()->getDevice(L"cpu")->get_default_queue();
    return cpu_queue;
}

static inline bool is_cpu_queue(const std::shared_ptr<KalmarQueue>& Queue) {
    return Queue->getDev()->get_path() == L"cpu";
}

static inline void copy_helper(std::shared_ptr<KalmarQueue>& srcQueue, void* src,
                               std::shared_ptr<KalmarQueue>& dstQueue, void* dst,
                               size_t cnt, bool block,
                               size_t src_offset = 0, size_t dst_offset = 0) {
    if (src == dst)
        return ;
    
    if (is_cpu_queue(dstQueue))
        srcQueue->read(src, (char*)dst + dst_offset, cnt, src_offset);
    else
        dstQueue->write(dst, (char*)src + src_offset, cnt, dst_offset, block);
}

enum states
{
    modified,
    shared,
    // not able to read and write
    invalid
};

struct dev_info
{
    void* data; 
    states state; 
};

struct rw_info
{
    void *data;
    const size_t count;
    std::shared_ptr<KalmarQueue> curr;
    std::shared_ptr<KalmarQueue> master;
    std::shared_ptr<KalmarQueue> stage;
    std::map<KalmarDevice*, dev_info> devs;
    access_type mode;
    unsigned int HostPtr : 1;

    bool toReleaseDevPointer;


    rw_info(const size_t count, void* ptr)
        : data(ptr), count(count), curr(nullptr), master(nullptr), stage(nullptr),
        devs(), mode(access_type_none), HostPtr(ptr != nullptr), toReleaseDevPointer(true) {
#if __KALMAR_ACCELERATOR__ == 2 || __KALMAR_CPU__ == 2
            if (CLAMP::in_cpu_kernel() && ptr == nullptr) {
                data = kalmar_aligned_alloc(0x1000, count);
                return;
            }
#endif
            if (ptr) {
                mode = access_type_read_write;
                curr = master = get_cpu_queue();
                devs[curr->getDev()] = {ptr, modified};
            }
        }

    rw_info(const std::shared_ptr<KalmarQueue>& Queue, const std::shared_ptr<KalmarQueue>& Stage,
            const size_t count, access_type mode_) : data(nullptr), count(count),
    curr(Queue), master(Queue), stage(nullptr), devs(), mode(mode_), HostPtr(false), toReleaseDevPointer(true) {
#if __KALMAR_ACCELERATOR__ == 2 || __KALMAR_CPU__ == 2
        if (CLAMP::in_cpu_kernel() && data == nullptr) {
            data = kalmar_aligned_alloc(0x1000, count);
            return;
        }
#endif
        if (mode == access_type_auto)
            mode = curr->getDev()->get_access();
        devs[curr->getDev()] = {curr->getDev()->create(count, this), modified};

        if (is_cpu_queue(curr) || (curr->getDev()->is_unified() && mode != access_type_none))
            data = devs[curr->getDev()].data;
        if (is_cpu_queue(curr)) {
            stage = Stage;
            if (Stage != curr)
                devs[stage->getDev()] = {stage->getDev()->create(count, this), invalid};
        } else
            stage = curr;
    }

    rw_info(const std::shared_ptr<KalmarQueue>& Queue, const std::shared_ptr<KalmarQueue>& Stage,
            const size_t count,
            void* device_pointer,
            access_type mode_) : data(nullptr), count(count), curr(Queue), master(Queue), stage(nullptr), devs(), mode(mode_), HostPtr(false), toReleaseDevPointer(false) {
         if (mode == access_type_auto)
             mode = curr->getDev()->get_access();
         devs[curr->getDev()] = { device_pointer, modified };

         if (is_cpu_queue(curr) || (curr->getDev()->is_unified() && mode != access_type_none))
             data = devs[curr->getDev()].data;
         if (is_cpu_queue(curr)) {
             stage = Stage;
             if (Stage != curr)
                 devs[stage->getDev()] = {stage->getDev()->create(count, this), invalid};
         } else
             stage = curr;
    }

    void* get_device_pointer() {
        return devs[curr->getDev()].data;
    }

    void construct(std::shared_ptr<KalmarQueue> pQueue) {
        curr = pQueue;
        devs[pQueue->getDev()] = {pQueue->getDev()->create(count, this), invalid};
        if (is_cpu_queue(pQueue))
            data = devs[pQueue->getDev()].data;
    }

    void disc() {
        for (auto& it : devs)
            it.second.state = invalid;
    }

    void try_switch_to_cpu() {
        if (is_cpu_queue(curr))
            return;
        auto cpu_queue = get_cpu_queue();
        if (devs.find(cpu_queue->getDev()) != std::end(devs))
            if (devs[cpu_queue->getDev()].state == shared)
                curr = cpu_queue;
    }

    void sync(std::shared_ptr<KalmarQueue> pQueue, bool modify, bool block = true) {
#if __KALMAR_ACCELERATOR__ == 2 || __KALMAR_CPU__ == 2
        if (CLAMP::in_cpu_kernel())
            return;
#endif
        if (!curr) {
            dev_info dev = {pQueue->getDev()->create(count, this),
                modify ? modified : shared};
            devs[pQueue->getDev()] = dev;
            if (is_cpu_queue(pQueue))
                data = dev.data;
            curr = pQueue;
            return;
        }

        if (curr == pQueue)
            return;

        if (curr->getDev() == pQueue->getDev()) {
            // curr->wait();
            curr = pQueue;
            if (modify) {
                disc();
                devs[curr->getDev()].state = modified;
            }
            return;
        }

        if (devs.find(pQueue->getDev()) == std::end(devs)) {
            dev_info dev = {pQueue->getDev()->create(count, this), invalid};
            devs[pQueue->getDev()] = dev;
            if (is_cpu_queue(pQueue))
                data = dev.data;
        }

        try_switch_to_cpu();
        dev_info& dst = devs[pQueue->getDev()];
        dev_info& src = devs[curr->getDev()];
        if (dst.state == invalid && src.state != invalid)
            copy_helper(curr, src.data, pQueue, dst.data, count, block);
        curr = pQueue;
        if (modify) {
            disc();
            dst.state = modified;
        } else {
            dst.state = shared;
            if (src.state == modified)
                src.state = shared;
        }
    }

    void* map(size_t cnt, size_t offset, bool modify) {
        if (cnt == 0)
            cnt = count;
        if (!curr) {
            curr = getContext()->auto_select();
            devs[curr->getDev()] = {curr->getDev()->create(count, this), modify ? modified : shared};
            return curr->map(data, cnt, offset, modify);
        }
        try_switch_to_cpu();
        dev_info& info = devs[curr->getDev()];
        if (info.state == shared && modify) {
            disc();
            info.state = modified;
        }
        return curr->map(info.data, cnt, offset, modify);
    }

    void unmap(void* addr, size_t cnt, size_t offset, bool modify) { curr->unmap(devs[curr->getDev()].data, addr, cnt, offset, modify); }

    void synchronize(bool modify) { sync(master, modify); }

    void get_cpu_access(bool modify) { sync(get_cpu_queue(), modify); }

    void write(const void* src, int cnt, int offset, bool blocking) {
        curr->write(devs[curr->getDev()].data, src, cnt, offset, blocking);
        dev_info& dev = devs[curr->getDev()];
        if (dev.state != modified) {
            disc();
            dev.state = modified;
        }
    }

    void read(void* dst, int cnt, int offset) {
        curr->read(devs[curr->getDev()].data, dst, cnt, offset);
    }

    void copy(rw_info* other, int src_offset, int dst_offset, int cnt) {
        if (cnt == 0)
            cnt = count;
        if (!curr) {
            if (!other->curr)
                return;
            else
                construct(other->curr);
        } else {
            if (!other->curr)
                other->construct(curr);
        }
        dev_info& dst = other->devs[other->curr->getDev()];
        dev_info& src = devs[curr->getDev()];
        if (src.state == invalid) {
            src.state = shared;
            if (is_cpu_queue(curr))
                memset((char*)src.data + src_offset, 0, cnt);
            else {
                void *ptr = kalmar_aligned_alloc(0x1000, cnt);
                memset(ptr, 0, cnt);
                curr->write(src.data, ptr, cnt, src_offset, true);
                kalmar_aligned_free(ptr);
            }
        }
        copy_helper(curr, src.data, other->curr, dst.data, cnt, true, src_offset, dst_offset);
        other->disc();
        dst.state = modified;
    }

    ~rw_info() {
#if __KALMAR_ACCELERATOR__ == 2 || __KALMAR_CPU__ == 2
        if (CLAMP::in_cpu_kernel()) {
            if (data && !HostPtr)
                kalmar_aligned_free(data);
            return;
        }
#endif

        if (HostPtr)
            synchronize(false);
        if (curr) {
            // Wait issues a system-scope release:
            // Need to make sure we write-back cache contents before deallocating the memory those writes might eventually touch
            curr->wait();
        }
        auto cpu_dev = get_cpu_queue()->getDev();
        if (devs.find(cpu_dev) != std::end(devs)) {
            if (!HostPtr)
                cpu_dev->release(devs[cpu_dev].data, this);
            devs.erase(cpu_dev);
        }
        KalmarDevice* pDev;
        dev_info info;
        for (const auto it : devs) {
            std::tie(pDev, info) = it;
            if (toReleaseDevPointer)
                pDev->release(info.data, this);
        }
    }
};


//--- Implementation:
//

inline void KalmarAsyncOp::setSeqNumFromQueue()  { seqNum = queue->assign_op_seq_num(); };

} // namespace Kalmar