/* SCE CONFIDENTIAL
PLAYSTATION(R)3 Programmer Tool Runtime Library 084.006
 * Copyright (C) 2006 Sony Computer Entertainment Inc.
 * All Rights Reserved.
 */

#ifndef __CELL_DMA_H__
#define __CELL_DMA_H__

#include <spu_mfcio.h>
#include <cellstatus.h>
#include <cell/error.h>
#include <stdint.h>

typedef mfc_list_element_t CellDmaListElement;

#ifdef NO_CELL_DMA_ASSERT
#define cellDmaAssert(cond)
#else /* NO_CELL_DMA_ASSERT */
#define cellDmaAssert(cond) spu_hcmpeq((cond), 0)
#endif /* NO_CELL_DMA_ASSERT */

/***************************************************************
 * DMA Assertions
 ***************************************************************/
#define cellDmaNormalAssert(ls,ea,size,tag) \
	cellDmaAssert((((uintptr_t)(ls) & 0xf) == 0)	& \
				  (((uintptr_t)(ea) & 0xf) == 0)	& \
				  (((size) & 0xf) == 0)				& \
				  ((size) <= (16<<10))				& \
				  ((tag) < 32))

#define cellDmaSmallAssert(ls,ea,size,tag) \
	cellDmaAssert((((uintptr_t)(ls) & 0xf) == ((uintptr_t)(ea) & 0xf))	& \
				  (((uintptr_t)(ls) & (size - 1)) == 0)					& \
				  (((size)==1)||((size)==2)||((size)==4)||((size)==8))	& \
				  ((tag) < 32))

#define cellDmaListAssert(ls,ea,la,lsize,tag) \
	cellDmaAssert((((uintptr_t)(ls) & 0xf) == 0)	& \
				  (((uintptr_t)(ea) & 0xf) == 0)	& \
				  (((uintptr_t)(la) &   7) == 0)	& \
				  (((lsize) & 7) == 0)				& \
				  ((lsize) <= (16<<10))				& \
				  ((tag) < 32))

#define cellDmaAtomicAssert(ls,ea) \
	cellDmaAssert((((uintptr_t)(ls) & 0x7f) == 0)	& \
				  (((uintptr_t)(ea) & 0x7f) == 0))

#define cellDmaPutqllucAssert(ls,ea,tag) \
	cellDmaAssert((((uintptr_t)(ls) & 0x7f) == 0)	& \
				  (((uintptr_t)(ea) & 0x7f) == 0)	& \
				  ((tag) < 32))

#define cellDmaLargeAssert(ls,ea,tag) \
	cellDmaAssert((((uintptr_t)(ls) & 0xf) == 0)	& \
				  (((uintptr_t)(ea) & 0xf) == 0)	& \
				  ((tag) < 32))

/***************************************************************
 * DMA which transfer size is a multiple of 16bytes
 ***************************************************************/
__attribute__((always_inline))
static inline void
cellDmaPut(const void *ls, uint64_t ea, uint32_t size, uint32_t tag, uint32_t tid, uint32_t rid)
{
	cellDmaNormalAssert(ls, ea, size, tag);
	mfc_put((volatile void*)(uintptr_t)ls, ea, size, tag, tid, rid);
}

__attribute__((always_inline))
static inline void
cellDmaPutf(const void *ls, uint64_t ea, uint32_t size, uint32_t tag, uint32_t tid, uint32_t rid)
{
	cellDmaNormalAssert(ls, ea, size, tag);
	mfc_putf((volatile void*)(uintptr_t)ls, ea, size, tag, tid, rid);
}

__attribute__((always_inline))
static inline void
cellDmaPutb(const void *ls, uint64_t ea, uint32_t size, uint32_t tag, uint32_t tid, uint32_t rid)
{
	cellDmaNormalAssert(ls, ea, size, tag);
	mfc_putb((volatile void*)(uintptr_t)ls, ea, size, tag, tid, rid);
}

__attribute__((always_inline))
static inline void
cellDmaGet(void *ls, uint64_t ea, uint32_t size, uint32_t tag, uint32_t tid, uint32_t rid)
{
	cellDmaNormalAssert(ls, ea, size, tag);
	mfc_get(ls, ea, size, tag, tid, rid);
}

__attribute__((always_inline))
static inline void
cellDmaGetf(void *ls, uint64_t ea, uint32_t size, uint32_t tag, uint32_t tid, uint32_t rid)
{
	cellDmaNormalAssert(ls, ea, size, tag);
	mfc_getf(ls, ea, size, tag, tid, rid);
}

__attribute__((always_inline))
static inline void
cellDmaGetb(void *ls, uint64_t ea, uint32_t size, uint32_t tag, uint32_t tid, uint32_t rid)
{
	cellDmaNormalAssert(ls, ea, size, tag);
	mfc_getb(ls, ea, size, tag, tid, rid);
}

/***************************************************************
 * DMA which transfer size is within 16bytes
 ***************************************************************/
__attribute__((always_inline))
static inline void
cellDmaSmallPut(const void *ls, uint64_t ea, uint32_t size, uint32_t tag, uint32_t tid, uint32_t rid)
{
	cellDmaSmallAssert(ls, ea, size, tag);
	mfc_put((volatile void*)(uintptr_t)ls, ea, size, tag, tid, rid);
}

__attribute__((always_inline))
static inline void
cellDmaSmallPutf(const void *ls, uint64_t ea, uint32_t size, uint32_t tag, uint32_t tid, uint32_t rid)
{
	cellDmaSmallAssert(ls, ea, size, tag);
	mfc_putf((volatile void*)(uintptr_t)ls, ea, size, tag, tid, rid);
}

__attribute__((always_inline))
static inline void
cellDmaSmallPutb(const void *ls, uint64_t ea, uint32_t size, uint32_t tag, uint32_t tid, uint32_t rid)
{
	cellDmaSmallAssert(ls, ea, size, tag);
	mfc_putb((volatile void*)(uintptr_t)ls, ea, size, tag, tid, rid);
}

__attribute__((always_inline))
static inline void
cellDmaSmallGet(void *ls, uint64_t ea, uint32_t size, uint32_t tag, uint32_t tid, uint32_t rid)
{
	cellDmaSmallAssert(ls, ea, size, tag);
	mfc_get(ls, ea, size, tag, tid, rid);
}

__attribute__((always_inline))
static inline void
cellDmaSmallGetf(void *ls, uint64_t ea, uint32_t size, uint32_t tag, uint32_t tid, uint32_t rid)
{
	cellDmaSmallAssert(ls, ea, size, tag);
	mfc_getf(ls, ea, size, tag, tid, rid);
}

__attribute__((always_inline))
static inline void
cellDmaSmallGetb(void *ls, uint64_t ea, uint32_t size, uint32_t tag, uint32_t tid, uint32_t rid)
{
	cellDmaSmallAssert(ls, ea, size, tag);
	mfc_getb(ls, ea, size, tag, tid, rid);
}

/***************************************************************
 * List DMA
 ***************************************************************/
__attribute__((always_inline))
static inline void
cellDmaListPut(const void *ls, uint64_t ea, const CellDmaListElement *list, uint32_t lsize, uint32_t tag, uint32_t tid, uint32_t rid)
{
	cellDmaListAssert(ls, ea, list, lsize, tag);
	mfc_putl((volatile void*)(uintptr_t)ls, ea, list, lsize, tag, tid, rid);
}

__attribute__((always_inline))
static inline void
cellDmaListPutf(const void *ls, uint64_t ea, const CellDmaListElement *list, uint32_t lsize, uint32_t tag, uint32_t tid, uint32_t rid)
{
	cellDmaListAssert(ls, ea, list, lsize, tag);
	mfc_putlf((volatile void*)(uintptr_t)ls, ea, list, lsize, tag, tid, rid);
}

__attribute__((always_inline))
static inline void
cellDmaListPutb(const void *ls, uint64_t ea, const CellDmaListElement *list, uint32_t lsize, uint32_t tag, uint32_t tid, uint32_t rid)
{
	cellDmaListAssert(ls, ea, list, lsize, tag);
	mfc_putlb((volatile void*)(uintptr_t)ls, ea, list, lsize, tag, tid, rid);
}

__attribute__((always_inline))
static inline void
cellDmaListGet(void *ls, uint64_t ea, const CellDmaListElement *list, uint32_t lsize, uint32_t tag, uint32_t tid, uint32_t rid)
{
	cellDmaListAssert(ls, ea, list, lsize, tag);
	mfc_getl(ls, ea, list, lsize, tag, tid, rid);
}

__attribute__((always_inline))
static inline void
cellDmaListGetf(void *ls, uint64_t ea, const CellDmaListElement *list, uint32_t lsize, uint32_t tag, uint32_t tid, uint32_t rid)
{
	cellDmaListAssert(ls, ea, list, lsize, tag);
	mfc_getlf(ls, ea, list, lsize, tag, tid, rid);
}

__attribute__((always_inline))
static inline void
cellDmaListGetb(void *ls, uint64_t ea, const CellDmaListElement *list, uint32_t lsize, uint32_t tag, uint32_t tid, uint32_t rid)
{
	cellDmaListAssert(ls, ea, list, lsize, tag);
	mfc_getlb(ls, ea, list, lsize, tag, tid, rid);
}

/***************************************************************
 * Atomic DMA
 ***************************************************************/
__attribute__((always_inline))
static inline void
cellDmaGetllar(void *ls, uint64_t ea, uint32_t tid, uint32_t rid)
{
	cellDmaAtomicAssert(ls, ea);
	mfc_getllar(ls, ea, tid, rid);
}

__attribute__((always_inline))
static inline void
cellDmaPutllc(const void *ls, uint64_t ea, uint32_t tid, uint32_t rid)
{
	cellDmaAtomicAssert(ls, ea);
	mfc_putllc((volatile void*)(uintptr_t)ls, ea, tid, rid);
}

__attribute__((always_inline))
static inline void
cellDmaPutlluc(const void *ls, uint64_t ea, uint32_t tid, uint32_t rid)
{
	cellDmaAtomicAssert(ls, ea);
	mfc_putlluc((volatile void*)(uintptr_t)ls, ea, tid, rid);
}

__attribute__((always_inline))
static inline void
cellDmaPutqlluc(const void *ls, uint64_t ea, uint32_t tag, uint32_t tid, uint32_t rid)
{
	cellDmaPutqllucAssert(ls, ea, tag);
	mfc_putqlluc((volatile void*)(uintptr_t)ls, ea, tag, tid, rid);
}

/***************************************************************
 * DMA Utilities - Data typed DMA
 ***************************************************************/
#ifdef __cplusplus
extern "C"
#endif
void cellDmaAndWait(void *ls, uint64_t ea, uint32_t size, uint32_t tag, uint32_t cmd);
static inline void cellDmaPutUint8( uint8_t  value, uint64_t ea, uint32_t tag, uint32_t tid, uint32_t rid) __attribute__((always_inline));
static inline void cellDmaPutUint16(uint16_t value, uint64_t ea, uint32_t tag, uint32_t tid, uint32_t rid) __attribute__((always_inline));
static inline void cellDmaPutUint32(uint32_t value, uint64_t ea, uint32_t tag, uint32_t tid, uint32_t rid) __attribute__((always_inline));
static inline void cellDmaPutUint64(uint64_t value, uint64_t ea, uint32_t tag, uint32_t tid, uint32_t rid) __attribute__((always_inline));
static inline uint8_t  cellDmaGetUint8( uint64_t ea, uint32_t tag, uint32_t tid, uint32_t rid) __attribute__((always_inline));
static inline uint16_t cellDmaGetUint16(uint64_t ea, uint32_t tag, uint32_t tid, uint32_t rid) __attribute__((always_inline));
static inline uint32_t cellDmaGetUint32(uint64_t ea, uint32_t tag, uint32_t tid, uint32_t rid) __attribute__((always_inline));
static inline uint64_t cellDmaGetUint64(uint64_t ea, uint32_t tag, uint32_t tid, uint32_t rid) __attribute__((always_inline));

#define cellDmaEa2Ls(ea, ls) (void*)((uintptr_t)(ls)+((uint32_t)(ea)&15))

#define cellDmaPutUintTemplate(SIZE)																			\
__attribute__((always_inline))																					\
static inline void																								\
cellDmaPutUint##SIZE(uint##SIZE##_t value, uint64_t ea, uint32_t tag, uint32_t tid, uint32_t rid)				\
{																												\
	qword buf = (qword)spu_splats(value);																		\
	cellDmaSmallAssert(ea, ea, sizeof(uint##SIZE##_t), tag);													\
	cellDmaAndWait(cellDmaEa2Ls(ea,&buf), ea, sizeof(uint##SIZE##_t), tag, MFC_CMD_WORD(tid,rid,MFC_PUT_CMD));	\
}
cellDmaPutUintTemplate(8)
cellDmaPutUintTemplate(16)
cellDmaPutUintTemplate(32)
cellDmaPutUintTemplate(64)

#define cellDmaGetUintTemplate(SIZE)																			\
__attribute__((always_inline))																					\
static inline uint##SIZE##_t																					\
cellDmaGetUint##SIZE(uint64_t ea, uint32_t tag, uint32_t tid, uint32_t rid)										\
{																												\
	qword buf;																									\
	cellDmaSmallAssert(ea, ea, sizeof(uint##SIZE##_t), tag);													\
	cellDmaAndWait(cellDmaEa2Ls(ea,&buf), ea, sizeof(uint##SIZE##_t), tag, MFC_CMD_WORD(tid,rid,MFC_GET_CMD));	\
	return *(uint##SIZE##_t*)((uintptr_t)&buf + ((uintptr_t)ea&15));											\
}
cellDmaGetUintTemplate(8)
cellDmaGetUintTemplate(16)
cellDmaGetUintTemplate(32)
cellDmaGetUintTemplate(64)

/***************************************************************
 * DMA Utilities - Any size DMA
 ***************************************************************/
#ifdef __cplusplus
extern "C"
#endif
void cellDmaLargeCmd(uintptr_t ls, uint64_t ea, uint32_t size, uint32_t tag, uint32_t cmd);

__attribute__((always_inline))
static inline void
cellDmaLargePut(const void *ls, uint64_t ea, uint32_t size, uint32_t tag, uint32_t tid, uint32_t rid)
{
	cellDmaLargeAssert(ls, ea, tag);
	cellDmaLargeCmd((uintptr_t)ls,ea,size,tag,MFC_CMD_WORD(tid,rid,MFC_PUT_CMD));
}

__attribute__((always_inline))
static inline void
cellDmaLargePutf(const void *ls, uint64_t ea, uint32_t size, uint32_t tag, uint32_t tid, uint32_t rid)
{
	cellDmaLargeAssert(ls, ea, tag);
	cellDmaLargeCmd((uintptr_t)ls,ea,size,tag,MFC_CMD_WORD(tid,rid,MFC_PUTF_CMD));
}

__attribute__((always_inline))
static inline void
cellDmaLargePutb(const void *ls, uint64_t ea, uint32_t size, uint32_t tag, uint32_t tid, uint32_t rid)
{
	cellDmaLargeAssert(ls, ea, tag);
	cellDmaLargeCmd((uintptr_t)ls,ea,size,tag,MFC_CMD_WORD(tid,rid,MFC_PUTB_CMD));
}

__attribute__((always_inline))
static inline void
cellDmaLargeGet(void *ls, uint64_t ea, uint32_t size, uint32_t tag, uint32_t tid, uint32_t rid)
{
	cellDmaLargeAssert(ls, ea, tag);
	cellDmaLargeCmd((uintptr_t)ls,ea,size,tag,MFC_CMD_WORD(tid,rid,MFC_GET_CMD));
}

__attribute__((always_inline))
static inline void
cellDmaLargeGetf(void *ls, uint64_t ea, uint32_t size, uint32_t tag, uint32_t tid, uint32_t rid)
{
	cellDmaLargeAssert(ls, ea, tag);
	cellDmaLargeCmd((uintptr_t)ls,ea,size,tag,MFC_CMD_WORD(tid,rid,MFC_GETF_CMD));
}

__attribute__((always_inline))
static inline void
cellDmaLargeGetb(void *ls, uint64_t ea, uint32_t size, uint32_t tag, uint32_t tid, uint32_t rid)
{
	cellDmaLargeAssert(ls, ea, tag);
	cellDmaLargeCmd((uintptr_t)ls,ea,size,tag,MFC_CMD_WORD(tid,rid,MFC_GETB_CMD));
}

/***************************************************************
 * DMA Utilities - Tag wait
 ***************************************************************/
__attribute__((always_inline))
static inline void
cellDmaCancelTagStatusUpdate(void)
{
	mfc_write_tag_update_immediate();
	do {} while(__builtin_expect(mfc_stat_tag_update() == 0,0));
	mfc_read_tag_status();
}

__attribute__((always_inline))
static inline uint32_t
cellDmaCancelAndWaitTagStatusAny(uint32_t tagmask)
{
	cellDmaCancelTagStatusUpdate();
	mfc_write_tag_mask(tagmask);
	return mfc_read_tag_status_any();
}

__attribute__((always_inline))
static inline uint32_t
cellDmaCancelAndWaitTagStatusAll(uint32_t tagmask)
{
	cellDmaCancelTagStatusUpdate();
	mfc_write_tag_mask(tagmask);
	return mfc_read_tag_status_all();
}

__attribute__((always_inline))
static inline uint32_t
cellDmaWaitTagStatusImmediate(uint32_t tagmask)
{
	mfc_write_tag_mask(tagmask);
	return mfc_read_tag_status_immediate();
}

__attribute__((always_inline))
static inline uint32_t
cellDmaWaitTagStatusAny(uint32_t tagmask)
{
	mfc_write_tag_mask(tagmask);
	return mfc_read_tag_status_any();
}

__attribute__((always_inline))
static inline uint32_t
cellDmaWaitTagStatusAll(uint32_t tagmask)
{
	mfc_write_tag_mask(tagmask);
	return mfc_read_tag_status_all();
}

#define cellDmaWaitAtomicStatus() mfc_read_atomic_status()

/***************************************************************
 * Error codes
 ***************************************************************/
#define CELL_ERROR_MINOR_FACILITY_DMA          0x0
#define CELL_ERROR_MAKE_DMA_ERROR(id) \
	(CELL_ERROR_MAKE_ERROR(CELL_ERROR_FACILITY_SPU, (CELL_ERROR_MINOR_FACILITY_DMA << 8) | (id)))

#define CELL_DMA_ERROR_INVAL CELL_ERROR_CAST(0x80410002)  

#endif /* CELL_DMA_H */

/*
 * Local Variables:
 * mode: C
 * c-file-style: "stroustrup"
 * tab-width: 4
 * End:
 * vim:sw=4:sts=4:ts=4
 */
